From 14700ba21046b1aff3eb56eda13aefbfc69f8f7b Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Thu, 28 Feb 2019 13:01:30 +0100 Subject: [PATCH 001/130] Initial checkin of new whitesource step --- .../templates/whitesourceVulnerabilities.html | 41 + resources/default_pipeline_environment.yml | 61 ++ src/com/sap/piper/JsonUtils.groovy | 6 + .../WhitesourceConfigurationHelper.groovy | 255 ++++++ .../WhitesourceOrgAdminRepository.groovy | 84 ++ .../integration/WhitesourceRepository.groovy | 252 ++++++ test/groovy/WhitesourceExecuteScanTest.groovy | 808 ++++++++++++++++++ vars/whitesourceExecuteScan.groovy | 347 ++++++++ 8 files changed, 1854 insertions(+) create mode 100644 resources/com.sap.piper/templates/whitesourceVulnerabilities.html create mode 100644 src/com/sap/piper/WhitesourceConfigurationHelper.groovy create mode 100644 src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy create mode 100644 src/com/sap/piper/integration/WhitesourceRepository.groovy create mode 100644 test/groovy/WhitesourceExecuteScanTest.groovy create mode 100644 vars/whitesourceExecuteScan.groovy diff --git a/resources/com.sap.piper/templates/whitesourceVulnerabilities.html b/resources/com.sap.piper/templates/whitesourceVulnerabilities.html new file mode 100644 index 000000000..209b559ea --- /dev/null +++ b/resources/com.sap.piper/templates/whitesourceVulnerabilities.html @@ -0,0 +1,41 @@ + + + + ${reportTitle} + + + +

${reportTitle}

+

+ + WhiteSource product name: ${whitesourceProductName}
+ Filtered project names: ${whitesourceProjectNames?:''} + +

+
+

total number of vulnerabilities: ${totalVulnerabilities}
+ total number of high/critical vulnerabilities with CVSS score >= 7.0: ${totalSevereVulnerabilities} +

+
+

Snapshot taken:${now} GMT

+ + + + + + + + + + + + + + + + + ${vulnerabilityTable} + +
Entry #DateCVECVSS ScoreCVSS VersionProjectLibrary file nameLibrary group IDLibrary artifact IDLibrary versionDescriptionTop fix
+ + diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 1a7b95c89..6f6080e3c 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -254,6 +254,67 @@ steps: - 'tests' npmExecute: dockerImage: 'node:8-stretch' + whitesourceExecuteScan: + buildDescriptorExcludeList: [] + parallelLimit: 0 + licensingVulnerabilities: true + securityVulnerabilities: false + reporting: true + vulnerabilityReportFileName: 'piper_whitesource_vulnerability_report' + vulnerabilityReportTitle: 'Whitesource Security Vulnerability Report' + projectNames: [] + jreDownloadUrl: 'https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz' + whitesourceAccessor: 'com.sap.piper.internal.integration.InternalWhitesource' + mta: + stashContent: + - 'buildDescriptor' + - 'opensourceConfiguration' + maven: + buildDescriptorFile: './pom.xml' + dockerImage: 'docker.wdf.sap.corp:50000/piper/maven' + dockerWorkspace: '/home/piper' + agentFileName: 'wss-unified-agent.jar' + agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' + agentParameters: '' + configFilePath: './wss-unified-agent.config' + stashContent: + - 'buildDescriptor' + - 'opensourceConfiguration' + npm: + buildDescriptorFile: './package.json' + dockerImage: 'docker.wdf.sap.corp:50000/piper/node' + dockerWorkspace: '/home/piper' + agentFileName: 'wss-unified-agent.jar' + agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' + agentParameters: '' + configFilePath: './wss-unified-agent.config' + stashContent: + - 'buildDescriptor' + - 'opensourceConfiguration' + pip: + buildDescriptorFile: './setup.py' + dockerImage: 'docker.wdf.sap.corp:50000/piper/node' + dockerWorkspace: '/home/piper' + agentFileName: 'wss-unified-agent.jar' + agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' + agentParameters: '' + configFilePath: './wss-unified-agent.config' + stashContent: + - 'buildDescriptor' + - 'opensourceConfiguration' + sbt: + buildDescriptorFile: './build.sbt' + dockerImage: 'docker.wdf.sap.corp:50000/piper/sbt' + dockerWorkspace: '/home/piper' + agentFileName: 'wss-unified-agent.jar' + agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' + agentParameters: '' + configFilePath: './wss-unified-agent.config' + stashContent: + - 'buildDescriptor' + - 'opensourceConfiguration' + verbose: false + timeout: 0 pipelineRestartSteps: sendMail: true timeoutInSeconds: 900 diff --git a/src/com/sap/piper/JsonUtils.groovy b/src/com/sap/piper/JsonUtils.groovy index 59b61664d..09db031d6 100644 --- a/src/com/sap/piper/JsonUtils.groovy +++ b/src/com/sap/piper/JsonUtils.groovy @@ -1,8 +1,14 @@ package com.sap.piper import com.cloudbees.groovy.cps.NonCPS +import groovy.json.JsonSlurperClassic @NonCPS String getPrettyJsonString(object) { return groovy.json.JsonOutput.prettyPrint(groovy.json.JsonOutput.toJson(object)) } + +@NonCPS +def parseJsonSerializable(text) { + return new JsonSlurperClassic().parseText(text) +} diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy new file mode 100644 index 000000000..b940f9ea0 --- /dev/null +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -0,0 +1,255 @@ +package com.sap.piper + +import com.cloudbees.groovy.cps.NonCPS + +import java.security.MessageDigest + +class WhitesourceConfigurationHelper implements Serializable { + + private static def SCALA_CONTENT_KEY = "@__content" + + static def extendUAConfigurationFile(script, Utils utils, config, path) { + def mapping = [] + def parsingClosure = { fileReadPath -> return script.readProperties (file: fileReadPath) } + def serializationClosure = { configuration -> serializeUAConfig(configuration) } + def inputFile = config.configFilePath.replaceFirst('\\./', '') + def suffix = MessageDigest.getInstance("MD5").digest(config.configFilePath.bytes).encodeHex().toString() + def targetFile = "${inputFile}.${suffix}" + if(config.productName.startsWith('DIST - ')) { + mapping += [ + [name: 'checkPolicies', value: false], + [name: 'forceCheckAllDependencies', value: false] + ] + } else if(config.productName.startsWith('SHC - ')) { + mapping += [ + [name: 'checkPolicies', value: true], + [name: 'forceCheckAllDependencies', value: true] + ] + } + if(config.verbose) + mapping += [name: 'log.level', value: 'debug'] + + mapping += [ + [name: 'apiKey', value: config.orgToken, warnIfPresent: true], + [name: 'productName', value: config.productName], + [name: 'productVersion', value: config.productVersion], + [name: 'projectName', value: config.projectName], + [name: 'projectVersion', value: config.productVersion], + [name: 'productToken', value: config.productToken, omitIfPresent: 'projectToken'], + [name: 'userKey', value: config.userKey, warnIfPresent: true], + [name: 'forceUpdate', value: true], + [name: 'offline', value: false], + [name: 'ignoreSourceFiles', value: true], + [name: 'resolveAllDependencies', value: false] + ] + switch (config.scanType) { + case 'maven': + mapping += [ + + ] + break + case 'npm': + mapping += [ + + ] + break + case 'pip': + mapping += [ + [name: 'python.resolveDependencies', value: true], + [name: 'python.ignoreSourceFiles', value: true], + [name: 'python.ignorePipInstallErrors', value: false], + [name: 'python.installVirtualenv', value: true], + [name: 'python.resolveHierarchyTree', value: true], + [name: 'python.requirementsFileIncludes', value: 'requirements.txt'], + [name: 'python.resolveSetupPyFiles', value: true], + [name: 'python.runPipenvPreStep', value: true], + [name: 'python.pipenvDevDependencies', value: true], + [name: 'python.IgnorePipenvInstallErrors', value: false], + [name: 'includes', value: '**/*.py **/*.txt'], + [name: 'excludes', value: '**/*sources.jar **/*javadoc.jar'], + [name: 'case.sensitive.glob', value: false], + [name: 'followSymbolicLinks', value: true] + ] + break + case 'sbt': + mapping += [ + + ] + break + } + + rewriteConfiguration(script, config, utils, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) + } + + static def extendConfigurationFile(script, Utils utils, config, path) { + def mapping = [:] + def parsingClosure + def serializationClosure + def inputFile = config.configFilePath.replaceFirst('\\./', '') + def suffix = MessageDigest.getInstance("MD5").digest(config.configFilePath.bytes).encodeHex().toString() + def targetFile = "${inputFile}.${suffix}" + switch (config.scanType) { + case 'unifiedAgent': + case 'fileAgent': + mapping = [ + [name: 'apiKey', value: config.orgToken, warnIfPresent: true], + [name: 'productName', value: config.productName], + [name: 'productToken', value: config.productToken, omitIfPresent: 'projectToken'], + [name: 'userKey', value: config.userKey, warnIfPresent: true] + ] + parsingClosure = { fileReadPath -> return script.readProperties (file: fileReadPath) } + serializationClosure = { Properties configuration -> new StringWriter().with{ w -> configuration.store(w, null); w }.toString() } + break + case 'npm': + mapping = [ + [name: 'apiKey', value: config.orgToken, warnIfPresent: true], + [name: 'productName', value: config.productName], + [name: 'productToken', value: config.productToken, omitIfPresent: 'projectToken'], + [name: 'userKey', value: config.userKey, warnIfPresent: true] + ] + parsingClosure = { fileReadPath -> return script.readJSON (file: fileReadPath) } + serializationClosure = { configuration -> return utils.jsonToString(configuration) } + break + case 'pip': + mapping = [ + [name: "'org_token'", value: "\'${config.orgToken}\'", warnIfPresent: true], + [name: "'product_name'", value: "\'${config.productName}\'"], + [name: "'product_token'", value: "\'${config.productToken}\'"], + [name: "'user_key'", value: "\'${config.userKey}\'", warnIfPresent: true] + ] + parsingClosure = { fileReadPath -> return readPythonConfig (script, fileReadPath) } + serializationClosure = { configuration -> serializePythonConfig(configuration) } + targetFile = "${inputFile}.${suffix}.py" + break + case 'sbt': + mapping = [ + [name: "whitesourceOrgToken in ThisBuild", value: "\"${config.orgToken}\"", warnIfPresent: true], + [name: "whitesourceProduct in ThisBuild", value: "\"${config.productName}\""], + [name: "whitesourceServiceUrl in ThisBuild", value: "uri(\"${config.agentUrl}\")"] + // actually not supported [name: "whitesourceUserKey in ThisBuild", value: config.userKey] + ] + parsingClosure = { fileReadPath -> return readScalaConfig (script, mapping, fileReadPath) } + serializationClosure = { configuration -> serializeScalaConfig (configuration) } + targetFile = inputFile + break + } + + rewriteConfiguration(script, config, utils, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) + } + + static private def rewriteConfiguration(script, config, utils, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) { + def inputFilePath = "${path}${inputFile}" + def outputFilePath = "${path}${targetFile}" + def moduleSpecificFile = parsingClosure(inputFilePath) + if (!moduleSpecificFile) + moduleSpecificFile = parsingClosure(config.configFilePath) + if (!moduleSpecificFile) + moduleSpecificFile = [:] + + mapping.each { + entry -> + //if (entry.warnIfPresent && moduleSpecificFile[entry.name]) + //Notify.warning(script, "Obsolete configuration ${entry.name} detected, please omit its use and rely on configuration via Piper.", 'WhitesourceConfigurationHelper') + def dependentValue = entry.omitIfPresent ? moduleSpecificFile[entry.omitIfPresent] : null + if ((entry.omitIfPresent && !dependentValue || !entry.omitIfPresent) && entry.value && entry.value != 'null' && entry.value != '') + moduleSpecificFile[entry.name] = entry.value.toString() + } + + def output = serializationClosure(moduleSpecificFile) + + if(config.verbose) + script.echo "Writing config file ${outputFilePath} with content:\n${output}" + script.writeFile file: outputFilePath, text: output + if(config.stashContent && config.stashContent.size() > 0) { + def stashName = "modified whitesource config ${suffix}".toString() + utils.stashWithMessage ( + stashName, + "Stashing modified Whitesource configuration", + outputFilePath.replaceFirst('\\./', '') + ) + config.stashContent += [stashName] + } + config.configFilePath = outputFilePath + } + + static private def readPythonConfig(script, filePath) { + def contents = script.readFile file: filePath + def lines = contents.split('\n') + def resultMap = [:] + lines.each { + line -> + List parts = line?.replaceAll(',$', '')?.split(':') + def key = parts[0]?.trim() + parts.removeAt(0) + resultMap[key] = parts.size() > 0 ? (parts as String[]).join(':').trim() : null + } + return resultMap + } + + static private def serializePythonConfig(configuration) { + StringBuilder result = new StringBuilder() + configuration.each { + entry -> + if(entry.key != '}') + result.append(entry.value ? ' ' : '').append(entry.key).append(entry.value ? ': ' : '').append(entry.value ?: '').append(entry.value ? ',' : '').append('\r\n') + } + return result.toString().replaceAll(',$', '\r\n}') + } + + static private def readScalaConfig(script, mapping, filePath) { + def contents = script.readFile file: filePath + def lines = contents.split('\n') + def resultMap = [:] + resultMap[SCALA_CONTENT_KEY] = [] + def keys = mapping.collect( { it.name } ) + lines.each { + line -> + def parts = line?.split(':=').toList() + def key = parts[0]?.trim() + if (keys.contains(key)) { + resultMap[key] = parts[1]?.trim() + } else if (line != null) { + resultMap[SCALA_CONTENT_KEY].add(line) + } + } + return resultMap + } + + static private def serializeScalaConfig(configuration) { + StringBuilder result = new StringBuilder() + + // write the general content + configuration[SCALA_CONTENT_KEY].each { + line -> + result.append(line) + result.append('\r\n') + } + + // write the mappings + def confKeys = configuration.keySet() + confKeys.remove(SCALA_CONTENT_KEY) + + confKeys.each { + key -> + def value = configuration[key] + result.append(key) + if (value != null) { + result.append(' := ').append(value) + } + result.append('\r\n') + } + + return result.toString() + } + + @NonCPS + static private def serializeUAConfig(configuration) { + Properties p = new Properties() + configuration.each { + entry -> + p.setProperty(entry.key, entry.value) + } + + new StringWriter().with{ w -> p.store(w, null); w }.toString() + } +} diff --git a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy new file mode 100644 index 000000000..ab3efc055 --- /dev/null +++ b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy @@ -0,0 +1,84 @@ +package com.sap.piper.integration + +import com.cloudbees.groovy.cps.NonCPS +import com.sap.piper.JsonUtils +import com.sap.piper.Utils + +class WhitesourceOrgAdminRepository implements Serializable { + + final Script script + final internalWhitesource + final Map config + + WhitesourceOrgAdminRepository(Script script, Map config) { + this.script = script + this.config = config + if(!this.config.serviceUrl && !this.config.whitesourceAccessor) + script.error "Parameter 'serviceUrl' must be provided as part of the configuration." + if(this.config.whitesourceAccessor instanceof String) { + def clazz = this.class.classLoader.loadClass(this.config.whitesourceAccessor) + this.internalWhitesource = clazz?.newInstance(this.script, this.config) + } + } + + def fetchProductMetaInfo() { + def requestBody = [ + requestType: "getOrganizationProductVitals", + orgToken: config.orgToken + ] + def response = internalWhitesource ? internalWhitesource.httpWhitesource(requestBody) : httpWhitesource(requestBody) + def parsedResponse = new JsonUtils().parseJsonSerializable(response.content) + + findProductMeta(parsedResponse) + } + + def findProductMeta(parsedResponse) { + def foundMetaProduct = null + for (product in parsedResponse.productVitals) { + if (product.name == config.productName) { + foundMetaProduct = product + break + } + } + + if (!foundMetaProduct) + script.error "[WhiteSource] Could not fetch/find requested product '${config.productName}'" + + return foundMetaProduct + } + + + + @NonCPS + protected def httpWhitesource(requestBody) { + script.withCredentials ([script.string( + credentialsId: config.orgAdminUserTokenCredentialsId, + variable: 'adminUserKey' + )]) { + requestBody["userKey"] = adminUserKey + def serializedBody = new JsonUtils().getPrettyJsonString(requestBody) + def params = [ + url : config.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: serializedBody, + quiet : !config.verbose, + timeout : config.timeout + ] + + if (script.env.HTTP_PROXY && !config.serviceUrl.matches('http(s)*://.*\\.sap\\.corp.*')) + params["httpProxy"] = script.env.HTTP_PROXY + + if (config.verbose) + script.echo "Sending http request with parameters ${params}" + + def response = script.httpRequest(params) + + if (config.verbose) + script.echo "Received response ${reponse}" + + return response + } + } +} diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy new file mode 100644 index 000000000..8820cfc76 --- /dev/null +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -0,0 +1,252 @@ +package com.sap.piper.integration + +import com.cloudbees.groovy.cps.NonCPS +import com.sap.piper.JsonUtils +import com.sap.piper.Utils + +class WhitesourceRepository implements Serializable { + + final Script script + final Map config + + WhitesourceRepository(Script script, Map config) { + this.script = script + this.config = config + + if(!config.serviceUrl) + script.error "Parameter 'serviceUrl' must be provided as part of the configuration." + } + + List fetchVulnerabilities(whitesourceProjectsMetaInformation) { + def fetchedVulnerabilities = [] + if (config.projectNames) { + for (int i = 0; i < whitesourceProjectsMetaInformation.size(); i++) { + def metaInfo = whitesourceProjectsMetaInformation[i] + + def requestBody = [ + requestType : "getProjectAlertsByType", + alertType : "SECURITY_VULNERABILITY", + projectToken: metaInfo.token + ] + + def response = fetchWhitesourceResource(requestBody) + fetchedVulnerabilities.addAll(response.alerts) + } + } else { + def requestBody = [ + requestType : "getProductAlertsByType", + alertType : "SECURITY_VULNERABILITY", + productToken: config.productToken, + ] + + def response = fetchWhitesourceResource(requestBody) + fetchedVulnerabilities.addAll(response.alerts) + } + + sortVulnerabilitiesByScore(fetchedVulnerabilities) + + return fetchedVulnerabilities + } + + List fetchLibraries(whitesourceProjectsMetaInformation) { + def fetchedLibraries = [] + if (config.projectNames) { + for (int i = 0; i < whitesourceProjectsMetaInformation.size(); i++) { + def metaInfo = whitesourceProjectsMetaInformation[i] + def requestBody = [ + requestType : "getProjectLicenses", + projectToken: metaInfo.token + ] + + def response = fetchWhitesourceResource(requestBody) + fetchedLibraries.addAll(response.libraries) + } + } else { + def requestBody = [ + requestType : "getProductLicenses", + productToken: config.productToken + ] + + def response = fetchWhitesourceResource(requestBody) + fetchedLibraries.addAll(response.libraries) + } + + sortLibrariesAlphabeticallyGAV(fetchedLibraries) + + return listUniqueEntries(fetchedLibraries) + } + + @NonCPS + def listUniqueEntries(Collection list) { + return list.unique() + } + + + protected def fetchWhitesourceResource(Map requestBody) { + final def response = httpWhitesource(requestBody) + def parsedResponse = new JsonUtils().parseJsonSerializable(response.content) + + if(parsedResponse?.errorCode){ + script.error "[WhiteSource] Request failed with error message '${parsedResponse.errorMessage}' (${parsedResponse.errorCode})." + } + + return parsedResponse + } + + @NonCPS + void sortLibrariesAlphabeticallyGAV(List libraries) { + script.echo "found a total of ${libraries.size()} dependencies (direct and indirect)" + libraries.sort { o1, o2 -> + String groupID1 = o1.groupId + String groupID2 = o2.groupId + def comparisionResult = groupID1 <=> groupID2; + + if (comparisionResult != 0) { + comparisionResult + } else { + String artifactID1 = o1.artifactId + String artifactID2 = o2.artifactId + + artifactID1 <=> artifactID2 + } + } + } + + @NonCPS + void sortVulnerabilitiesByScore(List vulnerabilities) { + script.echo "${vulnerabilities.size() > 0 ? 'WARNING: ' : ''}found a total of ${vulnerabilities.size()} vulnerabilities" + vulnerabilities.sort { o1, o2 -> + def cvss3score1 = o1.vulnerability.cvss3_score != 0 ? o1.vulnerability.cvss3_score : o1.vulnerability.score + def cvss3score2 = o2.vulnerability.cvss3_score != 0 ? o2.vulnerability.cvss3_score : o2.vulnerability.score + + def comparisionResult = cvss3score1 <=> cvss3score2 + + if (comparisionResult != 0) { + -comparisionResult + } else { + def score1 = o1.vulnerability.score + def score2 = o2.vulnerability.score + + -(score1 <=> score2) + } + } + } + + List fetchProjectsMetaInfo() { + def projectsMetaInfo = [] + if(config.projectNames){ + def requestBody = [ + requestType: "getProductProjectVitals", + productToken: config.productToken + ] + def response = fetchWhitesourceResource(requestBody) + + if(response?.projectVitals) { + projectsMetaInfo.addAll(findProjectsMeta(response.projectVitals)) + } else { + script.error "[WhiteSource] Could not fetch any projects for product '${config.productName}' from backend, response was ${response}" + } + } + return projectsMetaInfo + } + + List findProjectsMeta(projectVitals) { + def matchedProjects = [] + for (int i = 0; i < config.projectNames?.size(); i++) { + def requestedProjectName = config.projectNames[i].trim() + def matchedProjectInfo = null + + for (int j = 0; j < projectVitals.size(); j++) { + def projectResponse = projectVitals[j] + if (projectResponse.name == requestedProjectName) { + matchedProjectInfo = projectResponse + break + } + } + + if (matchedProjectInfo != null) { + matchedProjects.add(matchedProjectInfo) + } else { + script.error "[WhiteSource] Could not fetch/find requested project '${requestedProjectName}' for product '${config.productName}'" + } + } + + return matchedProjects + } + + void fetchReportForProduct() { + def requestContent = [ + requestType: "getProductRiskReport", + productToken: config.productToken + ] + + fetchFileFromWhiteSource("whitesource-riskReport.pdf", requestContent) + + } + + def fetchProductLicenseAlerts() { + def requestContent = [ + requestType: "getProductAlertsByType", + alertType: "REJECTED_BY_POLICY_RESOURCE", + productToken: config.productToken + ] + def parsedResponse = fetchWhitesourceResource(requestContent) + + return parsedResponse + } + + def fetchProjectLicenseAlerts(String projectToken) { + def requestContent = [ + requestType: "getProjectAlertsByType", + alertType: "REJECTED_BY_POLICY_RESOURCE", + projectToken: projectToken + ] + def parsedResponse = fetchWhitesourceResource(requestContent) + + return parsedResponse + } + + @NonCPS + protected def httpWhitesource(requestBody) { + handleAdditionalRequestParameters(requestBody) + def serializedBody = new JsonUtils().getPrettyJsonString(requestBody) + def params = [ + url : config.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: serializedBody, + quiet : !config.verbose, + timeout : config.timeout + ] + + if (script.env.HTTP_PROXY && !config.serviceUrl.matches('http(s)*://.*\\.sap\\.corp.*')) + params["httpProxy"] = script.env.HTTP_PROXY + + if(config.verbose) + script.echo "Sending http request with parameters ${params}" + + def response = script.httpRequest(params) + + if(config.verbose) + script.echo "Received response ${reponse}" + + return response + } + + @NonCPS + private void fetchFileFromWhiteSource(String fileName, Map params) { + handleAdditionalRequestParameters(params) + def serializedContent = new JsonUtils().getPrettyJsonString(params) + + if(config.verbose) + script.echo "Sending curl request with parameters ${params}" + + script.sh "${config.verbose ? '' : '#!/bin/sh -e\n'}curl -o ${fileName} -X POST ${config.serviceUrl} -H 'Content-Type: application/json' -d \'${serializedContent}\'" + } + + private void handleAdditionalRequestParameters(params) { + if(config.userKey) + params["userKey"] = config.userKey + } +} diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy new file mode 100644 index 000000000..f125e2b13 --- /dev/null +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -0,0 +1,808 @@ +#!groovy +import com.sap.piper.JsonUtils +import com.sap.piper.integration.WhitesourceOrgAdminRepository +import com.sap.piper.integration.WhitesourceRepository +import hudson.AbortException +import org.hamcrest.Matchers +import org.junit.Assert +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.ExpectedException +import org.junit.rules.RuleChain +import util.* + +import static org.hamcrest.Matchers.* +import static org.junit.Assert.assertThat + +class WhitesourceExecuteScanTest extends BasePiperTest { + + private ExpectedException thrown = ExpectedException.none() + private JenkinsDockerExecuteRule dockerExecuteRule = new JenkinsDockerExecuteRule(this) + private JenkinsShellCallRule shellRule = new JenkinsShellCallRule(this) + private JenkinsLoggingRule loggingRule = new JenkinsLoggingRule(this) + private JenkinsWriteFileRule writeFileRule = new JenkinsWriteFileRule(this) + private JenkinsStepRule stepRule = new JenkinsStepRule(this) + + @Rule + public RuleChain ruleChain = Rules + .getCommonRules(this) + .around(new JenkinsReadYamlRule(this)) + .around(thrown) + .around(dockerExecuteRule) + .around(shellRule) + .around(loggingRule) + .around(writeFileRule) + .around(stepRule) + + def whitesourceOrgAdminRepositoryStub + def whitesourceStub + + @Before + void init() { + def credentialsStore = ['ID-123456789': 'token-0815', 'ID-abcdefg': ['testUser', 'testPassword']] + def withCredentialsBindings + helper.registerAllowedMethod('string', [Map], { + m -> + withCredentialsBindings = ["${m.credentialsId}": "${m.variable}"] + return m + }) + helper.registerAllowedMethod('usernamePassword', [Map], { + m -> + withCredentialsBindings = ["${m.credentialsId}": ["${m.usernameVariable}", "${m.passwordVariable}"]] + return m + }) + helper.registerAllowedMethod('withCredentials', [List.class, Closure.class], { + l, body -> + def index = 0 + withCredentialsBindings.each { + entry -> + if(entry.value instanceof List) { + entry.value.each { + subEntry -> + def value = credentialsStore[entry.key] + getBinding().setProperty(subEntry, value[index]) + index++ + + } + } else { + getBinding().setProperty(entry.value, credentialsStore[entry.key]) + } + } + try { + body() + } finally { + withCredentialsBindings.each { + entry -> + if(entry.value instanceof List) { + entry.value.each { + subEntry -> + getBinding().setProperty(subEntry, null) + + } + } else { + getBinding().setProperty(entry.value, null) + } + } + } + }) + helper.registerAllowedMethod("archiveArtifacts", [Map.class], { m -> + if (m.artifacts == null) { + throw new Exception('artifacts cannot be null') + } + return null + }) + helper.registerAllowedMethod("findFiles", [Map.class], { map -> + return [].toArray() + }) + + whitesourceOrgAdminRepositoryStub = new WhitesourceOrgAdminRepository(nullScript, [serviceUrl: "http://some.host.whitesource.com/api/"]) + LibraryLoadingTestExecutionListener.prepareObjectInterceptors(whitesourceOrgAdminRepositoryStub) + + whitesourceStub = new WhitesourceRepository(nullScript, [serviceUrl: "http://some.host.whitesource.com/api/"]) + LibraryLoadingTestExecutionListener.prepareObjectInterceptors(whitesourceStub) + + helper.registerAllowedMethod("fetchProductMetaInfo", [], { + return new JsonUtils().parseJsonSerializable("{ \"productVitals\": [{ \"id\": 59639, \"name\": \"SHC - Piper\", \"token\": \"e30132d8e8f04a4c8be6332c75a0ff0580ab326fa7534540ad326e97a74d945b\", \"creationDate\": \"2017-09-20 09:22:46 +0000\", \"lastUpdatedDate\": \"2018-09-19 09:44:40 +0000\" }]}") + }) + helper.registerAllowedMethod("fetchProjectsMetaInfo", [], { + return new JsonUtils().parseJsonSerializable("{ \"projectVitals\": [{ \"id\": 261964, \"name\": \"piper-demo - 0.0.1\", \"token\": \"a2a62e5d7beb4170ad4dccfa3316b5a4cd3fadefc56c49f88fbf9400a09f7d94\", \"creationDate\": \"2017-09-21 00:28:06 +0000\", \"lastUpdatedDate\": \"2017-10-12 01:03:05 +0000\" }]}").projectVitals + }) + helper.registerAllowedMethod("fetchReportForProduct", [], { }) + helper.registerAllowedMethod( "fetchProjectLicenseAlerts", [Object.class], { + return new JsonUtils().parseJsonSerializable("{ \"alerts\": [] }").alerts + }) + helper.registerAllowedMethod( "fetchProductLicenseAlerts", [], { + return new JsonUtils().parseJsonSerializable("{ \"alerts\": [] }").alerts + }) + helper.registerAllowedMethod( "publishHTML", [Map], {}) + + helper.registerAllowedMethod( "getNpmGAV", [String], {return [group: 'com.sap.node', artifact: 'test-node', version: '1.2.3']}) + helper.registerAllowedMethod( "getSbtGAV", [String], {return [group: 'com.sap.sbt', artifact: 'test-scala', version: '1.2.3']}) + helper.registerAllowedMethod( "getPipGAV", [String], {return [artifact: 'test-python', version: '1.2.3']}) + helper.registerAllowedMethod( "readMavenGAV", [String], {return [group: 'com.sap.maven', artifact: 'test-java', version: '1.2.3']}) + + nullScript.commonPipelineEnvironment.configuration = nullScript.commonPipelineEnvironment.configuration ?: [:] + nullScript.commonPipelineEnvironment.configuration['steps'] = nullScript.commonPipelineEnvironment.configuration['steps'] ?: [:] + nullScript.commonPipelineEnvironment.configuration['steps']['whitesourceExecuteScan'] = nullScript.commonPipelineEnvironment.configuration['steps']['whitesourceExecuteScan'] ?: [:] + nullScript.commonPipelineEnvironment.configuration['general'] = nullScript.commonPipelineEnvironment.configuration['general'] ?: [:] + nullScript.commonPipelineEnvironment.setConfigProperty('userTokenCredentialsId', 'ID-123456789') + nullScript.commonPipelineEnvironment.configuration['steps']['whitesourceExecuteScan']['userTokenCredentialsId'] = 'ID-123456789' + } + + @Test + void testMaven() { + helper.registerAllowedMethod("readProperties", [Map], { + def result = new Properties() + result.putAll([ + "apiKey": "b39d1328-52e2-42e3-98f0-932709daf3f0", + "productName": "SHC - Piper", + "checkPolicies": "true", + "projectName": "python-test", + "projectVersion": "1.0.0" + ]) + return result + }) + + def publishHtmlMap = [:] + helper.registerAllowedMethod("publishHTML", [Map.class], { m -> + publishHtmlMap = m + return null + }) + + stepRule.step.whitesourceExecuteScan([ + script : nullScript, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + scanType : 'maven', + juStabUtils : utils, + orgToken : 'testOrgToken', + productName : 'testProduct' + ]) + + assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) + assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) + + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'docker.wdf.sap.corp:50000/piper/maven')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config 420a1bc5c82f57e80307205d8625304f'])) + + assertThat(shellRule.shell, Matchers.hasItems( + is('curl --location --output wss-unified-agent.jar https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/wss-unified-agent.jar'), + is('./bin/java -jar wss-unified-agent.jar -c \'./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProduct\'') + )) + + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('apiKey=testOrgToken')) + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('productName=testProduct')) + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('userKey=token-0815')) + } + + + @Test + void testNpm() { + helper.registerAllowedMethod("readProperties", [Map], { + def result = new Properties() + result.putAll([ + "apiKey": "b39d1328-52e2-42e3-98f0-932709daf3f0", + "productName": "SHC - Piper", + "devDep": "false", + "checkPolicies": "true", + "projectName": "pipeline-test-node", + "projectVersion": "1.0.0" + ]) + return result + }) + stepRule.step.whitesourceExecuteScan([ + script : nullScript, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + scanType : 'npm', + juStabUtils : utils, + orgToken : 'testOrgToken', + productName : 'testProductName', + productToken : 'testProductToken', + reporting : false + ]) + + assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) + assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) + + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'docker.wdf.sap.corp:50000/piper/node')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config 420a1bc5c82f57e80307205d8625304f'])) + assertThat(shellRule.shell, Matchers.hasItems( + is('curl --location --output wss-unified-agent.jar https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/wss-unified-agent.jar'), + is('curl --location --output jvm.tar.gz https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz && tar --strip-components=1 -xzf jvm.tar.gz'), + is('./bin/java -jar wss-unified-agent.jar -c \'./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProductName\'') + )) + + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('apiKey=testOrgToken')) + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('productName=testProductName')) + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('productToken=testProductToken')) + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('productVersion=1.2.3')) + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('projectName=com.sap.node.test-node')) + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('userKey=token-0815')) + } + + @Test + void testNpmWithCustomConfigAndFilePath() { + helper.registerAllowedMethod("readProperties", [Map], { + def result = new Properties() + result.putAll([ + "apiKey": "b39d1328-52e2-42e3-98f0-932709daf3f0", + "productName": "SHC - Piper", + "devDep": "false", + "checkPolicies": "true", + "projectName": "pipeline-test-node", + "projectVersion": "1.0.0" + ]) + return result + }) + stepRule.step.whitesourceExecuteScan([ + script : nullScript, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + scanType : 'npm', + productName : 'SHC - Piper', + configFilePath : './../../testConfigPath', + file : 'package.json', + juStabUtils : utils + ]) + + assertThat(shellRule.shell, Matchers.hasItems( + is('curl --location --output wss-unified-agent.jar https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/wss-unified-agent.jar'), + is('curl --location --output jvm.tar.gz https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz && tar --strip-components=1 -xzf jvm.tar.gz'), + is('./bin/java -jar wss-unified-agent.jar -c \'./../../testConfigPath.092aaffe7a79d11da13593b63b929754\' -userKey \'token-0815\' -product \'SHC - Piper\'') + )) + } + + @Test + void testPip() { + helper.registerAllowedMethod("readProperties", [Map], { + def result = new Properties() + result.putAll([ + "apiKey": "b39d1328-52e2-42e3-98f0-932709daf3f0", + "productName": "SHC - Piper", + "checkPolicies": "true", + "projectName": "python-test", + "projectVersion": "1.0.0" + ]) + return result + }) + + stepRule.step.whitesourceExecuteScan([ + script : nullScript, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + scanType : 'pip', + juStabUtils : utils, + orgToken : 'testOrgToken', + productName : 'testProductName', + reporting : false + ]) + + assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) + + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'docker.wdf.sap.corp:50000/piper/node')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config 420a1bc5c82f57e80307205d8625304f'])) + + assertThat(shellRule.shell, Matchers.hasItems( + is('curl --location --output wss-unified-agent.jar https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/wss-unified-agent.jar'), + is('curl --location --output jvm.tar.gz https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz && tar --strip-components=1 -xzf jvm.tar.gz'), + is('./bin/java -jar wss-unified-agent.jar -c \'./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProductName\'') + )) + + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('apiKey=testOrgToken')) + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('productName=testProductName')) + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('userKey=token-0815')) + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('productVersion=1.2.3')) + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('projectName=test-python')) + } + + @Test + void testSbt() { + helper.registerAllowedMethod("readProperties", [Map], { + def result = new Properties() + result.putAll([ + "apiKey": "b39d1328-52e2-42e3-98f0-932709daf3f0", + "productName": "SHC - Piper", + "checkPolicies": "true", + "projectName": "python-test", + "projectVersion": "1.0.0" + ]) + return result + }) + + stepRule.step.whitesourceExecuteScan([ + script : nullScript, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + scanType : 'sbt', + juStabUtils : utils, + productName : 'testProductName', + orgToken : 'testOrgToken', + reporting : false + ]) + + assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) + assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) + + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'docker.wdf.sap.corp:50000/piper/sbt')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config 420a1bc5c82f57e80307205d8625304f'])) + + assertThat(shellRule.shell, Matchers.hasItems( + is('curl --location --output wss-unified-agent.jar https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/wss-unified-agent.jar'), + is('./bin/java -jar wss-unified-agent.jar -c \'./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProductName\'') + )) + + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('apiKey=testOrgToken')) + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('productName=testProductName')) + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('userKey=token-0815')) + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('productVersion=1.2.3')) + assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('projectName=com.sap.sbt.test-scala')) + } + + @Test + void testAgentNoDownloads() { + helper.registerAllowedMethod("readProperties", [Map], { + def result = new Properties() + result.putAll([ + "apiKey": "b39d1328-52e2-42e3-98f0-932709daf3f0", + "productName": "SHC - Piper", + "checkPolicies": "true", + "projectName": "python-test", + "projectVersion": "1.0.0" + ]) + return result + }) + stepRule.step.whitesourceExecuteScan([ + script : nullScript, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + scanType : 'maven', + agentDownloadUrl : '', + jreDownloadUrl : '', + agentParameters : 'testParams', + juStabUtils : utils, + orgToken : 'testOrgToken', + productName : 'testProductName' + ]) + + assertThat(shellRule.shell[0], is('java -jar wss-unified-agent.jar -c \'./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProductName\' testParams')) + } + + @Test + void testMta() { + helper.registerAllowedMethod("readProperties", [Map], { + def result = new Properties() + result.putAll([ + "apiKey": "b39d1328-52e2-42e3-98f0-932709daf3f0", + "productName": "SHC - Piper", + "checkPolicies": "true", + "projectName": "python-test", + "projectVersion": "1.0.0" + ]) + return result + }) + helper.registerAllowedMethod("findFiles", [Map.class], { map -> + if (map.glob == "**${File.separator}pom.xml") { + return [new File('maven1/pom.xml'), new File('maven2/pom.xml')].toArray() + } + if (map.glob == "**${File.separator}package.json") { + return [new File('npm1/package.json'), new File('npm2/package.json')].toArray() + } + return [].toArray() + }) + + def whitesourceCalls = [] + helper.registerAllowedMethod("whitesourceExecuteScan", [Map.class], { map -> + whitesourceCalls.add(map) + stepRule.step.call(map) + }) + + def parallelMap = [:] + helper.registerAllowedMethod("parallel", [Map.class], { map -> + parallelMap = map + parallelMap['Whitesource - maven1']() + parallelMap['Whitesource - npm1']() + }) + + //need to use call due to mock above + stepRule.step.call([ + script : nullScript, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + scanType : 'mta', + productName : 'SHC - Piper', + buildDescriptorExcludeList : ["maven2${File.separator}pom.xml".toString(), "npm2${File.separator}package.json".toString()], + reporting : true, + juStabUtils : utils + ]) + + assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) + assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) + + assertThat(parallelMap, hasKey('Whitesource - maven1')) + assertThat(parallelMap, hasKey('Whitesource - npm1')) + assertThat(parallelMap.keySet(), hasSize(3)) + + assertThat(whitesourceCalls, hasItem(allOf( + hasEntry('scanType', 'maven'), + hasEntry('buildDescriptorFile', "maven1${File.separator}pom.xml".toString()) + ))) + assertThat(whitesourceCalls, hasItem(allOf( + hasEntry('scanType', 'npm'), + hasEntry('projectNames', ["com.sap.maven.test-java - 1.2.3", "com.sap.node.test-node - 1.2.3"]), + hasEntry('buildDescriptorFile', "npm1${File.separator}package.json".toString()) + ))) + } + + @Test + void testMtaBlocks() { + helper.registerAllowedMethod("findFiles", [Map.class], { map -> + if (map.glob == "**${File.separator}pom.xml") { + return [new File('maven1/pom.xml'), new File('maven2/pom.xml')].toArray() + } + if (map.glob == "**${File.separator}package.json") { + return [new File('npm1/package.json'), new File('npm2/package.json'), new File('npm3/package.json'), new File('npm4/package.json')].toArray() + } + return [].toArray() + }) + + def whitesourceCalls = [] + helper.registerAllowedMethod("whitesourceExecuteScan", [Map.class], { map -> + whitesourceCalls.add(map) + }) + + def invocation = 0 + def parallelMap = [:] + helper.registerAllowedMethod("parallel", [Map.class], { map -> + parallelMap[invocation] = map + for(i = 0; i < parallelMap[invocation].keySet().size(); i++) { + def key = parallelMap[invocation].keySet()[i] + if(key != "failFast") + parallelMap[invocation][key]() + } + invocation++ + }) + + //need to use call due to mock above + stepRule.step.call([ + script : nullScript, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + scanType : 'mta', + productName : 'SHC - Piper', + buildDescriptorExcludeList : ["maven2${File.separator}pom.xml".toString()], + juStabUtils : utils, + parallelLimit : 3 + ]) + + assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) + assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) + + assertThat(invocation, is(2)) + assertThat(parallelMap[0], hasKey('Whitesource - maven1')) + assertThat(parallelMap[0], hasKey('Whitesource - npm1')) + assertThat(parallelMap[0], hasKey('Whitesource - npm2')) + assertThat(parallelMap[0].keySet(), hasSize(4)) + assertThat(parallelMap[1], hasKey('Whitesource - npm3')) + assertThat(parallelMap[1], hasKey('Whitesource - npm4')) + assertThat(parallelMap[1].keySet(), hasSize(3)) + + assertThat(whitesourceCalls, hasItem(allOf( + hasEntry('scanType', 'maven'), + hasEntry('buildDescriptorFile', "maven1${File.separator}pom.xml".toString()) + ))) + assertThat(whitesourceCalls, hasItem(allOf( + hasEntry('scanType', 'npm'), + hasEntry('buildDescriptorFile', "npm1${File.separator}package.json".toString()) + ))) + assertThat(whitesourceCalls, hasItem(allOf( + hasEntry('scanType', 'npm'), + hasEntry('buildDescriptorFile', "npm2${File.separator}package.json".toString()) + ))) + assertThat(whitesourceCalls, hasItem(allOf( + hasEntry('scanType', 'npm'), + hasEntry('buildDescriptorFile', "npm3${File.separator}package.json".toString()) + ))) + assertThat(whitesourceCalls, hasItem(allOf( + hasEntry('scanType', 'npm'), + hasEntry('buildDescriptorFile', "npm4${File.separator}package.json".toString()) + ))) + } + + @Test + void testMtaSingleExclude() { + + helper.registerAllowedMethod("findFiles", [Map.class], { map -> + if (map.glob == "**${File.separator}pom.xml") { + return [new File('maven1/pom.xml'), new File('maven2/pom.xml')].toArray() + } + if (map.glob == "**${File.separator}package.json") { + return [new File('npm1/package.json'), new File('npm2/package.json')].toArray() + } + return [].toArray() + }) + + def parallelMap = [:] + helper.registerAllowedMethod("parallel", [Map.class], { map -> + parallelMap = map + }) + + stepRule.step.whitesourceExecuteScan([ + script : nullScript, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + scanType : 'mta', + productName : 'SHC - Piper', + buildDescriptorExcludeList : "maven2${File.separator}pom.xml", + juStabUtils : utils + ]) + + assertThat(parallelMap.keySet(), hasSize(4)) + + } + + @Test + void testNPMStatusCheckScanException() { + thrown.expect(AbortException.class) + stepRule.step.checkStatus(-1 & 0xFF, [licensingVulnerabilities: true]) + } + + @Test + void testNPMStatusCheckPolicyViolation() { + thrown.expect(AbortException.class) + stepRule.step.checkStatus(-2 & 0xFF, [licensingVulnerabilities: true]) + } + + @Test + void testNPMStatusCheckNoPolicyViolation() { + stepRule.step.checkStatus(-2 & 0xFF, [licensingVulnerabilities: false]) + } + + @Test + void testNPMStatusCheckClientException() { + thrown.expect(AbortException.class) + stepRule.step.checkStatus(-3 & 0xFF, [licensingVulnerabilities: true]) + } + + @Test + void testNPMStatusCheckConnectionException() { + thrown.expect(AbortException.class) + stepRule.step.checkStatus(-4 & 0xFF, [licensingVulnerabilities: true]) + } + + @Test + void testNPMStatusCheckServerException() { + thrown.expect(AbortException.class) + stepRule.step.checkStatus(-3 & 0xFF, [licensingVulnerabilities: true]) + } + + @Test + void testViolationPolicy() { + stepRule.step.checkViolationStatus(0) + } + + @Test + void testViolationPolicyException() { + thrown.expect(AbortException.class) + thrown.expectMessage("1") + stepRule.step.checkViolationStatus(1) + } + + @Test + void testFetchViolationCountProject() { + + def config = [projectNames: ["piper-java-cc - 0.0.1", "pipeline-test - 0.0.1"], productToken: "product-token"] + + def projectTokens = [[token:"abc-project-token"],[token:"def-project-token"]] + helper.registerAllowedMethod('fetchProjectsMetaInfo', [], { return projectTokens }) + helper.registerAllowedMethod('fetchProjectLicenseAlerts', [String], { projectToken -> + if(projectToken == projectTokens[0].token) + return [alerts: []] + if(projectToken == projectTokens[1].token) + return [alerts: []] + }) + + def violationCount = stepRule.step.fetchViolationCount(config, whitesourceStub) + + Assert.assertTrue(violationCount == 0) + } + + + @Test + void testFetchViolationCountProjectNotZero() { + + def config = [projectNames: ["piper-java-cc - 0.0.1", "pipeline-test - 0.0.1"]] + + def projectTokens = [[token:"abc-project-token"],[token:"def-project-token"]] + helper.registerAllowedMethod('fetchProjectsMetaInfo', [], { return projectTokens }) + helper.registerAllowedMethod('fetchProjectLicenseAlerts', [String], { projectToken -> + if(projectToken == projectTokens[0].token) + return [alerts: [{}, {}]] + if(projectToken == projectTokens[1].token) + return [alerts: [{}, {}, {}]] + }) + + def violationCount = stepRule.step.fetchViolationCount(config, whitesourceStub) + + Assert.assertTrue(violationCount == 5) + } + + @Test + void testFetchViolationCountProduct() { + + def config = [:] + + helper.registerAllowedMethod('fetchProductLicenseAlerts', [], { return [alerts: []] }) + + def violationCount = stepRule.step.fetchViolationCount(config, whitesourceStub) + + Assert.assertTrue(violationCount == 0) + } + + @Test + void testFetchViolationCountProductNotZero() { + + def config = [:] + + helper.registerAllowedMethod('fetchProductLicenseAlerts', [], { return [alerts: [{}, {}]]}) + + def violationCount = stepRule.step.fetchViolationCount(config, whitesourceStub) + + Assert.assertTrue(violationCount == 2) + } + + @Test + void testCheckException() { + helper.registerAllowedMethod("readProperties", [Map], { + def result = new Properties() + result.putAll([ + "apiKey": "b39d1328-52e2-42e3-98f0-932709daf3f0", + "productName": "SHC - Piper", + "checkPolicies": "true", + "projectName": "python-test", + "projectVersion": "1.0.0" + ]) + return result + }) + helper.registerAllowedMethod("fetchVulnerabilities", [Object.class], { + return new JsonUtils().parseJsonSerializable("{ \"alerts\": [ { \"vulnerability\": { \"name\": \"CVE-2017-15095\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 7.5, \"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15095\", \"description\": \"A deserialization flaw was discovered in the jackson-databind in versions before 2.8.10 and 2.9.1, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper. This issue extends the previous flaw CVE-2017-7525 by blacklisting more classes that could be used maliciously.\", \"topFix\": { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\"," + + "\"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\", \"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\"," + + "\"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e865a7a4464da63ded9f4b1a2328ad85c9ded78b#diff-98084d808198119d550a9211e128a16f\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-12-12\", \"message\": \"Fix #1737 (#1857)\", \"extraData\": \"key=e865a7a&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e8f043d1\"," + + "\"fixResolution\": \"release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-06-30\", \"message\": \"Fix #1680\", \"extraData\": \"key=e8f043d&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" } ], \"fixResolutionText\": \"Replace or update the following files: IllegalTypesCheckTest.java, VERSION, BeanDeserializerFactory.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\", \"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\"," + + "\"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-databind\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:5,\", \"date\": \"2017-11-15\" }, { \"vulnerability\": { \"name\": \"CVE-2017-17485\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 7.5, \"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-01-10\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-17485\"," + + "\"description\": \"FasterXML jackson-databind through 2.8.10 and 2.9.x through 2.9.3 allows unauthenticated remote code execution because of an incomplete fix for the CVE-2017-7525 deserialization flaw. This is exploitable by sending maliciously crafted JSON input to the readValue method of the ObjectMapper, bypassing a blacklist that is ineffective if the Spring libraries are available in the classpath.\", \"topFix\": { \"vulnerability\": \"CVE-2017-17485\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/bb45fb16709018842f858f1a6e1118676aaa34bd#diff-727a6e8db3603b95f185697108af6c48\", \"fixResolution\": \"src/test/java/org/springframework/jacksontest/AbstractApplicationContext.java,src/test/java/org/springframework/jacksontest/AbstractPointcutAdvisor.java,src/test/java/org/springframework/jacksontest/BogusApplicationContext.java,src/main/java/com/fasterxml/jackson/databind/jsontype/impl/SubTypeValidator.java,src/test/java/org/springframework/jacksontest/BogusPointcutAdvisor.java,src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java\"," + + "\"date\": \"2017-12-19\", \"message\": \"Fix issues with earlier fix for #1855\", \"extraData\": \"key=bb45fb1&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-17485\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/bb45fb16709018842f858f1a6e1118676aaa34bd#diff-727a6e8db3603b95f185697108af6c48\", \"fixResolution\": \"src/test/java/org/springframework/jacksontest/AbstractApplicationContext.java,src/test/java/org/springframework/jacksontest/AbstractPointcutAdvisor.java,src/test/java/org/springframework/jacksontest/BogusApplicationContext.java,src/main/java/com/fasterxml/jackson/databind/jsontype/impl/SubTypeValidator.java,src/test/java/org/springframework/jacksontest/BogusPointcutAdvisor.java,src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java\", \"date\": \"2017-12-19\", \"message\": \"Fix issues with earlier fix for #1855\"," + + "\"extraData\": \"key=bb45fb1&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-17485\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/bb45fb16709018842f858f1a6e1118676aaa34bd\", \"fixResolution\": \"src/test/java/org/springframework/jacksontest/AbstractApplicationContext.java,src/test/java/org/springframework/jacksontest/AbstractPointcutAdvisor.java,src/test/java/org/springframework/jacksontest/BogusApplicationContext.java,src/main/java/com/fasterxml/jackson/databind/jsontype/impl/SubTypeValidator.java,src/test/java/org/springframework/jacksontest/BogusPointcutAdvisor.java,src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java\", \"date\": \"2017-12-19\", \"message\": \"Fix issues with earlier fix for #1855\", \"extraData\": \"key=bb45fb1&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" } ]," + + "\"fixResolutionText\": \"Replace or update the following files: AbstractApplicationContext.java, AbstractPointcutAdvisor.java, BogusApplicationContext.java, SubTypeValidator.java, BogusPointcutAdvisor.java, IllegalTypesCheckTest.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\", \"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-databind\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\"," + + "\"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:5,\", \"date\": \"2017-11-15\" }, { \"vulnerability\": { \"name\": \"CVE-2017-7525\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 7.5, \"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-7525\", \"description\": \"A deserialization flaw was discovered in the jackson-databind, versions before 2.6.7.1, 2.7.9.1 and 2.8.9, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper.\", \"topFix\": { \"vulnerability\": \"CVE-2017-7525\"," + + "\"type\": \"UPGRADE_VERSION\", \"origin\": \"BUGZILLA\", \"url\": \"https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2017-7525\", \"fixResolution\": \"jackson-databind 2.8.9,jackson-databind 2.9.0\", \"message\": \"CVE-2017-7525 jackson-databind: Deserialization vulnerability via readValue method of ObjectMapper\", \"extraData\": \"key=1462702&assignee=Red Hat Product Security\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-7525\", \"type\": \"UPGRADE_VERSION\", \"origin\": \"BUGZILLA\", \"url\": \"https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2017-7525\", \"fixResolution\": \"jackson-databind 2.8.9,jackson-databind 2.9.0\", \"message\": \"CVE-2017-7525 jackson-databind: Deserialization vulnerability via readValue method of ObjectMapper\", \"extraData\": \"key=1462702&assignee=Red Hat Product Security\" }, { \"vulnerability\": \"CVE-2017-7525\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459cedcf079c6106ae7da2ac562bc32dcabe1#diff-98084d808198119d550a9211e128a16f\"," + + "\"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\", \"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-7525\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459cedcf079c6106ae7da2ac562bc32dcabe1\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\"," + + "\"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-7525\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/6ce32ffd18facac6abdbbf559c817b47fcb622c1\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.7(.10)\", \"extraData\": \"key=6ce32ff&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" } ], \"fixResolutionText\": \"Upgrade to version jackson-databind 2.8.9, jackson-databind 2.9.0 or greater\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\"," + + "\"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\", \"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-databind\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }," + + "\"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:5,\", \"date\": \"2017-11-15\" }, { \"vulnerability\": { \"name\": \"CVE-2018-5968\", \"type\": \"CVE\", \"severity\": \"medium\", \"score\": 5.1, \"cvss3_severity\": \"high\", \"cvss3_score\": 8.1, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-01-22\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-5968\", \"description\": \"FasterXML jackson-databind through 2.8.11 and 2.9.x through 2.9.3 allows unauthenticated remote code execution because of an incomplete fix for the CVE-2017-7525 and CVE-2017-17485 deserialization flaws. This is exploitable via two different gadgets that bypass a blacklist.\", \"topFix\": { \"vulnerability\": \"CVE-2018-5968\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\"," + + "\"url\": \"https://github.com/FasterXML/jackson-databind/commit/038b471e2efde2e8f96b4e0be958d3e5a1ff1d05\", \"fixResolution\": \"src/main/java/com/fasterxml/jackson/databind/jsontype/impl/SubTypeValidator.java,release-notes/VERSION\", \"date\": \"2018-01-22\", \"message\": \"Fix #1899\", \"extraData\": \"key=038b471&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [], \"fixResolutionText\": \"Replace or update the following files: SubTypeValidator.java, VERSION\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\"," + + "\"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-databind\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:5,\", \"date\": \"2017-11-15\" }, { \"vulnerability\": { \"name\": \"CVE-2018-7489\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 7.5," + + "\"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-26\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-7489\", \"description\": \"FasterXML jackson-databind before 2.8.11.1 and 2.9.x before 2.9.5 allows unauthenticated remote code execution because of an incomplete fix for the CVE-2017-7525 deserialization flaw. This is exploitable by sending maliciously crafted JSON input to the readValue method of the ObjectMapper, bypassing a blacklist that is ineffective if the c3p0 libraries are available in the classpath.\", \"topFix\": { \"vulnerability\": \"CVE-2018-7489\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/6799f8f10cc78e9af6d443ed6982d00a13f2e7d2\", \"fixResolution\": \"src/main/java/com/fasterxml/jackson/databind/jsontype/impl/SubTypeValidator.java,src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,src/test/java/com/mchange/v2/c3p0/jacksontest/ComboPooledDataSource.java,release-notes/VERSION\"," + + "\"date\": \"2018-02-11\", \"message\": \"Fix #1931\", \"extraData\": \"key=6799f8f&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [], \"fixResolutionText\": \"Replace or update the following files: SubTypeValidator.java, IllegalTypesCheckTest.java, ComboPooledDataSource.java, VERSION\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\", \"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\"," + + "\"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-databind\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:5,\", \"date\": \"2017-11-15\" }, { \"vulnerability\": { \"name\": \"CVE-2016-3720\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 7.5, \"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2016-06-10\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-3720\"," + + "\"description\": \"XML external entity (XXE) vulnerability in XmlMapper in the Data format extension for Jackson (aka jackson-dataformat-xml) allows attackers to have unspecified impact via unknown vectors.\", \"allFixes\": [], \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"9c7e7805-0ba0-480a-8647-ed0a457b4a88\", \"keyId\": 441802, \"filename\": \"jackson-dataformat-xml-2.4.2.jar\", \"name\": \"Jackson-dataformat-XML\", \"groupId\": \"com.fasterxml.jackson.dataformat\", \"artifactId\": \"jackson-dataformat-xml\", \"version\": \"2.4.2\", \"sha1\": \"02f2d96f68b2d3475452d95dde7a3fbee225f6ae\", \"type\": \"Java\", \"references\": { \"url\": \"http://wiki.fasterxml.com/JacksonExtensionXmlDataBinding\", \"issueUrl\": \"https://github.com/FasterXML/jackson-dataformat-xml/issues\", \"pomUrl\": \"http://maven.ibiblio.org/maven2/com/fasterxml/jackson/dataformat/jackson-dataformat-xml/2.4.2/jackson-dataformat-xml-2.4.2.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-dataformat-xml\" }," + + "\"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:1,\", \"date\": \"2017-10-26\" }, { \"vulnerability\": { \"name\": \"CVE-2017-5929\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 7.5, \"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2017-03-13\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-5929\", \"description\": \"QOS.ch Logback before 1.2.0 has a serialization vulnerability affecting the SocketServer and ServerSocketReceiver components.\", \"topFix\": { \"vulnerability\": \"CVE-2017-5929\"," + + "\"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/qos-ch/logback/commit/f46044b805bca91efe5fd6afe52257cd02f775f8\", \"fixResolution\": \"logback-classic/src/main/java/ch/qos/logback/classic/net/server/LogbackClassicSerializationHelper.java,logback-core/src/test/java/ch/qos/logback/core/net/Innocent.java,logback-classic/src/main/java/ch/qos/logback/classic/net/SimpleSocketServer.java,logback-core/src/main/java/ch/qos/logback/core/net/HardenedObjectInputStream.java,logback-classic/src/test/java/ch/qos/logback/classic/LoggerSerializationTest.java,logback-core/src/test/java/ch/qos/logback/core/net/HardenedObjectInputStreamTest.java\", \"date\": \"2017-02-07\", \"message\": \"harden serialization\", \"extraData\": \"key=f46044b&committerName=ceki&committerUrl=https://github.com/ceki&committerAvatar=https://avatars1.githubusercontent.com/u/115476?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-5929\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\"," + + "\"url\": \"https://github.com/qos-ch/logback/commit/f46044b805bca91efe5fd6afe52257cd02f775f8\", \"fixResolution\": \"logback-classic/src/main/java/ch/qos/logback/classic/net/server/LogbackClassicSerializationHelper.java,logback-core/src/test/java/ch/qos/logback/core/net/Innocent.java,logback-classic/src/main/java/ch/qos/logback/classic/net/SimpleSocketServer.java,logback-core/src/main/java/ch/qos/logback/core/net/HardenedObjectInputStream.java,logback-classic/src/test/java/ch/qos/logback/classic/LoggerSerializationTest.java,logback-core/src/test/java/ch/qos/logback/core/net/HardenedObjectInputStreamTest.java\", \"date\": \"2017-02-07\", \"message\": \"harden serialization\", \"extraData\": \"key=f46044b&committerName=ceki&committerUrl=https://github.com/ceki&committerAvatar=https://avatars1.githubusercontent.com/u/115476?v=4\" }, { \"vulnerability\": \"CVE-2017-5929\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/victims/victims-cve-db/commit/94745e07d4b10a58d98ffc3916d9711fa407b018\"," + + "\"fixResolution\": \"database/java/2017/5929.yaml\", \"date\": \"2017-03-15\", \"message\": \"Added CVE-2017-5929 per issue #76\", \"extraData\": \"key=94745e0&committerName=cplvic&committerUrl=https://github.com/cplvic&committerAvatar=https://avatars0.githubusercontent.com/u/11528385?v=4\" } ], \"fixResolutionText\": \"Replace or update the following files: LogbackClassicSerializationHelper.java, Innocent.java, SimpleSocketServer.java, HardenedObjectInputStream.java, LoggerSerializationTest.java, HardenedObjectInputStreamTest.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"1259a151-8d7c-4921-b293-3b4b8708ac5e\", \"keyId\": 754707, \"filename\": \"logback-classic-1.1.3.jar\", \"name\": \"Logback Classic Module\", \"groupId\": \"ch.qos.logback\", \"artifactId\": \"logback-classic\", \"version\": \"1.1.3\", \"sha1\": \"d90276fff414f06cb375f2057f6778cd63c6082f\", \"type\": \"Java\", \"references\": { \"url\": \"http://logback.qos.ch/logback-classic\"," + + "\"pomUrl\": \"http://maven.ibiblio.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom\", \"scmUrl\": \"https://github.com/ceki/logback/logback-classic\" }, \"licenses\": [ { \"name\": \"LGPL 2.1\", \"url\": \"http://opensource.org/licenses/lgpl-2.1\", \"profileInfo\": { \"copyrightRiskScore\": \"FIVE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"PARTIAL\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } }, { \"name\": \"Eclipse 1.0\", \"url\": \"http://opensource.org/licenses/eclipse-1.0.php\", \"profileInfo\": { \"copyrightRiskScore\": \"SIX\", \"patentRiskScore\": \"FOUR\", \"copyleft\": \"PARTIAL\", \"linking\": \"VIRAL\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": true, \"description\": \"High:1,\", \"date\": \"2017-10-26\" }, { \"vulnerability\": { \"name\": \"CVE-2017-5929\", \"type\": \"CVE\", \"severity\": \"high\"," + + "\"score\": 7.5, \"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2017-03-13\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-5929\", \"description\": \"QOS.ch Logback before 1.2.0 has a serialization vulnerability affecting the SocketServer and ServerSocketReceiver components.\", \"topFix\": { \"vulnerability\": \"CVE-2017-5929\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/qos-ch/logback/commit/f46044b805bca91efe5fd6afe52257cd02f775f8\", \"fixResolution\": \"logback-classic/src/main/java/ch/qos/logback/classic/net/server/LogbackClassicSerializationHelper.java,logback-core/src/test/java/ch/qos/logback/core/net/Innocent.java,logback-classic/src/main/java/ch/qos/logback/classic/net/SimpleSocketServer.java,logback-core/src/main/java/ch/qos/logback/core/net/HardenedObjectInputStream.java,logback-classic/src/test/java/ch/qos/logback/classic/LoggerSerializationTest.java,logback-core/src/test/java/ch/qos/logback/core/net/HardenedObjectInputStreamTest.java\", \"date\": \"2017-02-07\"," + + "\"message\": \"harden serialization\", \"extraData\": \"key=f46044b&committerName=ceki&committerUrl=https://github.com/ceki&committerAvatar=https://avatars1.githubusercontent.com/u/115476?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-5929\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/qos-ch/logback/commit/f46044b805bca91efe5fd6afe52257cd02f775f8\", \"fixResolution\": \"logback-classic/src/main/java/ch/qos/logback/classic/net/server/LogbackClassicSerializationHelper.java,logback-core/src/test/java/ch/qos/logback/core/net/Innocent.java,logback-classic/src/main/java/ch/qos/logback/classic/net/SimpleSocketServer.java,logback-core/src/main/java/ch/qos/logback/core/net/HardenedObjectInputStream.java,logback-classic/src/test/java/ch/qos/logback/classic/LoggerSerializationTest.java,logback-core/src/test/java/ch/qos/logback/core/net/HardenedObjectInputStreamTest.java\", \"date\": \"2017-02-07\", \"message\": \"harden serialization\", \"extraData\": \"key=f46044b&committerName=ceki&committerUrl=https://github.com/ceki&committerAvatar=https://avatars1.githubusercontent.com/u/115476?v=4\" }," + + "{ \"vulnerability\": \"CVE-2017-5929\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/victims/victims-cve-db/commit/94745e07d4b10a58d98ffc3916d9711fa407b018\", \"fixResolution\": \"database/java/2017/5929.yaml\", \"date\": \"2017-03-15\", \"message\": \"Added CVE-2017-5929 per issue #76\", \"extraData\": \"key=94745e0&committerName=cplvic&committerUrl=https://github.com/cplvic&committerAvatar=https://avatars0.githubusercontent.com/u/11528385?v=4\" } ], \"fixResolutionText\": \"Replace or update the following files: LogbackClassicSerializationHelper.java, Innocent.java, SimpleSocketServer.java, HardenedObjectInputStream.java, LoggerSerializationTest.java, HardenedObjectInputStreamTest.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"d2b236f1-ac7d-4413-92fa-848cc7c3d3e5\", \"keyId\": 754708, \"filename\": \"logback-core-1.1.3.jar\", \"name\": \"Logback Core Module\", \"groupId\": \"ch.qos.logback\", \"artifactId\": \"logback-core\", \"version\": \"1.1.3\", \"sha1\": \"e3c02049f2dbbc764681b40094ecf0dcbc99b157\", \"type\": \"Java\"," + + "\"references\": { \"url\": \"http://logback.qos.ch/logback-core\", \"pomUrl\": \"http://maven.ibiblio.org/maven2/ch/qos/logback/logback-core/1.1.3/logback-core-1.1.3.pom\", \"scmUrl\": \"https://github.com/ceki/logback/logback-core\" }, \"licenses\": [ { \"name\": \"Eclipse 1.0\", \"url\": \"http://opensource.org/licenses/eclipse-1.0.php\", \"profileInfo\": { \"copyrightRiskScore\": \"SIX\", \"patentRiskScore\": \"FOUR\", \"copyleft\": \"PARTIAL\", \"linking\": \"VIRAL\", \"royaltyFree\": \"CONDITIONAL\" } }, { \"name\": \"LGPL 2.1\", \"url\": \"http://opensource.org/licenses/lgpl-2.1\", \"profileInfo\": { \"copyrightRiskScore\": \"FIVE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"PARTIAL\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:1,\", \"date\": \"2017-10-26\" }, { \"vulnerability\": { \"name\": \"CVE-2012-4529\", \"type\": \"CVE\", \"severity\": \"medium\", \"score\": 4.3, \"cvss3_score\": 0," + + "\"publishDate\": \"2013-10-28\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2012-4529\", \"description\": \"The org.apache.catalina.connector.Response.encodeURL method in Red Hat JBoss Web 7.1.x and earlier, when the tracking mode is set to COOKIE, sends the jsessionid in the URL of the first response of a session, which allows remote attackers to obtain the session id (1) via a man-in-the-middle attack or (2) by reading a log.\", \"allFixes\": [], \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"aa3a10d7-4e2c-46fe-bbf9-3c2d06e43b02\", \"keyId\": 24276785, \"filename\": \"tomcat-embed-core-7.0.78.jar\", \"name\": \"tomcat-embed-core\", \"groupId\": \"org.apache.tomcat.embed\", \"artifactId\": \"tomcat-embed-core\", \"version\": \"7.0.78\", \"sha1\": \"ddb63d615ec3944b4394aed6dc825cd0cbb16b21\", \"type\": \"Java\", \"references\": { \"url\": \"http://tomcat.apache.org/\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/org/apache/tomcat/embed/tomcat-embed-core/7.0.78/tomcat-embed-core-7.0.78.pom\" }, \"licenses\": [ { \"name\": \"Apache 2.0\"," + + "\"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": true, \"description\": \"High:3,Medium:2,\", \"date\": \"2017-10-26\" }, { \"vulnerability\": { \"name\": \"CVE-2017-12615\", \"type\": \"CVE\", \"severity\": \"medium\", \"score\": 6.8, \"cvss3_severity\": \"high\", \"cvss3_score\": 8.1, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2017-09-19\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-12615\", \"description\": \"When running Apache Tomcat 7.0.0 to 7.0.79 on Windows with HTTP PUTs enabled (e.g. via setting the readonly initialisation parameter of the Default to false) it was possible to upload a JSP file to the server via a specially crafted request. This JSP could then be requested and any code it contained would be executed by the server.\"," + + "\"topFix\": { \"vulnerability\": \"CVE-2017-12615\", \"type\": \"UPGRADE_VERSION\", \"origin\": \"SECURITY_TRACKER\", \"url\": \"http://www.securitytracker.com/id/1039392\", \"fixResolution\": \"The vendor has issued a fix (7.0.81).\\n\\nThe vendor advisory is available at:\\n\\nhttps://tomcat.apache.org/security-7.html#Fixed_in_Apache_Tomcat_7.0.81\", \"date\": \"2017-12-31\", \"message\": \"Apache Tomcat on Windows HTTP PUT Request Processing Flaw Lets Remote Users Execute Arbitrary Code on the Target System\", \"extraData\": \"key=1039392\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-12615\", \"type\": \"UPGRADE_VERSION\", \"origin\": \"SECURITY_TRACKER\", \"url\": \"http://www.securitytracker.com/id/1039392\", \"fixResolution\": \"The vendor has issued a fix (7.0.81).\\n\\nThe vendor advisory is available at:\\n\\nhttps://tomcat.apache.org/security-7.html#Fixed_in_Apache_Tomcat_7.0.81\", \"date\": \"2017-12-31\", \"message\": \"Apache Tomcat on Windows HTTP PUT Request Processing Flaw Lets Remote Users Execute Arbitrary Code on the Target System\", \"extraData\": \"key=1039392\" }, { \"vulnerability\": \"CVE-2017-12615\", \"type\": \"CHANGE_FILES\"," + + "\"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat70/commit/07dc0ea2745f0afab6415f22b16a29f1c6de5727\", \"fixResolution\": \"java/org/apache/naming/resources/VirtualDirContext.java,webapps/docs/changelog.xml,java/org/apache/naming/resources/FileDirContext.java\", \"date\": \"2017-08-10\", \"message\": \"Correct regression in r1804604 that broke WebDAV.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc7.0.x/trunk@1804729 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=07dc0ea&committerName=markt-asf&committerUrl=https://github.com/markt-asf&committerAvatar=https://avatars3.githubusercontent.com/u/4690029?v=4\" } ], \"fixResolutionText\": \"The vendor has issued a fix (7.0.81).\\n\\nThe vendor advisory is available at:\\n\\nhttps://tomcat.apache.org/security-7.html#Fixed_in_Apache_Tomcat_7.0.81\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"aa3a10d7-4e2c-46fe-bbf9-3c2d06e43b02\", \"keyId\": 24276785, \"filename\": \"tomcat-embed-core-7.0.78.jar\", \"name\": \"tomcat-embed-core\", \"groupId\": \"org.apache.tomcat.embed\", \"artifactId\": \"tomcat-embed-core\"," + + "\"version\": \"7.0.78\", \"sha1\": \"ddb63d615ec3944b4394aed6dc825cd0cbb16b21\", \"type\": \"Java\", \"references\": { \"url\": \"http://tomcat.apache.org/\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/org/apache/tomcat/embed/tomcat-embed-core/7.0.78/tomcat-embed-core-7.0.78.pom\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": true, \"description\": \"High:3,Medium:2,\", \"date\": \"2017-10-26\" }, { \"vulnerability\": { \"name\": \"CVE-2017-12616\", \"type\": \"CVE\", \"severity\": \"medium\", \"score\": 5, \"cvss3_severity\": \"high\", \"cvss3_score\": 7.5, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N\", \"publishDate\": \"2017-09-19\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-12616\"," + + "\"description\": \"When using a VirtualDirContext with Apache Tomcat 7.0.0 to 7.0.80 it was possible to bypass security constraints and/or view the source code of JSPs for resources served by the VirtualDirContext using a specially crafted request.\", \"topFix\": { \"vulnerability\": \"CVE-2017-12616\", \"type\": \"UPGRADE_VERSION\", \"origin\": \"SECURITY_TRACKER\", \"url\": \"http://www.securitytracker.com/id/1039393\", \"fixResolution\": \"The vendor has issued a fix (7.0.81).\\n\\nThe vendor advisory is available at:\\n\\nhttps://tomcat.apache.org/security-7.html#Fixed_in_Apache_Tomcat_7.0.81\", \"date\": \"2017-12-31\", \"message\": \"Apache Tomcat VirtualDirContext Flaw Lets Remote Users View JSP Source Code for the Affected Resource\", \"extraData\": \"key=1039393\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-12616\", \"type\": \"UPGRADE_VERSION\", \"origin\": \"SECURITY_TRACKER\", \"url\": \"http://www.securitytracker.com/id/1039393\", \"fixResolution\": \"The vendor has issued a fix (7.0.81).\\n\\nThe vendor advisory is available at:\\n\\nhttps://tomcat.apache.org/security-7.html#Fixed_in_Apache_Tomcat_7.0.81\", \"date\": \"2017-12-31\"," + + "\"message\": \"Apache Tomcat VirtualDirContext Flaw Lets Remote Users View JSP Source Code for the Affected Resource\", \"extraData\": \"key=1039393\" }, { \"vulnerability\": \"CVE-2017-12616\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat70/commit/07dc0ea2745f0afab6415f22b16a29f1c6de5727\", \"fixResolution\": \"java/org/apache/naming/resources/VirtualDirContext.java,webapps/docs/changelog.xml,java/org/apache/naming/resources/FileDirContext.java\", \"date\": \"2017-08-10\", \"message\": \"Correct regression in r1804604 that broke WebDAV.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc7.0.x/trunk@1804729 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=07dc0ea&committerName=markt-asf&committerUrl=https://github.com/markt-asf&committerAvatar=https://avatars3.githubusercontent.com/u/4690029?v=4\" } ], \"fixResolutionText\": \"The vendor has issued a fix (7.0.81).\\n\\nThe vendor advisory is available at:\\n\\nhttps://tomcat.apache.org/security-7.html#Fixed_in_Apache_Tomcat_7.0.81\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\"," + + "\"library\": { \"keyUuid\": \"aa3a10d7-4e2c-46fe-bbf9-3c2d06e43b02\", \"keyId\": 24276785, \"filename\": \"tomcat-embed-core-7.0.78.jar\", \"name\": \"tomcat-embed-core\", \"groupId\": \"org.apache.tomcat.embed\", \"artifactId\": \"tomcat-embed-core\", \"version\": \"7.0.78\", \"sha1\": \"ddb63d615ec3944b4394aed6dc825cd0cbb16b21\", \"type\": \"Java\", \"references\": { \"url\": \"http://tomcat.apache.org/\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/org/apache/tomcat/embed/tomcat-embed-core/7.0.78/tomcat-embed-core-7.0.78.pom\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": true, \"description\": \"High:3,Medium:2,\", \"date\": \"2017-10-26\" }, { \"vulnerability\": { \"name\": \"CVE-2017-7674\", \"type\": \"CVE\", \"severity\": \"medium\", \"score\": 4.3," + + "\"cvss3_severity\": \"medium\", \"cvss3_score\": 4.3, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:L/A:N\", \"publishDate\": \"2017-08-11\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-7674\", \"description\": \"The CORS Filter in Apache Tomcat 9.0.0.M1 to 9.0.0.M21, 8.5.0 to 8.5.15, 8.0.0.RC1 to 8.0.44 and 7.0.41 to 7.0.78 did not add an HTTP Vary header indicating that the response varies depending on Origin. This permitted client and server side cache poisoning in some circumstances.\", \"topFix\": { \"vulnerability\": \"CVE-2017-7674\", \"type\": \"UPGRADE_VERSION\", \"origin\": \"BUGZILLA\", \"url\": \"https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2017-7674\", \"fixResolution\": \"tomcat 7.0.79,tomcat 8.0.45,tomcat 8.5.16\", \"message\": \"CVE-2017-7674 tomcat: Vary header not added by CORS filter leading to cache poisoning\", \"extraData\": \"key=1480618&assignee=Red Hat Product Security\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-7674\", \"type\": \"UPGRADE_VERSION\", \"origin\": \"BUGZILLA\", \"url\": \"https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2017-7674\"," + + "\"fixResolution\": \"tomcat 7.0.79,tomcat 8.0.45,tomcat 8.5.16\", \"message\": \"CVE-2017-7674 tomcat: Vary header not added by CORS filter leading to cache poisoning\", \"extraData\": \"key=1480618&assignee=Red Hat Product Security\" }, { \"vulnerability\": \"CVE-2017-7674\", \"type\": \"UPGRADE_VERSION\", \"origin\": \"BUGZILLA\", \"url\": \"https://bugzilla.redhat.com/show_bug.cgi?id=1480618\", \"fixResolution\": \"tomcat 7.0.79,tomcat 8.0.45,tomcat 8.5.16\", \"message\": \"CVE-2017-7674 tomcat: Vary header not added by CORS filter leading to cache poisoning\", \"extraData\": \"key=1480618&assignee=Red Hat Product Security\" }, { \"vulnerability\": \"CVE-2017-7674\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat/commit/b94478d45b7e1fc06134a785571f78772fa30fed\", \"fixResolution\": \"java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml\", \"date\": \"2017-05-22\", \"message\": \"BZ61101: CORS filter should set Vary header in response. Submitted by Rick Riemer.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/trunk@1795813 13f79535-47bb-0310-9956-ffa450edef68\"," + + "\"extraData\": \"key=b94478d&committerName=rmaucher&committerUrl=https://github.com/rmaucher&committerAvatar=https://avatars2.githubusercontent.com/u/324250?v=4\" }, { \"vulnerability\": \"CVE-2017-7674\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat85/commit/9044c1672bbe4b2cf4c55028cc8b977cc62650e7\", \"fixResolution\": \"java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml\", \"date\": \"2017-05-22\", \"message\": \"BZ61101: CORS filter should set Vary header in response. Submitted by Rick Riemer.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc8.5.x/trunk@1795814 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=9044c16&committerName=rmaucher&committerUrl=https://github.com/rmaucher&committerAvatar=https://avatars2.githubusercontent.com/u/324250?v=4\" }, { \"vulnerability\": \"CVE-2017-7674\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat70/commit/52382ebfbce20a98b01cd9d37184a12703987a5a\", \"fixResolution\": \"java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml\"," + + "\"date\": \"2017-05-22\", \"message\": \"BZ61101: CORS filter should set Vary header in response. Submitted by Rick Riemer.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc7.0.x/trunk@1795816 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=52382eb&committerName=rmaucher&committerUrl=https://github.com/rmaucher&committerAvatar=https://avatars2.githubusercontent.com/u/324250?v=4\" }, { \"vulnerability\": \"CVE-2017-7674\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat80/commit/f52c242d92d4563dd1226dcc993ec37370ba9ce3\", \"fixResolution\": \"java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml\", \"date\": \"2017-05-22\", \"message\": \"BZ61101: CORS filter should set Vary header in response. Submitted by Rick Riemer.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc8.0.x/trunk@1795815 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=f52c242&committerName=rmaucher&committerUrl=https://github.com/rmaucher&committerAvatar=https://avatars2.githubusercontent.com/u/324250?v=4\" } ], \"fixResolutionText\": \"Upgrade to version tomcat 7.0.79, tomcat 8.0.45, tomcat 8.5.16 or greater\"," + + "\"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"aa3a10d7-4e2c-46fe-bbf9-3c2d06e43b02\", \"keyId\": 24276785, \"filename\": \"tomcat-embed-core-7.0.78.jar\", \"name\": \"tomcat-embed-core\", \"groupId\": \"org.apache.tomcat.embed\", \"artifactId\": \"tomcat-embed-core\", \"version\": \"7.0.78\", \"sha1\": \"ddb63d615ec3944b4394aed6dc825cd0cbb16b21\", \"type\": \"Java\", \"references\": { \"url\": \"http://tomcat.apache.org/\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/org/apache/tomcat/embed/tomcat-embed-core/7.0.78/tomcat-embed-core-7.0.78.pom\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": true, \"description\": \"High:3,Medium:2,\", \"date\": \"2017-10-26\" }, { \"vulnerability\": { \"name\": \"CVE-2018-8014\"," + + "\"type\": \"CVE\", \"severity\": \"high\", \"score\": 7.5, \"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-05-16\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-8014\", \"description\": \"The defaults settings for the CORS filter provided in Apache Tomcat 9.0.0.M1 to 9.0.8, 8.5.0 to 8.5.31, 8.0.0.RC1 to 8.0.52, 7.0.41 to 7.0.88 are insecure and enable 'supportsCredentials' for all origins. It is expected that users of the CORS filter will have configured it appropriately for their environment rather than using it in the default configuration. Therefore, it is expected that most users will not be impacted by this issue.\", \"topFix\": { \"vulnerability\": \"CVE-2018-8014\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat70/commit/5877390a9605f56d9bd6859a54ccbfb16374a78b\", \"fixResolution\": \"java/org/apache/catalina/filters/LocalStrings.properties,test/org/apache/catalina/filters/TestCorsFilter.java,java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml,test/org/apache/catalina/filters/TesterFilterConfigs.java\"," + + "\"date\": \"2018-05-16\", \"message\": \"Fix https://bz.apache.org/bugzilla/show_bug.cgi?id=62343\\nMake CORS filter defaults more secure.\\nThis is the fix for CVE-2018-8014.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc7.0.x/trunk@1831730 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=5877390&committerName=markt-asf&committerUrl=https://github.com/markt-asf&committerAvatar=https://avatars3.githubusercontent.com/u/4690029?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2018-8014\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat70/commit/5877390a9605f56d9bd6859a54ccbfb16374a78b\", \"fixResolution\": \"java/org/apache/catalina/filters/LocalStrings.properties,test/org/apache/catalina/filters/TestCorsFilter.java,java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml,test/org/apache/catalina/filters/TesterFilterConfigs.java\", \"date\": \"2018-05-16\", \"message\": \"Fix https://bz.apache.org/bugzilla/show_bug.cgi?id=62343\\nMake CORS filter defaults more secure.\\nThis is the fix for CVE-2018-8014.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc7.0.x/trunk@1831730 13f79535-47bb-0310-9956-ffa450edef68\"," + + "\"extraData\": \"key=5877390&committerName=markt-asf&committerUrl=https://github.com/markt-asf&committerAvatar=https://avatars3.githubusercontent.com/u/4690029?v=4\" }, { \"vulnerability\": \"CVE-2018-8014\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat80/commit/2c9d8433bd3247a2856d4b2555447108758e813e#diff-32f241c95d21b1b224601e52f83af334\", \"fixResolution\": \"java/org/apache/catalina/filters/LocalStrings.properties,test/org/apache/catalina/filters/TestCorsFilter.java,java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml,test/org/apache/catalina/filters/TesterFilterConfigs.java\", \"date\": \"2018-05-16\", \"message\": \"Fix https://bz.apache.org/bugzilla/show_bug.cgi?id=62343\\nMake CORS filter defaults more secure.\\nThis is the fix for CVE-2018-8014.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc8.0.x/trunk@1831729 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=2c9d843&committerName=markt-asf&committerUrl=https://github.com/markt-asf&committerAvatar=https://avatars3.githubusercontent.com/u/4690029?v=4\" }, { \"vulnerability\": \"CVE-2018-8014\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\"," + + "\"url\": \"https://github.com/apache/tomcat/commit/d83a76732e6804739b81d8b2056365307637b42d\", \"fixResolution\": \"java/org/apache/catalina/filters/LocalStrings.properties,test/org/apache/catalina/filters/TestCorsFilter.java,java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml,test/org/apache/catalina/filters/TesterFilterConfigs.java\", \"date\": \"2018-05-16\", \"message\": \"Fix https://bz.apache.org/bugzilla/show_bug.cgi?id=62343\\nMake CORS filter defaults more secure.\\nThis is the fix for CVE-2018-8014.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/trunk@1831726 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=d83a767&committerName=markt-asf&committerUrl=https://github.com/markt-asf&committerAvatar=https://avatars3.githubusercontent.com/u/4690029?v=4\" }, { \"vulnerability\": \"CVE-2018-8014\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat85/commit/60f596a21fd6041335a3a1a4015d4512439cecb5\", \"fixResolution\": \"java/org/apache/catalina/filters/LocalStrings.properties,test/org/apache/catalina/filters/TestCorsFilter.java,java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml,test/org/apache/catalina/filters/TesterFilterConfigs.java\"," + + "\"date\": \"2018-05-16\", \"message\": \"Fix https://bz.apache.org/bugzilla/show_bug.cgi?id=62343\\nMake CORS filter defaults more secure.\\nThis is the fix for CVE-2018-8014.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc8.5.x/trunk@1831728 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=60f596a&committerName=markt-asf&committerUrl=https://github.com/markt-asf&committerAvatar=https://avatars3.githubusercontent.com/u/4690029?v=4\" } ], \"fixResolutionText\": \"Replace or update the following files: LocalStrings.properties, TestCorsFilter.java, CorsFilter.java, changelog.xml, TesterFilterConfigs.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"aa3a10d7-4e2c-46fe-bbf9-3c2d06e43b02\", \"keyId\": 24276785, \"filename\": \"tomcat-embed-core-7.0.78.jar\", \"name\": \"tomcat-embed-core\", \"groupId\": \"org.apache.tomcat.embed\", \"artifactId\": \"tomcat-embed-core\", \"version\": \"7.0.78\", \"sha1\": \"ddb63d615ec3944b4394aed6dc825cd0cbb16b21\", \"type\": \"Java\", \"references\": { \"url\": \"http://tomcat.apache.org/\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/org/apache/tomcat/embed/tomcat-embed-core/7.0.78/tomcat-embed-core-7.0.78.pom\" }," + + "\"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": true, \"description\": \"High:3,Medium:2,\", \"date\": \"2017-10-26\" } ] }").alerts + }) + + thrown.expect(AbortException.class) + + stepRule.step.whitesourceExecuteScan([ + script : nullScript, + scanType : 'npm', + juStabUtils : utils, + securityVulnerabilities : true, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + orgToken : 'testOrgToken', + productName : 'SHC - Piper', + projectNames : [ 'piper-demo - 0.0.1' ] + ]) + } + + @Test + void testCheckFindingBelowSeven() { + helper.registerAllowedMethod("readProperties", [Map], { + def result = new Properties() + result.putAll([ + "apiKey": "b39d1328-52e2-42e3-98f0-932709daf3f0", + "productName": "SHC - Piper", + "checkPolicies": "true", + "projectName": "pipeline-test-node", + "projectVersion": "1.0.0" + ]) + return result + }) + helper.registerAllowedMethod("fetchVulnerabilities", [Object.class], { + return new JsonUtils().parseJsonSerializable("{ \"alerts\": [ { \"vulnerability\": { \"name\": \"CVE-2017-15095\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 2.1, \"cvss3_severity\": \"high\", \"cvss3_score\": 5.3, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15095\", \"description\": \"A deserialization flaw was discovered in the jackson-databind in versions before 2.8.10 and 2.9.1, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper. This issue extends the previous flaw CVE-2017-7525 by blacklisting more classes that could be used maliciously.\", \"topFix\": { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\"," + + "\"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\", \"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\"," + + "\"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e865a7a4464da63ded9f4b1a2328ad85c9ded78b#diff-98084d808198119d550a9211e128a16f\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-12-12\", \"message\": \"Fix #1737 (#1857)\", \"extraData\": \"key=e865a7a&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\"," + + "\"url\": \"https://github.com/FasterXML/jackson-databind/commit/e8f043d1\", \"fixResolution\": \"release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-06-30\", \"message\": \"Fix #1680\", \"extraData\": \"key=e8f043d&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" } ], \"fixResolutionText\": \"Replace or update the following files: IllegalTypesCheckTest.java, VERSION, BeanDeserializerFactory.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\"," + + "\"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-databind\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:5,\", \"date\": \"2017-11-15\" } ] }").alerts + }) + + stepRule.step.whitesourceExecuteScan([ + script : nullScript, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + scanType : 'npm', + juStabUtils : utils, + securityVulnerabilities : true, + orgToken : 'testOrgToken', + productName : 'SHC - Piper', + projectNames : [ 'piper-demo - 0.0.1' ] + ]) + + assertThat(loggingRule.log, containsString('WARNING: 1 Open Source Software Security vulnerabilities with CVSS score below 7.0 detected.')) + assertThat(writeFileRule.files['piper_whitesource_vulnerability_report.json'], not(isEmptyOrNullString())) + } + + @Test + void testCheckNoFindings() { + helper.registerAllowedMethod("readProperties", [Map], { + def result = new Properties() + result.putAll([ + "apiKey": "b39d1328-52e2-42e3-98f0-932709daf3f0", + "productName": "SHC - Piper", + "checkPolicies": "true", + "projectName": "pipeline-test-node", + "projectVersion": "1.0.0" + ]) + return result + }) + helper.registerAllowedMethod("fetchVulnerabilities", [Object.class], { + return new JsonUtils().parseJsonSerializable("{ \"alerts\": [] }").alerts + }) + + stepRule.step.whitesourceExecuteScan([ + script : nullScript, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + scanType : 'npm', + juStabUtils : utils, + securityVulnerabilities : true, + orgToken : 'testOrgToken', + productName : 'SHC - Piper', + projectNames : [ 'piper-demo - 0.0.1' ] + ]) + + assertThat(loggingRule.log, containsString('No Open Source Software Security vulnerabilities detected.')) + assertThat(writeFileRule.files['piper_whitesource_vulnerability_report.json'], not(isEmptyOrNullString())) + } +} diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy new file mode 100644 index 000000000..48b2c1eb1 --- /dev/null +++ b/vars/whitesourceExecuteScan.groovy @@ -0,0 +1,347 @@ +import com.sap.piper.JsonUtils +import com.sap.piper.Utils +import com.sap.piper.integration.WhitesourceOrgAdminRepository +import com.sap.piper.integration.WhitesourceRepository +import com.sap.piper.ConfigurationHelper +import com.sap.piper.WhitesourceConfigurationHelper +import com.sap.piper.mta.MtaMultiplexer +import groovy.text.GStringTemplateEngine +import groovy.transform.Field +import groovy.text.SimpleTemplateEngine + +import static com.sap.piper.Prerequisites.checkScript + +@Field String STEP_NAME = 'whitesourceExecuteScan' +@Field Set GENERAL_CONFIG_KEYS = [ + 'agentDownloadUrl', + 'agentFileName', + 'agentParameters', + 'artifactUrl', + 'buildDescriptorExcludeList', + 'buildDescriptorFile', + 'configFilePath', + 'dockerImage', + 'dockerWorkspace', + 'internalServiceUrl', + 'jreDownloadUrl', + 'licensingVulnerabilities', + 'orgToken', + 'parallelLimit', + 'reporting', + 'scanType', + 'securityVulnerabilities', + 'stashContent', + 'timeout', + 'productName', + 'productVersion', + 'productToken', + 'projectNames', + 'serviceUrl', + 'userTokenCredentialsId', + 'vulnerabilityReportFileName', + 'vulnerabilityReportTitle', + 'whitesourceAccessor', + 'verbose' +] +@Field Set STEP_CONFIG_KEYS = GENERAL_CONFIG_KEYS +@Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS + +void call(Map parameters = [:]) { + handlePipelineStepErrors(stepName: STEP_NAME, stepParameters: parameters) { + def script = checkScript(this, parameters) ?: this + def utils = parameters.juStabUtils ?: new Utils() + def statusCode = 1 + + // load default & individual configuration + Map config = ConfigurationHelper.newInstance(this) + .loadStepDefaults() + .mixinGeneralConfig(script.commonPipelineEnvironment, GENERAL_CONFIG_KEYS) + .mixinStepConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS) + .mixinStageConfig(script.commonPipelineEnvironment, parameters.stageName ?: env.STAGE_NAME, STEP_CONFIG_KEYS) + .mixin(parameters, PARAMETER_KEYS) + .dependingOn('scanType').mixin('buildDescriptorFile') + .dependingOn('scanType').mixin('configFilePath') + .dependingOn('scanType').mixin('dockerImage') + .dependingOn('scanType').mixin('dockerWorkspace') + .dependingOn('scanType').mixin('stashContent') + .dependingOn('scanType').mixin('agentDownloadUrl') + .dependingOn('scanType').mixin('agentFileName') + .dependingOn('scanType').mixin('agentParameters') + .withMandatoryProperty('userTokenCredentialsId') + .withMandatoryProperty('productName') + .use() + + config.stashContent = utils.unstashAll(config.stashContent) + config.projectNames = (config.projectNames instanceof List) ? config.projectNames : config.projectNames?.tokenize(',') + parameters.projectNames = config.projectNames + + script.commonPipelineEnvironment.setInfluxStepData('whitesource', false) + + utils.pushToSWA([ + step: STEP_NAME, + stepParamKey1: 'scanType', + stepParam1: config.scanType + ], config) + + echo "Parameters: scanType: ${config.scanType}" + + def whitesourceRepository = parameters.whitesourceRepositoryStub ?: new WhitesourceRepository(this, config) + def whitesourceOrgAdminRepository = parameters.whitesourceOrgAdminRepositoryStub ?: new WhitesourceOrgAdminRepository(this, config) + + statusCode = triggerWhitesourceScanWithUserKey(script, config, utils, parameters, whitesourceRepository, whitesourceOrgAdminRepository) + + checkStatus(statusCode, config) + + script.commonPipelineEnvironment.setInfluxStepData('whitesource', true) + } +} + +private def triggerWhitesourceScanWithUserKey(script, config, utils, parameters, repository, orgAdminRepository) { + withCredentials ([string( + credentialsId: config.userTokenCredentialsId, + variable: 'userKey' + )]) { + config.userKey = userKey + def statusCode = 1 + echo "Triggering Whitesource scan on product '${config.productName}' with token '${config.productToken}' using credentials with ID '${config.userTokenCredentialsId}'" + switch (config.scanType) { + case 'mta': + def scanJobs = [:] + def mtaParameters = [:] + parameters + [reporting: false] + // harmonize buildDescriptorExcludeList + config.buildDescriptorExcludeList = config.buildDescriptorExcludeList instanceof List ? config.buildDescriptorExcludeList : config.buildDescriptorExcludeList?.replaceAll(', ', ',').replaceAll(' ,', ',').tokenize(',') + // create job for each pom.xml with scanType: 'maven' + scanJobs.putAll(MtaMultiplexer.createJobs( + this, mtaParameters, config.buildDescriptorExcludeList, 'Whitesource', 'pom.xml', 'maven' + ) { options -> return whitesourceExecuteScan(options) }) + // create job for each pom.xml with scanType: 'maven' + scanJobs.putAll(MtaMultiplexer.createJobs( + this, mtaParameters, config.buildDescriptorExcludeList, 'Whitesource', 'package.json', 'npm' + ) { options -> whitesourceExecuteScan(options) }) + // create job for each setup.py with scanType: 'pip' + scanJobs.putAll(MtaMultiplexer.createJobs( + this, mtaParameters, config.buildDescriptorExcludeList, 'Whitesource', 'setup.py', 'pip' + ) { options -> whitesourceExecuteScan(options) }) + // execute scan jobs + if (config.parallelLimit > 0 && config.parallelLimit < scanJobs.keySet().size()) { + // block wise + def scanJobsAll = scanJobs + scanJobs = [failFast: false] + for (int i = 1; i <= scanJobsAll.keySet().size(); i++) { + def index = i - 1 + def key = scanJobsAll.keySet()[index] + scanJobs[key] = scanJobsAll[key] + if (i % config.parallelLimit == 0 || i == scanJobsAll.keySet().size()) { + parallel scanJobs + scanJobs = [failFast: false] + } + } + } else { + // in parallel + scanJobs += [failFast: false] + parallel scanJobs + } + statusCode = 0 + break + default: + def path = config.buildDescriptorFile.substring(0, config.buildDescriptorFile.lastIndexOf('/') + 1) + def gav + switch (config.scanType) { + case 'npm': + gav = utils.getNpmGAV(config.buildDescriptorFile) + config.projectName = gav.group + "." + gav.artifact + config.productVersion = gav.version + break + case 'sbt': + gav = utils.getSbtGAV(config.buildDescriptorFile) + config.projectName = gav.group + "." + gav.artifact + config.productVersion = gav.version + break + case 'pip': + gav = utils.getPipGAV(config.buildDescriptorFile) + config.projectName = gav.artifact + config.productVersion = gav.version + break + default: + gav = utils.readMavenGAV(config.buildDescriptorFile) + config.projectName = gav.group + "." + gav.artifact + config.productVersion = gav.version + break + } + config.projectNames.add("${config.projectName} - ${config.productVersion}".toString()) + WhitesourceConfigurationHelper.extendUAConfigurationFile(script, utils, config, path) + dockerExecute(script: script, dockerImage: config.dockerImage, dockerWorkspace: config.dockerWorkspace, stashContent: config.stashContent) { + if (config.agentDownloadUrl) { + def agentDownloadUrl = new GStringTemplateEngine().createTemplate(config.agentDownloadUrl).make([config: config]).toString() + //if agentDownloadUrl empty, rely on dockerImage to contain unifiedAgent correctly set up and available + sh "curl ${script.env.HTTP_PROXY ? '--proxy ' + script.env.HTTP_PROXY + ' ' : ''}--location --output ${config.agentFileName} ${agentDownloadUrl}".toString() + } + + def javaCmd = 'java' + if (config.jreDownloadUrl) { + //if jreDownloadUrl empty, rely on dockerImage to contain java correctly set up and available on the path + sh "curl ${script.env.HTTP_PROXY ? '--proxy ' + script.env.HTTP_PROXY + ' ' : ''}--location --output jvm.tar.gz ${config.jreDownloadUrl} && tar --strip-components=1 -xzf jvm.tar.gz".toString() + javaCmd = './bin/java' + } + + def options = ["-jar ${config.agentFileName} -c \'${config.configFilePath}\'"] + if (config.orgToken) options.push("-apiKey '${config.orgToken}'") + if (config.userKey) options.push("-userKey '${config.userKey}'") + if (config.productName) options.push("-product '${config.productName}'") + + statusCode = sh(script: "${javaCmd} ${options.join(' ')} ${config.agentParameters}", returnStatus: true) + + // archive whitesource result files + archiveArtifacts artifacts: "whitesource/*.*", allowEmptyArchive: true + } + break + } + + if (config.reporting) { + analyseWhitesourceResults(utils, config, repository, orgAdminRepository) + } + + return statusCode + } +} + +void analyseWhitesourceResults(Utils utils, Map config, WhitesourceRepository repository, WhitesourceOrgAdminRepository orgAdminRepository) { + if (!config.productToken) { + def metaInfo = orgAdminRepository.fetchProductMetaInfo() + echo "Meta Information: $metaInfo" + config.productToken = metaInfo.token + } + + repository.fetchReportForProduct() + def pdfName = "whitesource-riskReport.pdf" + archiveArtifacts artifacts: pdfName + echo "A summary of the Whitesource findings was stored as artifact under the name $pdfName" + + int violationCount = fetchViolationCount(config, repository) + checkViolationStatus(violationCount) + + if (config.securityVulnerabilities) + config.severeVulnerabilities = checkSecurityViolations(config, repository) +} + +int fetchViolationCount(Map config, WhitesourceRepository repository) { + int violationCount = 0 + if (config.projectNames) { + def projectsMeta = repository.fetchProjectsMetaInfo() + for (int i = 0; i < projectsMeta.size(); i++) { + def project = projectsMeta[i] + def responseAlertsProject = repository.fetchProjectLicenseAlerts(project.token) + violationCount += responseAlertsProject.alerts.size() + } + } else { + def responseAlerts = repository.fetchProductLicenseAlerts() + violationCount += responseAlerts.alerts.size() + } + return violationCount +} + +void checkViolationStatus(int violationCount) { + if (violationCount == 0) { + echo "****\r\n[${STEP_NAME}] No policy violations found. You can deploy to production, and set the \"Intellectual Property (IP) Scan Plan\" in Sirius to completed. \r\n****" + } else { + error "[${STEP_NAME}] Whitesource found $violationCount policy violations for your product" + } +} + +int checkSecurityViolations(Map config, WhitesourceRepository repository) { + def whitesourceProjectsMetaInformation = repository.fetchProjectsMetaInfo() + def whitesourceVulnerabilities = repository.fetchVulnerabilities(whitesourceProjectsMetaInformation) + def severeVulnerabilities = 0 + whitesourceVulnerabilities.each { + item -> + if (item.vulnerability.score >= 7 || item.vulnerability.cvss3_score >= 7) + severeVulnerabilities++ + } + + writeFile(file: "${config.vulnerabilityReportFileName}.json", text: new JsonUtils().getPrettyJsonString(whitesourceVulnerabilities)) + writeFile(file: "${config.vulnerabilityReportFileName}.html", text: getReportHtml(config, whitesourceVulnerabilities, severeVulnerabilities)) + archiveArtifacts(artifacts: "${config.vulnerabilityReportFileName}.*") + + if (whitesourceVulnerabilities.size() - severeVulnerabilities > 0) + echo "[${STEP_NAME}] WARNING: ${whitesourceVulnerabilities.size() - severeVulnerabilities} Open Source Software Security vulnerabilities with CVSS score below 7.0 detected." + if (whitesourceVulnerabilities.size() == 0) + echo "[${STEP_NAME}] No Open Source Software Security vulnerabilities detected." + + return severeVulnerabilities +} + +// ExitCodes: https://whitesource.atlassian.net/wiki/spaces/WD/pages/34209870/NPM+Plugin#NPMPlugin-ExitCode +void checkStatus(int statusCode, config) { + def errorMessage = "" + if(config.securityVulnerabilities && config.severeVulnerabilities > 0) + errorMessage += "${config.severeVulnerabilities} Open Source Software Security vulnerabilities with CVSS score greater or equal 7.0 detected. - " + if (config.licensingVulnerabilities) + switch (statusCode) { + case 0: + break + case 255: + errorMessage += "The scan resulted in an error" + break + case 254: + errorMessage += "Whitesource found one or multiple policy violations" + break + case 253: + errorMessage += "The local scan client failed to execute the scan" + break + case 252: + errorMessage += "There was a failure in the connection to the WhiteSource servers" + break + case 251: + errorMessage += "The server failed to analyze the scan" + break + default: + errorMessage += "Whitesource scan failed with unknown error code '${statusCode}'" + } + + if (errorMessage) + error "[${STEP_NAME}] " + errorMessage +} + +def getReportHtml(config, vulnerabilityList, numSevereVulns) { + def now = new Date().format('MMM dd, yyyy - HH:mm:ss') + def vulnerabilityTable = '' + if (vulnerabilityList.size() == 0) { + vulnerabilityTable += ''' + + No publicly known vulnerabilities detected + ''' + } else { + for (int i = 0; i < vulnerabilityList.size(); i++) { + def item = vulnerabilityList[i] + def score = item.vulnerability.cvss3_score > 0 ? item.vulnerability.cvss3_score : item.vulnerability.score + def topFix = item.vulnerability.topFix ? "${item.vulnerability.topFix?.message}
${item.vulnerability.topFix?.fixResolution}
${item.vulnerability.topFix?.url}}" : '' + vulnerabilityTable += """ + + ${i + 1} + ${item.date} + ${item.vulnerability.name} + ${score} + ${item.vulnerability.cvss3_score > 0 ? 'v3' : 'v2'} + ${item.project} + ${item.library.filename} + ${item.library.groupId} + ${item.library.artifactId} + ${item.library.version} + ${item.vulnerability.description} + ${topFix} + """ + } + } + + return SimpleTemplateEngine.newInstance().createTemplate(libraryResource('com.sap.piper/templates/whitesourceVulnerabilities.html')).make( + [ + now : now, + reportTitle : config.vulnerabilityReportTitle, + style : config.style, + totalSevereVulnerabilities : numSevereVulns, + totalVulnerabilities : vulnerabilityList.size(), + vulnerabilityTable : vulnerabilityTable, + whitesourceProductName : config.whitesourceProductName, + whitesourceProjectNames : config.whitesourceProjectNames + ]).toString() +} From 2094d95fcc0c116e1d635b5d6cc21144bc8cf44b Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Fri, 1 Mar 2019 15:35:08 +0100 Subject: [PATCH 002/130] fix indentation --- .../templates/whitesourceVulnerabilities.html | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/resources/com.sap.piper/templates/whitesourceVulnerabilities.html b/resources/com.sap.piper/templates/whitesourceVulnerabilities.html index 209b559ea..a5fea28e9 100644 --- a/resources/com.sap.piper/templates/whitesourceVulnerabilities.html +++ b/resources/com.sap.piper/templates/whitesourceVulnerabilities.html @@ -8,30 +8,30 @@

${reportTitle}

- WhiteSource product name: ${whitesourceProductName}
- Filtered project names: ${whitesourceProjectNames?:''} + WhiteSource product name: ${whitesourceProductName}
+ Filtered project names: ${whitesourceProjectNames?:''}

total number of vulnerabilities: ${totalVulnerabilities}
total number of high/critical vulnerabilities with CVSS score >= 7.0: ${totalSevereVulnerabilities} -

+

Snapshot taken:${now} GMT

- - - - - - - - - - - - + + + + + + + + + + + + ${vulnerabilityTable} From 29a54db7c8a09684fd504adc4f52253ea7a3604d Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 10:32:47 +0100 Subject: [PATCH 003/130] Addressing PR feedback --- resources/default_pipeline_environment.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 6f6080e3c..52ae4a47e 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -256,15 +256,14 @@ steps: dockerImage: 'node:8-stretch' whitesourceExecuteScan: buildDescriptorExcludeList: [] - parallelLimit: 0 + parallelLimit: 15 licensingVulnerabilities: true - securityVulnerabilities: false + securityVulnerabilities: true reporting: true vulnerabilityReportFileName: 'piper_whitesource_vulnerability_report' - vulnerabilityReportTitle: 'Whitesource Security Vulnerability Report' + vulnerabilityReportTitle: 'WhiteSource Security Vulnerability Report' projectNames: [] jreDownloadUrl: 'https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz' - whitesourceAccessor: 'com.sap.piper.internal.integration.InternalWhitesource' mta: stashContent: - 'buildDescriptor' From 617256cc09c3b5a3c835109b5b028a77cfcefe52 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 10:40:58 +0100 Subject: [PATCH 004/130] Switching to official docker images --- resources/default_pipeline_environment.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 52ae4a47e..dd1bf8883 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -270,7 +270,7 @@ steps: - 'opensourceConfiguration' maven: buildDescriptorFile: './pom.xml' - dockerImage: 'docker.wdf.sap.corp:50000/piper/maven' + dockerImage: 'maven:3.5-jdk-7' dockerWorkspace: '/home/piper' agentFileName: 'wss-unified-agent.jar' agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' @@ -281,7 +281,7 @@ steps: - 'opensourceConfiguration' npm: buildDescriptorFile: './package.json' - dockerImage: 'docker.wdf.sap.corp:50000/piper/node' + dockerImage: 'node:11.10.1-stretch-slim' dockerWorkspace: '/home/piper' agentFileName: 'wss-unified-agent.jar' agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' @@ -292,7 +292,7 @@ steps: - 'opensourceConfiguration' pip: buildDescriptorFile: './setup.py' - dockerImage: 'docker.wdf.sap.corp:50000/piper/node' + dockerImage: 'python:3.7.2-slim-stretch' dockerWorkspace: '/home/piper' agentFileName: 'wss-unified-agent.jar' agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' @@ -303,7 +303,7 @@ steps: - 'opensourceConfiguration' sbt: buildDescriptorFile: './build.sbt' - dockerImage: 'docker.wdf.sap.corp:50000/piper/sbt' + dockerImage: 'hseeberger/scala-sbt:8u181_2.12.8_1.2.8' dockerWorkspace: '/home/piper' agentFileName: 'wss-unified-agent.jar' agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' From 2c00d8cdd05501343df9518524670e019f0b778d Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 10:57:06 +0100 Subject: [PATCH 005/130] Switching to official docker images + fixing tests --- resources/default_pipeline_environment.yml | 10 ++++---- .../WhitesourceOrgAdminRepository.groovy | 1 - .../integration/WhitesourceRepository.groovy | 1 - test/groovy/WhitesourceExecuteScanTest.groovy | 25 +++++++++++-------- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index dd1bf8883..73981bcaa 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -271,7 +271,7 @@ steps: maven: buildDescriptorFile: './pom.xml' dockerImage: 'maven:3.5-jdk-7' - dockerWorkspace: '/home/piper' + dockerWorkspace: '/home/java' agentFileName: 'wss-unified-agent.jar' agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' agentParameters: '' @@ -281,8 +281,8 @@ steps: - 'opensourceConfiguration' npm: buildDescriptorFile: './package.json' - dockerImage: 'node:11.10.1-stretch-slim' - dockerWorkspace: '/home/piper' + dockerImage: 'node:8-stretch' + dockerWorkspace: '/home/node' agentFileName: 'wss-unified-agent.jar' agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' agentParameters: '' @@ -293,7 +293,7 @@ steps: pip: buildDescriptorFile: './setup.py' dockerImage: 'python:3.7.2-slim-stretch' - dockerWorkspace: '/home/piper' + dockerWorkspace: '/home/python' agentFileName: 'wss-unified-agent.jar' agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' agentParameters: '' @@ -304,7 +304,7 @@ steps: sbt: buildDescriptorFile: './build.sbt' dockerImage: 'hseeberger/scala-sbt:8u181_2.12.8_1.2.8' - dockerWorkspace: '/home/piper' + dockerWorkspace: '/home/scala' agentFileName: 'wss-unified-agent.jar' agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' agentParameters: '' diff --git a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy index ab3efc055..25b179307 100644 --- a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy @@ -2,7 +2,6 @@ package com.sap.piper.integration import com.cloudbees.groovy.cps.NonCPS import com.sap.piper.JsonUtils -import com.sap.piper.Utils class WhitesourceOrgAdminRepository implements Serializable { diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy index 8820cfc76..66dafc059 100644 --- a/src/com/sap/piper/integration/WhitesourceRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -2,7 +2,6 @@ package com.sap.piper.integration import com.cloudbees.groovy.cps.NonCPS import com.sap.piper.JsonUtils -import com.sap.piper.Utils class WhitesourceRepository implements Serializable { diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index f125e2b13..87f0b358c 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -115,6 +115,9 @@ class WhitesourceExecuteScanTest extends BasePiperTest { helper.registerAllowedMethod( "fetchProductLicenseAlerts", [], { return new JsonUtils().parseJsonSerializable("{ \"alerts\": [] }").alerts }) + helper.registerAllowedMethod( "fetchVulnerabilities", [List], { + return new JsonUtils().parseJsonSerializable("{ \"alerts\": [] }").alerts + }) helper.registerAllowedMethod( "publishHTML", [Map], {}) helper.registerAllowedMethod( "getNpmGAV", [String], {return [group: 'com.sap.node', artifact: 'test-node', version: '1.2.3']}) @@ -163,8 +166,8 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'docker.wdf.sap.corp:50000/piper/maven')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'maven:3.5-jdk-7')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/java')) assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config 420a1bc5c82f57e80307205d8625304f'])) assertThat(shellRule.shell, Matchers.hasItems( @@ -207,8 +210,8 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'docker.wdf.sap.corp:50000/piper/node')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'node:8-stretch')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/node')) assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config 420a1bc5c82f57e80307205d8625304f'])) assertThat(shellRule.shell, Matchers.hasItems( is('curl --location --output wss-unified-agent.jar https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/wss-unified-agent.jar'), @@ -283,8 +286,8 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'docker.wdf.sap.corp:50000/piper/node')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'python:3.7.2-slim-stretch')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/python')) assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config 420a1bc5c82f57e80307205d8625304f'])) assertThat(shellRule.shell, Matchers.hasItems( @@ -328,8 +331,8 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'docker.wdf.sap.corp:50000/piper/sbt')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/piper')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'hseeberger/scala-sbt:8u181_2.12.8_1.2.8')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/scala')) assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config 420a1bc5c82f57e80307205d8625304f'])) assertThat(shellRule.shell, Matchers.hasItems( @@ -363,7 +366,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, scanType : 'maven', agentDownloadUrl : '', - jreDownloadUrl : '', + jreDownloadUrl : '', agentParameters : 'testParams', juStabUtils : utils, orgToken : 'testOrgToken', @@ -669,7 +672,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { ]) return result }) - helper.registerAllowedMethod("fetchVulnerabilities", [Object.class], { + helper.registerAllowedMethod("fetchVulnerabilities", [List], { return new JsonUtils().parseJsonSerializable("{ \"alerts\": [ { \"vulnerability\": { \"name\": \"CVE-2017-15095\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 7.5, \"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15095\", \"description\": \"A deserialization flaw was discovered in the jackson-databind in versions before 2.8.10 and 2.9.1, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper. This issue extends the previous flaw CVE-2017-7525 by blacklisting more classes that could be used maliciously.\", \"topFix\": { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\"," + "\"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\", \"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\"," + "\"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e865a7a4464da63ded9f4b1a2328ad85c9ded78b#diff-98084d808198119d550a9211e128a16f\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-12-12\", \"message\": \"Fix #1737 (#1857)\", \"extraData\": \"key=e865a7a&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e8f043d1\"," + @@ -749,7 +752,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { ]) return result }) - helper.registerAllowedMethod("fetchVulnerabilities", [Object.class], { + helper.registerAllowedMethod("fetchVulnerabilities", [List], { return new JsonUtils().parseJsonSerializable("{ \"alerts\": [ { \"vulnerability\": { \"name\": \"CVE-2017-15095\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 2.1, \"cvss3_severity\": \"high\", \"cvss3_score\": 5.3, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15095\", \"description\": \"A deserialization flaw was discovered in the jackson-databind in versions before 2.8.10 and 2.9.1, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper. This issue extends the previous flaw CVE-2017-7525 by blacklisting more classes that could be used maliciously.\", \"topFix\": { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\"," + "\"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\", \"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\"," + "\"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e865a7a4464da63ded9f4b1a2328ad85c9ded78b#diff-98084d808198119d550a9211e128a16f\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-12-12\", \"message\": \"Fix #1737 (#1857)\", \"extraData\": \"key=e865a7a&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\"," + From fc9adfd536355777a1385f199df784d028acf53d Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 11:51:43 +0100 Subject: [PATCH 006/130] Added stylesheet for report rendering --- resources/piper.css | 60 ++++++++++++++++++++++++++++++ vars/whitesourceExecuteScan.groovy | 7 +++- 2 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 resources/piper.css diff --git a/resources/piper.css b/resources/piper.css new file mode 100644 index 000000000..ee5ed56b9 --- /dev/null +++ b/resources/piper.css @@ -0,0 +1,60 @@ +body { + font-family: Arial, Verdana; +} +table { + border-collapse: collapse; +} +div.code { + font-family: "Courier New", "Lucida Console"; +} +th { + border-top: 1px solid #ddd; +} +th, td { + padding: 12px; + text-align: left; + border-bottom: 1px solid #ddd; + border-right: 1px solid #ddd; +} +tr:nth-child(even) { + background-color: #f2f2f2; +} +.bold { + font-weight: bold; +} +.nobullets { + list-style-type:none; + padding-left: 0; + padding-bottom: 0; + margin: 0; +} +.notok { + background-color: #ffe5e5; + padding: 5px +} +.warn { + background-color: #ffff99; + padding: 5px +} +.ok { + background-color: #e1f5a9; + padding: 5px +} +.green{ + color: olivedrab; +} +.red{ + color: orangered; +} +.risk-yellow{ + padding: 5px; + color: rgba(255, 255, 0, 0.6); +} +.risk-grey{ + background-color: rgba(212, 212, 212, 0.7); + padding: 5px; +} +.risk-black{ + background-color: rgba(0, 0, 0, 0.75); + padding: 5px; +} diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 48b2c1eb1..4429cf704 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -58,6 +58,9 @@ void call(Map parameters = [:]) { .mixinGeneralConfig(script.commonPipelineEnvironment, GENERAL_CONFIG_KEYS) .mixinStepConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS) .mixinStageConfig(script.commonPipelineEnvironment, parameters.stageName ?: env.STAGE_NAME, STEP_CONFIG_KEYS) + .mixin([ + style : libraryResource('piper.css') + ]) .mixin(parameters, PARAMETER_KEYS) .dependingOn('scanType').mixin('buildDescriptorFile') .dependingOn('scanType').mixin('configFilePath') @@ -341,7 +344,7 @@ def getReportHtml(config, vulnerabilityList, numSevereVulns) { totalSevereVulnerabilities : numSevereVulns, totalVulnerabilities : vulnerabilityList.size(), vulnerabilityTable : vulnerabilityTable, - whitesourceProductName : config.whitesourceProductName, - whitesourceProjectNames : config.whitesourceProjectNames + whitesourceProductName : config.productName, + whitesourceProjectNames : config.projectNames ]).toString() } From 58bd5ce7fcd9cb48e066333ea77a31f90c14d537 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 14:08:10 +0100 Subject: [PATCH 007/130] Additional tests --- .../WhitesourceConfigurationHelper.groovy | 16 +-- test/groovy/WhitesourceExecuteScanTest.groovy | 1 + .../WhiteSourceConfigurationHelperTest.groovy | 105 ++++++++++++++++++ test/resources/utilsTest/build.sbt | 53 +++++++++ test/resources/utilsTest/setup.py | 15 +++ 5 files changed, 182 insertions(+), 8 deletions(-) create mode 100644 test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy create mode 100644 test/resources/utilsTest/build.sbt create mode 100644 test/resources/utilsTest/setup.py diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index b940f9ea0..be7a49a15 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -8,7 +8,7 @@ class WhitesourceConfigurationHelper implements Serializable { private static def SCALA_CONTENT_KEY = "@__content" - static def extendUAConfigurationFile(script, Utils utils, config, path) { + static def extendUAConfigurationFile(script, config, path) { def mapping = [] def parsingClosure = { fileReadPath -> return script.readProperties (file: fileReadPath) } def serializationClosure = { configuration -> serializeUAConfig(configuration) } @@ -78,10 +78,10 @@ class WhitesourceConfigurationHelper implements Serializable { break } - rewriteConfiguration(script, config, utils, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) + rewriteConfiguration(script, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) } - static def extendConfigurationFile(script, Utils utils, config, path) { + static def extendConfigurationFile(script, config, path) { def mapping = [:] def parsingClosure def serializationClosure @@ -98,7 +98,7 @@ class WhitesourceConfigurationHelper implements Serializable { [name: 'userKey', value: config.userKey, warnIfPresent: true] ] parsingClosure = { fileReadPath -> return script.readProperties (file: fileReadPath) } - serializationClosure = { Properties configuration -> new StringWriter().with{ w -> configuration.store(w, null); w }.toString() } + serializationClosure = { configuration -> serializeUAConfig(configuration) } break case 'npm': mapping = [ @@ -108,7 +108,7 @@ class WhitesourceConfigurationHelper implements Serializable { [name: 'userKey', value: config.userKey, warnIfPresent: true] ] parsingClosure = { fileReadPath -> return script.readJSON (file: fileReadPath) } - serializationClosure = { configuration -> return utils.jsonToString(configuration) } + serializationClosure = { configuration -> return new JsonUtils().getPrettyJsonString(configuration) } break case 'pip': mapping = [ @@ -134,10 +134,10 @@ class WhitesourceConfigurationHelper implements Serializable { break } - rewriteConfiguration(script, config, utils, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) + rewriteConfiguration(script, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) } - static private def rewriteConfiguration(script, config, utils, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) { + static private def rewriteConfiguration(script, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) { def inputFilePath = "${path}${inputFile}" def outputFilePath = "${path}${targetFile}" def moduleSpecificFile = parsingClosure(inputFilePath) @@ -162,7 +162,7 @@ class WhitesourceConfigurationHelper implements Serializable { script.writeFile file: outputFilePath, text: output if(config.stashContent && config.stashContent.size() > 0) { def stashName = "modified whitesource config ${suffix}".toString() - utils.stashWithMessage ( + new Utils().stashWithMessage ( stashName, "Stashing modified Whitesource configuration", outputFilePath.replaceFirst('\\./', '') diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index 87f0b358c..f30fa68d0 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -1,4 +1,5 @@ #!groovy + import com.sap.piper.JsonUtils import com.sap.piper.integration.WhitesourceOrgAdminRepository import com.sap.piper.integration.WhitesourceRepository diff --git a/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy b/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy new file mode 100644 index 000000000..01c802604 --- /dev/null +++ b/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy @@ -0,0 +1,105 @@ +package com.sap.piper + +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.RuleChain +import util.BasePiperTest +import util.JenkinsReadFileRule +import util.JenkinsWriteFileRule +import util.Rules + +import static org.hamcrest.Matchers.* +import static org.junit.Assert.assertThat + +class WhiteSourceConfigurationHelperTest extends BasePiperTest { + JenkinsReadFileRule jrfr = new JenkinsReadFileRule(this, 'test/resources/utilsTest/') + JenkinsWriteFileRule jwfr = new JenkinsWriteFileRule(this) + + @Rule + public RuleChain ruleChain = Rules + .getCommonRules(this) + .around(jrfr) + .around(jwfr) + + private static getMapping() { + return [ + [name: "whitesourceOrgToken in ThisBuild", value: "config.orgToken", warnIfPresent: true], + [name: "whitesourceProduct in ThisBuild", value: "config.whitesourceProductName"] + ] + } + + @Before + void init() { + helper.registerAllowedMethod('readJSON', [Map], {return [:]}) + helper.registerAllowedMethod('readProperties', [Map], {return new Properties()}) + } + + @Test + void testReadScalaConfig() { + def resMap = WhitesourceConfigurationHelper.readScalaConfig(nullScript, getMapping(), "build.sbt") + assertThat(resMap, hasKey(WhitesourceConfigurationHelper.SCALA_CONTENT_KEY)) + + // mapping tokens should be removed from parsed content + assertThat(resMap[WhitesourceConfigurationHelper.SCALA_CONTENT_KEY], not(hasItem(containsString("whitesourceOrgToken in ThisBuild")))) + + assertThat(resMap, hasEntry("whitesourceOrgToken in ThisBuild", "\"org-token\"")) + assertThat(resMap, hasEntry("whitesourceProduct in ThisBuild", "\"PRODUCT VERSION\"")) + } + + @Test + void testSerializeScalaConfig() { + def resMap = [ + "whitesourceOrgToken in ThisBuild": "\"some-long-hash-value\"", + "whitesourceProduct in ThisBuild": "\"PRODUCT IDENTIFIER\"", + "whitesourceServiceUrl in ThisBuild": "uri(\"http://mo-393ef744d.mo.sap.corp:8080/wsui/wspluginProxy.jsp\")" + ] + resMap[WhitesourceConfigurationHelper.SCALA_CONTENT_KEY] = ["// build.sbt -- scala build file", "name := \"minimal-scala\"", "libraryDependencies += \"org.scalatest\" %% \"scalatest\" % \"2.2.4\" % \"test\""] + def fileContent = WhitesourceConfigurationHelper.serializeScalaConfig(resMap) + + resMap[WhitesourceConfigurationHelper.SCALA_CONTENT_KEY].each { + line -> + assertThat(fileContent, containsString("${line}\r")) + } + + assertThat(fileContent, containsString("whitesourceOrgToken in ThisBuild := \"some-long-hash-value\"")) + assertThat(fileContent, containsString("whitesourceProduct in ThisBuild := \"PRODUCT IDENTIFIER\"")) + assertThat(fileContent, containsString("whitesourceServiceUrl in ThisBuild := uri(\"http://mo-393ef744d.mo.sap.corp:8080/wsui/wspluginProxy.jsp\")")) + } + + @Test + void testExtendConfigurationFileUnifiedAgent() { + WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, [scanType: 'unifiedAgent', configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") + assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("apiKey=abcd")) + assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("productName=name")) + assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("productToken=1234")) + assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("userKey=0000")) + } + + @Test + void testExtendConfigurationFileNpm() { + WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, [scanType: 'npm', configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") + assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("\"apiKey\": \"abcd\",")) + assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("\"productName\": \"name\",")) + assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("\"productToken\": \"1234\",")) + assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("\"userKey\": \"0000\"")) + } + + @Test + void testExtendConfigurationFilePip() { + WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, [scanType: 'pip', configFilePath: './setup.py', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") + assertThat(jwfr.files['./setup.py.8813e60e0d9f7cacf0c414ae4964816f.py'], containsString("'org_token': 'abcd',")) + assertThat(jwfr.files['./setup.py.8813e60e0d9f7cacf0c414ae4964816f.py'], containsString("'product_name': 'name',")) + assertThat(jwfr.files['./setup.py.8813e60e0d9f7cacf0c414ae4964816f.py'], containsString("'product_token': '1234',")) + assertThat(jwfr.files['./setup.py.8813e60e0d9f7cacf0c414ae4964816f.py'], containsString("'user_key': '0000'")) + } + + @Test + void testExtendConfigurationFileSbt() { + WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, [scanType: 'sbt', configFilePath: './build.sbt', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000', agentUrl: 'http://mo-393ef744d.mo.sap.corp:8080/wsui/wspluginProxy.jsp'], "./") + assertThat(jwfr.files['./build.sbt'], containsString("whitesourceOrgToken in ThisBuild := \"abcd\"")) + assertThat(jwfr.files['./build.sbt'], containsString("whitesourceProduct in ThisBuild := \"name\"")) + assertThat(jwfr.files['./build.sbt'], containsString("whitesourceServiceUrl in ThisBuild := uri(\"http://mo-393ef744d.mo.sap.corp:8080/wsui/wspluginProxy.jsp\")")) + } +} + diff --git a/test/resources/utilsTest/build.sbt b/test/resources/utilsTest/build.sbt new file mode 100644 index 000000000..8b3fa5a46 --- /dev/null +++ b/test/resources/utilsTest/build.sbt @@ -0,0 +1,53 @@ +import scala.io.Source + +val buildDescriptorMap = JSON + .parseFull(Source.fromFile("sbtDescriptor.json").mkString) + .get + .asInstanceOf[Map[String, String]] + +lazy val buildSettings = Seq( + scalaVersion := "2.11.11", +) + +lazy val root = (project in file(".")) + .settings(buildSettings) + +libraryDependencies ++= Seq( + jdbc, + "org.scalatestplus.play" % "scalatestplus-play_2.11" % "2.0.0" % Test +) + +dependencyOverrides += "com.fasterxml.jackson.core" % "jackson-databind" % "2.8.11.2" + +resolvers ++= Seq( + Resolver.url("Typesafe Ivy releases", + url("https://repo.typesafe.com/typesafe/ivy-releases"))(Resolver.ivyStylePatterns) +) + +// Play provides two styles of routers, one expects its actions to be injected, the +// other, legacy style, accesses its actions statically. +routesGenerator := InjectedRoutesGenerator + +javaOptions in run ++= Seq( + "-Xmx12G" +) + +javaOptions in Universal ++= Seq( + "-Dpidfile.path=/dev/null" +) + +javaOptions in Test += "-Dconfig.file=conf/application.test.conf" + +// Do not add API documentation into generated package +sources in (Compile, doc) := Seq.empty +publishArtifact in (Universal, packageBin) := true + +// scala style +scalastyleConfig := baseDirectory.value / "scalastyle-production-config.xml" + +// Whitesource +whitesourceProduct in ThisBuild := "PRODUCT VERSION" +whitesourceOrgToken in ThisBuild := "org-token" +whitesourceAggregateProjectName in ThisBuild := "project-name" +whitesourceAggregateProjectToken in ThisBuild := "project-token" +whitesourceFailOnError in ThisBuild := false diff --git a/test/resources/utilsTest/setup.py b/test/resources/utilsTest/setup.py new file mode 100644 index 000000000..bf0a51216 --- /dev/null +++ b/test/resources/utilsTest/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup + +setup( + name='py_connect', + version='1.0', + + description='This is a python package to handle some ci-connect payload parts', + + url='https://github.wdf.sap.corp/sap-production/py_connect', + + # Author details + author='Some Author', + author_email='some.author@sap.com', + packages=['payload'] +) From 05fe2fb758b879ba5d7e96a4ab069254a7af1f6c Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 14:40:17 +0100 Subject: [PATCH 008/130] Cleanup --- resources/{piper.css => piper-os.css} | 0 .../sap/piper/WhitesourceConfigurationHelper.groovy | 12 ++++++------ .../piper/WhiteSourceConfigurationHelperTest.groovy | 8 ++++---- vars/whitesourceExecuteScan.groovy | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) rename resources/{piper.css => piper-os.css} (100%) diff --git a/resources/piper.css b/resources/piper-os.css similarity index 100% rename from resources/piper.css rename to resources/piper-os.css diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index be7a49a15..0a89a8c11 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -8,7 +8,7 @@ class WhitesourceConfigurationHelper implements Serializable { private static def SCALA_CONTENT_KEY = "@__content" - static def extendUAConfigurationFile(script, config, path) { + static def extendUAConfigurationFile(script, utils, config, path) { def mapping = [] def parsingClosure = { fileReadPath -> return script.readProperties (file: fileReadPath) } def serializationClosure = { configuration -> serializeUAConfig(configuration) } @@ -78,10 +78,10 @@ class WhitesourceConfigurationHelper implements Serializable { break } - rewriteConfiguration(script, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) + rewriteConfiguration(script, utils, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) } - static def extendConfigurationFile(script, config, path) { + static def extendConfigurationFile(script, utils, config, path) { def mapping = [:] def parsingClosure def serializationClosure @@ -134,10 +134,10 @@ class WhitesourceConfigurationHelper implements Serializable { break } - rewriteConfiguration(script, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) + rewriteConfiguration(script, utils, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) } - static private def rewriteConfiguration(script, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) { + static private def rewriteConfiguration(script, utils, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) { def inputFilePath = "${path}${inputFile}" def outputFilePath = "${path}${targetFile}" def moduleSpecificFile = parsingClosure(inputFilePath) @@ -162,7 +162,7 @@ class WhitesourceConfigurationHelper implements Serializable { script.writeFile file: outputFilePath, text: output if(config.stashContent && config.stashContent.size() > 0) { def stashName = "modified whitesource config ${suffix}".toString() - new Utils().stashWithMessage ( + utils.stashWithMessage ( stashName, "Stashing modified Whitesource configuration", outputFilePath.replaceFirst('\\./', '') diff --git a/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy b/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy index 01c802604..1012cc0c1 100644 --- a/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy +++ b/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy @@ -69,7 +69,7 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { @Test void testExtendConfigurationFileUnifiedAgent() { - WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, [scanType: 'unifiedAgent', configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") + WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, utils, [scanType: 'unifiedAgent', configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("apiKey=abcd")) assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("productName=name")) assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("productToken=1234")) @@ -78,7 +78,7 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { @Test void testExtendConfigurationFileNpm() { - WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, [scanType: 'npm', configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") + WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, utils, [scanType: 'npm', configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("\"apiKey\": \"abcd\",")) assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("\"productName\": \"name\",")) assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("\"productToken\": \"1234\",")) @@ -87,7 +87,7 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { @Test void testExtendConfigurationFilePip() { - WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, [scanType: 'pip', configFilePath: './setup.py', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") + WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, utils, [scanType: 'pip', configFilePath: './setup.py', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") assertThat(jwfr.files['./setup.py.8813e60e0d9f7cacf0c414ae4964816f.py'], containsString("'org_token': 'abcd',")) assertThat(jwfr.files['./setup.py.8813e60e0d9f7cacf0c414ae4964816f.py'], containsString("'product_name': 'name',")) assertThat(jwfr.files['./setup.py.8813e60e0d9f7cacf0c414ae4964816f.py'], containsString("'product_token': '1234',")) @@ -96,7 +96,7 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { @Test void testExtendConfigurationFileSbt() { - WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, [scanType: 'sbt', configFilePath: './build.sbt', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000', agentUrl: 'http://mo-393ef744d.mo.sap.corp:8080/wsui/wspluginProxy.jsp'], "./") + WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, utils, [scanType: 'sbt', configFilePath: './build.sbt', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000', agentUrl: 'http://mo-393ef744d.mo.sap.corp:8080/wsui/wspluginProxy.jsp'], "./") assertThat(jwfr.files['./build.sbt'], containsString("whitesourceOrgToken in ThisBuild := \"abcd\"")) assertThat(jwfr.files['./build.sbt'], containsString("whitesourceProduct in ThisBuild := \"name\"")) assertThat(jwfr.files['./build.sbt'], containsString("whitesourceServiceUrl in ThisBuild := uri(\"http://mo-393ef744d.mo.sap.corp:8080/wsui/wspluginProxy.jsp\")")) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 4429cf704..632d9706f 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -59,7 +59,7 @@ void call(Map parameters = [:]) { .mixinStepConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS) .mixinStageConfig(script.commonPipelineEnvironment, parameters.stageName ?: env.STAGE_NAME, STEP_CONFIG_KEYS) .mixin([ - style : libraryResource('piper.css') + style : libraryResource('piper-os.css') ]) .mixin(parameters, PARAMETER_KEYS) .dependingOn('scanType').mixin('buildDescriptorFile') From 03d3e2b2411f48b22243296ebab4a8758b95b0ab Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 15:02:01 +0100 Subject: [PATCH 009/130] Fix missing descriptor support --- src/com/sap/piper/DescriptorUtils.groovy | 97 +++++++++++++++++++ test/groovy/WhitesourceExecuteScanTest.groovy | 18 +++- test/groovy/util/BasePiperTestContext.groovy | 8 ++ vars/whitesourceExecuteScan.groovy | 14 +-- 4 files changed, 129 insertions(+), 8 deletions(-) create mode 100644 src/com/sap/piper/DescriptorUtils.groovy diff --git a/src/com/sap/piper/DescriptorUtils.groovy b/src/com/sap/piper/DescriptorUtils.groovy new file mode 100644 index 000000000..0f3c99c0e --- /dev/null +++ b/src/com/sap/piper/DescriptorUtils.groovy @@ -0,0 +1,97 @@ +package com.sap.piper + +def getMavenGAV(fileName) { + def result = [:] + def descriptor = readMavenPom(file: fileName) + def group = descriptor.getGroupId() + def artifact = descriptor.getArtifactId() + def version = descriptor.getVersion() + result['packaging'] = descriptor.getPackaging() + result['group'] = (null != group && group.length() > 0) ? group : sh(returnStdout: true, script: "mvn -f ${fileName} help:evaluate -Dexpression=project.groupId | grep -Ev '(^\\s*\\[|Download|Java\\w+:)'").trim() + result['artifact'] = (null != artifact && artifact.length() > 0) ? artifact : sh(returnStdout: true, script: "mvn -f ${fileName} help:evaluate -Dexpression=project.artifactId | grep -Ev '(^\\s*\\[|Download|Java\\w+:)'").trim() + result['version'] = (null != version && version.length() > 0) ? version : sh(returnStdout: true, script: "mvn -f ${fileName} help:evaluate -Dexpression=project.version | grep ^[0-9].*").trim() + echo "loaded ${result} from ${fileName}" + return result +} + +def getNpmGAV(file = 'package.json') { + def result = [:] + def descriptor = readJSON(file: file) + + if (descriptor.name.startsWith('@')) { + def packageNameArray = descriptor.name.split('/') + if (packageNameArray.length != 2) + error "Unable to parse package name '${descriptor.name}'" + result['group'] = packageNameArray[0] + result['artifact'] = packageNameArray[1] + } else { + result['group'] = '' + result['artifact'] = descriptor.name + } + result['version'] = descriptor.version + echo "loaded ${result} from ${file}" + return result +} + +def getDlangGAV(file = 'dub.json') { + def result = [:] + def descriptor = readJSON(file: file) + + result['group'] = 'com.sap.dlang' + result['artifact'] = descriptor.name + result['version'] = descriptor.version + result['packaging'] = 'tar.gz' + echo "loaded ${result} from ${file}" + return result +} + +def getSbtGAV(file = 'sbtDescriptor.json') { + def result = [:] + def descriptor = readJSON(file: file) + + result['group'] = descriptor.group + result['artifact'] = descriptor.artifactId + result['version'] = descriptor.version + result['packaging'] = descriptor.packaging + echo "loaded ${result} from ${file}" + return result +} + +def getMtaGAV(file = 'mta.yaml', xmakeConfigFile = '.xmake.cfg') { + def result = [:] + def descriptor = readYaml(file: file) + def xmakeConfig = readProperties(file: xmakeConfigFile) + + result['group'] = xmakeConfig['mtar-group'] + result['artifact'] = descriptor.ID + result['version'] = descriptor.version + result['packaging'] = 'mtar' + // using default value: https://github.wdf.sap.corp/dtxmake/xmake-mta-plugin#complete-list-of-default-values + if(!result['group']){ + result['group'] = 'com.sap.prd.xmake.example.mtars' + Notify.warning(this, "No groupID set in '.xmake.cfg', using default groupID '${result['group']}'.", 'com.sap.icd.jenkins.Utils') + } + echo "loaded ${result} from ${file} and ${xmakeConfigFile}" + return result +} + +def getPipGAV(file = 'setup.py') { + def result = [:] + def descriptor = sh(returnStdout: true, script: "cat ${file}") + + result['group'] = '' + result['packaging'] = '' + result['artifact'] = matches(name, descriptor) + result['version'] = matches(version, descriptor) + + if (result['version'] == '' || matches(method, result['version'])) { + file = file.replace('setup.py', 'version.txt') + def versionString = sh(returnStdout: true, script: "cat ${file}") + if (versionString) { + result['version'] = versionString.trim() + } + } + + echo "loaded ${result} from ${file}" + return result +} diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index f30fa68d0..83e3837c9 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -1,5 +1,5 @@ #!groovy - +import com.sap.piper.DescriptorUtils import com.sap.piper.JsonUtils import com.sap.piper.integration.WhitesourceOrgAdminRepository import com.sap.piper.integration.WhitesourceRepository @@ -11,6 +11,7 @@ import org.junit.Rule import org.junit.Test import org.junit.rules.ExpectedException import org.junit.rules.RuleChain +import org.springframework.beans.factory.annotation.Autowired import util.* import static org.hamcrest.Matchers.* @@ -39,6 +40,9 @@ class WhitesourceExecuteScanTest extends BasePiperTest { def whitesourceOrgAdminRepositoryStub def whitesourceStub + @Autowired + DescriptorUtils descriptorUtilsStub + @Before void init() { def credentialsStore = ['ID-123456789': 'token-0815', 'ID-abcdefg': ['testUser', 'testPassword']] @@ -124,7 +128,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { helper.registerAllowedMethod( "getNpmGAV", [String], {return [group: 'com.sap.node', artifact: 'test-node', version: '1.2.3']}) helper.registerAllowedMethod( "getSbtGAV", [String], {return [group: 'com.sap.sbt', artifact: 'test-scala', version: '1.2.3']}) helper.registerAllowedMethod( "getPipGAV", [String], {return [artifact: 'test-python', version: '1.2.3']}) - helper.registerAllowedMethod( "readMavenGAV", [String], {return [group: 'com.sap.maven', artifact: 'test-java', version: '1.2.3']}) + helper.registerAllowedMethod( "getMavenGAV", [String], {return [group: 'com.sap.maven', artifact: 'test-java', version: '1.2.3']}) nullScript.commonPipelineEnvironment.configuration = nullScript.commonPipelineEnvironment.configuration ?: [:] nullScript.commonPipelineEnvironment.configuration['steps'] = nullScript.commonPipelineEnvironment.configuration['steps'] ?: [:] @@ -158,6 +162,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { script : nullScript, whitesourceRepositoryStub : whitesourceStub, whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + descriptorUtilsStub : descriptorUtilsStub, scanType : 'maven', juStabUtils : utils, orgToken : 'testOrgToken', @@ -200,6 +205,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { script : nullScript, whitesourceRepositoryStub : whitesourceStub, whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + descriptorUtilsStub : descriptorUtilsStub, scanType : 'npm', juStabUtils : utils, orgToken : 'testOrgToken', @@ -246,6 +252,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { script : nullScript, whitesourceRepositoryStub : whitesourceStub, whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + descriptorUtilsStub : descriptorUtilsStub, scanType : 'npm', productName : 'SHC - Piper', configFilePath : './../../testConfigPath', @@ -278,6 +285,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { script : nullScript, whitesourceRepositoryStub : whitesourceStub, whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + descriptorUtilsStub : descriptorUtilsStub, scanType : 'pip', juStabUtils : utils, orgToken : 'testOrgToken', @@ -322,6 +330,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { script : nullScript, whitesourceRepositoryStub : whitesourceStub, whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + descriptorUtilsStub : descriptorUtilsStub, scanType : 'sbt', juStabUtils : utils, productName : 'testProductName', @@ -365,6 +374,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { script : nullScript, whitesourceRepositoryStub : whitesourceStub, whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + descriptorUtilsStub : descriptorUtilsStub, scanType : 'maven', agentDownloadUrl : '', jreDownloadUrl : '', @@ -418,6 +428,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { script : nullScript, whitesourceRepositoryStub : whitesourceStub, whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + descriptorUtilsStub : descriptorUtilsStub, scanType : 'mta', productName : 'SHC - Piper', buildDescriptorExcludeList : ["maven2${File.separator}pom.xml".toString(), "npm2${File.separator}package.json".toString()], @@ -734,6 +745,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { securityVulnerabilities : true, whitesourceRepositoryStub : whitesourceStub, whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + descriptorUtilsStub : descriptorUtilsStub, orgToken : 'testOrgToken', productName : 'SHC - Piper', projectNames : [ 'piper-demo - 0.0.1' ] @@ -765,6 +777,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { script : nullScript, whitesourceRepositoryStub : whitesourceStub, whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + descriptorUtilsStub : descriptorUtilsStub, scanType : 'npm', juStabUtils : utils, securityVulnerabilities : true, @@ -798,6 +811,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { script : nullScript, whitesourceRepositoryStub : whitesourceStub, whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + descriptorUtilsStub : descriptorUtilsStub, scanType : 'npm', juStabUtils : utils, securityVulnerabilities : true, diff --git a/test/groovy/util/BasePiperTestContext.groovy b/test/groovy/util/BasePiperTestContext.groovy index f27a840b2..290bc502a 100644 --- a/test/groovy/util/BasePiperTestContext.groovy +++ b/test/groovy/util/BasePiperTestContext.groovy @@ -2,6 +2,7 @@ package util +import com.sap.piper.DescriptorUtils import com.sap.piper.GitUtils import com.sap.piper.JenkinsUtils import com.sap.piper.Utils @@ -45,4 +46,11 @@ class BasePiperTestContext { LibraryLoadingTestExecutionListener.prepareObjectInterceptors(mockJenkinsUtils) return mockJenkinsUtils } + + @Bean + DescriptorUtils mockDescriptorUtils() { + def mockDescriptorUtils = new DescriptorUtils() + LibraryLoadingTestExecutionListener.prepareObjectInterceptors(mockDescriptorUtils) + return mockDescriptorUtils + } } diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 632d9706f..146989bda 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -1,3 +1,4 @@ +import com.sap.piper.DescriptorUtils import com.sap.piper.JsonUtils import com.sap.piper.Utils import com.sap.piper.integration.WhitesourceOrgAdminRepository @@ -50,6 +51,7 @@ void call(Map parameters = [:]) { handlePipelineStepErrors(stepName: STEP_NAME, stepParameters: parameters) { def script = checkScript(this, parameters) ?: this def utils = parameters.juStabUtils ?: new Utils() + def descriptorUtils = parameters.descriptorUtilsStub ?: new DescriptorUtils() def statusCode = 1 // load default & individual configuration @@ -91,7 +93,7 @@ void call(Map parameters = [:]) { def whitesourceRepository = parameters.whitesourceRepositoryStub ?: new WhitesourceRepository(this, config) def whitesourceOrgAdminRepository = parameters.whitesourceOrgAdminRepositoryStub ?: new WhitesourceOrgAdminRepository(this, config) - statusCode = triggerWhitesourceScanWithUserKey(script, config, utils, parameters, whitesourceRepository, whitesourceOrgAdminRepository) + statusCode = triggerWhitesourceScanWithUserKey(script, config, utils, descriptorUtils, parameters, whitesourceRepository, whitesourceOrgAdminRepository) checkStatus(statusCode, config) @@ -99,7 +101,7 @@ void call(Map parameters = [:]) { } } -private def triggerWhitesourceScanWithUserKey(script, config, utils, parameters, repository, orgAdminRepository) { +private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorUtils, parameters, repository, orgAdminRepository) { withCredentials ([string( credentialsId: config.userTokenCredentialsId, variable: 'userKey' @@ -151,22 +153,22 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, parameters, def gav switch (config.scanType) { case 'npm': - gav = utils.getNpmGAV(config.buildDescriptorFile) + gav = descriptorUtils.getNpmGAV(config.buildDescriptorFile) config.projectName = gav.group + "." + gav.artifact config.productVersion = gav.version break case 'sbt': - gav = utils.getSbtGAV(config.buildDescriptorFile) + gav = descriptorUtils.getSbtGAV(config.buildDescriptorFile) config.projectName = gav.group + "." + gav.artifact config.productVersion = gav.version break case 'pip': - gav = utils.getPipGAV(config.buildDescriptorFile) + gav = descriptorUtils.getPipGAV(config.buildDescriptorFile) config.projectName = gav.artifact config.productVersion = gav.version break default: - gav = utils.readMavenGAV(config.buildDescriptorFile) + gav = descriptorUtils.getMavenGAV(config.buildDescriptorFile) config.projectName = gav.group + "." + gav.artifact config.productVersion = gav.version break From 43e9bc116708eb5acdf6dc3151b995dac54bdbc6 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 15:08:39 +0100 Subject: [PATCH 010/130] Ammend missing regex patterns --- src/com/sap/piper/DescriptorUtils.groovy | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/com/sap/piper/DescriptorUtils.groovy b/src/com/sap/piper/DescriptorUtils.groovy index 0f3c99c0e..527699e9e 100644 --- a/src/com/sap/piper/DescriptorUtils.groovy +++ b/src/com/sap/piper/DescriptorUtils.groovy @@ -1,5 +1,16 @@ package com.sap.piper +import groovy.transform.Field + +import java.util.regex.Pattern + +@Field +def name = Pattern.compile("(.*)name=['\"](.*?)['\"](.*)", Pattern.DOTALL) +@Field +def version = Pattern.compile("(.*)version=['\"](.*?)['\"](.*)", Pattern.DOTALL) +@Field +def method = Pattern.compile("(.*)\\(\\)", Pattern.DOTALL) + def getMavenGAV(fileName) { def result = [:] def descriptor = readMavenPom(file: fileName) From e4cef8dba45c3d28c085e256d00a572830c2d5d1 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 15:15:43 +0100 Subject: [PATCH 011/130] Add missing NonCPE encapsulated matches --- src/com/sap/piper/DescriptorUtils.groovy | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/com/sap/piper/DescriptorUtils.groovy b/src/com/sap/piper/DescriptorUtils.groovy index 527699e9e..3756baefd 100644 --- a/src/com/sap/piper/DescriptorUtils.groovy +++ b/src/com/sap/piper/DescriptorUtils.groovy @@ -1,7 +1,9 @@ package com.sap.piper +import com.cloudbees.groovy.cps.NonCPS import groovy.transform.Field +import java.util.regex.Matcher import java.util.regex.Pattern @Field @@ -106,3 +108,9 @@ def getPipGAV(file = 'setup.py') { echo "loaded ${result} from ${file}" return result } + +@NonCPS +private def matches(regex, input) { + def m = new Matcher(regex, input) + return m.matches() ? m.group(2) : '' +} From 863831c9b3f47e59e0530888726834caa87b4fd7 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 15:30:50 +0100 Subject: [PATCH 012/130] Move to python image containing curl --- resources/default_pipeline_environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 73981bcaa..22b0b7b95 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -292,7 +292,7 @@ steps: - 'opensourceConfiguration' pip: buildDescriptorFile: './setup.py' - dockerImage: 'python:3.7.2-slim-stretch' + dockerImage: 'python:3.7.2-stretch' dockerWorkspace: '/home/python' agentFileName: 'wss-unified-agent.jar' agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' From 22096b881af7d92c30fa3a0c4e1f3ad416a2410e Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 15:32:34 +0100 Subject: [PATCH 013/130] Fix test --- test/groovy/WhitesourceExecuteScanTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index 83e3837c9..c3e9b7ec9 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -295,7 +295,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'python:3.7.2-slim-stretch')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'python:3.7.2-stretch')) assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/python')) assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config 420a1bc5c82f57e80307205d8625304f'])) From 606658507200a4c039f5a655aa13e8bc582b0b65 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 15:45:30 +0100 Subject: [PATCH 014/130] Improve report pulling method --- src/com/sap/piper/integration/WhitesourceRepository.groovy | 4 ++-- vars/whitesourceExecuteScan.groovy | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy index 66dafc059..8602acfe1 100644 --- a/src/com/sap/piper/integration/WhitesourceRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -173,13 +173,13 @@ class WhitesourceRepository implements Serializable { return matchedProjects } - void fetchReportForProduct() { + void fetchReportForProduct(reportName) { def requestContent = [ requestType: "getProductRiskReport", productToken: config.productToken ] - fetchFileFromWhiteSource("whitesource-riskReport.pdf", requestContent) + fetchFileFromWhiteSource(reportName, requestContent) } diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 146989bda..fc739295e 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -217,10 +217,10 @@ void analyseWhitesourceResults(Utils utils, Map config, WhitesourceRepository re config.productToken = metaInfo.token } - repository.fetchReportForProduct() def pdfName = "whitesource-riskReport.pdf" + repository.fetchReportForProduct(pdfName) archiveArtifacts artifacts: pdfName - echo "A summary of the Whitesource findings was stored as artifact under the name $pdfName" + echo "A summary of the Whitesource findings was stored as artifact under the name ${pdfName}" int violationCount = fetchViolationCount(config, repository) checkViolationStatus(violationCount) From bffb57137a85eb9f56f4a8847e4b391a6b16e240 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 15:47:19 +0100 Subject: [PATCH 015/130] Adapt test --- test/groovy/WhitesourceExecuteScanTest.groovy | 2 +- vars/whitesourceExecuteScan.groovy | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index c3e9b7ec9..bdf6c0d29 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -113,7 +113,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { helper.registerAllowedMethod("fetchProjectsMetaInfo", [], { return new JsonUtils().parseJsonSerializable("{ \"projectVitals\": [{ \"id\": 261964, \"name\": \"piper-demo - 0.0.1\", \"token\": \"a2a62e5d7beb4170ad4dccfa3316b5a4cd3fadefc56c49f88fbf9400a09f7d94\", \"creationDate\": \"2017-09-21 00:28:06 +0000\", \"lastUpdatedDate\": \"2017-10-12 01:03:05 +0000\" }]}").projectVitals }) - helper.registerAllowedMethod("fetchReportForProduct", [], { }) + helper.registerAllowedMethod("fetchReportForProduct", [String], { }) helper.registerAllowedMethod( "fetchProjectLicenseAlerts", [Object.class], { return new JsonUtils().parseJsonSerializable("{ \"alerts\": [] }").alerts }) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index fc739295e..6331cca63 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -203,14 +203,14 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU } if (config.reporting) { - analyseWhitesourceResults(utils, config, repository, orgAdminRepository) + analyseWhitesourceResults(config, repository, orgAdminRepository) } return statusCode } } -void analyseWhitesourceResults(Utils utils, Map config, WhitesourceRepository repository, WhitesourceOrgAdminRepository orgAdminRepository) { +void analyseWhitesourceResults(Map config, WhitesourceRepository repository, WhitesourceOrgAdminRepository orgAdminRepository) { if (!config.productToken) { def metaInfo = orgAdminRepository.fetchProductMetaInfo() echo "Meta Information: $metaInfo" From c0df1077c16911b02f6995292cf8ae1bccc807e3 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 16:23:03 +0100 Subject: [PATCH 016/130] Verbose --- src/com/sap/piper/integration/WhitesourceRepository.groovy | 3 +-- vars/whitesourceExecuteScan.groovy | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy index 8602acfe1..799e6bc87 100644 --- a/src/com/sap/piper/integration/WhitesourceRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -180,7 +180,6 @@ class WhitesourceRepository implements Serializable { ] fetchFileFromWhiteSource(reportName, requestContent) - } def fetchProductLicenseAlerts() { @@ -234,7 +233,7 @@ class WhitesourceRepository implements Serializable { } @NonCPS - private void fetchFileFromWhiteSource(String fileName, Map params) { + protected void fetchFileFromWhiteSource(String fileName, Map params) { handleAdditionalRequestParameters(params) def serializedContent = new JsonUtils().getPrettyJsonString(params) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 6331cca63..798363035 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -213,11 +213,12 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU void analyseWhitesourceResults(Map config, WhitesourceRepository repository, WhitesourceOrgAdminRepository orgAdminRepository) { if (!config.productToken) { def metaInfo = orgAdminRepository.fetchProductMetaInfo() - echo "Meta Information: $metaInfo" + echo "Meta Information: ${metaInfo}" config.productToken = metaInfo.token } def pdfName = "whitesource-riskReport.pdf" + echo "Fetching summary report of the Whitesource" repository.fetchReportForProduct(pdfName) archiveArtifacts artifacts: pdfName echo "A summary of the Whitesource findings was stored as artifact under the name ${pdfName}" From 3eedbbb17c310b96ce8d6e9267b910da1ffc6d19 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 16:49:54 +0100 Subject: [PATCH 017/130] Fix serialization --- src/com/sap/piper/JsonUtils.groovy | 6 ++++++ src/com/sap/piper/integration/WhitesourceRepository.groovy | 2 +- vars/whitesourceExecuteScan.groovy | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/com/sap/piper/JsonUtils.groovy b/src/com/sap/piper/JsonUtils.groovy index 09db031d6..cbc8e4602 100644 --- a/src/com/sap/piper/JsonUtils.groovy +++ b/src/com/sap/piper/JsonUtils.groovy @@ -1,8 +1,14 @@ package com.sap.piper import com.cloudbees.groovy.cps.NonCPS +import groovy.json.JsonBuilder import groovy.json.JsonSlurperClassic +@NonCPS +def jsonToString(content) { + return new JsonBuilder(content).toPrettyString() +} + @NonCPS String getPrettyJsonString(object) { return groovy.json.JsonOutput.prettyPrint(groovy.json.JsonOutput.toJson(object)) diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy index 799e6bc87..2eb1e64d6 100644 --- a/src/com/sap/piper/integration/WhitesourceRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -235,7 +235,7 @@ class WhitesourceRepository implements Serializable { @NonCPS protected void fetchFileFromWhiteSource(String fileName, Map params) { handleAdditionalRequestParameters(params) - def serializedContent = new JsonUtils().getPrettyJsonString(params) + def serializedContent = new JsonUtils().jsonToString(params) if(config.verbose) script.echo "Sending curl request with parameters ${params}" diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 798363035..e33542f22 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -218,7 +218,6 @@ void analyseWhitesourceResults(Map config, WhitesourceRepository repository, Whi } def pdfName = "whitesource-riskReport.pdf" - echo "Fetching summary report of the Whitesource" repository.fetchReportForProduct(pdfName) archiveArtifacts artifacts: pdfName echo "A summary of the Whitesource findings was stored as artifact under the name ${pdfName}" From 32b9c95cdbfb659d2bf51fedad999dba9307552e Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 22:55:43 +0100 Subject: [PATCH 018/130] Added tests --- .../WhitesourceOrgAdminRepository.groovy | 10 +- .../WhitesourceOrgAdminRepositoryTest.groovy | 104 ++++++ .../WhitesourceRepositoryTest.groovy | 344 ++++++++++++++++++ vars/whitesourceExecuteScan.groovy | 1 + 4 files changed, 455 insertions(+), 4 deletions(-) create mode 100644 test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy create mode 100644 test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy diff --git a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy index 25b179307..52b056646 100644 --- a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy @@ -9,6 +9,8 @@ class WhitesourceOrgAdminRepository implements Serializable { final internalWhitesource final Map config + def orgAdminUserKey + WhitesourceOrgAdminRepository(Script script, Map config) { this.script = script this.config = config @@ -52,10 +54,10 @@ class WhitesourceOrgAdminRepository implements Serializable { protected def httpWhitesource(requestBody) { script.withCredentials ([script.string( credentialsId: config.orgAdminUserTokenCredentialsId, - variable: 'adminUserKey' + variable: 'orgAdminUserKey' )]) { - requestBody["userKey"] = adminUserKey - def serializedBody = new JsonUtils().getPrettyJsonString(requestBody) + requestBody["userKey"] = orgAdminUserKey + def serializedBody = new JsonUtils().jsonToString(requestBody) def params = [ url : config.serviceUrl, httpMode : 'POST', @@ -75,7 +77,7 @@ class WhitesourceOrgAdminRepository implements Serializable { def response = script.httpRequest(params) if (config.verbose) - script.echo "Received response ${reponse}" + script.echo "Received response ${response}" return response } diff --git a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy new file mode 100644 index 000000000..20700163b --- /dev/null +++ b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy @@ -0,0 +1,104 @@ +package com.sap.piper.integration + +import com.sap.piper.JsonUtils +import org.junit.After +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.RuleChain +import util.BasePiperTest +import util.JenkinsEnvironmentRule +import util.JenkinsLoggingRule +import util.LibraryLoadingTestExecutionListener +import util.Rules + +import static org.assertj.core.api.Assertions.assertThat +import static org.hamcrest.Matchers.is + +class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { + + private JenkinsLoggingRule jlr = new JenkinsLoggingRule(this) + private JenkinsEnvironmentRule jer = new JenkinsEnvironmentRule(this) + + @Rule + public RuleChain ruleChain = Rules + .getCommonRules(this) + .around(jlr) + .around(jer) + + WhitesourceOrgAdminRepository repository + + @Before + void init() throws Exception { + nullScript.env['HTTP_PROXY'] = "http://proxy.wdf.sap.corp:8080" + + repository = new WhitesourceOrgAdminRepository(nullScript, [serviceUrl: "http://some.host.whitesource.com/api/", verbose: true]) + LibraryLoadingTestExecutionListener.prepareObjectInterceptors(repository) + } + + @After + void tearDown() { + printCallStack() + nullScript.env = [:] + } + + @Test + void testResolveProductMeta() { + + def whitesourceMetaResponse = [ + productVitals: [ + [ + token: '410389ae-0269-4719-9cbf-fb5e299c8415', + name : 'NW' + ], + [ + token: '2892f1db-4361-4e83-a89d-d28a262d65b9', + name : 'XS UAA' + ], + [ + token: '1111111-1111-1111-1111-111111111111', + name : 'Correct Name Cloud' + ] + ] + ] + + repository.config['productName'] = "Correct Name Cloud" + + def result = repository.findProductMeta(whitesourceMetaResponse) + + assertThat(result).isEqualTo([ + token: '1111111-1111-1111-1111-111111111111', + name : 'Correct Name Cloud' + ]) + } + + @Test + void testHttpWhitesourceCallUserKey() { + def config = [serviceUrl: "http://some.host.whitesource.com/api/"] + def requestBody = [ "\"someJson\"" : [ "\"someObject\"" : "\"abcdef\"" ] ] + + def requestParams + helper.registerAllowedMethod('httpRequest', [Map], { p -> + requestParams = p + }) + helper.registerAllowedMethod('withCredentials', [List, Closure], { l, c -> + repository.setProperty("orgAdminUserKey", "4711") + c() + }) + + repository.httpWhitesource(requestBody) + + assertThat(requestParams, is( + [ + url : config.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: requestBody, + quiet : false, + proxy : "http://proxy.wdf.sap.corp:8080", + userKey : "4711" + ] + )) + } +} diff --git a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy new file mode 100644 index 000000000..b63860c82 --- /dev/null +++ b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy @@ -0,0 +1,344 @@ +package com.sap.piper.integration + + +import hudson.AbortException +import org.junit.After +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.ExpectedException +import org.junit.rules.RuleChain +import util.BasePiperTest +import util.JenkinsEnvironmentRule +import util.JenkinsLoggingRule +import util.LibraryLoadingTestExecutionListener +import util.Rules + +import static org.assertj.core.api.Assertions.assertThat +import static org.hamcrest.Matchers.is +import static org.hamcrest.Matchers.isA + +class WhitesourceRepositoryTest extends BasePiperTest { + + private ExpectedException exception = ExpectedException.none() + private JenkinsLoggingRule jlr = new JenkinsLoggingRule(this) + private JenkinsEnvironmentRule jer = new JenkinsEnvironmentRule(this) + + @Rule + public RuleChain ruleChain = Rules + .getCommonRules(this) + .around(exception) + .around(jlr) + .around(jer) + + WhitesourceRepository repository + + @Before + void init() throws Exception { + nullScript.env['HTTP_PROXY'] = "http://proxy.wdf.sap.corp:8080" + + repository = new WhitesourceRepository(nullScript, [serviceUrl: "http://some.host.whitesource.com/api/"]) + LibraryLoadingTestExecutionListener.prepareObjectInterceptors(repository) + } + + @After + void tearDown() { + printCallStack() + nullScript.env = [:] + } + + @Test + void testResolveProjectsMeta() { + + + def whitesourceMetaResponse = [ + projectVitals: [ + [ + token: '410389ae-0269-4719-9cbf-fb5e299c8415', + name : 'NW' + ], + [ + token: '2892f1db-4361-4e83-a89d-d28a262d65b9', + name : 'Correct Project Name2' + ], + [ + token: '1111111-1111-1111-1111-111111111111', + name : 'Correct Project Name' + ] + ] + ] + + repository.config['productName'] = "Correct Name Cloud" + repository.config['projectNames'] = ["Correct Project Name", "Correct Project Name2"] + + def result = repository.findProjectsMeta(whitesourceMetaResponse.projectVitals) + + assertThat(result, is( + [ + { + token: '1111111-1111-1111-1111-111111111111' + name: 'Correct Name Cloud' + }, + { + token: '2892f1db-4361-4e83-a89d-d28a262d65b9' + name: 'Correct Project Name2' + } + ])) + + assertThat(result.size(), 2) + } + + @Test + void testResolveProjectsMetaFailNotFound() { + + + def whitesourceMetaResponse = [ + projectVitals: [ + [ + token: '410389ae-0269-4719-9cbf-fb5e299c8415', + name : 'NW' + ], + [ + token: '2892f1db-4361-4e83-a89d-d28a262d65b9', + name : 'Product Name' + ], + [ + token: '1111111-1111-1111-1111-111111111111', + name : 'Product Name2' + ] + ] + ] + + exception.expect(AbortException.class) + + exception.expectMessage("Correct Project Name") + + repository.config['projectNames'] = ["Correct Project Name"] + + repository.findProjectsMeta(whitesourceMetaResponse.projectVitals) + } + + @Test + void testSortLibrariesAlphabeticallyGAV() { + + def librariesResponse = [ + [ + groupId : 'xyz', + artifactId: 'abc' + ], + [ + groupId : 'abc', + artifactId: 'abc-def' + ], + [ + groupId : 'abc', + artifactId: 'def-abc' + ], + [ + groupId : 'def', + artifactId: 'test' + ] + ] + + repository.sortLibrariesAlphabeticallyGAV(librariesResponse) + + assertThat(librariesResponse, is( + [ + { + groupId: 'abc' + artifactId: 'abc-def' + }, + { + groupId: 'abc' + artifactId: 'def-abc' + }, + { + groupId: 'def' + artifactId: 'test' + }, + { + groupId: 'xyz' + artifactId: 'abc' + } + ])) + } + + @Test + void testSortVulnerabilitiesByScore() { + + def vulnerabilitiesResponse = [ + [ + vulnerability: [ + score : 6.9, + cvss3_score: 8.5 + ] + ], + [ + vulnerability: [ + score : 7.5, + cvss3_score: 9.8 + ] + ], + [ + vulnerability: [ + score : 4, + cvss3_score: 0 + ] + ], + [ + vulnerability: [ + score : 9.8, + cvss3_score: 0 + ] + ], + [ + vulnerability: [ + score : 0, + cvss3_score: 5 + ] + ] + ] + + repository.sortVulnerabilitiesByScore(vulnerabilitiesResponse) + + assertThat(vulnerabilitiesResponse, is( + [ + {vulnerability: { + score: 9.8 + cvss3_score: 0 + }} +, + {vulnerability: { + score : 7.5 + cvss3_score: 9.8 + }} +, + {vulnerability: { + score : 6.9 + cvss3_score: 8.5 + }} +, + {vulnerability: { + score : 0 + cvss3_score: 5 + }} +, + {vulnerability: { + score : 4 + cvss3_score: 0 + }} + ])) + } + + @Test + void testHttpWhitesourceExternalCallNoUserKey() { + def config = [ whitesourceServiceUrl: "https://saas.whitesource.com/api", verbose: true] + def requestBody = "{ \"someJson\" : { \"someObject\" : \"abcdef\" } }" + + def requestParams + helper.registerAllowedMethod('httpRequest', [Map], { p -> + requestParams = p + }) + + repository.httpWhitesource(requestBody) + + assertThat(requestParams, is( + [ + url : config.whitesourceServiceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: requestBody, + quiet : false, + proxy : "http://proxy.wdf.sap.corp:8080" + ] + )) + } + + @Test + void testHttpWhitesourceExternalCallUserKey() { + def config = [ serviceUrl: "https://saas.whitesource.com/api", verbose: true, userKey: "4711"] + def requestBody = "{ \"someJson\" : { \"someObject\" : \"abcdef\" } }" + + def requestParams + helper.registerAllowedMethod('httpRequest', [Map], { p -> + requestParams = p + }) + + repository.httpWhitesource(requestBody) + + assertThat(requestParams, is( + [ + url : config.whitesourceServiceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: requestBody, + quiet : false, + proxy : "http://proxy.wdf.sap.corp:8080", + userKey : "4711" + ] + )) + } + + @Test + void testHttpWhitesourceInternalCallUserKey() { + def config = [ whitesourceServiceUrl: "http://mo-323123123.sap.corp/some", verbose: false, userKey: "4711"] + def requestBody = "{ \"someJson\" : { \"someObject\" : \"abcdef\" } }" + + def requestParams + helper.registerAllowedMethod('httpRequest', [Map], { p -> + requestParams = p + }) + + repository.httpWhitesource(requestBody) + + assertThat(requestParams, is( + [ + url : config.whitesourceServiceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: requestBody, + quiet : true + ] + )) + } + + @Test + void testHttpCallWithError() { + def responseBody = """{ + \"errorCode\": 5001, + \"errorMessage\": \"User is not allowed to perform this action\" + }""" + + exception.expect(isA(AbortException.class)) + exception.expectMessage("[WhiteSource] Request failed with error message 'User is not allowed to perform this action' (5001)") + + helper.registerAllowedMethod('httpRequest', [Map], { p -> + requestParams = p + return [content: responseBody] + }) + + repository.fetchWhitesourceResource([httpMode: 'POST']) + + } + + @Test + void testFetchReportForProduct() { + repository.config.putAll([ whitesourceServiceUrl: "http://mo-323123123.sap.corp/some", verbose: false, productToken: "4711"]) + def command + helper.registerAllowedMethod('sh', [String], { cmd -> + command = cmd + }) + + repository.fetchReportForProduct("test.file") + + assertThat(command, equals('''#!/bin/sh -e +curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type: application/json' -d '{ + "requestType": "getProductRiskReport", + "productToken": "4711" +}''' + )) + } +} diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index e33542f22..7dd674f61 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -26,6 +26,7 @@ import static com.sap.piper.Prerequisites.checkScript 'internalServiceUrl', 'jreDownloadUrl', 'licensingVulnerabilities', + 'orgAdminUserTokenCredentialsId', 'orgToken', 'parallelLimit', 'reporting', From 3bf3afd7acd69bd4fabef88f37e17d0d9dd37a61 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 23:09:40 +0100 Subject: [PATCH 019/130] Added debug output --- src/com/sap/piper/integration/WhitesourceRepository.groovy | 5 ++++- vars/whitesourceExecuteScan.groovy | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy index 2eb1e64d6..3498d013d 100644 --- a/src/com/sap/piper/integration/WhitesourceRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -234,6 +234,8 @@ class WhitesourceRepository implements Serializable { @NonCPS protected void fetchFileFromWhiteSource(String fileName, Map params) { + if(config.verbose) + script.echo "Entered routine to send file download request" handleAdditionalRequestParameters(params) def serializedContent = new JsonUtils().jsonToString(params) @@ -243,7 +245,8 @@ class WhitesourceRepository implements Serializable { script.sh "${config.verbose ? '' : '#!/bin/sh -e\n'}curl -o ${fileName} -X POST ${config.serviceUrl} -H 'Content-Type: application/json' -d \'${serializedContent}\'" } - private void handleAdditionalRequestParameters(params) { + @NonCPS + protected void handleAdditionalRequestParameters(params) { if(config.userKey) params["userKey"] = config.userKey } diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 7dd674f61..1b35cb267 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -197,6 +197,11 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU statusCode = sh(script: "${javaCmd} ${options.join(' ')} ${config.agentParameters}", returnStatus: true) + if (config.jreDownloadUrl) { + sh "rm -r ./bin ./conf ./legal ./lib ./man" + javaCmd = './bin/java' + } + // archive whitesource result files archiveArtifacts artifacts: "whitesource/*.*", allowEmptyArchive: true } From 34857f112f1cc29dc3aa643fea2761ed3598de7e Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Mar 2019 23:22:15 +0100 Subject: [PATCH 020/130] Cleanup --- .../integration/WhitesourceRepository.groovy | 2 -- .../WhitesourceOrgAdminRepositoryTest.groovy | 30 ------------------- 2 files changed, 32 deletions(-) diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy index 3498d013d..b128f7f9b 100644 --- a/src/com/sap/piper/integration/WhitesourceRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -234,8 +234,6 @@ class WhitesourceRepository implements Serializable { @NonCPS protected void fetchFileFromWhiteSource(String fileName, Map params) { - if(config.verbose) - script.echo "Entered routine to send file download request" handleAdditionalRequestParameters(params) def serializedContent = new JsonUtils().jsonToString(params) diff --git a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy index 20700163b..fba567e93 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy @@ -71,34 +71,4 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { name : 'Correct Name Cloud' ]) } - - @Test - void testHttpWhitesourceCallUserKey() { - def config = [serviceUrl: "http://some.host.whitesource.com/api/"] - def requestBody = [ "\"someJson\"" : [ "\"someObject\"" : "\"abcdef\"" ] ] - - def requestParams - helper.registerAllowedMethod('httpRequest', [Map], { p -> - requestParams = p - }) - helper.registerAllowedMethod('withCredentials', [List, Closure], { l, c -> - repository.setProperty("orgAdminUserKey", "4711") - c() - }) - - repository.httpWhitesource(requestBody) - - assertThat(requestParams, is( - [ - url : config.serviceUrl, - httpMode : 'POST', - acceptType : 'APPLICATION_JSON', - contentType: 'APPLICATION_JSON', - requestBody: requestBody, - quiet : false, - proxy : "http://proxy.wdf.sap.corp:8080", - userKey : "4711" - ] - )) - } } From 53491ce956a9e75924497456cb36fe1c576c39e3 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 5 Mar 2019 13:59:27 +0100 Subject: [PATCH 021/130] Cleanup and alignment with internal step --- resources/default_pipeline_environment.yml | 1 + .../WhitesourceConfigurationHelper.groovy | 130 +---------------- test/groovy/WhitesourceExecuteScanTest.groovy | 135 ++++++++++++++++++ .../WhiteSourceConfigurationHelperTest.groovy | 78 ++-------- vars/whitesourceExecuteScan.groovy | 44 +++--- 5 files changed, 175 insertions(+), 213 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 22b0b7b95..863ec4c14 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -259,6 +259,7 @@ steps: parallelLimit: 15 licensingVulnerabilities: true securityVulnerabilities: true + cvssSeverityLimit: -1 reporting: true vulnerabilityReportFileName: 'piper_whitesource_vulnerability_report' vulnerabilityReportTitle: 'WhiteSource Security Vulnerability Report' diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index 0a89a8c11..9e1b4590e 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -6,8 +6,6 @@ import java.security.MessageDigest class WhitesourceConfigurationHelper implements Serializable { - private static def SCALA_CONTENT_KEY = "@__content" - static def extendUAConfigurationFile(script, utils, config, path) { def mapping = [] def parsingClosure = { fileReadPath -> return script.readProperties (file: fileReadPath) } @@ -81,62 +79,6 @@ class WhitesourceConfigurationHelper implements Serializable { rewriteConfiguration(script, utils, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) } - static def extendConfigurationFile(script, utils, config, path) { - def mapping = [:] - def parsingClosure - def serializationClosure - def inputFile = config.configFilePath.replaceFirst('\\./', '') - def suffix = MessageDigest.getInstance("MD5").digest(config.configFilePath.bytes).encodeHex().toString() - def targetFile = "${inputFile}.${suffix}" - switch (config.scanType) { - case 'unifiedAgent': - case 'fileAgent': - mapping = [ - [name: 'apiKey', value: config.orgToken, warnIfPresent: true], - [name: 'productName', value: config.productName], - [name: 'productToken', value: config.productToken, omitIfPresent: 'projectToken'], - [name: 'userKey', value: config.userKey, warnIfPresent: true] - ] - parsingClosure = { fileReadPath -> return script.readProperties (file: fileReadPath) } - serializationClosure = { configuration -> serializeUAConfig(configuration) } - break - case 'npm': - mapping = [ - [name: 'apiKey', value: config.orgToken, warnIfPresent: true], - [name: 'productName', value: config.productName], - [name: 'productToken', value: config.productToken, omitIfPresent: 'projectToken'], - [name: 'userKey', value: config.userKey, warnIfPresent: true] - ] - parsingClosure = { fileReadPath -> return script.readJSON (file: fileReadPath) } - serializationClosure = { configuration -> return new JsonUtils().getPrettyJsonString(configuration) } - break - case 'pip': - mapping = [ - [name: "'org_token'", value: "\'${config.orgToken}\'", warnIfPresent: true], - [name: "'product_name'", value: "\'${config.productName}\'"], - [name: "'product_token'", value: "\'${config.productToken}\'"], - [name: "'user_key'", value: "\'${config.userKey}\'", warnIfPresent: true] - ] - parsingClosure = { fileReadPath -> return readPythonConfig (script, fileReadPath) } - serializationClosure = { configuration -> serializePythonConfig(configuration) } - targetFile = "${inputFile}.${suffix}.py" - break - case 'sbt': - mapping = [ - [name: "whitesourceOrgToken in ThisBuild", value: "\"${config.orgToken}\"", warnIfPresent: true], - [name: "whitesourceProduct in ThisBuild", value: "\"${config.productName}\""], - [name: "whitesourceServiceUrl in ThisBuild", value: "uri(\"${config.agentUrl}\")"] - // actually not supported [name: "whitesourceUserKey in ThisBuild", value: config.userKey] - ] - parsingClosure = { fileReadPath -> return readScalaConfig (script, mapping, fileReadPath) } - serializationClosure = { configuration -> serializeScalaConfig (configuration) } - targetFile = inputFile - break - } - - rewriteConfiguration(script, utils, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) - } - static private def rewriteConfiguration(script, utils, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) { def inputFilePath = "${path}${inputFile}" def outputFilePath = "${path}${targetFile}" @@ -148,7 +90,7 @@ class WhitesourceConfigurationHelper implements Serializable { mapping.each { entry -> - //if (entry.warnIfPresent && moduleSpecificFile[entry.name]) + if (entry.warnIfPresent && moduleSpecificFile[entry.name]) //Notify.warning(script, "Obsolete configuration ${entry.name} detected, please omit its use and rely on configuration via Piper.", 'WhitesourceConfigurationHelper') def dependentValue = entry.omitIfPresent ? moduleSpecificFile[entry.omitIfPresent] : null if ((entry.omitIfPresent && !dependentValue || !entry.omitIfPresent) && entry.value && entry.value != 'null' && entry.value != '') @@ -172,76 +114,6 @@ class WhitesourceConfigurationHelper implements Serializable { config.configFilePath = outputFilePath } - static private def readPythonConfig(script, filePath) { - def contents = script.readFile file: filePath - def lines = contents.split('\n') - def resultMap = [:] - lines.each { - line -> - List parts = line?.replaceAll(',$', '')?.split(':') - def key = parts[0]?.trim() - parts.removeAt(0) - resultMap[key] = parts.size() > 0 ? (parts as String[]).join(':').trim() : null - } - return resultMap - } - - static private def serializePythonConfig(configuration) { - StringBuilder result = new StringBuilder() - configuration.each { - entry -> - if(entry.key != '}') - result.append(entry.value ? ' ' : '').append(entry.key).append(entry.value ? ': ' : '').append(entry.value ?: '').append(entry.value ? ',' : '').append('\r\n') - } - return result.toString().replaceAll(',$', '\r\n}') - } - - static private def readScalaConfig(script, mapping, filePath) { - def contents = script.readFile file: filePath - def lines = contents.split('\n') - def resultMap = [:] - resultMap[SCALA_CONTENT_KEY] = [] - def keys = mapping.collect( { it.name } ) - lines.each { - line -> - def parts = line?.split(':=').toList() - def key = parts[0]?.trim() - if (keys.contains(key)) { - resultMap[key] = parts[1]?.trim() - } else if (line != null) { - resultMap[SCALA_CONTENT_KEY].add(line) - } - } - return resultMap - } - - static private def serializeScalaConfig(configuration) { - StringBuilder result = new StringBuilder() - - // write the general content - configuration[SCALA_CONTENT_KEY].each { - line -> - result.append(line) - result.append('\r\n') - } - - // write the mappings - def confKeys = configuration.keySet() - confKeys.remove(SCALA_CONTENT_KEY) - - confKeys.each { - key -> - def value = configuration[key] - result.append(key) - if (value != null) { - result.append(' := ').append(value) - } - result.append('\r\n') - } - - return result.toString() - } - @NonCPS static private def serializeUAConfig(configuration) { Properties p = new Properties() diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index bdf6c0d29..803cf9037 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -25,6 +25,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { private JenkinsLoggingRule loggingRule = new JenkinsLoggingRule(this) private JenkinsWriteFileRule writeFileRule = new JenkinsWriteFileRule(this) private JenkinsStepRule stepRule = new JenkinsStepRule(this) + private JenkinsErrorRule errorRule = new JenkinsErrorRule(this) @Rule public RuleChain ruleChain = Rules @@ -36,6 +37,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { .around(loggingRule) .around(writeFileRule) .around(stepRule) + .around(errorRule) def whitesourceOrgAdminRepositoryStub def whitesourceStub @@ -746,6 +748,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { whitesourceRepositoryStub : whitesourceStub, whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, descriptorUtilsStub : descriptorUtilsStub, + cvssSeverityLimit : 7, orgToken : 'testOrgToken', productName : 'SHC - Piper', projectNames : [ 'piper-demo - 0.0.1' ] @@ -823,4 +826,136 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(loggingRule.log, containsString('No Open Source Software Security vulnerabilities detected.')) assertThat(writeFileRule.files['piper_whitesource_vulnerability_report.json'], not(isEmptyOrNullString())) } + + @Test + void testCheckStatus_0() { + + def error = false + try { + stepRule.step.checkStatus(0, [licensingVulnerabilities: true]) + } catch (e) { + error = true + } + assertThat(error, is(false)) + } + + @Test + void testCheckStatus_255() { + def error = false + try { + stepRule.step.checkStatus(255, [licensingVulnerabilities: true]) + } catch (e) { + error = true + assertThat(e.getMessage(), is("[whitesourceExecuteScan] The scan resulted in an error")) + } + assertThat(error, is(true)) + } + + @Test + void testCheckStatus_254() { + def error = false + try { + stepRule.step.checkStatus(254, [licensingVulnerabilities: true]) + } catch (e) { + error = true + assertThat(e.getMessage(), is("[whitesourceExecuteScan] Whitesource found one or multiple policy violations")) + } + assertThat(error, is(true)) + } + + @Test + void testCheckStatus_253() { + def error = false + try { + stepRule.step.checkStatus(253, [licensingVulnerabilities: true]) + } catch (e) { + error = true + assertThat(e.getMessage(), is("[whitesourceExecuteScan] The local scan client failed to execute the scan")) + } + assertThat(error, is(true)) + } + + @Test + void testCheckStatus_252() { + def error = false + try { + stepRule.step.checkStatus(252, [licensingVulnerabilities: true]) + } catch (e) { + error = true + assertThat(e.getMessage(), is("[whitesourceExecuteScan] There was a failure in the connection to the WhiteSource servers")) + } + assertThat(error, is(true)) + } + + @Test + void testCheckStatus_251() { + def error = false + try { + stepRule.step.checkStatus(251, [licensingVulnerabilities: true]) + } catch (e) { + error = true + assertThat(e.getMessage(), is("[whitesourceExecuteScan] The server failed to analyze the scan")) + } + assertThat(error, is(true)) + } + + @Test + void testCheckStatus_250() { + def error = false + try { + stepRule.step.checkStatus(250, [licensingVulnerabilities: true]) + } catch (e) { + error = true + assertThat(e.getMessage(), is("[whitesourceExecuteScan] Pre-step failure")) + } + assertThat(error, is(true)) + } + + @Test + void testCheckStatus_127() { + def error = false + try { + stepRule.step.checkStatus(127, [licensingVulnerabilities: true]) + } catch (e) { + error = true + assertThat(e.getMessage(), is("[whitesourceExecuteScan] Whitesource scan failed with unknown error code '127'")) + } + assertThat(error, is(true)) + } + + @Test + void testCheckStatus_vulnerability() { + def error = false + try { + stepRule.step.checkStatus(0, [licensingVulnerabilities: false, securityVulnerabilities: true, severeVulnerabilities: 5]) + } catch (e) { + error = true + assertThat(e.getMessage(), is("[whitesourceExecuteScan] 5 Open Source Software Security vulnerabilities with CVSS score greater or equal 7.0 detected. - ")) + } + assertThat(error, is(true)) + } + + @Test + void testCheckViolationStatus_0() { + def error = false + try { + stepRule.step.checkViolationStatus(0) + } catch (e) { + error = true + } + assertThat(error, is(false)) + assertThat(loggingRule.log, containsString("****\r\n[whitesourceExecuteScan] No policy violations found. You can deploy to production, and set the \"Intellectual Property (IP) Scan Plan\" in Sirius to completed. \r\n****")) + } + + @Test + void testCheckViolationStatus_5() { + def error = false + try { + stepRule.step.checkViolationStatus(5) + } catch (e) { + error = true + assertThat(e.getMessage(), is("[whitesourceExecuteScan] Whitesource found 5 policy violations for your product")) + } + assertThat(error, is(true)) + } } diff --git a/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy b/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy index 1012cc0c1..42f41a4f9 100644 --- a/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy +++ b/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy @@ -9,7 +9,7 @@ import util.JenkinsReadFileRule import util.JenkinsWriteFileRule import util.Rules -import static org.hamcrest.Matchers.* +import static org.hamcrest.Matchers.containsString import static org.junit.Assert.assertThat class WhiteSourceConfigurationHelperTest extends BasePiperTest { @@ -22,84 +22,30 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { .around(jrfr) .around(jwfr) - private static getMapping() { - return [ - [name: "whitesourceOrgToken in ThisBuild", value: "config.orgToken", warnIfPresent: true], - [name: "whitesourceProduct in ThisBuild", value: "config.whitesourceProductName"] - ] - } - @Before void init() { - helper.registerAllowedMethod('readJSON', [Map], {return [:]}) helper.registerAllowedMethod('readProperties', [Map], {return new Properties()}) } @Test - void testReadScalaConfig() { - def resMap = WhitesourceConfigurationHelper.readScalaConfig(nullScript, getMapping(), "build.sbt") - assertThat(resMap, hasKey(WhitesourceConfigurationHelper.SCALA_CONTENT_KEY)) - - // mapping tokens should be removed from parsed content - assertThat(resMap[WhitesourceConfigurationHelper.SCALA_CONTENT_KEY], not(hasItem(containsString("whitesourceOrgToken in ThisBuild")))) - - assertThat(resMap, hasEntry("whitesourceOrgToken in ThisBuild", "\"org-token\"")) - assertThat(resMap, hasEntry("whitesourceProduct in ThisBuild", "\"PRODUCT VERSION\"")) - } - - @Test - void testSerializeScalaConfig() { - def resMap = [ - "whitesourceOrgToken in ThisBuild": "\"some-long-hash-value\"", - "whitesourceProduct in ThisBuild": "\"PRODUCT IDENTIFIER\"", - "whitesourceServiceUrl in ThisBuild": "uri(\"http://mo-393ef744d.mo.sap.corp:8080/wsui/wspluginProxy.jsp\")" - ] - resMap[WhitesourceConfigurationHelper.SCALA_CONTENT_KEY] = ["// build.sbt -- scala build file", "name := \"minimal-scala\"", "libraryDependencies += \"org.scalatest\" %% \"scalatest\" % \"2.2.4\" % \"test\""] - def fileContent = WhitesourceConfigurationHelper.serializeScalaConfig(resMap) - - resMap[WhitesourceConfigurationHelper.SCALA_CONTENT_KEY].each { - line -> - assertThat(fileContent, containsString("${line}\r")) - } - - assertThat(fileContent, containsString("whitesourceOrgToken in ThisBuild := \"some-long-hash-value\"")) - assertThat(fileContent, containsString("whitesourceProduct in ThisBuild := \"PRODUCT IDENTIFIER\"")) - assertThat(fileContent, containsString("whitesourceServiceUrl in ThisBuild := uri(\"http://mo-393ef744d.mo.sap.corp:8080/wsui/wspluginProxy.jsp\")")) - } - - @Test - void testExtendConfigurationFileUnifiedAgent() { - WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, utils, [scanType: 'unifiedAgent', configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") + void testExtendConfigurationFileUnifiedAgentPip() { + WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("apiKey=abcd")) assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("productName=name")) assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("productToken=1234")) assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("userKey=0000")) + assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("python.resolveDependencies=true")) } @Test - void testExtendConfigurationFileNpm() { - WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, utils, [scanType: 'npm', configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") - assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("\"apiKey\": \"abcd\",")) - assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("\"productName\": \"name\",")) - assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("\"productToken\": \"1234\",")) - assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("\"userKey\": \"0000\"")) - } - - @Test - void testExtendConfigurationFilePip() { - WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, utils, [scanType: 'pip', configFilePath: './setup.py', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") - assertThat(jwfr.files['./setup.py.8813e60e0d9f7cacf0c414ae4964816f.py'], containsString("'org_token': 'abcd',")) - assertThat(jwfr.files['./setup.py.8813e60e0d9f7cacf0c414ae4964816f.py'], containsString("'product_name': 'name',")) - assertThat(jwfr.files['./setup.py.8813e60e0d9f7cacf0c414ae4964816f.py'], containsString("'product_token': '1234',")) - assertThat(jwfr.files['./setup.py.8813e60e0d9f7cacf0c414ae4964816f.py'], containsString("'user_key': '0000'")) - } - - @Test - void testExtendConfigurationFileSbt() { - WhitesourceConfigurationHelper.extendConfigurationFile(nullScript, utils, [scanType: 'sbt', configFilePath: './build.sbt', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000', agentUrl: 'http://mo-393ef744d.mo.sap.corp:8080/wsui/wspluginProxy.jsp'], "./") - assertThat(jwfr.files['./build.sbt'], containsString("whitesourceOrgToken in ThisBuild := \"abcd\"")) - assertThat(jwfr.files['./build.sbt'], containsString("whitesourceProduct in ThisBuild := \"name\"")) - assertThat(jwfr.files['./build.sbt'], containsString("whitesourceServiceUrl in ThisBuild := uri(\"http://mo-393ef744d.mo.sap.corp:8080/wsui/wspluginProxy.jsp\")")) + void testExtendConfigurationFileUnifiedAgentVerbose() { + WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', verbose: true, configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") + assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("apiKey=abcd")) + assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("productName=name")) + assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("productToken=1234")) + assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("userKey=0000")) + assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("python.resolveDependencies=true")) + assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("log.level=debug")) } } diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 1b35cb267..8e84b1e99 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -14,6 +14,19 @@ import static com.sap.piper.Prerequisites.checkScript @Field String STEP_NAME = 'whitesourceExecuteScan' @Field Set GENERAL_CONFIG_KEYS = [ + 'orgAdminUserTokenCredentialsId', + 'orgToken', + 'productName', + 'productVersion', + 'productToken', + 'projectNames', + 'scanType', + 'serviceUrl', + 'internalServiceUrl', + 'userTokenCredentialsId', + 'verbose' +] +@Field Set STEP_CONFIG_KEYS = GENERAL_CONFIG_KEYS + [ 'agentDownloadUrl', 'agentFileName', 'agentParameters', @@ -23,29 +36,19 @@ import static com.sap.piper.Prerequisites.checkScript 'configFilePath', 'dockerImage', 'dockerWorkspace', - 'internalServiceUrl', 'jreDownloadUrl', 'licensingVulnerabilities', - 'orgAdminUserTokenCredentialsId', - 'orgToken', 'parallelLimit', 'reporting', - 'scanType', 'securityVulnerabilities', + 'cvssSeverityLimit', 'stashContent', 'timeout', - 'productName', - 'productVersion', - 'productToken', - 'projectNames', - 'serviceUrl', - 'userTokenCredentialsId', 'vulnerabilityReportFileName', 'vulnerabilityReportTitle', - 'whitesourceAccessor', - 'verbose' + 'whitesourceAccessor' ] -@Field Set STEP_CONFIG_KEYS = GENERAL_CONFIG_KEYS + @Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS void call(Map parameters = [:]) { @@ -77,6 +80,7 @@ void call(Map parameters = [:]) { .withMandatoryProperty('productName') .use() + config.cvssSeverityLimit = Integer.valueOf(config.cvssSeverityLimit) config.stashContent = utils.unstashAll(config.stashContent) config.projectNames = (config.projectNames instanceof List) ? config.projectNames : config.projectNames?.tokenize(',') parameters.projectNames = config.projectNames @@ -199,7 +203,6 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU if (config.jreDownloadUrl) { sh "rm -r ./bin ./conf ./legal ./lib ./man" - javaCmd = './bin/java' } // archive whitesource result files @@ -228,8 +231,10 @@ void analyseWhitesourceResults(Map config, WhitesourceRepository repository, Whi archiveArtifacts artifacts: pdfName echo "A summary of the Whitesource findings was stored as artifact under the name ${pdfName}" - int violationCount = fetchViolationCount(config, repository) - checkViolationStatus(violationCount) + if(config.licensingVulnerabilities) { + def violationCount = fetchViolationCount(config, repository) + checkViolationStatus(violationCount) + } if (config.securityVulnerabilities) config.severeVulnerabilities = checkSecurityViolations(config, repository) @@ -255,7 +260,7 @@ void checkViolationStatus(int violationCount) { if (violationCount == 0) { echo "****\r\n[${STEP_NAME}] No policy violations found. You can deploy to production, and set the \"Intellectual Property (IP) Scan Plan\" in Sirius to completed. \r\n****" } else { - error "[${STEP_NAME}] Whitesource found $violationCount policy violations for your product" + error "[${STEP_NAME}] Whitesource found ${violationCount} policy violations for your product" } } @@ -265,7 +270,7 @@ int checkSecurityViolations(Map config, WhitesourceRepository repository) { def severeVulnerabilities = 0 whitesourceVulnerabilities.each { item -> - if (item.vulnerability.score >= 7 || item.vulnerability.cvss3_score >= 7) + if ((item.vulnerability.score >= config.cvssSeverityLimit || item.vulnerability.cvss3_score >= config.cvssSeverityLimit) && config.cvssSeverityLimit >= 0) severeVulnerabilities++ } @@ -305,6 +310,9 @@ void checkStatus(int statusCode, config) { case 251: errorMessage += "The server failed to analyze the scan" break + case 250: + errorMessage += "Pre-step failure" + break default: errorMessage += "Whitesource scan failed with unknown error code '${statusCode}'" } From 78367c689ba6d3ceca49a843e5b7411517515ec0 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 5 Mar 2019 14:08:22 +0100 Subject: [PATCH 022/130] Fix code --- src/com/sap/piper/WhitesourceConfigurationHelper.groovy | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index 9e1b4590e..af5853629 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -28,13 +28,13 @@ class WhitesourceConfigurationHelper implements Serializable { mapping += [name: 'log.level', value: 'debug'] mapping += [ - [name: 'apiKey', value: config.orgToken, warnIfPresent: true], + [name: 'apiKey', value: config.orgToken], [name: 'productName', value: config.productName], [name: 'productVersion', value: config.productVersion], [name: 'projectName', value: config.projectName], [name: 'projectVersion', value: config.productVersion], [name: 'productToken', value: config.productToken, omitIfPresent: 'projectToken'], - [name: 'userKey', value: config.userKey, warnIfPresent: true], + [name: 'userKey', value: config.userKey], [name: 'forceUpdate', value: true], [name: 'offline', value: false], [name: 'ignoreSourceFiles', value: true], @@ -90,8 +90,6 @@ class WhitesourceConfigurationHelper implements Serializable { mapping.each { entry -> - if (entry.warnIfPresent && moduleSpecificFile[entry.name]) - //Notify.warning(script, "Obsolete configuration ${entry.name} detected, please omit its use and rely on configuration via Piper.", 'WhitesourceConfigurationHelper') def dependentValue = entry.omitIfPresent ? moduleSpecificFile[entry.omitIfPresent] : null if ((entry.omitIfPresent && !dependentValue || !entry.omitIfPresent) && entry.value && entry.value != 'null' && entry.value != '') moduleSpecificFile[entry.name] = entry.value.toString() From a0e241b7e4be5919d583ca9c4f63181b0d371a42 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 5 Mar 2019 14:34:57 +0100 Subject: [PATCH 023/130] Add deletion of agent file downloaded --- vars/whitesourceExecuteScan.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 8e84b1e99..374dd3ebd 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -201,8 +201,12 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU statusCode = sh(script: "${javaCmd} ${options.join(' ')} ${config.agentParameters}", returnStatus: true) + if (config.agentDownloadUrl) { + sh "rm -f ${config.agentFileName}" + } + if (config.jreDownloadUrl) { - sh "rm -r ./bin ./conf ./legal ./lib ./man" + sh "rm -rf ./bin ./conf ./legal ./lib ./man" } // archive whitesource result files From 84ae8832643246365485a09f9885c71cd62b473b Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 5 Mar 2019 14:40:36 +0100 Subject: [PATCH 024/130] Enforce orgToken, sanitize limit --- vars/whitesourceExecuteScan.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 374dd3ebd..fd296b72a 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -76,11 +76,12 @@ void call(Map parameters = [:]) { .dependingOn('scanType').mixin('agentDownloadUrl') .dependingOn('scanType').mixin('agentFileName') .dependingOn('scanType').mixin('agentParameters') + .withMandatoryProperty('orgToken') .withMandatoryProperty('userTokenCredentialsId') .withMandatoryProperty('productName') .use() - config.cvssSeverityLimit = Integer.valueOf(config.cvssSeverityLimit) + config.cvssSeverityLimit = config.cvssSeverityLimit ? Integer.valueOf(config.cvssSeverityLimit) : -1 config.stashContent = utils.unstashAll(config.stashContent) config.projectNames = (config.projectNames instanceof List) ? config.projectNames : config.projectNames?.tokenize(',') parameters.projectNames = config.projectNames From f7f33f38acd13579ea86fd502e60bfa753127831 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 6 Mar 2019 11:04:49 +0100 Subject: [PATCH 025/130] Address comments --- resources/default_pipeline_environment.yml | 22 ++---- .../WhitesourceConfigurationHelper.groovy | 4 +- test/groovy/WhitesourceExecuteScanTest.groovy | 72 ++++++++++--------- vars/whitesourceExecuteScan.groovy | 3 - 4 files changed, 44 insertions(+), 57 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 863ec4c14..0f9962c0d 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -265,18 +265,18 @@ steps: vulnerabilityReportTitle: 'WhiteSource Security Vulnerability Report' projectNames: [] jreDownloadUrl: 'https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz' + agentFileName: 'wss-unified-agent.jar' + agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' + agentParameters: '' + configFilePath: './wss-unified-agent.config' mta: stashContent: - 'buildDescriptor' - 'opensourceConfiguration' maven: buildDescriptorFile: './pom.xml' - dockerImage: 'maven:3.5-jdk-7' + dockerImage: 'maven:3.5-jdk-8' dockerWorkspace: '/home/java' - agentFileName: 'wss-unified-agent.jar' - agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' - agentParameters: '' - configFilePath: './wss-unified-agent.config' stashContent: - 'buildDescriptor' - 'opensourceConfiguration' @@ -284,10 +284,6 @@ steps: buildDescriptorFile: './package.json' dockerImage: 'node:8-stretch' dockerWorkspace: '/home/node' - agentFileName: 'wss-unified-agent.jar' - agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' - agentParameters: '' - configFilePath: './wss-unified-agent.config' stashContent: - 'buildDescriptor' - 'opensourceConfiguration' @@ -295,10 +291,6 @@ steps: buildDescriptorFile: './setup.py' dockerImage: 'python:3.7.2-stretch' dockerWorkspace: '/home/python' - agentFileName: 'wss-unified-agent.jar' - agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' - agentParameters: '' - configFilePath: './wss-unified-agent.config' stashContent: - 'buildDescriptor' - 'opensourceConfiguration' @@ -306,10 +298,6 @@ steps: buildDescriptorFile: './build.sbt' dockerImage: 'hseeberger/scala-sbt:8u181_2.12.8_1.2.8' dockerWorkspace: '/home/scala' - agentFileName: 'wss-unified-agent.jar' - agentDownloadUrl: 'https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}' - agentParameters: '' - configFilePath: './wss-unified-agent.config' stashContent: - 'buildDescriptor' - 'opensourceConfiguration' diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index af5853629..21b793b36 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -2,8 +2,6 @@ package com.sap.piper import com.cloudbees.groovy.cps.NonCPS -import java.security.MessageDigest - class WhitesourceConfigurationHelper implements Serializable { static def extendUAConfigurationFile(script, utils, config, path) { @@ -11,7 +9,7 @@ class WhitesourceConfigurationHelper implements Serializable { def parsingClosure = { fileReadPath -> return script.readProperties (file: fileReadPath) } def serializationClosure = { configuration -> serializeUAConfig(configuration) } def inputFile = config.configFilePath.replaceFirst('\\./', '') - def suffix = MessageDigest.getInstance("MD5").digest(config.configFilePath.bytes).encodeHex().toString() + def suffix = utils.generateSha1(config.configFilePath) def targetFile = "${inputFile}.${suffix}" if(config.productName.startsWith('DIST - ')) { mapping += [ diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index 803cf9037..73db25fd9 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -174,18 +174,18 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'maven:3.5-jdk-7')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'maven:3.5-jdk-8')) assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/java')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config 420a1bc5c82f57e80307205d8625304f'])) + assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config d3aa80454919391024374ba46b4df082d15ab9a3'])) assertThat(shellRule.shell, Matchers.hasItems( is('curl --location --output wss-unified-agent.jar https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/wss-unified-agent.jar'), - is('./bin/java -jar wss-unified-agent.jar -c \'./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProduct\'') + is('./bin/java -jar wss-unified-agent.jar -c \'./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProduct\'') )) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('apiKey=testOrgToken')) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('productName=testProduct')) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('userKey=token-0815')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('apiKey=testOrgToken')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productName=testProduct')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('userKey=token-0815')) } @@ -221,19 +221,19 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'node:8-stretch')) assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/node')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config 420a1bc5c82f57e80307205d8625304f'])) + assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config d3aa80454919391024374ba46b4df082d15ab9a3'])) assertThat(shellRule.shell, Matchers.hasItems( is('curl --location --output wss-unified-agent.jar https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/wss-unified-agent.jar'), is('curl --location --output jvm.tar.gz https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz && tar --strip-components=1 -xzf jvm.tar.gz'), - is('./bin/java -jar wss-unified-agent.jar -c \'./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProductName\'') + is('./bin/java -jar wss-unified-agent.jar -c \'./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProductName\'') )) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('apiKey=testOrgToken')) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('productName=testProductName')) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('productToken=testProductToken')) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('productVersion=1.2.3')) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('projectName=com.sap.node.test-node')) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('userKey=token-0815')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('apiKey=testOrgToken')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productName=testProductName')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productToken=testProductToken')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1.2.3')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=com.sap.node.test-node')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('userKey=token-0815')) } @Test @@ -259,13 +259,14 @@ class WhitesourceExecuteScanTest extends BasePiperTest { productName : 'SHC - Piper', configFilePath : './../../testConfigPath', file : 'package.json', - juStabUtils : utils + juStabUtils : utils, + orgToken : 'b39d1328-52e2-42e3-98f0-932709daf3f0' ]) assertThat(shellRule.shell, Matchers.hasItems( is('curl --location --output wss-unified-agent.jar https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/wss-unified-agent.jar'), is('curl --location --output jvm.tar.gz https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz && tar --strip-components=1 -xzf jvm.tar.gz'), - is('./bin/java -jar wss-unified-agent.jar -c \'./../../testConfigPath.092aaffe7a79d11da13593b63b929754\' -userKey \'token-0815\' -product \'SHC - Piper\'') + is('./bin/java -jar wss-unified-agent.jar -c \'./../../testConfigPath.2766cacc0cf1449dd4034385f4a9f0a6fdb755cf\' -apiKey \'b39d1328-52e2-42e3-98f0-932709daf3f0\' -userKey \'token-0815\' -product \'SHC - Piper\'') )) } @@ -299,19 +300,19 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'python:3.7.2-stretch')) assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/python')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config 420a1bc5c82f57e80307205d8625304f'])) + assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config d3aa80454919391024374ba46b4df082d15ab9a3'])) assertThat(shellRule.shell, Matchers.hasItems( is('curl --location --output wss-unified-agent.jar https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/wss-unified-agent.jar'), is('curl --location --output jvm.tar.gz https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz && tar --strip-components=1 -xzf jvm.tar.gz'), - is('./bin/java -jar wss-unified-agent.jar -c \'./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProductName\'') + is('./bin/java -jar wss-unified-agent.jar -c \'./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProductName\'') )) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('apiKey=testOrgToken')) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('productName=testProductName')) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('userKey=token-0815')) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('productVersion=1.2.3')) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('projectName=test-python')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('apiKey=testOrgToken')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productName=testProductName')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('userKey=token-0815')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1.2.3')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=test-python')) } @Test @@ -345,18 +346,18 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'hseeberger/scala-sbt:8u181_2.12.8_1.2.8')) assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/scala')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config 420a1bc5c82f57e80307205d8625304f'])) + assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config d3aa80454919391024374ba46b4df082d15ab9a3'])) assertThat(shellRule.shell, Matchers.hasItems( is('curl --location --output wss-unified-agent.jar https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/wss-unified-agent.jar'), - is('./bin/java -jar wss-unified-agent.jar -c \'./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProductName\'') + is('./bin/java -jar wss-unified-agent.jar -c \'./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProductName\'') )) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('apiKey=testOrgToken')) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('productName=testProductName')) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('userKey=token-0815')) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('productVersion=1.2.3')) - assertThat(writeFileRule.files['./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f'], containsString('projectName=com.sap.sbt.test-scala')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('apiKey=testOrgToken')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productName=testProductName')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('userKey=token-0815')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1.2.3')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=com.sap.sbt.test-scala')) } @Test @@ -386,7 +387,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { productName : 'testProductName' ]) - assertThat(shellRule.shell[0], is('java -jar wss-unified-agent.jar -c \'./wss-unified-agent.config.420a1bc5c82f57e80307205d8625304f\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProductName\' testParams')) + assertThat(shellRule.shell[0], is('java -jar wss-unified-agent.jar -c \'./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProductName\' testParams')) } @Test @@ -435,7 +436,8 @@ class WhitesourceExecuteScanTest extends BasePiperTest { productName : 'SHC - Piper', buildDescriptorExcludeList : ["maven2${File.separator}pom.xml".toString(), "npm2${File.separator}package.json".toString()], reporting : true, - juStabUtils : utils + juStabUtils : utils, + orgToken : 'b39d1328-52e2-42e3-98f0-932709daf3f0' ]) assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) @@ -494,7 +496,8 @@ class WhitesourceExecuteScanTest extends BasePiperTest { productName : 'SHC - Piper', buildDescriptorExcludeList : ["maven2${File.separator}pom.xml".toString()], juStabUtils : utils, - parallelLimit : 3 + parallelLimit : 3, + orgToken : 'b39d1328-52e2-42e3-98f0-932709daf3f0' ]) assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) @@ -556,7 +559,8 @@ class WhitesourceExecuteScanTest extends BasePiperTest { scanType : 'mta', productName : 'SHC - Piper', buildDescriptorExcludeList : "maven2${File.separator}pom.xml", - juStabUtils : utils + juStabUtils : utils, + orgToken : 'b39d1328-52e2-42e3-98f0-932709daf3f0' ]) assertThat(parallelMap.keySet(), hasSize(4)) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index fd296b72a..54deceb96 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -73,9 +73,6 @@ void call(Map parameters = [:]) { .dependingOn('scanType').mixin('dockerImage') .dependingOn('scanType').mixin('dockerWorkspace') .dependingOn('scanType').mixin('stashContent') - .dependingOn('scanType').mixin('agentDownloadUrl') - .dependingOn('scanType').mixin('agentFileName') - .dependingOn('scanType').mixin('agentParameters') .withMandatoryProperty('orgToken') .withMandatoryProperty('userTokenCredentialsId') .withMandatoryProperty('productName') From df01a88b34eb439b0fa71aaea366dcc217898943 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 6 Mar 2019 12:14:38 +0100 Subject: [PATCH 026/130] Fix tests --- .../WhiteSourceConfigurationHelperTest.groovy | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy b/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy index 42f41a4f9..9d7374af0 100644 --- a/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy +++ b/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy @@ -30,22 +30,22 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { @Test void testExtendConfigurationFileUnifiedAgentPip() { WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") - assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("apiKey=abcd")) - assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("productName=name")) - assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("productToken=1234")) - assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("userKey=0000")) - assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("python.resolveDependencies=true")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=name")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("python.resolveDependencies=true")) } @Test void testExtendConfigurationFileUnifiedAgentVerbose() { WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', verbose: true, configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") - assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("apiKey=abcd")) - assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("productName=name")) - assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("productToken=1234")) - assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("userKey=0000")) - assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("python.resolveDependencies=true")) - assertThat(jwfr.files['./config.c92a71303bcc841344e07d1bf49d1f9b'], containsString("log.level=debug")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=name")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("python.resolveDependencies=true")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("log.level=debug")) } } From 93b3afe67d7a29af2968ebc220eea5c69f11921d Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 6 Mar 2019 12:16:51 +0100 Subject: [PATCH 027/130] Add default for whitesource backend URL --- resources/default_pipeline_environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 0f9962c0d..5afbe760e 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -23,6 +23,7 @@ general: jnlpAgent: 's4sdk/jenkins-agent-k8s:latest' manualConfirmation: true productiveBranch: 'master' + serviceUrl: 'https://saas.whitesourcesoftware.com/api' #Steps Specific Configuration steps: From 4a05b317b3ac58839d3850b0da36c4ed1c45029a Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Wed, 6 Mar 2019 13:07:55 +0100 Subject: [PATCH 028/130] Update vars/whitesourceExecuteScan.groovy Co-Authored-By: nevskrem <33895725+nevskrem@users.noreply.github.com> --- vars/whitesourceExecuteScan.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 54deceb96..abe35c773 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -12,7 +12,7 @@ import groovy.text.SimpleTemplateEngine import static com.sap.piper.Prerequisites.checkScript -@Field String STEP_NAME = 'whitesourceExecuteScan' +@Field String STEP_NAME = getClass().getName() @Field Set GENERAL_CONFIG_KEYS = [ 'orgAdminUserTokenCredentialsId', 'orgToken', From 39ef78c219699d2d882de7792de12f66912b3e82 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Wed, 6 Mar 2019 13:10:31 +0100 Subject: [PATCH 029/130] Update vars/whitesourceExecuteScan.groovy Co-Authored-By: nevskrem <33895725+nevskrem@users.noreply.github.com> --- vars/whitesourceExecuteScan.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index abe35c773..f4297bd36 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -281,7 +281,7 @@ int checkSecurityViolations(Map config, WhitesourceRepository repository) { archiveArtifacts(artifacts: "${config.vulnerabilityReportFileName}.*") if (whitesourceVulnerabilities.size() - severeVulnerabilities > 0) - echo "[${STEP_NAME}] WARNING: ${whitesourceVulnerabilities.size() - severeVulnerabilities} Open Source Software Security vulnerabilities with CVSS score below 7.0 detected." + echo "[${STEP_NAME}] WARNING: ${whitesourceVulnerabilities.size() - severeVulnerabilities} Open Source Software Security vulnerabilities with CVSS score below ${config.cvssSeverityLimit} detected." if (whitesourceVulnerabilities.size() == 0) echo "[${STEP_NAME}] No Open Source Software Security vulnerabilities detected." From ebbc373d566afc223093d56e2bb4e8368519ee58 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Wed, 6 Mar 2019 13:10:49 +0100 Subject: [PATCH 030/130] Update vars/whitesourceExecuteScan.groovy Co-Authored-By: nevskrem <33895725+nevskrem@users.noreply.github.com> --- vars/whitesourceExecuteScan.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index f4297bd36..a313f2d47 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -341,7 +341,7 @@ def getReportHtml(config, vulnerabilityList, numSevereVulns) { - + From d000c25addc23bb18399bb7f4e28409ca413e16b Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Wed, 6 Mar 2019 13:11:05 +0100 Subject: [PATCH 031/130] Update vars/whitesourceExecuteScan.groovy Co-Authored-By: nevskrem <33895725+nevskrem@users.noreply.github.com> --- vars/whitesourceExecuteScan.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index a313f2d47..d2166dfcb 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -292,7 +292,7 @@ int checkSecurityViolations(Map config, WhitesourceRepository repository) { void checkStatus(int statusCode, config) { def errorMessage = "" if(config.securityVulnerabilities && config.severeVulnerabilities > 0) - errorMessage += "${config.severeVulnerabilities} Open Source Software Security vulnerabilities with CVSS score greater or equal 7.0 detected. - " + errorMessage += "${config.severeVulnerabilities} Open Source Software Security vulnerabilities with CVSS score greater or equal ${config.cvssSeverityLimit} detected. - " if (config.licensingVulnerabilities) switch (statusCode) { case 0: From 6a907cff2332188bc29699aec83f6460c9ae178c Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 6 Mar 2019 13:12:23 +0100 Subject: [PATCH 032/130] Add creation capabilities for whitesource products --- resources/default_pipeline_environment.yml | 2 ++ .../WhitesourceOrgAdminRepository.groovy | 33 +++++++++++++++++-- vars/whitesourceExecuteScan.groovy | 11 +++++-- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 5afbe760e..c81e2f407 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -256,6 +256,8 @@ steps: npmExecute: dockerImage: 'node:8-stretch' whitesourceExecuteScan: + createProductFromPipeline: true + emailAddressesOfInitialProductAdmins: [] buildDescriptorExcludeList: [] parallelLimit: 15 licensingVulnerabilities: true diff --git a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy index 52b056646..eb1593d44 100644 --- a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy @@ -42,13 +42,40 @@ class WhitesourceOrgAdminRepository implements Serializable { } } - if (!foundMetaProduct) - script.error "[WhiteSource] Could not fetch/find requested product '${config.productName}'" - return foundMetaProduct } + def createProduct() { + def requestBody = [ + requestType: "createProduct", + orgToken: config.orgToken, + productName: config.productName + ] + def response = issueHttpRequest(requestBody) + def parsedResponse = new JsonUtils().parseJsonSerializable(response.content) + def metaInfo = parsedResponse + def groups = [] + def users = [] + config.emailAddressesOfInitialProductAdmins.each { + email -> users.add(["email": config.emailOfInitialProductAdmin]) + } + + requestBody = [ + "requestType" : "setProductAssignments", + "productToken" : metaInfo.productToken, + "productMembership" : ["userAssignments":[], "groupAssignments":groups], + "productAdmins" : ["userAssignments":users], + "alertsEmailReceivers" : ["userAssignments":[]] + ] + issueHttpRequest(requestBody) + + return metaInfo + } + + def issueHttpRequest(requestBody) { + internalWhitesource ? internalWhitesource.httpWhitesource(requestBody) : httpWhitesource(requestBody) + } @NonCPS protected def httpWhitesource(requestBody) { diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 54deceb96..1ffb2871c 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -224,8 +224,15 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU void analyseWhitesourceResults(Map config, WhitesourceRepository repository, WhitesourceOrgAdminRepository orgAdminRepository) { if (!config.productToken) { def metaInfo = orgAdminRepository.fetchProductMetaInfo() - echo "Meta Information: ${metaInfo}" - config.productToken = metaInfo.token + def key = "token" + if(!metaInfo && config.createProductFromPipeline) { + metaInfo = orgAdminRepository.createProduct() + key = "productToken" + } else if(!metaInfo) { + error "[WhiteSource] Could not fetch/find requested product '${config.productName}' and automatic creation has been disabled" + } + echo "Meta Info: ${metaInfo}" + config.productToken = metaInfo[key] } def pdfName = "whitesource-riskReport.pdf" From cfc3a1528a98048f93a59dafa781c182328d7cca Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 6 Mar 2019 14:03:00 +0100 Subject: [PATCH 033/130] Implement comments --- resources/default_pipeline_environment.yml | 3 +- .../WhitesourceOrgAdminRepository.groovy | 2 +- .../integration/WhitesourceRepository.groovy | 2 +- test/groovy/WhitesourceExecuteScanTest.groovy | 55 ++++++++++++++++--- vars/whitesourceExecuteScan.groovy | 31 ++++++++--- 5 files changed, 76 insertions(+), 17 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index c81e2f407..c478181e5 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -23,7 +23,8 @@ general: jnlpAgent: 's4sdk/jenkins-agent-k8s:latest' manualConfirmation: true productiveBranch: 'master' - serviceUrl: 'https://saas.whitesourcesoftware.com/api' + whitesource: + serviceUrl: 'https://saas.whitesourcesoftware.com/api' #Steps Specific Configuration steps: diff --git a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy index eb1593d44..b8c3a309c 100644 --- a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy @@ -95,7 +95,7 @@ class WhitesourceOrgAdminRepository implements Serializable { timeout : config.timeout ] - if (script.env.HTTP_PROXY && !config.serviceUrl.matches('http(s)*://.*\\.sap\\.corp.*')) + if (script.env.HTTP_PROXY) params["httpProxy"] = script.env.HTTP_PROXY if (config.verbose) diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy index b128f7f9b..b7f8569b0 100644 --- a/src/com/sap/piper/integration/WhitesourceRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -218,7 +218,7 @@ class WhitesourceRepository implements Serializable { timeout : config.timeout ] - if (script.env.HTTP_PROXY && !config.serviceUrl.matches('http(s)*://.*\\.sap\\.corp.*')) + if (script.env.HTTP_PROXY) params["httpProxy"] = script.env.HTTP_PROXY if(config.verbose) diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index 73db25fd9..987119bbb 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -760,7 +760,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { } @Test - void testCheckFindingBelowSeven() { + void testCheckFindingBelowThreshold() { helper.registerAllowedMethod("readProperties", [Map], { def result = new Properties() result.putAll([ @@ -790,13 +790,55 @@ class WhitesourceExecuteScanTest extends BasePiperTest { securityVulnerabilities : true, orgToken : 'testOrgToken', productName : 'SHC - Piper', - projectNames : [ 'piper-demo - 0.0.1' ] + projectNames : [ 'piper-demo - 0.0.1' ], + cvssSeverityLimit : 7 ]) - assertThat(loggingRule.log, containsString('WARNING: 1 Open Source Software Security vulnerabilities with CVSS score below 7.0 detected.')) + assertThat(loggingRule.log, containsString('WARNING: 1 Open Source Software Security vulnerabilities with CVSS score below 7 detected.')) assertThat(writeFileRule.files['piper_whitesource_vulnerability_report.json'], not(isEmptyOrNullString())) } + @Test + void testCheckFindingAbove() { + helper.registerAllowedMethod("readProperties", [Map], { + def result = new Properties() + result.putAll([ + "apiKey": "b39d1328-52e2-42e3-98f0-932709daf3f0", + "productName": "SHC - Piper", + "checkPolicies": "true", + "projectName": "pipeline-test-node", + "projectVersion": "1.0.0" + ]) + return result + }) + helper.registerAllowedMethod("fetchVulnerabilities", [List], { + return new JsonUtils().parseJsonSerializable("{ \"alerts\": [ { \"vulnerability\": { \"name\": \"CVE-2017-15095\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 2.1, \"cvss3_severity\": \"high\", \"cvss3_score\": 5.3, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15095\", \"description\": \"A deserialization flaw was discovered in the jackson-databind in versions before 2.8.10 and 2.9.1, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper. This issue extends the previous flaw CVE-2017-7525 by blacklisting more classes that could be used maliciously.\", \"topFix\": { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\"," + + "\"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\", \"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\"," + + "\"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e865a7a4464da63ded9f4b1a2328ad85c9ded78b#diff-98084d808198119d550a9211e128a16f\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-12-12\", \"message\": \"Fix #1737 (#1857)\", \"extraData\": \"key=e865a7a&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\"," + + "\"url\": \"https://github.com/FasterXML/jackson-databind/commit/e8f043d1\", \"fixResolution\": \"release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-06-30\", \"message\": \"Fix #1680\", \"extraData\": \"key=e8f043d&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" } ], \"fixResolutionText\": \"Replace or update the following files: IllegalTypesCheckTest.java, VERSION, BeanDeserializerFactory.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\"," + + "\"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-databind\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:5,\", \"date\": \"2017-11-15\" } ] }").alerts + }) + + try { + stepRule.step.whitesourceExecuteScan([ + script : nullScript, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub: whitesourceOrgAdminRepositoryStub, + descriptorUtilsStub : descriptorUtilsStub, + scanType : 'npm', + juStabUtils : utils, + securityVulnerabilities : true, + orgToken : 'testOrgToken', + productName : 'SHC - Piper', + projectNames : ['piper-demo - 0.0.1'], + cvssSeverityLimit : 0 + ]) + } catch (e) { + assertThat(e.getMessage(), containsString('[whitesourceExecuteScan] 1 Open Source Software Security vulnerabilities with CVSS score greater or equal 0 detected. - ')) + assertThat(writeFileRule.files['piper_whitesource_vulnerability_report.json'], not(isEmptyOrNullString())) + } + } + @Test void testCheckNoFindings() { helper.registerAllowedMethod("readProperties", [Map], { @@ -833,7 +875,6 @@ class WhitesourceExecuteScanTest extends BasePiperTest { @Test void testCheckStatus_0() { - def error = false try { stepRule.step.checkStatus(0, [licensingVulnerabilities: true]) @@ -931,10 +972,10 @@ class WhitesourceExecuteScanTest extends BasePiperTest { void testCheckStatus_vulnerability() { def error = false try { - stepRule.step.checkStatus(0, [licensingVulnerabilities: false, securityVulnerabilities: true, severeVulnerabilities: 5]) + stepRule.step.checkStatus(0, [licensingVulnerabilities: false, securityVulnerabilities: true, severeVulnerabilities: 5, cvssSeverityLimit: 7]) } catch (e) { error = true - assertThat(e.getMessage(), is("[whitesourceExecuteScan] 5 Open Source Software Security vulnerabilities with CVSS score greater or equal 7.0 detected. - ")) + assertThat(e.getMessage(), is("[whitesourceExecuteScan] 5 Open Source Software Security vulnerabilities with CVSS score greater or equal 7 detected. - ")) } assertThat(error, is(true)) } @@ -948,7 +989,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { error = true } assertThat(error, is(false)) - assertThat(loggingRule.log, containsString("****\r\n[whitesourceExecuteScan] No policy violations found. You can deploy to production, and set the \"Intellectual Property (IP) Scan Plan\" in Sirius to completed. \r\n****")) + assertThat(loggingRule.log, containsString("[whitesourceExecuteScan] No policy violations found")) } @Test diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 3d32d1600..9058e0de5 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -45,12 +45,29 @@ import static com.sap.piper.Prerequisites.checkScript 'stashContent', 'timeout', 'vulnerabilityReportFileName', - 'vulnerabilityReportTitle', - 'whitesourceAccessor' + 'vulnerabilityReportTitle' ] @Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS +@Field Map CONFIG_KEY_COMPATIBILITY = [ + whitesource : [ + 'orgAdminUserTokenCredentialsId' : 'orgAdminUserTokenCredentialsId', + 'orgToken' : 'orgToken', + 'productName' : 'productName', + 'productVersion' : 'productVersion', + 'productToken' : 'productToken', + 'projectNames' : 'projectNames', + 'scanType' : 'scanType', + 'serviceUrl' : 'serviceUrl', + 'userTokenCredentialsId' : 'userTokenCredentialsId' + ], + whitesourceUserTokenCredentialsId : 'userTokenCredentialsId', + whitesourceProductName : 'productName', + whitesourceProjectNames : 'projectNames', + whitesourceProductToken : 'productToken' +] + void call(Map parameters = [:]) { handlePipelineStepErrors(stepName: STEP_NAME, stepParameters: parameters) { def script = checkScript(this, parameters) ?: this @@ -61,9 +78,9 @@ void call(Map parameters = [:]) { // load default & individual configuration Map config = ConfigurationHelper.newInstance(this) .loadStepDefaults() - .mixinGeneralConfig(script.commonPipelineEnvironment, GENERAL_CONFIG_KEYS) - .mixinStepConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS) - .mixinStageConfig(script.commonPipelineEnvironment, parameters.stageName ?: env.STAGE_NAME, STEP_CONFIG_KEYS) + .mixinGeneralConfig(script.commonPipelineEnvironment, GENERAL_CONFIG_KEYS, CONFIG_KEY_COMPATIBILITY) + .mixinStepConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS, CONFIG_KEY_COMPATIBILITY) + .mixinStageConfig(script.commonPipelineEnvironment, parameters.stageName ?: env.STAGE_NAME, STEP_CONFIG_KEYS, CONFIG_KEY_COMPATIBILITY) .mixin([ style : libraryResource('piper-os.css') ]) @@ -78,7 +95,7 @@ void call(Map parameters = [:]) { .withMandatoryProperty('productName') .use() - config.cvssSeverityLimit = config.cvssSeverityLimit ? Integer.valueOf(config.cvssSeverityLimit) : -1 + config.cvssSeverityLimit = config.cvssSeverityLimit == null ? -1 : Integer.valueOf(config.cvssSeverityLimit) config.stashContent = utils.unstashAll(config.stashContent) config.projectNames = (config.projectNames instanceof List) ? config.projectNames : config.projectNames?.tokenize(',') parameters.projectNames = config.projectNames @@ -267,7 +284,7 @@ int fetchViolationCount(Map config, WhitesourceRepository repository) { void checkViolationStatus(int violationCount) { if (violationCount == 0) { - echo "****\r\n[${STEP_NAME}] No policy violations found. You can deploy to production, and set the \"Intellectual Property (IP) Scan Plan\" in Sirius to completed. \r\n****" + echo "[${STEP_NAME}] No policy violations found" } else { error "[${STEP_NAME}] Whitesource found ${violationCount} policy violations for your product" } From d3bfab50f6119278f29882d17b12da31e0a0a18e Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 6 Mar 2019 14:16:28 +0100 Subject: [PATCH 034/130] Remove confidential information --- .../WhitesourceOrgAdminRepository.groovy | 4 +-- .../WhitesourceOrgAdminRepositoryTest.groovy | 28 +++++++++++++++++-- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy index b8c3a309c..c3928ef81 100644 --- a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy @@ -9,8 +9,6 @@ class WhitesourceOrgAdminRepository implements Serializable { final internalWhitesource final Map config - def orgAdminUserKey - WhitesourceOrgAdminRepository(Script script, Map config) { this.script = script this.config = config @@ -83,7 +81,7 @@ class WhitesourceOrgAdminRepository implements Serializable { credentialsId: config.orgAdminUserTokenCredentialsId, variable: 'orgAdminUserKey' )]) { - requestBody["userKey"] = orgAdminUserKey + requestBody["userKey"] = script.env.orgAdminUserKey def serializedBody = new JsonUtils().jsonToString(requestBody) def params = [ url : config.serviceUrl, diff --git a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy index fba567e93..764d18201 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy @@ -30,8 +30,6 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { @Before void init() throws Exception { - nullScript.env['HTTP_PROXY'] = "http://proxy.wdf.sap.corp:8080" - repository = new WhitesourceOrgAdminRepository(nullScript, [serviceUrl: "http://some.host.whitesource.com/api/", verbose: true]) LibraryLoadingTestExecutionListener.prepareObjectInterceptors(repository) } @@ -71,4 +69,30 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { name : 'Correct Name Cloud' ]) } + + @Test + void testHttpWhitesourceInternalCallUserKey() { + nullScript.env.orgAdminUserKey = "4711" + def config = [ serviceUrl: "http://some.host.whitesource.com/api/", verbose: false, orgAdminUserKey: nullScript.env.orgAdminUserKey] + def requestBody = ["someJson" : [ "someObject" : "abcdef" ]] + + def requestParams + helper.registerAllowedMethod('httpRequest', [Map], { p -> + requestParams = p + }) + + repository.httpWhitesource(requestBody) + + assertThat(requestParams, is( + [ + url : config.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: requestBody, + quiet : true, + userKey : config.orgAdminUserKey + ] + )) + } } From bfbee88620966bdd9dcfef6685cff88a76dd9a75 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 6 Mar 2019 15:08:42 +0100 Subject: [PATCH 035/130] Added documentation of parameters and step --- documentation/bin/createDocu.groovy | 3 +- .../WhitesourceOrgAdminRepositoryTest.groovy | 1 - vars/whitesourceExecuteScan.groovy | 113 +++++++++++++++++- 3 files changed, 112 insertions(+), 5 deletions(-) diff --git a/documentation/bin/createDocu.groovy b/documentation/bin/createDocu.groovy index ee8516417..9e7225d08 100644 --- a/documentation/bin/createDocu.groovy +++ b/documentation/bin/createDocu.groovy @@ -1,8 +1,7 @@ -import groovy.io.FileType; +import groovy.io.FileType import org.yaml.snakeyaml.Yaml import org.codehaus.groovy.control.CompilerConfiguration import com.sap.piper.GenerateDocumentation -import com.sap.piper.DefaultValueCache import java.util.regex.Matcher // diff --git a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy index 764d18201..83a485e7d 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy @@ -1,6 +1,5 @@ package com.sap.piper.integration -import com.sap.piper.JsonUtils import org.junit.After import org.junit.Before import org.junit.Rule diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 9058e0de5..e360eed6c 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -1,4 +1,5 @@ import com.sap.piper.DescriptorUtils +import com.sap.piper.GenerateDocumentation import com.sap.piper.JsonUtils import com.sap.piper.Utils import com.sap.piper.integration.WhitesourceOrgAdminRepository @@ -14,37 +15,133 @@ import static com.sap.piper.Prerequisites.checkScript @Field String STEP_NAME = getClass().getName() @Field Set GENERAL_CONFIG_KEYS = [ + /** + * Jenkins credentials ID referring to the organization admin's token. + */ 'orgAdminUserTokenCredentialsId', + /** + * WhiteSource token identifying your organization. + */ 'orgToken', + /** + * Name of the WhiteSource product to be created and used for results aggregation. + */ 'productName', + /** + * Version of the WhiteSource product to be created and used for results aggregation, usually determined automatically. + */ 'productVersion', + /** + * Token of the WhiteSource product to be created and used for results aggregation, usually determined automatically. + */ 'productToken', + /** + * List of WhiteSource projects to be included in the assessment part of the step, usually determined automatically. + */ 'projectNames', + /** + * Type of development stack used to implement the solution. + * @possibleValues `maven`, `mta`, `npm`, `pip`, `sbt` + */ 'scanType', + /** + * URL to the WhiteSource server API used for communication, defaults to `https://saas.whitesourcesoftware.com/api`. + */ 'serviceUrl', - 'internalServiceUrl', + /** + * Jenkins credentials ID referring to the product admin's token. + */ 'userTokenCredentialsId', + /** + * Whether verbose output should be produced. + * @possibleValues `true`, `false` + */ 'verbose' ] @Field Set STEP_CONFIG_KEYS = GENERAL_CONFIG_KEYS + [ + /** + * URL used to download the latest version of the WhiteSource Unified Agent. + */ 'agentDownloadUrl', + /** + * Locally used name for the Unified Agent jar file after download. + */ 'agentFileName', + /** + * Additional parameters passed to the Unified Agent command line. + */ 'agentParameters', - 'artifactUrl', + /** + * List of build descriptors and therefore modules to exclude from the scan and assessment activities. + */ 'buildDescriptorExcludeList', + /** + * Explicit path to the build descriptor file. + */ 'buildDescriptorFile', + /** + * Explicit path to the WhiteSource Unified Agent configuration file. + */ 'configFilePath', + /** + * Whether to create the related WhiteSource product on the fly based on the supplied pipeline configuration. + */ + 'createProductFromPipeline', + /** + * The list of email addresses to assign as product admins for newly created WhiteSource products. + */ + 'emailAddressesOfInitialProductAdmins', + /** + * Docker image to be used for scanning. + */ 'dockerImage', + /** + * Docker workspace to be used for scanning. + */ 'dockerWorkspace', + /** + * URL used for downloading the Java Runtime Environment (JRE) required to run the WhiteSource Unified Agent. + */ 'jreDownloadUrl', + /** + * Whether license compliance is considered and reported as part of the assessment. + * @possibleValues `true`, `false` + */ 'licensingVulnerabilities', + /** + * Limit of parallel jobs being run at once in case of `scanType: 'mta'` based scenarios, defaults to `15`. + */ 'parallelLimit', + /** + * Whether assessment is being done at all, defaults to `true`. + * @possibleValues `true`, `false` + */ 'reporting', + /** + * Whether security compliance is considered and reported as part of the assessment. + * @possibleValues `true`, `false` + */ 'securityVulnerabilities', + /** + * Limit of tollerable CVSS v3 score upon assessment and in consequence fails the build, defaults to `-1`. + * @possibleValues `-1` to switch failing off, any `positive integer between 0 and 10` to fail on issues with the specified limit or above + */ 'cvssSeverityLimit', + /** + * List of stashes to be unstashed into the workspace before performing the scan. + */ 'stashContent', + /** + * Timeout in seconds until a HTTP call is forcefully terminated. + */ 'timeout', + /** + * Name of the file the vulnerability report is written to. + */ 'vulnerabilityReportFileName', + /** + * Title of vulnerability report written during the assessment phase. + */ 'vulnerabilityReportTitle' ] @@ -68,6 +165,18 @@ import static com.sap.piper.Prerequisites.checkScript whitesourceProductToken : 'productToken' ] +/** + * With this step [WhiteSource](https://www.whitesource.com) security and license compliance scans can be executed and assessed. + * + * WhiteSource is a Software as a Service offering based on a so called unified scanning agent that locally determines the dependency + * tree of a node.js, Java, Python, Ruby, or Scala based solution and sends it to the WhiteSource server for a policy based license compliance + * check and additional Free and Open Source Software Publicly Known Vulnerabilities assessment. + * + * !!! note "Docker Images" + * The underlying Docker images are public and specific to the solution's programming language(s) and may therefore be exchanged + * to fit and suite the relevant scenario. The default Python environment used is i.e. Python 3 based. + */ +@GenerateDocumentation void call(Map parameters = [:]) { handlePipelineStepErrors(stepName: STEP_NAME, stepParameters: parameters) { def script = checkScript(this, parameters) ?: this From fa33f1925c78c1aacb7e3b987654140839d1bc2e Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 6 Mar 2019 15:12:21 +0100 Subject: [PATCH 036/130] Fix compatibility mapping --- vars/whitesourceExecuteScan.groovy | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index e360eed6c..99ac6f2a3 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -149,15 +149,15 @@ import static com.sap.piper.Prerequisites.checkScript @Field Map CONFIG_KEY_COMPATIBILITY = [ whitesource : [ - 'orgAdminUserTokenCredentialsId' : 'orgAdminUserTokenCredentialsId', - 'orgToken' : 'orgToken', - 'productName' : 'productName', - 'productVersion' : 'productVersion', - 'productToken' : 'productToken', - 'projectNames' : 'projectNames', - 'scanType' : 'scanType', - 'serviceUrl' : 'serviceUrl', - 'userTokenCredentialsId' : 'userTokenCredentialsId' + orgAdminUserTokenCredentialsId : 'orgAdminUserTokenCredentialsId', + orgToken : 'orgToken', + productName : 'productName', + productVersion : 'productVersion', + productToken : 'productToken', + projectNames : 'projectNames', + scanType : 'scanType', + serviceUrl : 'serviceUrl', + userTokenCredentialsId : 'userTokenCredentialsId' ], whitesourceUserTokenCredentialsId : 'userTokenCredentialsId', whitesourceProductName : 'productName', From 4344f279fa2ac65a5e0690ea27fa6fa43af681bc Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 6 Mar 2019 15:33:19 +0100 Subject: [PATCH 037/130] Fix compatibility feature --- src/com/sap/piper/ConfigurationHelper.groovy | 2 +- vars/whitesourceExecuteScan.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/com/sap/piper/ConfigurationHelper.groovy b/src/com/sap/piper/ConfigurationHelper.groovy index f13436fe1..e6abb255f 100644 --- a/src/com/sap/piper/ConfigurationHelper.groovy +++ b/src/com/sap/piper/ConfigurationHelper.groovy @@ -49,7 +49,7 @@ class ConfigurationHelper implements Serializable { final ConfigurationHelper mixin(Map parameters, Set filter = null, Map compatibleParameters = [:]){ if (parameters.size() > 0 && compatibleParameters.size() > 0) { - parameters = ConfigurationMerger.merge(handleCompatibility(compatibleParameters, parameters), null, parameters) + parameters = ConfigurationMerger.merge(handleCompatibility(compatibleParameters, parameters, config), null, parameters) } if (filter) { filter.add('collectTelemetryData') diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 99ac6f2a3..051f4544e 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -193,7 +193,7 @@ void call(Map parameters = [:]) { .mixin([ style : libraryResource('piper-os.css') ]) - .mixin(parameters, PARAMETER_KEYS) + .mixin(parameters, PARAMETER_KEYS, CONFIG_KEY_COMPATIBILITY) .dependingOn('scanType').mixin('buildDescriptorFile') .dependingOn('scanType').mixin('configFilePath') .dependingOn('scanType').mixin('dockerImage') From 28b370371fda8e276d8d1aaca97aef6ae621aee6 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 6 Mar 2019 15:46:04 +0100 Subject: [PATCH 038/130] Fix compatibility --- src/com/sap/piper/ConfigurationHelper.groovy | 6 ++--- vars/whitesourceExecuteScan.groovy | 28 +++++++++----------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/com/sap/piper/ConfigurationHelper.groovy b/src/com/sap/piper/ConfigurationHelper.groovy index e6abb255f..787b1168b 100644 --- a/src/com/sap/piper/ConfigurationHelper.groovy +++ b/src/com/sap/piper/ConfigurationHelper.groovy @@ -33,8 +33,8 @@ class ConfigurationHelper implements Serializable { } ConfigurationHelper mixinGeneralConfig(commonPipelineEnvironment, Set filter = null, Map compatibleParameters = [:]){ - Map stepConfiguration = ConfigurationLoader.generalConfiguration([commonPipelineEnvironment: commonPipelineEnvironment]) - return mixin(stepConfiguration, filter, compatibleParameters) + Map generalConfiguration = ConfigurationLoader.generalConfiguration([commonPipelineEnvironment: commonPipelineEnvironment]) + return mixin(generalConfiguration, filter, compatibleParameters) } ConfigurationHelper mixinStageConfig(commonPipelineEnvironment, stageName, Set filter = null, Map compatibleParameters = [:]){ @@ -49,7 +49,7 @@ class ConfigurationHelper implements Serializable { final ConfigurationHelper mixin(Map parameters, Set filter = null, Map compatibleParameters = [:]){ if (parameters.size() > 0 && compatibleParameters.size() > 0) { - parameters = ConfigurationMerger.merge(handleCompatibility(compatibleParameters, parameters, config), null, parameters) + parameters = ConfigurationMerger.merge(handleCompatibility(compatibleParameters, parameters), null, parameters) } if (filter) { filter.add('collectTelemetryData') diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 051f4544e..d932d2c08 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -148,21 +148,19 @@ import static com.sap.piper.Prerequisites.checkScript @Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS @Field Map CONFIG_KEY_COMPATIBILITY = [ - whitesource : [ - orgAdminUserTokenCredentialsId : 'orgAdminUserTokenCredentialsId', - orgToken : 'orgToken', - productName : 'productName', - productVersion : 'productVersion', - productToken : 'productToken', - projectNames : 'projectNames', - scanType : 'scanType', - serviceUrl : 'serviceUrl', - userTokenCredentialsId : 'userTokenCredentialsId' - ], - whitesourceUserTokenCredentialsId : 'userTokenCredentialsId', - whitesourceProductName : 'productName', - whitesourceProjectNames : 'projectNames', - whitesourceProductToken : 'productToken' + orgAdminUserTokenCredentialsId : 'whitesource.orgAdminUserTokenCredentialsId', + orgToken : 'whitesource.orgToken', + productName : 'whitesource.productName', + productVersion : 'whitesource.productVersion', + productToken : 'whitesource.productToken', + projectNames : 'whitesource.projectNames', + scanType : 'whitesource.scanType', + serviceUrl : 'whitesource.serviceUrl', + userTokenCredentialsId : 'whitesource.userTokenCredentialsId', + userTokenCredentialsId : 'whitesourceUserTokenCredentialsId', + productName : 'whitesourceProductName', + projectNames : 'whitesourceProjectNames', + productToken : 'whitesourceProductToken' ] /** From e6c3d6e4ba4144feb6968c2bbee3fbea422a60ab Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Thu, 7 Mar 2019 11:13:24 +0100 Subject: [PATCH 039/130] Enhance config compatibility feature --- src/com/sap/piper/ConfigurationHelper.groovy | 16 +++++++++++++++- .../com/sap/piper/ConfigurationHelperTest.groovy | 11 +++++++++++ vars/whitesourceExecuteScan.groovy | 8 ++++---- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/com/sap/piper/ConfigurationHelper.groovy b/src/com/sap/piper/ConfigurationHelper.groovy index 787b1168b..a23df489a 100644 --- a/src/com/sap/piper/ConfigurationHelper.groovy +++ b/src/com/sap/piper/ConfigurationHelper.groovy @@ -70,7 +70,9 @@ class ConfigurationHelper implements Serializable { configSubMap = configSubMap?.get(key) } if (configSubMap == null || (configSubMap != null && configSubMap[entry.getKey()] == null)) { - newConfig[entry.getKey()] = configMap[entry.getValue()] + def stages = entry.getValue()?.tokenize('.') + def configOldSubMap = configMap + newConfig[entry.getKey()] = resolveToFlat(stages, configOldSubMap) def paramName = (paramStructure ? paramStructure + '.' : '') + entry.getKey() if (configMap[entry.getValue()] != null) { this.step.echo ("[INFO] The parameter '${entry.getValue()}' is COMPATIBLE to the parameter '${paramName}'") @@ -81,6 +83,18 @@ class ConfigurationHelper implements Serializable { return newConfig } + private String resolveToFlat(List stages, configMap) { + def result + def configSubMap = configMap[stages[0]] + if(configSubMap instanceof Map) { + stages.remove(0) + result = resolveToFlat(stages, configSubMap) + } else { + result = configSubMap + } + return result + } + Map dependingOn(dependentKey){ return [ mixin: {key -> diff --git a/test/groovy/com/sap/piper/ConfigurationHelperTest.groovy b/test/groovy/com/sap/piper/ConfigurationHelperTest.groovy index b48e51745..33b9b6c2c 100644 --- a/test/groovy/com/sap/piper/ConfigurationHelperTest.groovy +++ b/test/groovy/com/sap/piper/ConfigurationHelperTest.groovy @@ -166,6 +166,17 @@ class ConfigurationHelperTest { Assert.assertThat(configuration.deep.deeper.newStructure.new2, is('oldValue2')) } + @Test + void testHandleCompatibilityFlatten() { + def configuration = ConfigurationHelper.newInstance(mockScript) + .mixin([old1: [deeper: 'oldValue1'], old2: [deeper: 'oldValue2'], test: 'testValue'], null, [new1: 'old1.deeper', new2: 'old2.deeper']) + .use() + + Assert.assertThat(configuration.size(), is(5)) + Assert.assertThat(configuration.new1, is('oldValue1')) + Assert.assertThat(configuration.new2, is('oldValue2')) + } + @Test void testHandleCompatibilityNewAvailable() { def configuration = ConfigurationHelper.newInstance(mockScript, [old1: 'oldValue1', newStructure: [new1: 'newValue1'], test: 'testValue']) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index d932d2c08..ee6e2815c 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -157,10 +157,10 @@ import static com.sap.piper.Prerequisites.checkScript scanType : 'whitesource.scanType', serviceUrl : 'whitesource.serviceUrl', userTokenCredentialsId : 'whitesource.userTokenCredentialsId', - userTokenCredentialsId : 'whitesourceUserTokenCredentialsId', - productName : 'whitesourceProductName', - projectNames : 'whitesourceProjectNames', - productToken : 'whitesourceProductToken' + userTokenCredentialsId : 'whitesourceUserTokenCredentialsId', + productName : 'whitesourceProductName', + projectNames : 'whitesourceProjectNames', + productToken : 'whitesourceProductToken' ] /** From 6fd4436494964cff3be017203a3df75b5e263ccb Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Thu, 7 Mar 2019 11:43:08 +0100 Subject: [PATCH 040/130] Fix flattening config compatibility feature --- src/com/sap/piper/ConfigurationHelper.groovy | 5 +++-- test/groovy/WhitesourceExecuteScanTest.groovy | 2 ++ vars/whitesourceExecuteScan.groovy | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/com/sap/piper/ConfigurationHelper.groovy b/src/com/sap/piper/ConfigurationHelper.groovy index a23df489a..e09f3cbed 100644 --- a/src/com/sap/piper/ConfigurationHelper.groovy +++ b/src/com/sap/piper/ConfigurationHelper.groovy @@ -72,9 +72,10 @@ class ConfigurationHelper implements Serializable { if (configSubMap == null || (configSubMap != null && configSubMap[entry.getKey()] == null)) { def stages = entry.getValue()?.tokenize('.') def configOldSubMap = configMap - newConfig[entry.getKey()] = resolveToFlat(stages, configOldSubMap) + def value = resolveToFlat(stages, configOldSubMap) def paramName = (paramStructure ? paramStructure + '.' : '') + entry.getKey() - if (configMap[entry.getValue()] != null) { + if (value != null) { + newConfig[entry.getKey()] = value this.step.echo ("[INFO] The parameter '${entry.getValue()}' is COMPATIBLE to the parameter '${paramName}'") } } diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index 987119bbb..0b827cb1e 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -136,6 +136,8 @@ class WhitesourceExecuteScanTest extends BasePiperTest { nullScript.commonPipelineEnvironment.configuration['steps'] = nullScript.commonPipelineEnvironment.configuration['steps'] ?: [:] nullScript.commonPipelineEnvironment.configuration['steps']['whitesourceExecuteScan'] = nullScript.commonPipelineEnvironment.configuration['steps']['whitesourceExecuteScan'] ?: [:] nullScript.commonPipelineEnvironment.configuration['general'] = nullScript.commonPipelineEnvironment.configuration['general'] ?: [:] + nullScript.commonPipelineEnvironment.configuration['general']['whitesource'] = nullScript.commonPipelineEnvironment.configuration['general']['whitesource'] ?: [:] + nullScript.commonPipelineEnvironment.configuration['general']['whitesource']['serviceUrl'] = "http://some.host.whitesource.com/api/" nullScript.commonPipelineEnvironment.setConfigProperty('userTokenCredentialsId', 'ID-123456789') nullScript.commonPipelineEnvironment.configuration['steps']['whitesourceExecuteScan']['userTokenCredentialsId'] = 'ID-123456789' } diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index ee6e2815c..97b2b4c0a 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -197,6 +197,7 @@ void call(Map parameters = [:]) { .dependingOn('scanType').mixin('dockerImage') .dependingOn('scanType').mixin('dockerWorkspace') .dependingOn('scanType').mixin('stashContent') + .withMandatoryProperty('serviceUrl') .withMandatoryProperty('orgToken') .withMandatoryProperty('userTokenCredentialsId') .withMandatoryProperty('productName') From 1f5f812a1a4069b79fcc508fb76d80c3aa9437b1 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Thu, 7 Mar 2019 12:14:25 +0100 Subject: [PATCH 041/130] Fix compatibility for defaults --- src/com/sap/piper/ConfigurationHelper.groovy | 17 +++++++++-------- vars/whitesourceExecuteScan.groovy | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/com/sap/piper/ConfigurationHelper.groovy b/src/com/sap/piper/ConfigurationHelper.groovy index e09f3cbed..a5a4e0a68 100644 --- a/src/com/sap/piper/ConfigurationHelper.groovy +++ b/src/com/sap/piper/ConfigurationHelper.groovy @@ -9,10 +9,11 @@ class ConfigurationHelper implements Serializable { new ConfigurationHelper(step, config) } - ConfigurationHelper loadStepDefaults() { + ConfigurationHelper loadStepDefaults(Map compatibleParameters = [:]) { this.step.prepareDefaultValues() this.config = ConfigurationLoader.defaultGeneralConfiguration() - mixin(ConfigurationLoader.defaultStepConfiguration(null, name)) + mixin(ConfigurationLoader.defaultGeneralConfiguration(), null, compatibleParameters) + mixin(ConfigurationLoader.defaultStepConfiguration(null, name), null, compatibleParameters) } private Map config @@ -71,11 +72,10 @@ class ConfigurationHelper implements Serializable { } if (configSubMap == null || (configSubMap != null && configSubMap[entry.getKey()] == null)) { def stages = entry.getValue()?.tokenize('.') - def configOldSubMap = configMap - def value = resolveToFlat(stages, configOldSubMap) - def paramName = (paramStructure ? paramStructure + '.' : '') + entry.getKey() + def value = resolveToFlat(stages, configMap) if (value != null) { newConfig[entry.getKey()] = value + def paramName = (paramStructure ? paramStructure + '.' : '') + entry.getKey() this.step.echo ("[INFO] The parameter '${entry.getValue()}' is COMPATIBLE to the parameter '${paramName}'") } } @@ -84,11 +84,12 @@ class ConfigurationHelper implements Serializable { return newConfig } - private String resolveToFlat(List stages, configMap) { + private String resolveToFlat(stages, configMap) { + def first = 0 def result - def configSubMap = configMap[stages[0]] + def configSubMap = configMap[stages[first]] if(configSubMap instanceof Map) { - stages.remove(0) + stages.remove(first) result = resolveToFlat(stages, configSubMap) } else { result = configSubMap diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 97b2b4c0a..8147268a8 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -184,7 +184,7 @@ void call(Map parameters = [:]) { // load default & individual configuration Map config = ConfigurationHelper.newInstance(this) - .loadStepDefaults() + .loadStepDefaults(CONFIG_KEY_COMPATIBILITY) .mixinGeneralConfig(script.commonPipelineEnvironment, GENERAL_CONFIG_KEYS, CONFIG_KEY_COMPATIBILITY) .mixinStepConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS, CONFIG_KEY_COMPATIBILITY) .mixinStageConfig(script.commonPipelineEnvironment, parameters.stageName ?: env.STAGE_NAME, STEP_CONFIG_KEYS, CONFIG_KEY_COMPATIBILITY) From 3e8df1fb685dd204e4a4f96e3a211862c19d51ec Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Thu, 7 Mar 2019 12:21:35 +0100 Subject: [PATCH 042/130] Formatting --- vars/whitesourceExecuteScan.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 8147268a8..e8a7e8378 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -157,10 +157,10 @@ import static com.sap.piper.Prerequisites.checkScript scanType : 'whitesource.scanType', serviceUrl : 'whitesource.serviceUrl', userTokenCredentialsId : 'whitesource.userTokenCredentialsId', - userTokenCredentialsId : 'whitesourceUserTokenCredentialsId', - productName : 'whitesourceProductName', - projectNames : 'whitesourceProjectNames', - productToken : 'whitesourceProductToken' + userTokenCredentialsId : 'whitesourceUserTokenCredentialsId', + productName : 'whitesourceProductName', + projectNames : 'whitesourceProjectNames', + productToken : 'whitesourceProductToken' ] /** From 78b41ce7d828fdcff48d8bb9327e7d0e65dd84b5 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Thu, 7 Mar 2019 13:16:03 +0100 Subject: [PATCH 043/130] Fix documentation link --- vars/whitesourceExecuteScan.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index e8a7e8378..8a9abc1aa 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -164,7 +164,7 @@ import static com.sap.piper.Prerequisites.checkScript ] /** - * With this step [WhiteSource](https://www.whitesource.com) security and license compliance scans can be executed and assessed. + * With this step [WhiteSource](https://www.whitesourcesoftware.com) security and license compliance scans can be executed and assessed. * * WhiteSource is a Software as a Service offering based on a so called unified scanning agent that locally determines the dependency * tree of a node.js, Java, Python, Ruby, or Scala based solution and sends it to the WhiteSource server for a policy based license compliance From 6b61bc1b98a0ee3bd7410df068499321dbfb046b Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 8 Mar 2019 12:33:31 +0100 Subject: [PATCH 044/130] Updated documentation --- documentation/bin/createDocu.groovy | 22 +-- .../docs/steps/checkChangeInDevelopment.md | 47 ++++++- documentation/docs/steps/dockerExecute.md | 75 ++++++++++- .../docs/steps/dockerExecuteOnKubernetes.md | 68 +++++++++- documentation/docs/steps/npmExecute.md | 29 +++- .../docs/steps/uiVeri5ExecuteTests.md | 78 ++++++++++- .../docs/steps/whitesourceExecuteScan.md | 127 ++++++++++++++++++ documentation/mkdocs.yml | 1 + vars/whitesourceExecuteScan.groovy | 4 +- 9 files changed, 424 insertions(+), 27 deletions(-) create mode 100644 documentation/docs/steps/whitesourceExecuteScan.md diff --git a/documentation/bin/createDocu.groovy b/documentation/bin/createDocu.groovy index 9e7225d08..c380202c6 100644 --- a/documentation/bin/createDocu.groovy +++ b/documentation/bin/createDocu.groovy @@ -5,7 +5,7 @@ import com.sap.piper.GenerateDocumentation import java.util.regex.Matcher // -// Collects helper functions for rendering the docu +// Collects helper functions for rendering the documentation // class TemplateHelper { @@ -83,13 +83,15 @@ class TemplateHelper { // class Helper { + static projectRoot = new File(Helper.class.protectionDomain.codeSource.location.path).getParentFile().getParentFile().getParentFile() + static getConfigHelper(classLoader, roots, script) { def compilerConfig = new CompilerConfiguration() compilerConfig.setClasspathList( roots ) new GroovyClassLoader(classLoader, compilerConfig, true) - .parseClass(new File('src/com/sap/piper/ConfigurationHelper.groovy')) + .parseClass(new File(projectRoot, 'src/com/sap/piper/ConfigurationHelper.groovy')) .newInstance(script, [:]).loadStepDefaults() } @@ -101,7 +103,7 @@ class Helper { m, c -> c() } prepareDefaultValuesStep.metaClass.libraryResource { - f -> new File("resources/${f}").text + f -> new File(projectRoot,"resources/${f}").text } prepareDefaultValuesStep.metaClass.readYaml { m -> new Yaml().load(m.text) @@ -331,8 +333,8 @@ class Helper { stepsDir.traverse(type: FileType.FILES, maxDepth: 0) { if(it.getName().endsWith('.groovy')) { - def scriptName = (it =~ /vars\/(.*)\.groovy/)[0][1] - def stepScript = gse.createScript("${scriptName}.groovy", new Binding()) + def scriptName = (it.getName() =~ /(.*)\.groovy/)[0][1] + def stepScript = gse.createScript(it.getName(), new Binding()) for (def method in stepScript.getClass().getMethods()) { if(method.getName() == 'call' && method.getAnnotation(GenerateDocumentation) != null) { docuRelevantSteps << scriptName @@ -346,8 +348,8 @@ class Helper { } roots = [ - 'vars', - 'src', + new File(Helper.projectRoot, "vars").getAbsolutePath(), + new File(Helper.projectRoot, "src").getAbsolutePath() ] stepsDir = null @@ -361,12 +363,12 @@ steps = [] if(args.length >= 1) stepsDir = new File(args[0]) -stepsDir = stepsDir ?: new File('vars') +stepsDir = stepsDir ?: new File(Helper.projectRoot, "vars") if(args.length >= 2) stepsDocuDir = new File(args[1]) -stepsDocuDir = stepsDocuDir ?: new File('documentation/docs/steps') +stepsDocuDir = stepsDocuDir ?: new File(Helper.projectRoot, "documentation/docs/steps") if(args.length >= 3) @@ -393,7 +395,7 @@ if( !stepsDir.exists() ) { // sanity checks // -def gse = new GroovyScriptEngine( [ stepsDir.getName() ] as String[] , getClass().getClassLoader() ) +def gse = new GroovyScriptEngine([ stepsDir.getAbsolutePath() ] as String[], GenerateDocumentation.class.getClassLoader() ) // // find all the steps we have to document (if no step has been provided from outside) diff --git a/documentation/docs/steps/checkChangeInDevelopment.md b/documentation/docs/steps/checkChangeInDevelopment.md index b164dce27..7f94463f2 100644 --- a/documentation/docs/steps/checkChangeInDevelopment.md +++ b/documentation/docs/steps/checkChangeInDevelopment.md @@ -2,7 +2,11 @@ ## Description -Content here is generated from corresponding step, see `vars`. +Checks if a Change Document in SAP Solution Manager is in status 'in development'. The change document id is retrieved from the git commit history. The change document id +can also be provided via parameter `changeDocumentId`. Any value provided as parameter has a higher precedence than a value from the commit history. + +By default the git commit messages between `origin/master` and `HEAD` are scanned for a line like `ChangeDocument : `. The commit +range and the pattern can be configured. For details see 'parameters' table. ## Prerequisites @@ -10,11 +14,48 @@ Content here is generated from corresponding step, see `vars`. ## Parameters -Content here is generated from corresponding step, see `vars`. +| name | mandatory | default | possible values | +|------|-----------|---------|-----------------| +| `changeDocumentId` | yes | | | +| `changeManagement/changeDocumentLabel` | no | `ChangeDocument\s?:` | regex pattern | +| `changeManagement/clientOpts` | no | | | +| `changeManagement/credentialsId` | no | `CM` | | +| `changeManagement/endpoint` | yes | | | +| `changeManagement/git/format` | no | `%b` | see `git log --help` | +| `changeManagement/git/from` | no | `origin/master` | | +| `changeManagement/git/to` | no | `HEAD` | | +| `failIfStatusIsNotInDevelopment` | no | `true` | `true`, `false` | +| `script` | yes | | | + +* `changeDocumentId` - The id of the change document to transport. If not provided, it is retrieved from the git commit history. +* `changeManagement/changeDocumentLabel` - A pattern used for identifying lines holding the change document id. +* `changeManagement/clientOpts` - Additional options for cm command line client, e.g. like JAVA_OPTS. +* `changeManagement/credentialsId` - The id of the credentials to connect to the Solution Manager. The credentials needs to be maintained on Jenkins. +* `changeManagement/endpoint` - The address of the Solution Manager. +* `changeManagement/git/format` - Specifies what part of the commit is scanned. By default the body of the commit message is scanned. +* `changeManagement/git/from` - The starting point for retrieving the change document id +* `changeManagement/git/to` - The end point for retrieving the change document id +* `failIfStatusIsNotInDevelopment` - When set to `false` the step will not fail in case the step is not in status 'in development'. +* `script` - The common script environment of the Jenkinsfile running. Typically the reference to the script calling the pipeline step is provided with the this parameter, as in `script: this`. This allows the function to access the commonPipelineEnvironment for retrieving, for example, configuration parameters. ## Step configuration -Content here is generated from corresponding step, see `vars`. +We recommend to define values of step parameters via [config.yml file](../configuration.md). + +In following sections of the config.yml the configuration is possible: + +| parameter | general | step | stage | +|-----------|---------|------|-------| +| `changeDocumentId` | | | X | +| `changeManagement/changeDocumentLabel` | X | X | X | +| `changeManagement/clientOpts` | X | X | X | +| `changeManagement/credentialsId` | X | X | X | +| `changeManagement/endpoint` | X | X | X | +| `changeManagement/git/format` | X | X | X | +| `changeManagement/git/from` | X | X | X | +| `changeManagement/git/to` | X | X | X | +| `failIfStatusIsNotInDevelopment` | | X | X | +| `script` | | | | ## Exceptions diff --git a/documentation/docs/steps/dockerExecute.md b/documentation/docs/steps/dockerExecute.md index bb327eb1e..de9589a49 100644 --- a/documentation/docs/steps/dockerExecute.md +++ b/documentation/docs/steps/dockerExecute.md @@ -2,11 +2,55 @@ ## Description -Content here is generated from corresponding step, see `vars`. +Executes a closure inside a docker container with the specified docker image. +The workspace is mounted into the docker image. +Proxy environment variables defined on the Jenkins machine are also available in the Docker container. ## Parameters -Content here is generated from corresponding step, see `vars`. +| name | mandatory | default | possible values | +|------|-----------|---------|-----------------| +| `containerCommand` | no | | | +| `containerPortMappings` | no | | | +| `containerShell` | no | | | +| `dockerEnvVars` | no | | | +| `dockerImage` | no | | | +| `dockerName` | no | | | +| `dockerOptions` | no | | | +| `dockerPullImage` | no | `true` | | +| `dockerVolumeBind` | no | | | +| `dockerWorkspace` | no | | | +| `jenkinsKubernetes` | no | `[jnlpAgent:s4sdk/jenkins-agent-k8s:latest]` | | +| `script` | yes | | | +| `sidecarEnvVars` | no | | | +| `sidecarImage` | no | | | +| `sidecarName` | no | | | +| `sidecarOptions` | no | | | +| `sidecarPullImage` | no | `true` | | +| `sidecarVolumeBind` | no | | | +| `sidecarWorkspace` | no | | | +| `stashContent` | no | | | + +* `containerCommand` - Kubernetes only: Allows to specify start command for container created with dockerImage parameter to overwrite Piper default (`/usr/bin/tail -f /dev/null`). +* `containerPortMappings` - Map which defines per docker image the port mappings, e.g. `containerPortMappings: ['selenium/standalone-chrome': [[name: 'selPort', containerPort: 4444, hostPort: 4444]]]`. +* `containerShell` - Kubernetes only: Allows to specify the shell to be used for execution of commands. +* `dockerEnvVars` - Environment variables to set in the container, e.g. [http_proxy: 'proxy:8080']. +* `dockerImage` - Name of the docker image that should be used. If empty, Docker is not used and the command is executed directly on the Jenkins system. +* `dockerName` - Kubernetes only: Name of the container launching `dockerImage`. SideCar only: Name of the container in local network. +* `dockerOptions` - Docker options to be set when starting the container (List or String). +* `dockerPullImage` - Set this to 'false' to bypass a docker image pull. Usefull during development process. Allows testing of images which are available in the local registry only. +* `dockerVolumeBind` - Volumes that should be mounted into the container. +* `dockerWorkspace` - Kubernetes only: Specifies a dedicated user home directory for the container which will be passed as value for environment variable `HOME`. +* `jenkinsKubernetes` - +* `script` - The common script environment of the Jenkinsfile running. Typically the reference to the script calling the pipeline step is provided with the this parameter, as in `script: this`. This allows the function to access the commonPipelineEnvironment for retrieving, for example, configuration parameters. +* `sidecarEnvVars` - as `dockerEnvVars` for the sidecar container +* `sidecarImage` - as `dockerImage` for the sidecar container +* `sidecarName` - as `dockerName` for the sidecar container +* `sidecarOptions` - as `dockerOptions` for the sidecar container +* `sidecarPullImage` - Set this to 'false' to bypass a docker image pull. Usefull during development process. Allows testing of images which are available in the local registry only. +* `sidecarVolumeBind` - as `dockerVolumeBind` for the sidecar container +* `sidecarWorkspace` - as `dockerWorkspace` for the sidecar container +* `stashContent` - Specific stashes that should be considered for the step execution. ## Kubernetes support @@ -14,7 +58,32 @@ If the Jenkins is setup on a Kubernetes cluster, then you can execute the closur ## Step configuration -Content here is generated from corresponding step, see `vars`. +We recommend to define values of step parameters via [config.yml file](../configuration.md). + +In following sections of the config.yml the configuration is possible: + +| parameter | general | step | stage | +|-----------|---------|------|-------| +| `containerCommand` | | X | X | +| `containerPortMappings` | | X | X | +| `containerShell` | | X | X | +| `dockerEnvVars` | | X | X | +| `dockerImage` | | X | X | +| `dockerName` | | X | X | +| `dockerOptions` | | X | X | +| `dockerPullImage` | | X | X | +| `dockerVolumeBind` | | X | X | +| `dockerWorkspace` | | X | X | +| `jenkinsKubernetes` | X | X | X | +| `script` | | | | +| `sidecarEnvVars` | | X | X | +| `sidecarImage` | | X | X | +| `sidecarName` | | X | X | +| `sidecarOptions` | | X | X | +| `sidecarPullImage` | | X | X | +| `sidecarVolumeBind` | | X | X | +| `sidecarWorkspace` | | X | X | +| `stashContent` | | X | X | ## Side effects diff --git a/documentation/docs/steps/dockerExecuteOnKubernetes.md b/documentation/docs/steps/dockerExecuteOnKubernetes.md index 013ce042a..f202a2dc8 100644 --- a/documentation/docs/steps/dockerExecuteOnKubernetes.md +++ b/documentation/docs/steps/dockerExecuteOnKubernetes.md @@ -2,7 +2,8 @@ ## Description -Content here is generated from corresponding step, see `vars`. +Executes a closure inside a container in a kubernetes pod. +Proxy environment variables defined on the Jenkins machine are also available in the container. ## Prerequisites @@ -13,11 +14,72 @@ Content here is generated from corresponding step, see `vars`. ## Parameters -Content here is generated from corresponding step, see `vars`. +| name | mandatory | default | possible values | +|------|-----------|---------|-----------------| +| `containerCommand` | no | | | +| `containerCommands` | no | | | +| `containerEnvVars` | no | | | +| `containerMap` | no | | | +| `containerName` | no | | | +| `containerPortMappings` | no | | | +| `containerPullImageFlags` | no | | | +| `containerShell` | no | | | +| `containerWorkspaces` | no | | | +| `dockerEnvVars` | no | | | +| `dockerImage` | yes | | | +| `dockerPullImage` | no | `true` | | +| `dockerWorkspace` | no | | | +| `jenkinsKubernetes` | no | `[jnlpAgent:s4sdk/jenkins-agent-k8s:latest]` | | +| `script` | yes | | | +| `stashContent` | no | | | +| `stashExcludes` | no | `[workspace:nohup.out]` | | +| `stashIncludes` | no | `[workspace:**/*]` | | + +* `containerCommand` - Allows to specify start command for container created with dockerImage parameter to overwrite Piper default (`/usr/bin/tail -f /dev/null`). +* `containerCommands` - Specifies start command for containers to overwrite Piper default (`/usr/bin/tail -f /dev/null`). If container's defaultstart command should be used provide empty string like: `['selenium/standalone-chrome': '']`. +* `containerEnvVars` - Specifies environment variables per container. If not provided `dockerEnvVars` will be used. +* `containerMap` - A map of docker image to the name of the container. The pod will be created with all the images from this map and they are labled based on the value field of each map entry. Example: `['maven:3.5-jdk-8-alpine': 'mavenExecute', 'selenium/standalone-chrome': 'selenium', 'famiko/jmeter-base': 'checkJMeter', 's4sdk/docker-cf-cli': 'cloudfoundry']` +* `containerName` - Optional configuration in combination with containerMap to define the container where the commands should be executed in. +* `containerPortMappings` - Map which defines per docker image the port mappings, e.g. `containerPortMappings: ['selenium/standalone-chrome': [[name: 'selPort', containerPort: 4444, hostPort: 4444]]]`. +* `containerPullImageFlags` - Specifies the pullImage flag per container. +* `containerShell` - Allows to specify the shell to be executed for container with containerName. +* `containerWorkspaces` - Specifies a dedicated user home directory per container which will be passed as value for environment variable `HOME`. If not provided `dockerWorkspace` will be used. +* `dockerEnvVars` - Environment variables to set in the container, e.g. [http_proxy:'proxy:8080']. +* `dockerImage` - Name of the docker image that should be used. If empty, Docker is not used. +* `dockerPullImage` - Set this to 'false' to bypass a docker image pull. Usefull during development process. Allows testing of images which are available in the local registry only. +* `dockerWorkspace` - Specifies a dedicated user home directory for the container which will be passed as value for environment variable `HOME`. +* `jenkinsKubernetes` - +* `script` - The common script environment of the Jenkinsfile running. Typically the reference to the script calling the pipeline step is provided with the this parameter, as in `script: this`. This allows the function to access the commonPipelineEnvironment for retrieving, for example, configuration parameters. +* `stashContent` - Specific stashes that should be considered for the step execution. +* `stashExcludes` - +* `stashIncludes` - ## Step configuration -Content here is generated from corresponding step, see `vars`. +We recommend to define values of step parameters via [config.yml file](../configuration.md). + +In following sections of the config.yml the configuration is possible: + +| parameter | general | step | stage | +|-----------|---------|------|-------| +| `containerCommand` | | X | X | +| `containerCommands` | | X | X | +| `containerEnvVars` | | X | X | +| `containerMap` | | X | X | +| `containerName` | | X | X | +| `containerPortMappings` | | X | X | +| `containerPullImageFlags` | | X | X | +| `containerShell` | | X | X | +| `containerWorkspaces` | | X | X | +| `dockerEnvVars` | | X | X | +| `dockerImage` | | X | X | +| `dockerPullImage` | | X | X | +| `dockerWorkspace` | | X | X | +| `jenkinsKubernetes` | X | X | X | +| `script` | | | | +| `stashContent` | | X | X | +| `stashExcludes` | | X | | +| `stashIncludes` | | X | | ## Side effects diff --git a/documentation/docs/steps/npmExecute.md b/documentation/docs/steps/npmExecute.md index 619cb9cb1..c29ebdcec 100644 --- a/documentation/docs/steps/npmExecute.md +++ b/documentation/docs/steps/npmExecute.md @@ -2,15 +2,38 @@ ## Description -Content here is generated from corresponding step, see `vars`. +Executes NPM commands inside a docker container. +Docker image, docker options and npm commands can be specified or configured. ## Parameters -Content here is generated from corresponding step, see `vars`. +| name | mandatory | default | possible values | +|------|-----------|---------|-----------------| +| `defaultNpmRegistry` | no | | | +| `dockerImage` | no | `node:8-stretch` | | +| `dockerOptions` | no | | | +| `npmCommand` | no | | | +| `script` | yes | | | + +* `defaultNpmRegistry` - URL of default NPM registry +* `dockerImage` - Name of the docker image that should be used, in which node should be installed and configured. Default value is 'node:8-stretch'. +* `dockerOptions` - Docker options to be set when starting the container. +* `npmCommand` - Which NPM command should be executed. +* `script` - The common script environment of the Jenkinsfile running. Typically the reference to the script calling the pipeline step is provided with the this parameter, as in `script: this`. This allows the function to access the commonPipelineEnvironment for retrieving, for example, configuration parameters. ## Step configuration -Content here is generated from corresponding step, see `vars`. +We recommend to define values of step parameters via [config.yml file](../configuration.md). + +In following sections of the config.yml the configuration is possible: + +| parameter | general | step | stage | +|-----------|---------|------|-------| +| `defaultNpmRegistry` | | X | X | +| `dockerImage` | | X | X | +| `dockerOptions` | | | X | +| `npmCommand` | | X | X | +| `script` | | | | ## Exceptions diff --git a/documentation/docs/steps/uiVeri5ExecuteTests.md b/documentation/docs/steps/uiVeri5ExecuteTests.md index 25e4d9e39..d4c18f433 100644 --- a/documentation/docs/steps/uiVeri5ExecuteTests.md +++ b/documentation/docs/steps/uiVeri5ExecuteTests.md @@ -2,17 +2,89 @@ ## Description -Content here is generated from corresponding step, see `vars`. +With this step [UIVeri5](https://github.com/SAP/ui5-uiveri5) tests can be executed. + +UIVeri5 describes following benefits on its GitHub page: + +* Automatic synchronization with UI5 app rendering so there is no need to add waits and sleeps to your test. Tests are reliable by design. +* Tests are written in synchronous manner, no callbacks, no promise chaining so are really simple to write and maintain. +* Full power of webdriverjs, protractor and jasmine - deferred selectors, custom matchers, custom locators. +* Control locators (OPA5 declarative matchers) allow locating and interacting with UI5 controls. +* Does not depend on testability support in applications - works with autorefreshing views, resizing elements, animated transitions. +* Declarative authentications - authentication flow over OAuth2 providers, etc. +* Console operation, CI ready, fully configurable, no need for java (comming soon) or IDE. +* Covers full ui5 browser matrix - Chrome,Firefox,IE,Edge,Safari,iOS,Android. +* Open-source, modify to suite your specific neeeds. + +!!! note "Browser Matrix" + With this step and the underlying Docker image ([selenium/standalone-chrome](https://github.com/SeleniumHQ/docker-selenium/tree/master/StandaloneChrome)) only Chrome tests are possible. + + Testing of further browsers can be done with using a custom Docker image. ## Prerequisites ## Parameters -Content here is generated from corresponding step, see `vars`. +| name | mandatory | default | possible values | +|------|-----------|---------|-----------------| +| `dockerEnvVars` | no | | | +| `dockerImage` | no | | | +| `dockerWorkspace` | no | | | +| `failOnError` | no | | `true`, `false` | +| `gitBranch` | no | | | +| `gitSshKeyCredentialsId` | no | | Jenkins credentialId | +| `installCommand` | no | `npm install @ui5/uiveri5 --global --quiet` | | +| `runCommand` | no | `uiveri5 --seleniumAddress='http://${config.seleniumHost}:${config.seleniumPort}/wd/hub'` | | +| `script` | yes | | | +| `seleniumHost` | no | | | +| `seleniumPort` | no | `4444` | | +| `sidecarEnvVars` | no | | | +| `sidecarImage` | no | | | +| `stashContent` | no | `[buildDescriptor, tests]` | | +| `testOptions` | no | | | +| `testRepository` | no | | | + +* `dockerEnvVars` - A map of environment variables to set in the container, e.g. [http_proxy:'proxy:8080']. +* `dockerImage` - The name of the docker image that should be used. If empty, Docker is not used and the command is executed directly on the Jenkins system. +* `dockerWorkspace` - Only relevant for Kubernetes case: Specifies a dedicated user home directory for the container which will be passed as value for environment variable `HOME`. +* `failOnError` - With `failOnError` the behavior in case tests fail can be defined. +* `gitBranch` - In case a `testRepository` is provided the branch in this repository can be specified with `gitBranch`. +* `gitSshKeyCredentialsId` - In case a `testRepository` is provided and it is protected, access credentials (as Jenkins credentials) can be provided with `gitSshKeyCredentialsId`. **Note: In case of using a protected repository, `testRepository` should include the ssh link to the repository.** +* `installCommand` - The command that is executed to install the test tool. +* `runCommand` - The command that is executed to start the tests. +* `script` - The common script environment of the Jenkinsfile running. Typically the reference to the script calling the pipeline step is provided with the this parameter, as in `script: this`. This allows the function to access the commonPipelineEnvironment for retrieving, for example, configuration parameters. +* `seleniumHost` - The host of the selenium hub, this is set automatically to `localhost` in a Kubernetes environment (determined by the `ON_K8S` environment variable) of to `selenium` in any other case. The value is only needed for the `runCommand`. +* `seleniumPort` - The port of the selenium hub. The value is only needed for the `runCommand`. +* `sidecarEnvVars` - A map of environment variables to set in the sidecar container, similar to `dockerEnvVars`. +* `sidecarImage` - The name of the docker image of the sidecar container. If empty, no sidecar container is started. +* `stashContent` - If specific stashes should be considered for the tests, their names need to be passed via the parameter `stashContent`. +* `testOptions` - This allows to set specific options for the UIVeri5 execution. Details can be found [in the UIVeri5 documentation](https://github.com/SAP/ui5-uiveri5/blob/master/docs/config/config.md#configuration). +* `testRepository` - With `testRepository` the tests can be loaded from another reposirory. ## Step configuration -Content here is generated from corresponding step, see `vars`. +We recommend to define values of step parameters via [config.yml file](../configuration.md). + +In following sections of the config.yml the configuration is possible: + +| parameter | general | step | stage | +|-----------|---------|------|-------| +| `dockerEnvVars` | | X | X | +| `dockerImage` | | X | X | +| `dockerWorkspace` | | X | X | +| `failOnError` | | X | X | +| `gitBranch` | | X | X | +| `gitSshKeyCredentialsId` | X | X | X | +| `installCommand` | | X | X | +| `runCommand` | | X | X | +| `script` | | | | +| `seleniumHost` | | X | X | +| `seleniumPort` | | X | X | +| `sidecarEnvVars` | | X | X | +| `sidecarImage` | | X | X | +| `stashContent` | | X | X | +| `testOptions` | | X | X | +| `testRepository` | | X | X | ## Exceptions diff --git a/documentation/docs/steps/whitesourceExecuteScan.md b/documentation/docs/steps/whitesourceExecuteScan.md new file mode 100644 index 000000000..41bcc579e --- /dev/null +++ b/documentation/docs/steps/whitesourceExecuteScan.md @@ -0,0 +1,127 @@ +# whitesourceExecuteScan + +## Description + +With this step [WhiteSource](https://www.whitesourcesoftware.com) security and license compliance scans can be executed and assessed. + +WhiteSource is a Software as a Service offering based on a so called unified agent that locally determines the dependency +tree of a node.js, Java, Python, Ruby, or Scala based solution and sends it to the WhiteSource server for a policy based license compliance +check and additional Free and Open Source Software Publicly Known Vulnerabilities detection. + +!!! note "Docker Images" + The underlying Docker images are public and specific to the solution's programming language(s) and may therefore be exchanged + to fit and suite the relevant scenario. The default Python environment used is i.e. Python 3 based. + +## Prerequisites + +## Parameters + +| name | mandatory | default | possible values | +|------|-----------|---------|-----------------| +| `agentDownloadUrl` | no | `https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}` | | +| `agentFileName` | no | `wss-unified-agent.jar` | | +| `agentParameters` | no | | | +| `buildDescriptorExcludeList` | no | | | +| `buildDescriptorFile` | no | | | +| `configFilePath` | no | `./wss-unified-agent.config` | | +| `createProductFromPipeline` | no | `true` | | +| `cvssSeverityLimit` | no | `-1` | `-1` to switch failing off, any `positive integer between 0 and 10` to fail on issues with the specified limit or above | +| `dockerImage` | no | | | +| `dockerWorkspace` | no | | | +| `emailAddressesOfInitialProductAdmins` | no | | | +| `jreDownloadUrl` | no | `https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz` | | +| `licensingVulnerabilities` | no | `true` | `true`, `false` | +| `orgAdminUserTokenCredentialsId` | no | | | +| `orgToken` | yes | | | +| `parallelLimit` | no | `15` | | +| `productName` | yes | | | +| `productToken` | no | | | +| `productVersion` | no | | | +| `projectNames` | no | | | +| `reporting` | no | `true` | `true`, `false` | +| `scanType` | no | | `maven`, `mta`, `npm`, `pip`, `sbt` | +| `script` | yes | | | +| `securityVulnerabilities` | no | `true` | `true`, `false` | +| `serviceUrl` | yes | | | +| `stashContent` | no | | | +| `timeout` | no | | | +| `userTokenCredentialsId` | yes | | | +| `verbose` | no | | `true`, `false` | +| `vulnerabilityReportFileName` | no | `piper_whitesource_vulnerability_report` | | +| `vulnerabilityReportTitle` | no | `WhiteSource Security Vulnerability Report` | | + +* `agentDownloadUrl` - URL used to download the latest version of the WhiteSource Unified Agent. +* `agentFileName` - Locally used name for the Unified Agent jar file after download. +* `agentParameters` - Additional parameters passed to the Unified Agent command line. +* `buildDescriptorExcludeList` - List of build descriptors and therefore modules to exclude from the scan and assessment activities. +* `buildDescriptorFile` - Explicit path to the build descriptor file. +* `configFilePath` - Explicit path to the WhiteSource Unified Agent configuration file. +* `createProductFromPipeline` - Whether to create the related WhiteSource product on the fly based on the supplied pipeline configuration. +* `cvssSeverityLimit` - Limit of tollerable CVSS v3 score upon assessment and in consequence fails the build, defaults to `-1`. +* `dockerImage` - Docker image to be used for scanning. +* `dockerWorkspace` - Docker workspace to be used for scanning. +* `emailAddressesOfInitialProductAdmins` - The list of email addresses to assign as product admins for newly created WhiteSource products. +* `jreDownloadUrl` - URL used for downloading the Java Runtime Environment (JRE) required to run the WhiteSource Unified Agent. +* `licensingVulnerabilities` - Whether license compliance is considered and reported as part of the assessment. +* `orgAdminUserTokenCredentialsId` - Jenkins credentials ID referring to the organization admin's token. +* `orgToken` - WhiteSource token identifying your organization. +* `parallelLimit` - Limit of parallel jobs being run at once in case of `scanType: 'mta'` based scenarios, defaults to `15`. +* `productName` - Name of the WhiteSource product to be created and used for results aggregation. +* `productToken` - Token of the WhiteSource product to be created and used for results aggregation, usually determined automatically. +* `productVersion` - Version of the WhiteSource product to be created and used for results aggregation, usually determined automatically. +* `projectNames` - List of WhiteSource projects to be included in the assessment part of the step, usually determined automatically. +* `reporting` - Whether assessment is being done at all, defaults to `true`. +* `scanType` - Type of development stack used to implement the solution. +* `script` - The common script environment of the Jenkinsfile running. Typically the reference to the script calling the pipeline step is provided with the this parameter, as in `script: this`. This allows the function to access the commonPipelineEnvironment for retrieving, for example, configuration parameters. +* `securityVulnerabilities` - Whether security compliance is considered and reported as part of the assessment. +* `serviceUrl` - URL to the WhiteSource server API used for communication, defaults to `https://saas.whitesourcesoftware.com/api`. +* `stashContent` - List of stashes to be unstashed into the workspace before performing the scan. +* `timeout` - Timeout in seconds until a HTTP call is forcefully terminated. +* `userTokenCredentialsId` - Jenkins credentials ID referring to the product admin's token. +* `verbose` - Whether verbose output should be produced. +* `vulnerabilityReportFileName` - Name of the file the vulnerability report is written to. +* `vulnerabilityReportTitle` - Title of vulnerability report written during the assessment phase. + +## Step configuration + +We recommend to define values of step parameters via [config.yml file](../configuration.md). + +In following sections of the config.yml the configuration is possible: + +| parameter | general | step | stage | +|-----------|---------|------|-------| +| `agentDownloadUrl` | | X | X | +| `agentFileName` | | X | X | +| `agentParameters` | | X | X | +| `buildDescriptorExcludeList` | | X | X | +| `buildDescriptorFile` | | X | X | +| `configFilePath` | | X | X | +| `createProductFromPipeline` | | X | X | +| `cvssSeverityLimit` | | X | X | +| `dockerImage` | | X | X | +| `dockerWorkspace` | | X | X | +| `emailAddressesOfInitialProductAdmins` | | X | X | +| `jreDownloadUrl` | | X | X | +| `licensingVulnerabilities` | | X | X | +| `orgAdminUserTokenCredentialsId` | X | X | X | +| `orgToken` | X | X | X | +| `parallelLimit` | | X | X | +| `productName` | X | X | X | +| `productToken` | X | X | X | +| `productVersion` | X | X | X | +| `projectNames` | X | X | X | +| `reporting` | | X | X | +| `scanType` | X | X | X | +| `script` | | | | +| `securityVulnerabilities` | | X | X | +| `serviceUrl` | X | X | X | +| `stashContent` | | X | X | +| `timeout` | | X | X | +| `userTokenCredentialsId` | X | X | X | +| `verbose` | X | X | X | +| `vulnerabilityReportFileName` | | X | X | +| `vulnerabilityReportTitle` | | X | X | + +## Exceptions + +## Examples diff --git a/documentation/mkdocs.yml b/documentation/mkdocs.yml index 84af48355..93980e845 100644 --- a/documentation/mkdocs.yml +++ b/documentation/mkdocs.yml @@ -38,6 +38,7 @@ nav: - transportRequestRelease: steps/transportRequestRelease.md - transportRequestUploadFile: steps/transportRequestUploadFile.md - uiVeri5ExecuteTests: steps/uiVeri5ExecuteTests.md + - whitesourceExecuteScan: steps/whitesourceExecuteScan.md - 'Scenarios': - 'Build and Deploy Hybrid Applications with Jenkins and SAP Solution Manager': scenarios/changeManagement.md - 'Build and Deploy SAP UI5 or SAP Fiori Applications on SAP Cloud Platform with Jenkins': scenarios/ui5-sap-cp/Readme.md diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 8a9abc1aa..8f87050f6 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -166,9 +166,9 @@ import static com.sap.piper.Prerequisites.checkScript /** * With this step [WhiteSource](https://www.whitesourcesoftware.com) security and license compliance scans can be executed and assessed. * - * WhiteSource is a Software as a Service offering based on a so called unified scanning agent that locally determines the dependency + * WhiteSource is a Software as a Service offering based on a so called unified agent that locally determines the dependency * tree of a node.js, Java, Python, Ruby, or Scala based solution and sends it to the WhiteSource server for a policy based license compliance - * check and additional Free and Open Source Software Publicly Known Vulnerabilities assessment. + * check and additional Free and Open Source Software Publicly Known Vulnerabilities detection. * * !!! note "Docker Images" * The underlying Docker images are public and specific to the solution's programming language(s) and may therefore be exchanged From 4ae6675824a905815c5c99d8d8db470e32068bd8 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 8 Mar 2019 13:56:07 +0100 Subject: [PATCH 045/130] Revise docs --- .../docs/steps/checkChangeInDevelopment.md | 47 +------ documentation/docs/steps/dockerExecute.md | 75 +---------- .../docs/steps/dockerExecuteOnKubernetes.md | 68 +--------- documentation/docs/steps/npmExecute.md | 29 +---- .../docs/steps/uiVeri5ExecuteTests.md | 78 +---------- .../docs/steps/whitesourceExecuteScan.md | 122 ++---------------- vars/whitesourceExecuteScan.groovy | 6 +- 7 files changed, 27 insertions(+), 398 deletions(-) diff --git a/documentation/docs/steps/checkChangeInDevelopment.md b/documentation/docs/steps/checkChangeInDevelopment.md index 7f94463f2..ecf035b39 100644 --- a/documentation/docs/steps/checkChangeInDevelopment.md +++ b/documentation/docs/steps/checkChangeInDevelopment.md @@ -2,11 +2,7 @@ ## Description -Checks if a Change Document in SAP Solution Manager is in status 'in development'. The change document id is retrieved from the git commit history. The change document id -can also be provided via parameter `changeDocumentId`. Any value provided as parameter has a higher precedence than a value from the commit history. - -By default the git commit messages between `origin/master` and `HEAD` are scanned for a line like `ChangeDocument : `. The commit -range and the pattern can be configured. For details see 'parameters' table. +Content here is generated from corresponnding step, see `vars`. ## Prerequisites @@ -14,48 +10,11 @@ range and the pattern can be configured. For details see 'parameters' table. ## Parameters -| name | mandatory | default | possible values | -|------|-----------|---------|-----------------| -| `changeDocumentId` | yes | | | -| `changeManagement/changeDocumentLabel` | no | `ChangeDocument\s?:` | regex pattern | -| `changeManagement/clientOpts` | no | | | -| `changeManagement/credentialsId` | no | `CM` | | -| `changeManagement/endpoint` | yes | | | -| `changeManagement/git/format` | no | `%b` | see `git log --help` | -| `changeManagement/git/from` | no | `origin/master` | | -| `changeManagement/git/to` | no | `HEAD` | | -| `failIfStatusIsNotInDevelopment` | no | `true` | `true`, `false` | -| `script` | yes | | | - -* `changeDocumentId` - The id of the change document to transport. If not provided, it is retrieved from the git commit history. -* `changeManagement/changeDocumentLabel` - A pattern used for identifying lines holding the change document id. -* `changeManagement/clientOpts` - Additional options for cm command line client, e.g. like JAVA_OPTS. -* `changeManagement/credentialsId` - The id of the credentials to connect to the Solution Manager. The credentials needs to be maintained on Jenkins. -* `changeManagement/endpoint` - The address of the Solution Manager. -* `changeManagement/git/format` - Specifies what part of the commit is scanned. By default the body of the commit message is scanned. -* `changeManagement/git/from` - The starting point for retrieving the change document id -* `changeManagement/git/to` - The end point for retrieving the change document id -* `failIfStatusIsNotInDevelopment` - When set to `false` the step will not fail in case the step is not in status 'in development'. -* `script` - The common script environment of the Jenkinsfile running. Typically the reference to the script calling the pipeline step is provided with the this parameter, as in `script: this`. This allows the function to access the commonPipelineEnvironment for retrieving, for example, configuration parameters. +Content here is generated from corresponnding step, see `vars`. ## Step configuration -We recommend to define values of step parameters via [config.yml file](../configuration.md). - -In following sections of the config.yml the configuration is possible: - -| parameter | general | step | stage | -|-----------|---------|------|-------| -| `changeDocumentId` | | | X | -| `changeManagement/changeDocumentLabel` | X | X | X | -| `changeManagement/clientOpts` | X | X | X | -| `changeManagement/credentialsId` | X | X | X | -| `changeManagement/endpoint` | X | X | X | -| `changeManagement/git/format` | X | X | X | -| `changeManagement/git/from` | X | X | X | -| `changeManagement/git/to` | X | X | X | -| `failIfStatusIsNotInDevelopment` | | X | X | -| `script` | | | | +Content here is generated from corresponnding step, see `vars`. ## Exceptions diff --git a/documentation/docs/steps/dockerExecute.md b/documentation/docs/steps/dockerExecute.md index de9589a49..bb327eb1e 100644 --- a/documentation/docs/steps/dockerExecute.md +++ b/documentation/docs/steps/dockerExecute.md @@ -2,55 +2,11 @@ ## Description -Executes a closure inside a docker container with the specified docker image. -The workspace is mounted into the docker image. -Proxy environment variables defined on the Jenkins machine are also available in the Docker container. +Content here is generated from corresponding step, see `vars`. ## Parameters -| name | mandatory | default | possible values | -|------|-----------|---------|-----------------| -| `containerCommand` | no | | | -| `containerPortMappings` | no | | | -| `containerShell` | no | | | -| `dockerEnvVars` | no | | | -| `dockerImage` | no | | | -| `dockerName` | no | | | -| `dockerOptions` | no | | | -| `dockerPullImage` | no | `true` | | -| `dockerVolumeBind` | no | | | -| `dockerWorkspace` | no | | | -| `jenkinsKubernetes` | no | `[jnlpAgent:s4sdk/jenkins-agent-k8s:latest]` | | -| `script` | yes | | | -| `sidecarEnvVars` | no | | | -| `sidecarImage` | no | | | -| `sidecarName` | no | | | -| `sidecarOptions` | no | | | -| `sidecarPullImage` | no | `true` | | -| `sidecarVolumeBind` | no | | | -| `sidecarWorkspace` | no | | | -| `stashContent` | no | | | - -* `containerCommand` - Kubernetes only: Allows to specify start command for container created with dockerImage parameter to overwrite Piper default (`/usr/bin/tail -f /dev/null`). -* `containerPortMappings` - Map which defines per docker image the port mappings, e.g. `containerPortMappings: ['selenium/standalone-chrome': [[name: 'selPort', containerPort: 4444, hostPort: 4444]]]`. -* `containerShell` - Kubernetes only: Allows to specify the shell to be used for execution of commands. -* `dockerEnvVars` - Environment variables to set in the container, e.g. [http_proxy: 'proxy:8080']. -* `dockerImage` - Name of the docker image that should be used. If empty, Docker is not used and the command is executed directly on the Jenkins system. -* `dockerName` - Kubernetes only: Name of the container launching `dockerImage`. SideCar only: Name of the container in local network. -* `dockerOptions` - Docker options to be set when starting the container (List or String). -* `dockerPullImage` - Set this to 'false' to bypass a docker image pull. Usefull during development process. Allows testing of images which are available in the local registry only. -* `dockerVolumeBind` - Volumes that should be mounted into the container. -* `dockerWorkspace` - Kubernetes only: Specifies a dedicated user home directory for the container which will be passed as value for environment variable `HOME`. -* `jenkinsKubernetes` - -* `script` - The common script environment of the Jenkinsfile running. Typically the reference to the script calling the pipeline step is provided with the this parameter, as in `script: this`. This allows the function to access the commonPipelineEnvironment for retrieving, for example, configuration parameters. -* `sidecarEnvVars` - as `dockerEnvVars` for the sidecar container -* `sidecarImage` - as `dockerImage` for the sidecar container -* `sidecarName` - as `dockerName` for the sidecar container -* `sidecarOptions` - as `dockerOptions` for the sidecar container -* `sidecarPullImage` - Set this to 'false' to bypass a docker image pull. Usefull during development process. Allows testing of images which are available in the local registry only. -* `sidecarVolumeBind` - as `dockerVolumeBind` for the sidecar container -* `sidecarWorkspace` - as `dockerWorkspace` for the sidecar container -* `stashContent` - Specific stashes that should be considered for the step execution. +Content here is generated from corresponding step, see `vars`. ## Kubernetes support @@ -58,32 +14,7 @@ If the Jenkins is setup on a Kubernetes cluster, then you can execute the closur ## Step configuration -We recommend to define values of step parameters via [config.yml file](../configuration.md). - -In following sections of the config.yml the configuration is possible: - -| parameter | general | step | stage | -|-----------|---------|------|-------| -| `containerCommand` | | X | X | -| `containerPortMappings` | | X | X | -| `containerShell` | | X | X | -| `dockerEnvVars` | | X | X | -| `dockerImage` | | X | X | -| `dockerName` | | X | X | -| `dockerOptions` | | X | X | -| `dockerPullImage` | | X | X | -| `dockerVolumeBind` | | X | X | -| `dockerWorkspace` | | X | X | -| `jenkinsKubernetes` | X | X | X | -| `script` | | | | -| `sidecarEnvVars` | | X | X | -| `sidecarImage` | | X | X | -| `sidecarName` | | X | X | -| `sidecarOptions` | | X | X | -| `sidecarPullImage` | | X | X | -| `sidecarVolumeBind` | | X | X | -| `sidecarWorkspace` | | X | X | -| `stashContent` | | X | X | +Content here is generated from corresponding step, see `vars`. ## Side effects diff --git a/documentation/docs/steps/dockerExecuteOnKubernetes.md b/documentation/docs/steps/dockerExecuteOnKubernetes.md index f202a2dc8..013ce042a 100644 --- a/documentation/docs/steps/dockerExecuteOnKubernetes.md +++ b/documentation/docs/steps/dockerExecuteOnKubernetes.md @@ -2,8 +2,7 @@ ## Description -Executes a closure inside a container in a kubernetes pod. -Proxy environment variables defined on the Jenkins machine are also available in the container. +Content here is generated from corresponding step, see `vars`. ## Prerequisites @@ -14,72 +13,11 @@ Proxy environment variables defined on the Jenkins machine are also available in ## Parameters -| name | mandatory | default | possible values | -|------|-----------|---------|-----------------| -| `containerCommand` | no | | | -| `containerCommands` | no | | | -| `containerEnvVars` | no | | | -| `containerMap` | no | | | -| `containerName` | no | | | -| `containerPortMappings` | no | | | -| `containerPullImageFlags` | no | | | -| `containerShell` | no | | | -| `containerWorkspaces` | no | | | -| `dockerEnvVars` | no | | | -| `dockerImage` | yes | | | -| `dockerPullImage` | no | `true` | | -| `dockerWorkspace` | no | | | -| `jenkinsKubernetes` | no | `[jnlpAgent:s4sdk/jenkins-agent-k8s:latest]` | | -| `script` | yes | | | -| `stashContent` | no | | | -| `stashExcludes` | no | `[workspace:nohup.out]` | | -| `stashIncludes` | no | `[workspace:**/*]` | | - -* `containerCommand` - Allows to specify start command for container created with dockerImage parameter to overwrite Piper default (`/usr/bin/tail -f /dev/null`). -* `containerCommands` - Specifies start command for containers to overwrite Piper default (`/usr/bin/tail -f /dev/null`). If container's defaultstart command should be used provide empty string like: `['selenium/standalone-chrome': '']`. -* `containerEnvVars` - Specifies environment variables per container. If not provided `dockerEnvVars` will be used. -* `containerMap` - A map of docker image to the name of the container. The pod will be created with all the images from this map and they are labled based on the value field of each map entry. Example: `['maven:3.5-jdk-8-alpine': 'mavenExecute', 'selenium/standalone-chrome': 'selenium', 'famiko/jmeter-base': 'checkJMeter', 's4sdk/docker-cf-cli': 'cloudfoundry']` -* `containerName` - Optional configuration in combination with containerMap to define the container where the commands should be executed in. -* `containerPortMappings` - Map which defines per docker image the port mappings, e.g. `containerPortMappings: ['selenium/standalone-chrome': [[name: 'selPort', containerPort: 4444, hostPort: 4444]]]`. -* `containerPullImageFlags` - Specifies the pullImage flag per container. -* `containerShell` - Allows to specify the shell to be executed for container with containerName. -* `containerWorkspaces` - Specifies a dedicated user home directory per container which will be passed as value for environment variable `HOME`. If not provided `dockerWorkspace` will be used. -* `dockerEnvVars` - Environment variables to set in the container, e.g. [http_proxy:'proxy:8080']. -* `dockerImage` - Name of the docker image that should be used. If empty, Docker is not used. -* `dockerPullImage` - Set this to 'false' to bypass a docker image pull. Usefull during development process. Allows testing of images which are available in the local registry only. -* `dockerWorkspace` - Specifies a dedicated user home directory for the container which will be passed as value for environment variable `HOME`. -* `jenkinsKubernetes` - -* `script` - The common script environment of the Jenkinsfile running. Typically the reference to the script calling the pipeline step is provided with the this parameter, as in `script: this`. This allows the function to access the commonPipelineEnvironment for retrieving, for example, configuration parameters. -* `stashContent` - Specific stashes that should be considered for the step execution. -* `stashExcludes` - -* `stashIncludes` - +Content here is generated from corresponding step, see `vars`. ## Step configuration -We recommend to define values of step parameters via [config.yml file](../configuration.md). - -In following sections of the config.yml the configuration is possible: - -| parameter | general | step | stage | -|-----------|---------|------|-------| -| `containerCommand` | | X | X | -| `containerCommands` | | X | X | -| `containerEnvVars` | | X | X | -| `containerMap` | | X | X | -| `containerName` | | X | X | -| `containerPortMappings` | | X | X | -| `containerPullImageFlags` | | X | X | -| `containerShell` | | X | X | -| `containerWorkspaces` | | X | X | -| `dockerEnvVars` | | X | X | -| `dockerImage` | | X | X | -| `dockerPullImage` | | X | X | -| `dockerWorkspace` | | X | X | -| `jenkinsKubernetes` | X | X | X | -| `script` | | | | -| `stashContent` | | X | X | -| `stashExcludes` | | X | | -| `stashIncludes` | | X | | +Content here is generated from corresponding step, see `vars`. ## Side effects diff --git a/documentation/docs/steps/npmExecute.md b/documentation/docs/steps/npmExecute.md index c29ebdcec..619cb9cb1 100644 --- a/documentation/docs/steps/npmExecute.md +++ b/documentation/docs/steps/npmExecute.md @@ -2,38 +2,15 @@ ## Description -Executes NPM commands inside a docker container. -Docker image, docker options and npm commands can be specified or configured. +Content here is generated from corresponding step, see `vars`. ## Parameters -| name | mandatory | default | possible values | -|------|-----------|---------|-----------------| -| `defaultNpmRegistry` | no | | | -| `dockerImage` | no | `node:8-stretch` | | -| `dockerOptions` | no | | | -| `npmCommand` | no | | | -| `script` | yes | | | - -* `defaultNpmRegistry` - URL of default NPM registry -* `dockerImage` - Name of the docker image that should be used, in which node should be installed and configured. Default value is 'node:8-stretch'. -* `dockerOptions` - Docker options to be set when starting the container. -* `npmCommand` - Which NPM command should be executed. -* `script` - The common script environment of the Jenkinsfile running. Typically the reference to the script calling the pipeline step is provided with the this parameter, as in `script: this`. This allows the function to access the commonPipelineEnvironment for retrieving, for example, configuration parameters. +Content here is generated from corresponding step, see `vars`. ## Step configuration -We recommend to define values of step parameters via [config.yml file](../configuration.md). - -In following sections of the config.yml the configuration is possible: - -| parameter | general | step | stage | -|-----------|---------|------|-------| -| `defaultNpmRegistry` | | X | X | -| `dockerImage` | | X | X | -| `dockerOptions` | | | X | -| `npmCommand` | | X | X | -| `script` | | | | +Content here is generated from corresponding step, see `vars`. ## Exceptions diff --git a/documentation/docs/steps/uiVeri5ExecuteTests.md b/documentation/docs/steps/uiVeri5ExecuteTests.md index d4c18f433..25e4d9e39 100644 --- a/documentation/docs/steps/uiVeri5ExecuteTests.md +++ b/documentation/docs/steps/uiVeri5ExecuteTests.md @@ -2,89 +2,17 @@ ## Description -With this step [UIVeri5](https://github.com/SAP/ui5-uiveri5) tests can be executed. - -UIVeri5 describes following benefits on its GitHub page: - -* Automatic synchronization with UI5 app rendering so there is no need to add waits and sleeps to your test. Tests are reliable by design. -* Tests are written in synchronous manner, no callbacks, no promise chaining so are really simple to write and maintain. -* Full power of webdriverjs, protractor and jasmine - deferred selectors, custom matchers, custom locators. -* Control locators (OPA5 declarative matchers) allow locating and interacting with UI5 controls. -* Does not depend on testability support in applications - works with autorefreshing views, resizing elements, animated transitions. -* Declarative authentications - authentication flow over OAuth2 providers, etc. -* Console operation, CI ready, fully configurable, no need for java (comming soon) or IDE. -* Covers full ui5 browser matrix - Chrome,Firefox,IE,Edge,Safari,iOS,Android. -* Open-source, modify to suite your specific neeeds. - -!!! note "Browser Matrix" - With this step and the underlying Docker image ([selenium/standalone-chrome](https://github.com/SeleniumHQ/docker-selenium/tree/master/StandaloneChrome)) only Chrome tests are possible. - - Testing of further browsers can be done with using a custom Docker image. +Content here is generated from corresponding step, see `vars`. ## Prerequisites ## Parameters -| name | mandatory | default | possible values | -|------|-----------|---------|-----------------| -| `dockerEnvVars` | no | | | -| `dockerImage` | no | | | -| `dockerWorkspace` | no | | | -| `failOnError` | no | | `true`, `false` | -| `gitBranch` | no | | | -| `gitSshKeyCredentialsId` | no | | Jenkins credentialId | -| `installCommand` | no | `npm install @ui5/uiveri5 --global --quiet` | | -| `runCommand` | no | `uiveri5 --seleniumAddress='http://${config.seleniumHost}:${config.seleniumPort}/wd/hub'` | | -| `script` | yes | | | -| `seleniumHost` | no | | | -| `seleniumPort` | no | `4444` | | -| `sidecarEnvVars` | no | | | -| `sidecarImage` | no | | | -| `stashContent` | no | `[buildDescriptor, tests]` | | -| `testOptions` | no | | | -| `testRepository` | no | | | - -* `dockerEnvVars` - A map of environment variables to set in the container, e.g. [http_proxy:'proxy:8080']. -* `dockerImage` - The name of the docker image that should be used. If empty, Docker is not used and the command is executed directly on the Jenkins system. -* `dockerWorkspace` - Only relevant for Kubernetes case: Specifies a dedicated user home directory for the container which will be passed as value for environment variable `HOME`. -* `failOnError` - With `failOnError` the behavior in case tests fail can be defined. -* `gitBranch` - In case a `testRepository` is provided the branch in this repository can be specified with `gitBranch`. -* `gitSshKeyCredentialsId` - In case a `testRepository` is provided and it is protected, access credentials (as Jenkins credentials) can be provided with `gitSshKeyCredentialsId`. **Note: In case of using a protected repository, `testRepository` should include the ssh link to the repository.** -* `installCommand` - The command that is executed to install the test tool. -* `runCommand` - The command that is executed to start the tests. -* `script` - The common script environment of the Jenkinsfile running. Typically the reference to the script calling the pipeline step is provided with the this parameter, as in `script: this`. This allows the function to access the commonPipelineEnvironment for retrieving, for example, configuration parameters. -* `seleniumHost` - The host of the selenium hub, this is set automatically to `localhost` in a Kubernetes environment (determined by the `ON_K8S` environment variable) of to `selenium` in any other case. The value is only needed for the `runCommand`. -* `seleniumPort` - The port of the selenium hub. The value is only needed for the `runCommand`. -* `sidecarEnvVars` - A map of environment variables to set in the sidecar container, similar to `dockerEnvVars`. -* `sidecarImage` - The name of the docker image of the sidecar container. If empty, no sidecar container is started. -* `stashContent` - If specific stashes should be considered for the tests, their names need to be passed via the parameter `stashContent`. -* `testOptions` - This allows to set specific options for the UIVeri5 execution. Details can be found [in the UIVeri5 documentation](https://github.com/SAP/ui5-uiveri5/blob/master/docs/config/config.md#configuration). -* `testRepository` - With `testRepository` the tests can be loaded from another reposirory. +Content here is generated from corresponding step, see `vars`. ## Step configuration -We recommend to define values of step parameters via [config.yml file](../configuration.md). - -In following sections of the config.yml the configuration is possible: - -| parameter | general | step | stage | -|-----------|---------|------|-------| -| `dockerEnvVars` | | X | X | -| `dockerImage` | | X | X | -| `dockerWorkspace` | | X | X | -| `failOnError` | | X | X | -| `gitBranch` | | X | X | -| `gitSshKeyCredentialsId` | X | X | X | -| `installCommand` | | X | X | -| `runCommand` | | X | X | -| `script` | | | | -| `seleniumHost` | | X | X | -| `seleniumPort` | | X | X | -| `sidecarEnvVars` | | X | X | -| `sidecarImage` | | X | X | -| `stashContent` | | X | X | -| `testOptions` | | X | X | -| `testRepository` | | X | X | +Content here is generated from corresponding step, see `vars`. ## Exceptions diff --git a/documentation/docs/steps/whitesourceExecuteScan.md b/documentation/docs/steps/whitesourceExecuteScan.md index 41bcc579e..1d1bf77f7 100644 --- a/documentation/docs/steps/whitesourceExecuteScan.md +++ b/documentation/docs/steps/whitesourceExecuteScan.md @@ -2,126 +2,26 @@ ## Description -With this step [WhiteSource](https://www.whitesourcesoftware.com) security and license compliance scans can be executed and assessed. - -WhiteSource is a Software as a Service offering based on a so called unified agent that locally determines the dependency -tree of a node.js, Java, Python, Ruby, or Scala based solution and sends it to the WhiteSource server for a policy based license compliance -check and additional Free and Open Source Software Publicly Known Vulnerabilities detection. - -!!! note "Docker Images" - The underlying Docker images are public and specific to the solution's programming language(s) and may therefore be exchanged - to fit and suite the relevant scenario. The default Python environment used is i.e. Python 3 based. +Content here is generated from corresponding step, see `vars`. ## Prerequisites +None + ## Parameters -| name | mandatory | default | possible values | -|------|-----------|---------|-----------------| -| `agentDownloadUrl` | no | `https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/${config.agentFileName}` | | -| `agentFileName` | no | `wss-unified-agent.jar` | | -| `agentParameters` | no | | | -| `buildDescriptorExcludeList` | no | | | -| `buildDescriptorFile` | no | | | -| `configFilePath` | no | `./wss-unified-agent.config` | | -| `createProductFromPipeline` | no | `true` | | -| `cvssSeverityLimit` | no | `-1` | `-1` to switch failing off, any `positive integer between 0 and 10` to fail on issues with the specified limit or above | -| `dockerImage` | no | | | -| `dockerWorkspace` | no | | | -| `emailAddressesOfInitialProductAdmins` | no | | | -| `jreDownloadUrl` | no | `https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz` | | -| `licensingVulnerabilities` | no | `true` | `true`, `false` | -| `orgAdminUserTokenCredentialsId` | no | | | -| `orgToken` | yes | | | -| `parallelLimit` | no | `15` | | -| `productName` | yes | | | -| `productToken` | no | | | -| `productVersion` | no | | | -| `projectNames` | no | | | -| `reporting` | no | `true` | `true`, `false` | -| `scanType` | no | | `maven`, `mta`, `npm`, `pip`, `sbt` | -| `script` | yes | | | -| `securityVulnerabilities` | no | `true` | `true`, `false` | -| `serviceUrl` | yes | | | -| `stashContent` | no | | | -| `timeout` | no | | | -| `userTokenCredentialsId` | yes | | | -| `verbose` | no | | `true`, `false` | -| `vulnerabilityReportFileName` | no | `piper_whitesource_vulnerability_report` | | -| `vulnerabilityReportTitle` | no | `WhiteSource Security Vulnerability Report` | | - -* `agentDownloadUrl` - URL used to download the latest version of the WhiteSource Unified Agent. -* `agentFileName` - Locally used name for the Unified Agent jar file after download. -* `agentParameters` - Additional parameters passed to the Unified Agent command line. -* `buildDescriptorExcludeList` - List of build descriptors and therefore modules to exclude from the scan and assessment activities. -* `buildDescriptorFile` - Explicit path to the build descriptor file. -* `configFilePath` - Explicit path to the WhiteSource Unified Agent configuration file. -* `createProductFromPipeline` - Whether to create the related WhiteSource product on the fly based on the supplied pipeline configuration. -* `cvssSeverityLimit` - Limit of tollerable CVSS v3 score upon assessment and in consequence fails the build, defaults to `-1`. -* `dockerImage` - Docker image to be used for scanning. -* `dockerWorkspace` - Docker workspace to be used for scanning. -* `emailAddressesOfInitialProductAdmins` - The list of email addresses to assign as product admins for newly created WhiteSource products. -* `jreDownloadUrl` - URL used for downloading the Java Runtime Environment (JRE) required to run the WhiteSource Unified Agent. -* `licensingVulnerabilities` - Whether license compliance is considered and reported as part of the assessment. -* `orgAdminUserTokenCredentialsId` - Jenkins credentials ID referring to the organization admin's token. -* `orgToken` - WhiteSource token identifying your organization. -* `parallelLimit` - Limit of parallel jobs being run at once in case of `scanType: 'mta'` based scenarios, defaults to `15`. -* `productName` - Name of the WhiteSource product to be created and used for results aggregation. -* `productToken` - Token of the WhiteSource product to be created and used for results aggregation, usually determined automatically. -* `productVersion` - Version of the WhiteSource product to be created and used for results aggregation, usually determined automatically. -* `projectNames` - List of WhiteSource projects to be included in the assessment part of the step, usually determined automatically. -* `reporting` - Whether assessment is being done at all, defaults to `true`. -* `scanType` - Type of development stack used to implement the solution. -* `script` - The common script environment of the Jenkinsfile running. Typically the reference to the script calling the pipeline step is provided with the this parameter, as in `script: this`. This allows the function to access the commonPipelineEnvironment for retrieving, for example, configuration parameters. -* `securityVulnerabilities` - Whether security compliance is considered and reported as part of the assessment. -* `serviceUrl` - URL to the WhiteSource server API used for communication, defaults to `https://saas.whitesourcesoftware.com/api`. -* `stashContent` - List of stashes to be unstashed into the workspace before performing the scan. -* `timeout` - Timeout in seconds until a HTTP call is forcefully terminated. -* `userTokenCredentialsId` - Jenkins credentials ID referring to the product admin's token. -* `verbose` - Whether verbose output should be produced. -* `vulnerabilityReportFileName` - Name of the file the vulnerability report is written to. -* `vulnerabilityReportTitle` - Title of vulnerability report written during the assessment phase. +Content here is generated from corresponding step, see `vars`. ## Step configuration -We recommend to define values of step parameters via [config.yml file](../configuration.md). - -In following sections of the config.yml the configuration is possible: - -| parameter | general | step | stage | -|-----------|---------|------|-------| -| `agentDownloadUrl` | | X | X | -| `agentFileName` | | X | X | -| `agentParameters` | | X | X | -| `buildDescriptorExcludeList` | | X | X | -| `buildDescriptorFile` | | X | X | -| `configFilePath` | | X | X | -| `createProductFromPipeline` | | X | X | -| `cvssSeverityLimit` | | X | X | -| `dockerImage` | | X | X | -| `dockerWorkspace` | | X | X | -| `emailAddressesOfInitialProductAdmins` | | X | X | -| `jreDownloadUrl` | | X | X | -| `licensingVulnerabilities` | | X | X | -| `orgAdminUserTokenCredentialsId` | X | X | X | -| `orgToken` | X | X | X | -| `parallelLimit` | | X | X | -| `productName` | X | X | X | -| `productToken` | X | X | X | -| `productVersion` | X | X | X | -| `projectNames` | X | X | X | -| `reporting` | | X | X | -| `scanType` | X | X | X | -| `script` | | | | -| `securityVulnerabilities` | | X | X | -| `serviceUrl` | X | X | X | -| `stashContent` | | X | X | -| `timeout` | | X | X | -| `userTokenCredentialsId` | X | X | X | -| `verbose` | X | X | X | -| `vulnerabilityReportFileName` | | X | X | -| `vulnerabilityReportTitle` | | X | X | +Content here is generated from corresponding step, see `vars`. ## Exceptions +None + ## Examples + +```groovy +whitesourceExecuteScan script: this, scanType: 'pip', productName: 'My Whitesource Product', userTokenCredentialsId: 'companyAdminToken', orgAdminUserTokenCredentialsId: 'orgAdmiToken', orgToken: 'myWhitesourceOrganizationToken' +``` diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 8f87050f6..34d1f7645 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -156,11 +156,7 @@ import static com.sap.piper.Prerequisites.checkScript projectNames : 'whitesource.projectNames', scanType : 'whitesource.scanType', serviceUrl : 'whitesource.serviceUrl', - userTokenCredentialsId : 'whitesource.userTokenCredentialsId', - userTokenCredentialsId : 'whitesourceUserTokenCredentialsId', - productName : 'whitesourceProductName', - projectNames : 'whitesourceProjectNames', - productToken : 'whitesourceProductToken' + userTokenCredentialsId : 'whitesource.userTokenCredentialsId' ] /** From 61a9710fab664de7d1ad3d2b401bc241a2f180a8 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 8 Mar 2019 14:11:04 +0100 Subject: [PATCH 046/130] Fix org admin repo --- .../piper/integration/WhitesourceOrgAdminRepository.groovy | 4 +++- .../integration/WhitesourceOrgAdminRepositoryTest.groovy | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy index c3928ef81..b8c3a309c 100644 --- a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy @@ -9,6 +9,8 @@ class WhitesourceOrgAdminRepository implements Serializable { final internalWhitesource final Map config + def orgAdminUserKey + WhitesourceOrgAdminRepository(Script script, Map config) { this.script = script this.config = config @@ -81,7 +83,7 @@ class WhitesourceOrgAdminRepository implements Serializable { credentialsId: config.orgAdminUserTokenCredentialsId, variable: 'orgAdminUserKey' )]) { - requestBody["userKey"] = script.env.orgAdminUserKey + requestBody["userKey"] = orgAdminUserKey def serializedBody = new JsonUtils().jsonToString(requestBody) def params = [ url : config.serviceUrl, diff --git a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy index 83a485e7d..dc762d042 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy @@ -71,8 +71,8 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { @Test void testHttpWhitesourceInternalCallUserKey() { - nullScript.env.orgAdminUserKey = "4711" - def config = [ serviceUrl: "http://some.host.whitesource.com/api/", verbose: false, orgAdminUserKey: nullScript.env.orgAdminUserKey] + repository.orgAdminUserKey = "4711" + def config = [ serviceUrl: "http://some.host.whitesource.com/api/", verbose: false, orgAdminUserKey: "4711"] def requestBody = ["someJson" : [ "someObject" : "abcdef" ]] def requestParams From 61d6280c3a632447bc90a3115e9ae20c4b25fa35 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 11 Mar 2019 10:46:22 +0100 Subject: [PATCH 047/130] Moved admin auth handling into script --- .../WhitesourceOrgAdminRepository.groovy | 47 +++++++++---------- .../WhitesourceOrgAdminRepositoryTest.groovy | 4 +- vars/whitesourceExecuteScan.groovy | 17 ++++++- 3 files changed, 40 insertions(+), 28 deletions(-) diff --git a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy index b8c3a309c..ddf5c4e3a 100644 --- a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy @@ -9,7 +9,9 @@ class WhitesourceOrgAdminRepository implements Serializable { final internalWhitesource final Map config - def orgAdminUserKey + WhitesourceOrgAdminRepository() { + super() + } WhitesourceOrgAdminRepository(Script script, Map config) { this.script = script @@ -79,34 +81,29 @@ class WhitesourceOrgAdminRepository implements Serializable { @NonCPS protected def httpWhitesource(requestBody) { - script.withCredentials ([script.string( - credentialsId: config.orgAdminUserTokenCredentialsId, - variable: 'orgAdminUserKey' - )]) { - requestBody["userKey"] = orgAdminUserKey - def serializedBody = new JsonUtils().jsonToString(requestBody) - def params = [ - url : config.serviceUrl, - httpMode : 'POST', - acceptType : 'APPLICATION_JSON', - contentType: 'APPLICATION_JSON', - requestBody: serializedBody, - quiet : !config.verbose, - timeout : config.timeout - ] + requestBody["userKey"] = config.orgAdminUserKey + def serializedBody = new JsonUtils().jsonToString(requestBody) + def params = [ + url : config.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: serializedBody, + quiet : !config.verbose, + timeout : config.timeout + ] - if (script.env.HTTP_PROXY) - params["httpProxy"] = script.env.HTTP_PROXY + if (script.env.HTTP_PROXY) + params["httpProxy"] = script.env.HTTP_PROXY - if (config.verbose) - script.echo "Sending http request with parameters ${params}" + if (config.verbose) + script.echo "Sending http request with parameters ${params}" - def response = script.httpRequest(params) + def response = script.httpRequest(params) - if (config.verbose) - script.echo "Received response ${response}" + if (config.verbose) + script.echo "Received response ${response}" - return response - } + return response } } diff --git a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy index dc762d042..21b86bdaa 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy @@ -1,5 +1,7 @@ package com.sap.piper.integration +import com.lesfurets.jenkins.unit.PipelineTestHelper +import org.codehaus.groovy.runtime.InvokerHelper import org.junit.After import org.junit.Before import org.junit.Rule @@ -71,8 +73,8 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { @Test void testHttpWhitesourceInternalCallUserKey() { - repository.orgAdminUserKey = "4711" def config = [ serviceUrl: "http://some.host.whitesource.com/api/", verbose: false, orgAdminUserKey: "4711"] + repository.config.putAll(config) def requestBody = ["someJson" : [ "someObject" : "abcdef" ]] def requestParams diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 34d1f7645..6f89ae3f8 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -217,14 +217,27 @@ void call(Map parameters = [:]) { def whitesourceRepository = parameters.whitesourceRepositoryStub ?: new WhitesourceRepository(this, config) def whitesourceOrgAdminRepository = parameters.whitesourceOrgAdminRepositoryStub ?: new WhitesourceOrgAdminRepository(this, config) - statusCode = triggerWhitesourceScanWithUserKey(script, config, utils, descriptorUtils, parameters, whitesourceRepository, whitesourceOrgAdminRepository) - + if(config.orgAdminUserTokenCredentialsId) { + statusCode = triggerWhitesourceScanWithOrgAdminUserKey(script, config, utils, descriptorUtils, parameters, whitesourceRepository, whitesourceOrgAdminRepository) + } else { + statusCode = triggerWhitesourceScanWithUserKey(script, config, utils, descriptorUtils, parameters, whitesourceRepository, whitesourceOrgAdminRepository) + } checkStatus(statusCode, config) script.commonPipelineEnvironment.setInfluxStepData('whitesource', true) } } +private def triggerWhitesourceScanWithOrgAdminUserKey(script, config, utils, descriptorUtils, parameters, repository, orgAdminRepository) { + withCredentials ([script.string( + credentialsId: config.orgAdminUserTokenCredentialsId, + variable: 'orgAdminUserKey' + )]) { + config.orgAdminUserKey = orgAdminUserKey + triggerWhitesourceScanWithUserKey(script, config, utils, descriptorUtils, parameters, repository, orgAdminRepository) + } +} + private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorUtils, parameters, repository, orgAdminRepository) { withCredentials ([string( credentialsId: config.userTokenCredentialsId, From e1092937afef76cce586c01c2923cf825d756a9c Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 11 Mar 2019 11:06:31 +0100 Subject: [PATCH 048/130] Cleanup --- .../piper/integration/WhitesourceOrgAdminRepository.groovy | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy index ddf5c4e3a..1ff58e8a1 100644 --- a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy @@ -9,10 +9,6 @@ class WhitesourceOrgAdminRepository implements Serializable { final internalWhitesource final Map config - WhitesourceOrgAdminRepository() { - super() - } - WhitesourceOrgAdminRepository(Script script, Map config) { this.script = script this.config = config From 362bcb787f523a0326353b0686755fe9de1c4fc6 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 11 Mar 2019 11:44:39 +0100 Subject: [PATCH 049/130] Cleanup --- vars/whitesourceExecuteScan.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 6f89ae3f8..e52e281ff 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -359,10 +359,10 @@ void analyseWhitesourceResults(Map config, WhitesourceRepository repository, Whi if (!config.productToken) { def metaInfo = orgAdminRepository.fetchProductMetaInfo() def key = "token" - if(!metaInfo && config.createProductFromPipeline) { + if(null != metaInfo && !metaInfo[key] && config.createProductFromPipeline) { metaInfo = orgAdminRepository.createProduct() key = "productToken" - } else if(!metaInfo) { + } else if(null == metaInfo || !metaInfo[key]) { error "[WhiteSource] Could not fetch/find requested product '${config.productName}' and automatic creation has been disabled" } echo "Meta Info: ${metaInfo}" From 5555858acac95e46217db678ea17d70ec948294e Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 11 Mar 2019 13:14:48 +0100 Subject: [PATCH 050/130] Fix logic --- .../sap/piper/integration/WhitesourceOrgAdminRepository.groovy | 2 +- vars/whitesourceExecuteScan.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy index 1ff58e8a1..4e753e0ee 100644 --- a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy @@ -56,7 +56,7 @@ class WhitesourceOrgAdminRepository implements Serializable { def groups = [] def users = [] config.emailAddressesOfInitialProductAdmins.each { - email -> users.add(["email": config.emailOfInitialProductAdmin]) + email -> users.add(["email": email]) } requestBody = [ diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index e52e281ff..9d54e99f8 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -359,7 +359,7 @@ void analyseWhitesourceResults(Map config, WhitesourceRepository repository, Whi if (!config.productToken) { def metaInfo = orgAdminRepository.fetchProductMetaInfo() def key = "token" - if(null != metaInfo && !metaInfo[key] && config.createProductFromPipeline) { + if((null == metaInfo || !metaInfo[key]) && config.createProductFromPipeline) { metaInfo = orgAdminRepository.createProduct() key = "productToken" } else if(null == metaInfo || !metaInfo[key]) { From 28418eae7ccecd4902f85ddaba0ef6d8d6146f20 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 11 Mar 2019 15:27:06 +0100 Subject: [PATCH 051/130] Move creation of project before scan --- vars/whitesourceExecuteScan.groovy | 31 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 9d54e99f8..0ced3605b 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -246,6 +246,20 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU config.userKey = userKey def statusCode = 1 echo "Triggering Whitesource scan on product '${config.productName}' with token '${config.productToken}' using credentials with ID '${config.userTokenCredentialsId}'" + + if (!config.productToken) { + def metaInfo = orgAdminRepository.fetchProductMetaInfo() + def key = "token" + if((null == metaInfo || !metaInfo[key]) && config.createProductFromPipeline) { + metaInfo = orgAdminRepository.createProduct() + key = "productToken" + } else if(null == metaInfo || !metaInfo[key]) { + error "[WhiteSource] Could not fetch/find requested product '${config.productName}' and automatic creation has been disabled" + } + echo "Meta Info: ${metaInfo}" + config.productToken = metaInfo[key] + } + switch (config.scanType) { case 'mta': def scanJobs = [:] @@ -348,27 +362,14 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU } if (config.reporting) { - analyseWhitesourceResults(config, repository, orgAdminRepository) + analyseWhitesourceResults(config, repository) } return statusCode } } -void analyseWhitesourceResults(Map config, WhitesourceRepository repository, WhitesourceOrgAdminRepository orgAdminRepository) { - if (!config.productToken) { - def metaInfo = orgAdminRepository.fetchProductMetaInfo() - def key = "token" - if((null == metaInfo || !metaInfo[key]) && config.createProductFromPipeline) { - metaInfo = orgAdminRepository.createProduct() - key = "productToken" - } else if(null == metaInfo || !metaInfo[key]) { - error "[WhiteSource] Could not fetch/find requested product '${config.productName}' and automatic creation has been disabled" - } - echo "Meta Info: ${metaInfo}" - config.productToken = metaInfo[key] - } - +void analyseWhitesourceResults(Map config, WhitesourceRepository repository) { def pdfName = "whitesource-riskReport.pdf" repository.fetchReportForProduct(pdfName) archiveArtifacts artifacts: pdfName From d1a25f6ae5f1a79e07987274d2a0676b798dd10f Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 12 Mar 2019 11:04:03 +0100 Subject: [PATCH 052/130] Add error handling to org admin --- .../WhitesourceOrgAdminRepository.groovy | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy index 4e753e0ee..2c39441b2 100644 --- a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy @@ -25,8 +25,7 @@ class WhitesourceOrgAdminRepository implements Serializable { requestType: "getOrganizationProductVitals", orgToken: config.orgToken ] - def response = internalWhitesource ? internalWhitesource.httpWhitesource(requestBody) : httpWhitesource(requestBody) - def parsedResponse = new JsonUtils().parseJsonSerializable(response.content) + def parsedResponse = issueHttpRequest(requestBody) findProductMeta(parsedResponse) } @@ -49,8 +48,7 @@ class WhitesourceOrgAdminRepository implements Serializable { orgToken: config.orgToken, productName: config.productName ] - def response = issueHttpRequest(requestBody) - def parsedResponse = new JsonUtils().parseJsonSerializable(response.content) + def parsedResponse = issueHttpRequest(requestBody) def metaInfo = parsedResponse def groups = [] @@ -72,7 +70,12 @@ class WhitesourceOrgAdminRepository implements Serializable { } def issueHttpRequest(requestBody) { - internalWhitesource ? internalWhitesource.httpWhitesource(requestBody) : httpWhitesource(requestBody) + def response = internalWhitesource ? internalWhitesource.httpWhitesource(requestBody) : httpWhitesource(requestBody) + def parsedResponse = new JsonUtils().parseJsonSerializable(response.content) + if(parsedResponse?.errorCode){ + script.error "[WhiteSource] Request failed with error message '${parsedResponse.errorMessage}' (${parsedResponse.errorCode})." + } + return parsedResponse } @NonCPS From 74a06ef871fb36dff04f13e53b9d355b0d7c6615 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 12 Mar 2019 11:53:23 +0100 Subject: [PATCH 053/130] Fix test --- .../WhitesourceConfigurationHelper.groovy | 19 +++++++++++++++---- test/groovy/WhitesourceExecuteScanTest.groovy | 3 +++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index 21b793b36..e55195fa4 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -39,11 +39,7 @@ class WhitesourceConfigurationHelper implements Serializable { [name: 'resolveAllDependencies', value: false] ] switch (config.scanType) { - case 'maven': - mapping += [ - ] - break case 'npm': mapping += [ @@ -72,6 +68,21 @@ class WhitesourceConfigurationHelper implements Serializable { ] break + case 'golang': + mapping += [ + + ] + break + case 'dlang': + mapping += [ + + ] + break + default: + // maven + mapping += [ + + ] } rewriteConfiguration(script, utils, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index 0b827cb1e..a3e289751 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -125,6 +125,9 @@ class WhitesourceExecuteScanTest extends BasePiperTest { helper.registerAllowedMethod( "fetchVulnerabilities", [List], { return new JsonUtils().parseJsonSerializable("{ \"alerts\": [] }").alerts }) + helper.registerAllowedMethod( "createProduct", [], { + return new JsonUtils().parseJsonSerializable("{ \"productToken\": \"e30132d8e8f04a4c8be6332c75a0ff0580ab326fa7534540ad326e97a74d945b\" }") + }) helper.registerAllowedMethod( "publishHTML", [Map], {}) helper.registerAllowedMethod( "getNpmGAV", [String], {return [group: 'com.sap.node', artifact: 'test-node', version: '1.2.3']}) From c6909fa7839e6c9d7075a0db9f43fabb0c504982 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 12 Mar 2019 12:24:25 +0100 Subject: [PATCH 054/130] Improve info message --- vars/whitesourceExecuteScan.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 0ced3605b..2f9122d48 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -245,7 +245,7 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU )]) { config.userKey = userKey def statusCode = 1 - echo "Triggering Whitesource scan on product '${config.productName}' with token '${config.productToken}' using credentials with ID '${config.userTokenCredentialsId}'" + echo "Triggering Whitesource scan on product '${config.productName}'${config.productToken ? ' with token \'' + config.productToken + '\'' : ''} using product admin credentials with ID '${config.userTokenCredentialsId}'${config.orgAdminUserTokenCredentialsId ? ' and organization admin credentials with ID \'' + config.orgAdminUserTokenCredentialsId + '\'' : ''}" if (!config.productToken) { def metaInfo = orgAdminRepository.fetchProductMetaInfo() From 46ddaaf13794d9e0f5492a8299effc8ea51e6a62 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 12 Mar 2019 12:29:30 +0100 Subject: [PATCH 055/130] Add compatibility support for old values --- test/groovy/WhitesourceExecuteScanTest.groovy | 2 +- vars/whitesourceExecuteScan.groovy | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index a3e289751..38c773008 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -173,7 +173,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { scanType : 'maven', juStabUtils : utils, orgToken : 'testOrgToken', - productName : 'testProduct' + whitesourceProductName : 'testProduct' ]) assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 2f9122d48..c172fcfd4 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -156,7 +156,11 @@ import static com.sap.piper.Prerequisites.checkScript projectNames : 'whitesource.projectNames', scanType : 'whitesource.scanType', serviceUrl : 'whitesource.serviceUrl', - userTokenCredentialsId : 'whitesource.userTokenCredentialsId' + userTokenCredentialsId : 'whitesource.userTokenCredentialsId', + productName : 'whitesourceProductName', + productToken : 'whitesourceProductToken', + projectNames : 'whitesourceProjectNames', + userTokenCredentialsId : 'whitesourceUserTokenCredentialsId' ] /** From 63861b0dce0fb352e3c39cbb5809fb9a437bdf7b Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 12 Mar 2019 22:34:20 +0100 Subject: [PATCH 056/130] Add compatibility support for old values --- src/com/sap/piper/ConfigurationHelper.groovy | 10 ++++++---- vars/whitesourceExecuteScan.groovy | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/com/sap/piper/ConfigurationHelper.groovy b/src/com/sap/piper/ConfigurationHelper.groovy index a5a4e0a68..93464db87 100644 --- a/src/com/sap/piper/ConfigurationHelper.groovy +++ b/src/com/sap/piper/ConfigurationHelper.groovy @@ -63,8 +63,8 @@ class ConfigurationHelper implements Serializable { Map newConfig = [:] compatibleParameters.each {entry -> if (entry.getValue() instanceof Map) { - paramStructure = (paramStructure ? paramStructure + '.' : '') + entry.getKey() - newConfig[entry.getKey()] = handleCompatibility(entry.getValue(), paramStructure, configMap) + def internalParamStructure = (paramStructure ? paramStructure + '.' : '') + entry.getKey() + newConfig[entry.getKey()] = handleCompatibility(entry.getValue(), internalParamStructure, configMap) } else { def configSubMap = configMap for(String key in paramStructure.tokenize('.')){ @@ -73,6 +73,8 @@ class ConfigurationHelper implements Serializable { if (configSubMap == null || (configSubMap != null && configSubMap[entry.getKey()] == null)) { def stages = entry.getValue()?.tokenize('.') def value = resolveToFlat(stages, configMap) + if(null == value) + value = resolveToFlat(stages, newConfig) if (value != null) { newConfig[entry.getKey()] = value def paramName = (paramStructure ? paramStructure + '.' : '') + entry.getKey() @@ -87,9 +89,9 @@ class ConfigurationHelper implements Serializable { private String resolveToFlat(stages, configMap) { def first = 0 def result - def configSubMap = configMap[stages[first]] + def configSubMap = configMap[stages?.get(first)] if(configSubMap instanceof Map) { - stages.remove(first) + stages?.remove(first) result = resolveToFlat(stages, configSubMap) } else { result = configSubMap diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index c172fcfd4..d1fe2da6a 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -142,12 +142,22 @@ import static com.sap.piper.Prerequisites.checkScript /** * Title of vulnerability report written during the assessment phase. */ - 'vulnerabilityReportTitle' + 'vulnerabilityReportTitle', + 'whitesourceProductName', + 'whitesourceProductToken', + 'whitesourceProjectNames', + 'whitesourceUserTokenCredentialsId' ] @Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS @Field Map CONFIG_KEY_COMPATIBILITY = [ + whitesource : [ + productName : 'whitesourceProductName', + productToken : 'whitesourceProductToken', + projectNames : 'whitesourceProjectNames', + userTokenCredentialsId : 'whitesourceUserTokenCredentialsId' + ], orgAdminUserTokenCredentialsId : 'whitesource.orgAdminUserTokenCredentialsId', orgToken : 'whitesource.orgToken', productName : 'whitesource.productName', @@ -156,11 +166,7 @@ import static com.sap.piper.Prerequisites.checkScript projectNames : 'whitesource.projectNames', scanType : 'whitesource.scanType', serviceUrl : 'whitesource.serviceUrl', - userTokenCredentialsId : 'whitesource.userTokenCredentialsId', - productName : 'whitesourceProductName', - productToken : 'whitesourceProductToken', - projectNames : 'whitesourceProjectNames', - userTokenCredentialsId : 'whitesourceUserTokenCredentialsId' + userTokenCredentialsId : 'whitesource.userTokenCredentialsId' ] /** From 1202f4f55a3d1b0b1c82b88f4064cca541c852ee Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 12 Mar 2019 23:22:27 +0100 Subject: [PATCH 057/130] Added test --- .../com/sap/piper/DescriptorUtilsTest.groovy | 133 ++++++++++++++++++ ...LibraryLoadingTestExecutionListener.groovy | 11 -- test/resources/DescriptorUtils/dlang/dub.json | 4 + .../DescriptorUtils/npm/package.json | 4 + 4 files changed, 141 insertions(+), 11 deletions(-) create mode 100644 test/groovy/com/sap/piper/DescriptorUtilsTest.groovy create mode 100644 test/resources/DescriptorUtils/dlang/dub.json create mode 100644 test/resources/DescriptorUtils/npm/package.json diff --git a/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy new file mode 100644 index 000000000..a71100f54 --- /dev/null +++ b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy @@ -0,0 +1,133 @@ +package com.sap.piper + +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.RuleChain +import util.BasePiperTest +import util.JenkinsLoggingRule +import util.JenkinsSetupRule +import util.LibraryLoadingTestExecutionListener +import util.SharedLibraryCreator + +import static org.junit.Assert.assertEquals + +class DescriptorUtilsTest extends BasePiperTest { + + @Rule + public JenkinsSetupRule setUpRule = new JenkinsSetupRule(this, SharedLibraryCreator.lazyLoadedLibrary) + public JenkinsLoggingRule loggingRule = new JenkinsLoggingRule(this) + + @Rule + public RuleChain ruleChain = + RuleChain.outerRule(setUpRule) + .around(loggingRule) + + DescriptorUtils descriptorUtils + + @Before + void init() throws Exception { + nullScript.commonPipelineEnvironment = new Object() { + def reset() {} + } + descriptorUtils = new DescriptorUtils() + LibraryLoadingTestExecutionListener.prepareObjectInterceptors(descriptorUtils) + } + + @Test + void testGetNpmGAV() { + + helper.registerAllowedMethod("readJSON", [Map.class], { + searchConfig -> + def packageJsonFile = new File("test/resources/DescriptorUtils/npm/${searchConfig.file}") + return new JsonUtils().parseJsonSerializable(packageJsonFile.text) + }) + + def gav = descriptorUtils.getNpmGAV('package.json') + + assertEquals(gav.group, '@sap') + assertEquals(gav.artifact, 'hdi-deploy') + assertEquals(gav.version, '2.3.0') + } + + @Test + void testGetDlangGAV() { + + helper.registerAllowedMethod("readJSON", [Map.class], { + searchConfig -> + def packageJsonFile = new File("test/resources/DescriptorUtils/dlang/${searchConfig.file}") + return new JsonUtils().parseJsonSerializable(packageJsonFile.text) + }) + + def gav = descriptorUtils.getDlangGAV('dub.json') + + assertEquals(gav.group, 'com.sap.dlang') + assertEquals(gav.artifact, 'hdi-deploy') + assertEquals(gav.version, '2.3.0') + } + + @Test + void testGetPipGAV() { + + helper.registerAllowedMethod("sh", [Map.class], { + map -> + def descriptorFile = new File("test/resources/utilsTest/${map.script.substring(4, map.script.size())}") + return descriptorFile.text + }) + + def gav = descriptorUtils.getPipGAV('setup.py') + + assertEquals('', gav.group) + assertEquals('py_connect', gav.artifact) + assertEquals('1.0', gav.version) + } + + @Test + void testGetMavenGAVComplete() { + + helper.registerAllowedMethod("readMavenPom", [Map.class], { + searchConfig -> + return new Object(){ + def groupId = 'test.group', artifactId = 'test-artifact', version = '1.2.4', packaging = 'jar' + } + }) + + def gav = descriptorUtils.getMavenGAV('pom.xml') + + assertEquals(gav.group, 'test.group') + assertEquals(gav.artifact, 'test-artifact') + assertEquals(gav.version, '1.2.4') + assertEquals(gav.packaging, 'jar') + } + + @Test + void testGetMavenGAVPartial() { + def parameters = [] + + helper.registerAllowedMethod("readMavenPom", [Map.class], { + searchConfig -> + return new Object(){ + def groupId = null, artifactId = null, version = null, packaging = 'jar' + } + }) + + helper.registerAllowedMethod("sh", [Map.class], { + mvnHelpCommand -> + def scriptCommand = mvnHelpCommand['script'] + parameters.add(scriptCommand) + if(scriptCommand.contains('project.groupId')) + return 'test.group' + if(scriptCommand.contains('project.artifactId')) + return 'test-artifact' + if(scriptCommand.contains('project.version')) + return '1.2.4' + }) + + def gav = descriptorUtils.getMavenGAV('pom.xml') + + assertEquals(gav.group, 'test.group') + assertEquals(gav.artifact, 'test-artifact') + assertEquals(gav.version, '1.2.4') + assertEquals(gav.packaging, 'jar') + } +} diff --git a/test/groovy/util/LibraryLoadingTestExecutionListener.groovy b/test/groovy/util/LibraryLoadingTestExecutionListener.groovy index cc2e58675..87e263578 100644 --- a/test/groovy/util/LibraryLoadingTestExecutionListener.groovy +++ b/test/groovy/util/LibraryLoadingTestExecutionListener.groovy @@ -1,9 +1,7 @@ package util -import com.lesfurets.jenkins.unit.InterceptingGCL import com.lesfurets.jenkins.unit.MethodSignature import com.lesfurets.jenkins.unit.PipelineTestHelper -import org.codehaus.groovy.control.CompilerConfiguration import org.springframework.test.context.TestContext import org.springframework.test.context.support.AbstractTestExecutionListener import org.springframework.test.context.support.DependencyInjectionTestExecutionListener @@ -14,15 +12,6 @@ class LibraryLoadingTestExecutionListener extends AbstractTestExecutionListener static PipelineTestHelper singletonInstance - static CompilerConfiguration configuration - - static GroovyClassLoader cLoader - - static { - configuration = new CompilerConfiguration() - cLoader = new InterceptingGCL(singletonInstance, LibraryLoadingTestExecutionListener.class.getClassLoader(), configuration) - } - static List TRACKED_ON_CLASS = [] static List TRACKED_ON_METHODS = [] diff --git a/test/resources/DescriptorUtils/dlang/dub.json b/test/resources/DescriptorUtils/dlang/dub.json new file mode 100644 index 000000000..f4d7bf892 --- /dev/null +++ b/test/resources/DescriptorUtils/dlang/dub.json @@ -0,0 +1,4 @@ +{ + "name": "hdi-deploy", + "version": "2.3.0" +} diff --git a/test/resources/DescriptorUtils/npm/package.json b/test/resources/DescriptorUtils/npm/package.json new file mode 100644 index 000000000..212530865 --- /dev/null +++ b/test/resources/DescriptorUtils/npm/package.json @@ -0,0 +1,4 @@ +{ + "name": "@sap/hdi-deploy", + "version": "2.3.0" +} From 99f853e207637b2201d96ebdf23c69bbd3861296 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 12 Mar 2019 23:36:55 +0100 Subject: [PATCH 058/130] Improve coverage --- src/com/sap/piper/DescriptorUtils.groovy | 18 ------------------ .../com/sap/piper/DescriptorUtilsTest.groovy | 17 +++++++++++++++++ .../DescriptorUtils/sbt/sbtDescriptor.json | 6 ++++++ 3 files changed, 23 insertions(+), 18 deletions(-) create mode 100644 test/resources/DescriptorUtils/sbt/sbtDescriptor.json diff --git a/src/com/sap/piper/DescriptorUtils.groovy b/src/com/sap/piper/DescriptorUtils.groovy index 3756baefd..c6976353a 100644 --- a/src/com/sap/piper/DescriptorUtils.groovy +++ b/src/com/sap/piper/DescriptorUtils.groovy @@ -70,24 +70,6 @@ def getSbtGAV(file = 'sbtDescriptor.json') { return result } -def getMtaGAV(file = 'mta.yaml', xmakeConfigFile = '.xmake.cfg') { - def result = [:] - def descriptor = readYaml(file: file) - def xmakeConfig = readProperties(file: xmakeConfigFile) - - result['group'] = xmakeConfig['mtar-group'] - result['artifact'] = descriptor.ID - result['version'] = descriptor.version - result['packaging'] = 'mtar' - // using default value: https://github.wdf.sap.corp/dtxmake/xmake-mta-plugin#complete-list-of-default-values - if(!result['group']){ - result['group'] = 'com.sap.prd.xmake.example.mtars' - Notify.warning(this, "No groupID set in '.xmake.cfg', using default groupID '${result['group']}'.", 'com.sap.icd.jenkins.Utils') - } - echo "loaded ${result} from ${file} and ${xmakeConfigFile}" - return result -} - def getPipGAV(file = 'setup.py') { def result = [:] def descriptor = sh(returnStdout: true, script: "cat ${file}") diff --git a/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy index a71100f54..9f15ef9cb 100644 --- a/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy +++ b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy @@ -50,6 +50,23 @@ class DescriptorUtilsTest extends BasePiperTest { assertEquals(gav.version, '2.3.0') } + @Test + void testGetSbtGAV() { + + helper.registerAllowedMethod("readJSON", [Map.class], { + searchConfig -> + def packageJsonFile = new File("test/resources/DescriptorUtils/sbt/${searchConfig.file}") + return new JsonUtils().parseJsonSerializable(packageJsonFile.text) + }) + + def gav = descriptorUtils.getSbtGAV('sbtDescriptor.json') + + assertEquals(gav.group, 'sap') + assertEquals(gav.artifact, 'hdi-deploy') + assertEquals(gav.packaging, 'test') + assertEquals(gav.version, '2.3.0') + } + @Test void testGetDlangGAV() { diff --git a/test/resources/DescriptorUtils/sbt/sbtDescriptor.json b/test/resources/DescriptorUtils/sbt/sbtDescriptor.json new file mode 100644 index 000000000..8a6064f23 --- /dev/null +++ b/test/resources/DescriptorUtils/sbt/sbtDescriptor.json @@ -0,0 +1,6 @@ +{ + "group": "sap", + "artifactId": "hdi-deploy", + "version": "2.3.0", + "packaging": "test" +} From 7dba1dadc93f21d002ce538ba1999f447b7920d3 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 13 Mar 2019 09:58:47 +0100 Subject: [PATCH 059/130] Improve coverage --- .../integration/WhitesourceRepository.groovy | 34 --- .../WhitesourceOrgAdminRepositoryTest.groovy | 181 +++++++++++++++- .../WhitesourceRepositoryTest.groovy | 199 +++++++++++++++++- 3 files changed, 372 insertions(+), 42 deletions(-) diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy index b7f8569b0..9845f25d0 100644 --- a/src/com/sap/piper/integration/WhitesourceRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -47,40 +47,6 @@ class WhitesourceRepository implements Serializable { return fetchedVulnerabilities } - List fetchLibraries(whitesourceProjectsMetaInformation) { - def fetchedLibraries = [] - if (config.projectNames) { - for (int i = 0; i < whitesourceProjectsMetaInformation.size(); i++) { - def metaInfo = whitesourceProjectsMetaInformation[i] - def requestBody = [ - requestType : "getProjectLicenses", - projectToken: metaInfo.token - ] - - def response = fetchWhitesourceResource(requestBody) - fetchedLibraries.addAll(response.libraries) - } - } else { - def requestBody = [ - requestType : "getProductLicenses", - productToken: config.productToken - ] - - def response = fetchWhitesourceResource(requestBody) - fetchedLibraries.addAll(response.libraries) - } - - sortLibrariesAlphabeticallyGAV(fetchedLibraries) - - return listUniqueEntries(fetchedLibraries) - } - - @NonCPS - def listUniqueEntries(Collection list) { - return list.unique() - } - - protected def fetchWhitesourceResource(Map requestBody) { final def response = httpWhitesource(requestBody) def parsedResponse = new JsonUtils().parseJsonSerializable(response.content) diff --git a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy index 21b86bdaa..c59703e86 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy @@ -1,7 +1,6 @@ package com.sap.piper.integration -import com.lesfurets.jenkins.unit.PipelineTestHelper -import org.codehaus.groovy.runtime.InvokerHelper +import hudson.AbortException import org.junit.After import org.junit.Before import org.junit.Rule @@ -9,23 +8,28 @@ import org.junit.Test import org.junit.rules.RuleChain import util.BasePiperTest import util.JenkinsEnvironmentRule +import util.JenkinsErrorRule import util.JenkinsLoggingRule import util.LibraryLoadingTestExecutionListener import util.Rules import static org.assertj.core.api.Assertions.assertThat +import static org.hamcrest.Matchers.containsString import static org.hamcrest.Matchers.is +import static org.hamcrest.Matchers.isA class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { - private JenkinsLoggingRule jlr = new JenkinsLoggingRule(this) - private JenkinsEnvironmentRule jer = new JenkinsEnvironmentRule(this) + private JenkinsErrorRule thrown = new JenkinsErrorRule(this) + private JenkinsLoggingRule loggingRule = new JenkinsLoggingRule(this) + private JenkinsEnvironmentRule environmentRule = new JenkinsEnvironmentRule(this) @Rule public RuleChain ruleChain = Rules .getCommonRules(this) - .around(jlr) - .around(jer) + .around(thrown) + .around(loggingRule) + .around(environmentRule) WhitesourceOrgAdminRepository repository @@ -96,4 +100,169 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { ] )) } + + @Test + void testHttpWhitesourceInternalCallUserKeyVerboseProxy() { + def config = [ serviceUrl: "http://some.host.whitesource.com/api/", verbose: true, orgAdminUserKey: "4711"] + nullScript.env['HTTP_PROXY'] = "http://test.sap.com:8080" + repository.config.putAll(config) + def requestBody = ["someJson" : [ "someObject" : "abcdef" ]] + + def requestParams + helper.registerAllowedMethod('httpRequest', [Map], { p -> + requestParams = p + }) + + repository.httpWhitesource(requestBody) + + assertThat(requestParams, is( + [ + url : config.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: requestBody, + quiet : false, + userKey : config.orgAdminUserKey, + httpProxy : "http://test.sap.com:8080" + ] + )) + + assertThat(loggingRule.log, containsString("Sending http request with parameters")) + assertThat(loggingRule.log, containsString("Received response")) + } + + @Test + void testCreateProduct() { + def config = [ + serviceUrl: "http://some.host.whitesource.com/api/", + verbose: false, + orgAdminUserKey: "4711", + orgToken: "abcd1234", + productName: "testProduct", + emailAddressesOfInitialProductAdmins: ['some@somewhere.com', 'some2@somewhere.com'] + ] + repository.config.putAll(config) + def requestBody1 = [ + requestType: "getOrganizationProductVitals", + orgToken: config.orgToken, + userKey: "4711" + ] + + def requestBody2 = [ + "requestType" : "setProductAssignments", + "productToken" : "54785", + "productMembership" : ["userAssignments":[], "groupAssignments":[]], + "productAdmins" : ["userAssignments":[[ "email": "some@somewhere.com" ], ["email": "some2@somewhere.com"]]], + "alertsEmailReceivers" : ["userAssignments":[]], + "userKey": "4711" + ] + + def requestParams = [] + helper.registerAllowedMethod('httpRequest', [Map], { p -> + requestParams.add(p) + return [ content : "{ \"productToken\" : \"54785\" }" ] + }) + + repository.createProduct() + + assertThat(requestParams[0], is( + [ + url : config.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: requestBody1, + quiet : false, + userKey : config.orgAdminUserKey, + httpProxy : "http://test.sap.com:8080" + ] + )) + + assertThat(requestParams[1], is( + [ + url : config.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: requestBody2, + quiet : false, + userKey : config.orgAdminUserKey, + httpProxy : "http://test.sap.com:8080" + ] + )) + } + + @Test + void testIssueHttpRequestError() { + def config = [ serviceUrl: "http://some.host.whitesource.com/api/", verbose: false, orgAdminUserKey: "4711"] + repository.config.putAll(config) + def requestBody = ["someJson" : [ "someObject" : "abcdef" ]] + + def requestParams + helper.registerAllowedMethod('httpRequest', [Map], { p -> + requestParams = p + return [content: "{ \"error\" : \"4546\", \"errorMessage\" : \"some text\" } }"] + }) + + def errorCaught = false + try { + repository.issueHttpRequest(requestBody) + } catch (e) { + errorCaught = true + assertThat(e, isA(AbortException.class)) + assertThat(e.getMessage(), equals("[WhiteSource] Request failed with error message 'some text' (4546).")) + } + assertThat(errorCaught, is(true)) + + assertThat(requestParams, is( + [ + url : config.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: requestBody, + quiet : true, + userKey : config.orgAdminUserKey + ] + )) + } + + @Test + void testFetchProductMetaInfo() { + def config = [ serviceUrl: "http://some.host.whitesource.com/api/", verbose: true, orgAdminUserKey: "4711", orgToken: "12345", productName: "testProduct"] + nullScript.env['HTTP_PROXY'] = "http://test.sap.com:8080" + repository.config.putAll(config) + + def requestBody = [ + requestType: "getOrganizationProductVitals", + orgToken: config.orgToken, + userKey: "4711" + ] + + def requestParams + helper.registerAllowedMethod('httpRequest', [Map], { p -> + requestParams = p + return [ content: "{ \"productVitals\" : [ { \"name\": \"testProduct\"} ] }"] + }) + + def result = repository.fetchProductMetaInfo() + + assertThat(requestParams, is( + [ + url : config.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: requestBody, + quiet : false, + userKey : config.orgAdminUserKey, + httpProxy : "http://test.sap.com:8080" + ] + )) + + assertThat(result, is([ name: "testProduct"])) + assertThat(loggingRule.log, containsString("Sending http request with parameters")) + assertThat(loggingRule.log, containsString("Received response")) + } } diff --git a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy index b63860c82..7482e2461 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy @@ -15,6 +15,8 @@ import util.LibraryLoadingTestExecutionListener import util.Rules import static org.assertj.core.api.Assertions.assertThat +import static org.hamcrest.Matchers.containsString +import static org.hamcrest.Matchers.containsString import static org.hamcrest.Matchers.is import static org.hamcrest.Matchers.isA @@ -49,8 +51,6 @@ class WhitesourceRepositoryTest extends BasePiperTest { @Test void testResolveProjectsMeta() { - - def whitesourceMetaResponse = [ projectVitals: [ [ @@ -341,4 +341,199 @@ curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type }''' )) } + + @Test + void testFetchProductLicenseAlerts() { + def config = [ serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711", productToken: "8547"] + nullScript.env['HTTP_PROXY'] = "http://test.sap.com:8080" + repository.config.putAll(config) + + def requestBody = [ + requestType: "getProductAlertsByType", + alertType: "REJECTED_BY_POLICY_RESOURCE", + productToken: config.productToken + ] + + def requestParams + helper.registerAllowedMethod('httpRequest', [Map], { p -> + requestParams = p + return [ content: "{ \"alerts\" : [] }"] + }) + + repository.fetchProductLicenseAlerts() + + assertThat(requestParams, is( + [ + url : config.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: requestBody, + quiet : false, + userKey : config.userKey, + httpProxy : "http://test.sap.com:8080" + ] + )) + } + + @Test + void testFetchProjectLicenseAlerts() { + def projectToken = "8547" + def config = [ serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711"] + nullScript.env['HTTP_PROXY'] = "http://test.sap.com:8080" + repository.config.putAll(config) + + def requestBody = [ + requestType: "getProjectAlertsByType", + alertType: "REJECTED_BY_POLICY_RESOURCE", + projectToken: projectToken + ] + + def requestParams + helper.registerAllowedMethod('httpRequest', [Map], { p -> + requestParams = p + return [ content: "{ \"alerts\" : [] }"] + }) + + repository.fetchProjectLicenseAlerts(projectToken) + + assertThat(requestParams, is( + [ + url : config.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: requestBody, + quiet : false, + userKey : config.userKey, + httpProxy : "http://test.sap.com:8080" + ] + )) + } + + @Test + void testFetchProdjectsMetaInfo() { + def config = [ serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711", productToken: '8475', projectNames: ['testProject1', 'testProject2']] + nullScript.env['HTTP_PROXY'] = "http://test.sap.com:8080" + repository.config.putAll(config) + + def requestBody = [ + requestType: "getProductProjectVitals", + productToken: config.productToken + ] + + def requestParams + helper.registerAllowedMethod('httpRequest', [Map], { p -> + requestParams = p + return [ content: "{ \"projectVitals\" : [ { \"name\": \"testProject1\"}, { \"name\": \"testProject2\"} ] }"] + }) + + def result = repository.fetchProjectsMetaInfo() + + assertThat(requestParams, is( + [ + url : config.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: requestBody, + quiet : false, + userKey : config.userKey, + httpProxy : "http://test.sap.com:8080" + ] + )) + + assertThat(result, is([[ name: "testProduct1"], [ name: "testProduct2"]])) + } + + @Test + void testFetchVulnerabilitiesOnProjects() { + def config = [ serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711", productToken: '8475', projectNames: ['testProject1', 'testProject2']] + nullScript.env['HTTP_PROXY'] = "http://test.sap.com:8080" + repository.config.putAll(config) + + def requestBody1 = [ + requestType : "getProjectAlertsByType", + alertType : "SECURITY_VULNERABILITY", + projectToken: "1234" + ] + + def requestBody2 = [ + requestType : "getProjectAlertsByType", + alertType : "SECURITY_VULNERABILITY", + projectToken: "2345" + ] + + def requestParams = [] + helper.registerAllowedMethod('httpRequest', [Map], { p -> + requestParams.add(p) + return [ content: "{ \"alerts\" : [ { \"vulnerability\" : { \"cvss3_score\" : \"7\"} } ] }"] + }) + + def result = repository.fetchVulnerabilities([ [name: "testProject1", token: "1234"], [name: "testProject2", token: "2345"] ]) + + assertThat(requestParams[0], is( + [ + url : config.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: requestBody1, + quiet : false, + userKey : config.userKey, + httpProxy : "http://test.sap.com:8080" + ] + )) + + assertThat(requestParams[1], is( + [ + url : config.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: requestBody2, + quiet : false, + userKey : config.userKey, + httpProxy : "http://test.sap.com:8080" + ] + )) + + assertThat(result.size(), is(2)) + } + + @Test + void testFetchVulnerabilitiesOnProduct() { + def config = [ serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711", productToken: '8475', productName : 'testProduct'] + nullScript.env['HTTP_PROXY'] = "http://test.sap.com:8080" + repository.config.putAll(config) + + def requestBody = [ + requestType : "getProductAlertsByType", + alertType : "SECURITY_VULNERABILITY", + productToken: config.productToken, + ] + + def requestParams = [] + helper.registerAllowedMethod('httpRequest', [Map], { p -> + requestParams.add(p) + return [ content: "{ \"alerts\" : [ { \"vulnerability\" : { \"cvss3_score\" : \"7\"} } ] }"] + }) + + def result = repository.fetchVulnerabilities([ [name: "testProject1", token: "1234"], [name: "testProject2", token: "2345"] ]) + + assertThat(requestParams[0], is( + [ + url : config.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_JSON', + contentType: 'APPLICATION_JSON', + requestBody: requestBody, + quiet : false, + userKey : config.userKey, + httpProxy : "http://test.sap.com:8080" + ] + )) + + assertThat(result.size(), is(1)) + } } From dee6f61faee8706d5f549697bb07bca238a99e2b Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 13 Mar 2019 10:21:41 +0100 Subject: [PATCH 060/130] Improve coverage --- test/groovy/WhitesourceExecuteScanTest.groovy | 100 +++++++++++++++++- 1 file changed, 97 insertions(+), 3 deletions(-) diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index 38c773008..0e5269539 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -47,7 +47,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { @Before void init() { - def credentialsStore = ['ID-123456789': 'token-0815', 'ID-abcdefg': ['testUser', 'testPassword']] + def credentialsStore = ['ID-123456789': 'token-0815', 'ID-9876543': 'token-0816', 'ID-abcdefg': ['testUser', 'testPassword']] def withCredentialsBindings helper.registerAllowedMethod('string', [Map], { m -> @@ -319,6 +319,91 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1.2.3')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=test-python')) } + + @Test + void testWithOrgAdminCredentials() { + helper.registerAllowedMethod("readProperties", [Map], { + def result = new Properties() + result.putAll([ + "apiKey": "b39d1328-52e2-42e3-98f0-932709daf3f0", + "productName": "SHC - Piper", + "checkPolicies": "true", + "projectName": "python-test", + "projectVersion": "1.0.0" + ]) + return result + }) + + stepRule.step.whitesourceExecuteScan([ + script : nullScript, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + descriptorUtilsStub : descriptorUtilsStub, + scanType : 'pip', + juStabUtils : utils, + orgToken : 'testOrgToken', + productName : 'testProductName', + orgAdminUserTokenCredentialsId : 'ID-9876543', + reporting : false + ]) + + assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) + + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'python:3.7.2-stretch')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/python')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config d3aa80454919391024374ba46b4df082d15ab9a3'])) + + assertThat(shellRule.shell, Matchers.hasItems( + is('curl --location --output wss-unified-agent.jar https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/wss-unified-agent.jar'), + is('curl --location --output jvm.tar.gz https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz && tar --strip-components=1 -xzf jvm.tar.gz'), + is('./bin/java -jar wss-unified-agent.jar -c \'./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProductName\'') + )) + + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('apiKey=testOrgToken')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productName=testProductName')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('userKey=token-0815')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1.2.3')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=test-python')) + } + + @Test + void testNoProjectNoCreation() { + helper.registerAllowedMethod("readProperties", [Map], { + def result = new Properties() + result.putAll([ + "apiKey": "b39d1328-52e2-42e3-98f0-932709daf3f0", + "productName": "SHC - Piper", + "checkPolicies": "true", + "projectName": "python-test", + "projectVersion": "1.0.0" + ]) + return result + }) + + def errorCaught = false + try { + stepRule.step.whitesourceExecuteScan([ + script : nullScript, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub: whitesourceOrgAdminRepositoryStub, + descriptorUtilsStub : descriptorUtilsStub, + scanType : 'pip', + juStabUtils : utils, + orgToken : 'testOrgToken', + productName : 'testProductName', + createProductFromPipeline : false, + orgAdminUserTokenCredentialsId : 'ID-9876543', + reporting : false + ]) + } catch (e) { + errorCaught = true + assertThat(e, isA(AbortException.class)) + assertThat(e.getMessage(), is("[WhiteSource] Could not fetch/find requested product 'testProductName' and automatic creation has been disabled")) + } + + assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) + assertThat(errorCaught, is(true)) + } @Test void testSbt() { @@ -415,6 +500,9 @@ class WhitesourceExecuteScanTest extends BasePiperTest { if (map.glob == "**${File.separator}package.json") { return [new File('npm1/package.json'), new File('npm2/package.json')].toArray() } + if (map.glob == "**${File.separator}setup.py") { + return [new File('pip/setup.py')].toArray() + } return [].toArray() }) @@ -429,6 +517,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { parallelMap = map parallelMap['Whitesource - maven1']() parallelMap['Whitesource - npm1']() + parallelMap['Whitesource - pip']() }) //need to use call due to mock above @@ -450,7 +539,8 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(parallelMap, hasKey('Whitesource - maven1')) assertThat(parallelMap, hasKey('Whitesource - npm1')) - assertThat(parallelMap.keySet(), hasSize(3)) + assertThat(parallelMap, hasKey('Whitesource - pip')) + assertThat(parallelMap.keySet(), hasSize(4)) assertThat(whitesourceCalls, hasItem(allOf( hasEntry('scanType', 'maven'), @@ -458,9 +548,13 @@ class WhitesourceExecuteScanTest extends BasePiperTest { ))) assertThat(whitesourceCalls, hasItem(allOf( hasEntry('scanType', 'npm'), - hasEntry('projectNames', ["com.sap.maven.test-java - 1.2.3", "com.sap.node.test-node - 1.2.3"]), hasEntry('buildDescriptorFile', "npm1${File.separator}package.json".toString()) ))) + assertThat(whitesourceCalls, hasItem(allOf( + hasEntry('scanType', 'pip'), + hasEntry('projectNames', ["com.sap.maven.test-java - 1.2.3", "com.sap.node.test-node - 1.2.3", "test-python - 1.2.3"]), + hasEntry('buildDescriptorFile', "pip${File.separator}setup.py".toString()) + ))) } @Test From 0280fe56a9be22198da06b89a63299f1ec2032fe Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 13 Mar 2019 10:29:25 +0100 Subject: [PATCH 061/130] Fix typo --- documentation/docs/steps/checkChangeInDevelopment.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation/docs/steps/checkChangeInDevelopment.md b/documentation/docs/steps/checkChangeInDevelopment.md index ecf035b39..b164dce27 100644 --- a/documentation/docs/steps/checkChangeInDevelopment.md +++ b/documentation/docs/steps/checkChangeInDevelopment.md @@ -2,7 +2,7 @@ ## Description -Content here is generated from corresponnding step, see `vars`. +Content here is generated from corresponding step, see `vars`. ## Prerequisites @@ -10,11 +10,11 @@ Content here is generated from corresponnding step, see `vars`. ## Parameters -Content here is generated from corresponnding step, see `vars`. +Content here is generated from corresponding step, see `vars`. ## Step configuration -Content here is generated from corresponnding step, see `vars`. +Content here is generated from corresponding step, see `vars`. ## Exceptions From 39caa295677613879bdaf0072d0d464c50a3ea17 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 13 Mar 2019 10:41:11 +0100 Subject: [PATCH 062/130] Improve coverage --- .../com/sap/piper/DescriptorUtilsTest.groovy | 16 ++++++++++++++ test/resources/DescriptorUtils/pip/setup.py | 22 +++++++++++++++++++ .../resources/DescriptorUtils/pip/version.txt | 1 + 3 files changed, 39 insertions(+) create mode 100644 test/resources/DescriptorUtils/pip/setup.py create mode 100644 test/resources/DescriptorUtils/pip/version.txt diff --git a/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy index 9f15ef9cb..91fbe9f00 100644 --- a/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy +++ b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy @@ -99,6 +99,22 @@ class DescriptorUtilsTest extends BasePiperTest { assertEquals('1.0', gav.version) } + @Test + void testGetPipGAVFromVersionTxt() { + + helper.registerAllowedMethod("sh", [Map.class], { + map -> + def descriptorFile = new File("test/resources/DescriptorUtils/pip/${map.script.substring(4, map.script.size())}") + return descriptorFile.text + }) + + def gav = descriptorUtils.getPipGAV('setup.py') + + assertEquals('', gav.group) + assertEquals('some-test', gav.artifact) + assertEquals('1.0.0-SNAPSHOT', gav.version) + } + @Test void testGetMavenGAVComplete() { diff --git a/test/resources/DescriptorUtils/pip/setup.py b/test/resources/DescriptorUtils/pip/setup.py new file mode 100644 index 000000000..01b7ed4bc --- /dev/null +++ b/test/resources/DescriptorUtils/pip/setup.py @@ -0,0 +1,22 @@ +from setuptools import setup, find_packages +from codecs import open +from os import path + +def get_version(): + with open('version.txt') as ver_file: + version_str = ver_file.readline().rstrip() + return version_str + + +def get_install_requires(): + with open('requirements.txt') as reqs_file: + reqs = [line.rstrip() for line in reqs_file.readlines()] + return reqs + +setup(name="some-test", + version=get_version(), + python_requires='>=3', + packages=find_packages(exclude=['contrib', 'docs', 'tests*', 'coverage', 'bin']), + description="test", + install_requires=get_install_requires(), + ) diff --git a/test/resources/DescriptorUtils/pip/version.txt b/test/resources/DescriptorUtils/pip/version.txt new file mode 100644 index 000000000..f75514931 --- /dev/null +++ b/test/resources/DescriptorUtils/pip/version.txt @@ -0,0 +1 @@ +1.0.0-SNAPSHOT From cd8067b5d5c5873368bc49d38130df34d787df0b Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 13 Mar 2019 11:06:47 +0100 Subject: [PATCH 063/130] Improve coverage --- .../com/sap/piper/DescriptorUtilsTest.groovy | 51 +++++++++++++++++-- .../DescriptorUtils/npm/package.json | 4 +- .../DescriptorUtils/npm/package2.json | 4 ++ .../DescriptorUtils/npm/package3.json | 4 ++ 4 files changed, 56 insertions(+), 7 deletions(-) create mode 100644 test/resources/DescriptorUtils/npm/package2.json create mode 100644 test/resources/DescriptorUtils/npm/package3.json diff --git a/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy index 91fbe9f00..24b4881b7 100644 --- a/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy +++ b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy @@ -1,39 +1,63 @@ package com.sap.piper +import hudson.AbortException import org.junit.Before import org.junit.Rule import org.junit.Test import org.junit.rules.RuleChain import util.BasePiperTest +import util.JenkinsEnvironmentRule +import util.JenkinsErrorRule import util.JenkinsLoggingRule import util.JenkinsSetupRule import util.LibraryLoadingTestExecutionListener -import util.SharedLibraryCreator import static org.junit.Assert.assertEquals +import static org.junit.Assert.assertThat +import static org.hamcrest.core.Is.* class DescriptorUtilsTest extends BasePiperTest { @Rule - public JenkinsSetupRule setUpRule = new JenkinsSetupRule(this, SharedLibraryCreator.lazyLoadedLibrary) + public JenkinsErrorRule errorRule = new JenkinsErrorRule(this) + @Rule + public JenkinsEnvironmentRule environmentRule = new JenkinsEnvironmentRule(this) + @Rule + public JenkinsSetupRule setUpRule = new JenkinsSetupRule(this) + @Rule public JenkinsLoggingRule loggingRule = new JenkinsLoggingRule(this) @Rule public RuleChain ruleChain = RuleChain.outerRule(setUpRule) + .around(errorRule) + .around(environmentRule) .around(loggingRule) DescriptorUtils descriptorUtils @Before void init() throws Exception { - nullScript.commonPipelineEnvironment = new Object() { - def reset() {} - } descriptorUtils = new DescriptorUtils() LibraryLoadingTestExecutionListener.prepareObjectInterceptors(descriptorUtils) } + @Test + void testGetNpmGAVSapArtifact() { + + helper.registerAllowedMethod("readJSON", [Map.class], { + searchConfig -> + def packageJsonFile = new File("test/resources/DescriptorUtils/npm/${searchConfig.file}") + return new JsonUtils().parseJsonSerializable(packageJsonFile.text) + }) + + def gav = descriptorUtils.getNpmGAV('package2.json') + + assertEquals(gav.group, '') + assertEquals(gav.artifact, 'some-test') + assertEquals(gav.version, '1.2.3') + } + @Test void testGetNpmGAV() { @@ -50,6 +74,23 @@ class DescriptorUtilsTest extends BasePiperTest { assertEquals(gav.version, '2.3.0') } + @Test + void testGetNpmGAVSapArtifactError() { + + helper.registerAllowedMethod("readJSON", [Map.class], { + searchConfig -> + def packageJsonFile = new File("test/resources/DescriptorUtils/npm/${searchConfig.file}") + return new JsonUtils().parseJsonSerializable(packageJsonFile.text) + }) + + try { + descriptorUtils.getNpmGAV('package3.json') + } catch (e) { + assertThat(e, isA(AbortException.class)) + assertThat(e.getMessage(), is("Unable to parse package name '@someerror'")) + } + } + @Test void testGetSbtGAV() { diff --git a/test/resources/DescriptorUtils/npm/package.json b/test/resources/DescriptorUtils/npm/package.json index 212530865..d362917b4 100644 --- a/test/resources/DescriptorUtils/npm/package.json +++ b/test/resources/DescriptorUtils/npm/package.json @@ -1,4 +1,4 @@ { - "name": "@sap/hdi-deploy", - "version": "2.3.0" + "name": "@sap/hdi-deploy", + "version": "2.3.0" } diff --git a/test/resources/DescriptorUtils/npm/package2.json b/test/resources/DescriptorUtils/npm/package2.json new file mode 100644 index 000000000..3ef004f51 --- /dev/null +++ b/test/resources/DescriptorUtils/npm/package2.json @@ -0,0 +1,4 @@ +{ + "name": "some-test", + "version": "1.2.3" +} diff --git a/test/resources/DescriptorUtils/npm/package3.json b/test/resources/DescriptorUtils/npm/package3.json new file mode 100644 index 000000000..6d478abaf --- /dev/null +++ b/test/resources/DescriptorUtils/npm/package3.json @@ -0,0 +1,4 @@ +{ + "name": "@someerror", + "version": "1.2.3" +} From 088c5901e7ef9b3e3a40ab55e7642b8af21c123a Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 13 Mar 2019 11:38:44 +0100 Subject: [PATCH 064/130] Improve coverage --- .../integration/WhitesourceRepository.groovy | 2 +- .../com/sap/piper/DescriptorUtilsTest.groovy | 5 +++ .../WhitesourceOrgAdminRepositoryTest.groovy | 20 ++++++++- .../WhitesourceRepositoryTest.groovy | 44 +++++++++++++++++-- 4 files changed, 66 insertions(+), 5 deletions(-) diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy index 9845f25d0..6882f075f 100644 --- a/src/com/sap/piper/integration/WhitesourceRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -193,7 +193,7 @@ class WhitesourceRepository implements Serializable { def response = script.httpRequest(params) if(config.verbose) - script.echo "Received response ${reponse}" + script.echo "Received response ${response}" return response } diff --git a/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy index 24b4881b7..0a3159fff 100644 --- a/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy +++ b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy @@ -12,6 +12,8 @@ import util.JenkinsLoggingRule import util.JenkinsSetupRule import util.LibraryLoadingTestExecutionListener +import static org.assertj.core.api.Assertions.assertThat +import static org.hamcrest.Matchers.is import static org.junit.Assert.assertEquals import static org.junit.Assert.assertThat import static org.hamcrest.core.Is.* @@ -83,12 +85,15 @@ class DescriptorUtilsTest extends BasePiperTest { return new JsonUtils().parseJsonSerializable(packageJsonFile.text) }) + def errorCaught = false try { descriptorUtils.getNpmGAV('package3.json') } catch (e) { + errorCaught = true assertThat(e, isA(AbortException.class)) assertThat(e.getMessage(), is("Unable to parse package name '@someerror'")) } + assertThat(errorCaught, is(true)) } @Test diff --git a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy index c59703e86..1f4c56e19 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy @@ -45,6 +45,24 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { nullScript.env = [:] } + @Test + void testMissingConfig() { + def errorCaught = false + try { + new WhitesourceOrgAdminRepository(null, [:]) + } catch (e) { + errorCaught = true + assertThat(e, isA(AbortException.class)) + assertThat(e.getMessage(), is("Parameter 'serviceUrl' must be provided as part of the configuration.")) + } + assertThat(errorCaught, is(true)) + } + + @Test + void testAccessor() { + new WhitesourceOrgAdminRepository(nullScript, [whitesourceAccessor: "com.sap.piper.integration.WhitesourceRepository", serviceUrl: "http://test.com"]) + } + @Test void testResolveProductMeta() { @@ -202,7 +220,7 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { def requestParams helper.registerAllowedMethod('httpRequest', [Map], { p -> requestParams = p - return [content: "{ \"error\" : \"4546\", \"errorMessage\" : \"some text\" } }"] + return [content: "{ \"errorCode\" : \"4546\", \"errorMessage\" : \"some text\" } }"] }) def errorCaught = false diff --git a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy index 7482e2461..db2ea3f23 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy @@ -49,6 +49,19 @@ class WhitesourceRepositoryTest extends BasePiperTest { nullScript.env = [:] } + @Test + void testMissingConfig() { + def errorCaught = false + try { + new WhitesourceRepository(null, [:]) + } catch (e) { + errorCaught = true + assertThat(e, isA(AbortException.class)) + assertThat(e.getMessage(), is("Parameter 'serviceUrl' must be provided as part of the configuration.")) + } + assertThat(errorCaught, is(true)) + } + @Test void testResolveProjectsMeta() { def whitesourceMetaResponse = [ @@ -233,6 +246,7 @@ class WhitesourceRepositoryTest extends BasePiperTest { @Test void testHttpWhitesourceExternalCallNoUserKey() { def config = [ whitesourceServiceUrl: "https://saas.whitesource.com/api", verbose: true] + repository.config.putAll(config) def requestBody = "{ \"someJson\" : { \"someObject\" : \"abcdef\" } }" def requestParams @@ -326,7 +340,7 @@ class WhitesourceRepositoryTest extends BasePiperTest { @Test void testFetchReportForProduct() { - repository.config.putAll([ whitesourceServiceUrl: "http://mo-323123123.sap.corp/some", verbose: false, productToken: "4711"]) + repository.config.putAll([ whitesourceServiceUrl: "http://mo-323123123.sap.corp/some", verbose: true, productToken: "4711"]) def command helper.registerAllowedMethod('sh', [String], { cmd -> command = cmd @@ -340,6 +354,8 @@ curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type "productToken": "4711" }''' )) + + assertThat(jlr.log, containsString("Sending curl request with parameters [requestType:getProductRiskReport, productToken:4711]")) } @Test @@ -380,7 +396,6 @@ curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type void testFetchProjectLicenseAlerts() { def projectToken = "8547" def config = [ serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711"] - nullScript.env['HTTP_PROXY'] = "http://test.sap.com:8080" repository.config.putAll(config) def requestBody = [ @@ -412,7 +427,7 @@ curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type } @Test - void testFetchProdjectsMetaInfo() { + void testFetchProjectsMetaInfo() { def config = [ serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711", productToken: '8475', projectNames: ['testProject1', 'testProject2']] nullScript.env['HTTP_PROXY'] = "http://test.sap.com:8080" repository.config.putAll(config) @@ -446,6 +461,29 @@ curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type assertThat(result, is([[ name: "testProduct1"], [ name: "testProduct2"]])) } + @Test + void testFetchProjectsMetaInfoError() { + def config = [ serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711", productName: 'kjdkjkhd', productToken: '8475', projectNames: ['testProject1', 'testProject2']] + repository.config.putAll(config) + + def requestParams + helper.registerAllowedMethod('httpRequest', [Map], { p -> + requestParams = p + return [ content: "{ }"] + }) + + def errorCaught = false + try { + repository.fetchProjectsMetaInfo() + } catch (e) { + errorCaught = true + assertThat(e, isA(AbortException.class)) + assertThat(e.getMessage(), is("[WhiteSource] Could not fetch any projects for product '${config.productName}' from backend, response was {}")) + } + assertThat(errorCaught, is(true)) + } + + @Test void testFetchVulnerabilitiesOnProjects() { def config = [ serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711", productToken: '8475', projectNames: ['testProject1', 'testProject2']] From 06189d9ae05f1523bde991a21ba674cdbabada13 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 13 Mar 2019 12:10:23 +0100 Subject: [PATCH 065/130] Cleanup --- vars/whitesourceExecuteScan.groovy | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index d1fe2da6a..332bee554 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -142,11 +142,7 @@ import static com.sap.piper.Prerequisites.checkScript /** * Title of vulnerability report written during the assessment phase. */ - 'vulnerabilityReportTitle', - 'whitesourceProductName', - 'whitesourceProductToken', - 'whitesourceProjectNames', - 'whitesourceUserTokenCredentialsId' + 'vulnerabilityReportTitle' ] @Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS From c96f73866a07d76bf008ce8a52b78815af071741 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Thu, 14 Mar 2019 15:14:09 +0100 Subject: [PATCH 066/130] Refactored to generally migrate config to deep structure --- .../docs/steps/whitesourceExecuteScan.md | 2 +- src/com/sap/piper/ConfigurationHelper.groovy | 22 +-- .../WhitesourceConfigurationHelper.groovy | 61 ++++--- .../WhitesourceOrgAdminRepository.groovy | 26 +-- .../integration/WhitesourceRepository.groovy | 42 ++--- test/groovy/WhitesourceExecuteScanTest.groovy | 80 ++++---- .../sap/piper/ConfigurationHelperTest.groovy | 22 +-- .../WhiteSourceConfigurationHelperTest.groovy | 23 ++- .../WhitesourceOrgAdminRepositoryTest.groovy | 8 +- .../WhitesourceRepositoryTest.groovy | 6 +- vars/whitesourceExecuteScan.groovy | 171 ++++++++++-------- 11 files changed, 252 insertions(+), 211 deletions(-) diff --git a/documentation/docs/steps/whitesourceExecuteScan.md b/documentation/docs/steps/whitesourceExecuteScan.md index 1d1bf77f7..f8accedf1 100644 --- a/documentation/docs/steps/whitesourceExecuteScan.md +++ b/documentation/docs/steps/whitesourceExecuteScan.md @@ -23,5 +23,5 @@ None ## Examples ```groovy -whitesourceExecuteScan script: this, scanType: 'pip', productName: 'My Whitesource Product', userTokenCredentialsId: 'companyAdminToken', orgAdminUserTokenCredentialsId: 'orgAdmiToken', orgToken: 'myWhitesourceOrganizationToken' +whitesourceExecuteScan script: this, scanType: 'pip', whitesource: [ productName: 'My Whitesource Product', userTokenCredentialsId: 'companyAdminToken', orgAdminUserTokenCredentialsId: 'orgAdmiToken', orgToken: 'myWhitesourceOrganizationToken' ] ``` diff --git a/src/com/sap/piper/ConfigurationHelper.groovy b/src/com/sap/piper/ConfigurationHelper.groovy index 93464db87..a5439990b 100644 --- a/src/com/sap/piper/ConfigurationHelper.groovy +++ b/src/com/sap/piper/ConfigurationHelper.groovy @@ -59,22 +59,21 @@ class ConfigurationHelper implements Serializable { return this } - private Map handleCompatibility(Map compatibleParameters, String paramStructure = '', Map configMap ) { + private Map handleCompatibility(Map compatibleParameters, String paramStructure = '', Map configMap, Map newConfigMap = [:] ) { Map newConfig = [:] compatibleParameters.each {entry -> if (entry.getValue() instanceof Map) { def internalParamStructure = (paramStructure ? paramStructure + '.' : '') + entry.getKey() - newConfig[entry.getKey()] = handleCompatibility(entry.getValue(), internalParamStructure, configMap) + newConfig[entry.getKey()] = handleCompatibility(entry.getValue(), internalParamStructure, configMap, newConfig) } else { def configSubMap = configMap for(String key in paramStructure.tokenize('.')){ configSubMap = configSubMap?.get(key) } if (configSubMap == null || (configSubMap != null && configSubMap[entry.getKey()] == null)) { - def stages = entry.getValue()?.tokenize('.') - def value = resolveToFlat(stages, configMap) + def value = configMap[entry.getValue()] if(null == value) - value = resolveToFlat(stages, newConfig) + value = newConfigMap[entry.getValue()] if (value != null) { newConfig[entry.getKey()] = value def paramName = (paramStructure ? paramStructure + '.' : '') + entry.getKey() @@ -86,19 +85,6 @@ class ConfigurationHelper implements Serializable { return newConfig } - private String resolveToFlat(stages, configMap) { - def first = 0 - def result - def configSubMap = configMap[stages?.get(first)] - if(configSubMap instanceof Map) { - stages?.remove(first) - result = resolveToFlat(stages, configSubMap) - } else { - result = configSubMap - } - return result - } - Map dependingOn(dependentKey){ return [ mixin: {key -> diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index e55195fa4..410ff0926 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -11,35 +11,34 @@ class WhitesourceConfigurationHelper implements Serializable { def inputFile = config.configFilePath.replaceFirst('\\./', '') def suffix = utils.generateSha1(config.configFilePath) def targetFile = "${inputFile}.${suffix}" - if(config.productName.startsWith('DIST - ')) { + if(config.whitesource.productName.startsWith('DIST - ')) { mapping += [ - [name: 'checkPolicies', value: false], - [name: 'forceCheckAllDependencies', value: false] + [name: 'checkPolicies', value: false, force: true], + [name: 'forceCheckAllDependencies', value: false, force: true] ] - } else if(config.productName.startsWith('SHC - ')) { + } else if(config.whitesource.productName.startsWith('SHC - ')) { mapping += [ - [name: 'checkPolicies', value: true], - [name: 'forceCheckAllDependencies', value: true] + [name: 'checkPolicies', value: true, force: true], + [name: 'forceCheckAllDependencies', value: true, force: true] ] } - if(config.verbose) - mapping += [name: 'log.level', value: 'debug'] + if(config.whitesource.verbose) + mapping += [name: 'log.level', value: 'debug', force: true] mapping += [ - [name: 'apiKey', value: config.orgToken], - [name: 'productName', value: config.productName], - [name: 'productVersion', value: config.productVersion], - [name: 'projectName', value: config.projectName], - [name: 'projectVersion', value: config.productVersion], - [name: 'productToken', value: config.productToken, omitIfPresent: 'projectToken'], - [name: 'userKey', value: config.userKey], - [name: 'forceUpdate', value: true], - [name: 'offline', value: false], - [name: 'ignoreSourceFiles', value: true], - [name: 'resolveAllDependencies', value: false] + [name: 'apiKey', value: config.whitesource.orgToken, force: true], + [name: 'productName', value: config.whitesource.productName, force: true], + [name: 'productVersion', value: config.whitesource.productVersion, force: true], + [name: 'projectName', value: config.whitesource.projectName, force: true], + [name: 'projectVersion', value: config.whitesource.productVersion, force: true], + [name: 'productToken', value: config.whitesource.productToken, omitIfPresent: 'projectToken', force: true], + [name: 'userKey', value: config.whitesource.userKey, force: true], + [name: 'forceUpdate', value: true, force: true], + [name: 'offline', value: false, force: true], + [name: 'ignoreSourceFiles', value: true, force: true], + [name: 'resolveAllDependencies', value: false, force: true] ] switch (config.scanType) { - case 'npm': mapping += [ @@ -47,8 +46,8 @@ class WhitesourceConfigurationHelper implements Serializable { break case 'pip': mapping += [ - [name: 'python.resolveDependencies', value: true], - [name: 'python.ignoreSourceFiles', value: true], + [name: 'python.resolveDependencies', value: true, force: true], + [name: 'python.ignoreSourceFiles', value: true, force: true], [name: 'python.ignorePipInstallErrors', value: false], [name: 'python.installVirtualenv', value: true], [name: 'python.resolveHierarchyTree', value: true], @@ -70,7 +69,14 @@ class WhitesourceConfigurationHelper implements Serializable { break case 'golang': mapping += [ - + [name: 'go.resolveDependencies', value: true, force: true], + [name: 'go.ignoreSourceFiles', value: true, force: true], + [name: 'go.collectDependenciesAtRuntime', value: true], + [name: 'go.dependencyManager', value: ''], + [name: 'includes', value: '**/*.lock'], + [name: 'excludes', value: '**/*sources.jar **/*javadoc.jar'], + [name: 'case.sensitive.glob', value: false], + [name: 'followSymbolicLinks', value: true] ] break case 'dlang': @@ -100,13 +106,13 @@ class WhitesourceConfigurationHelper implements Serializable { mapping.each { entry -> def dependentValue = entry.omitIfPresent ? moduleSpecificFile[entry.omitIfPresent] : null - if ((entry.omitIfPresent && !dependentValue || !entry.omitIfPresent) && entry.value && entry.value != 'null' && entry.value != '') + if ((entry.omitIfPresent && !dependentValue || !entry.omitIfPresent) && (entry.force || moduleSpecificFile[entry.name] == null) && entry.value != 'null') moduleSpecificFile[entry.name] = entry.value.toString() } def output = serializationClosure(moduleSpecificFile) - if(config.verbose) + if(config.whitesource.verbose) script.echo "Writing config file ${outputFilePath} with content:\n${output}" script.writeFile file: outputFilePath, text: output if(config.stashContent && config.stashContent.size() > 0) { @@ -124,10 +130,7 @@ class WhitesourceConfigurationHelper implements Serializable { @NonCPS static private def serializeUAConfig(configuration) { Properties p = new Properties() - configuration.each { - entry -> - p.setProperty(entry.key, entry.value) - } + p.putAll(configuration) new StringWriter().with{ w -> p.store(w, null); w }.toString() } diff --git a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy index 2c39441b2..c5d34851a 100644 --- a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy @@ -12,8 +12,8 @@ class WhitesourceOrgAdminRepository implements Serializable { WhitesourceOrgAdminRepository(Script script, Map config) { this.script = script this.config = config - if(!this.config.serviceUrl && !this.config.whitesourceAccessor) - script.error "Parameter 'serviceUrl' must be provided as part of the configuration." + if(!this.config.whitesource.serviceUrl && !this.config.whitesourceAccessor) + script.error "Parameter 'whitesource.serviceUrl' must be provided as part of the configuration." if(this.config.whitesourceAccessor instanceof String) { def clazz = this.class.classLoader.loadClass(this.config.whitesourceAccessor) this.internalWhitesource = clazz?.newInstance(this.script, this.config) @@ -23,7 +23,7 @@ class WhitesourceOrgAdminRepository implements Serializable { def fetchProductMetaInfo() { def requestBody = [ requestType: "getOrganizationProductVitals", - orgToken: config.orgToken + orgToken: config.whitesource.orgToken ] def parsedResponse = issueHttpRequest(requestBody) @@ -33,7 +33,7 @@ class WhitesourceOrgAdminRepository implements Serializable { def findProductMeta(parsedResponse) { def foundMetaProduct = null for (product in parsedResponse.productVitals) { - if (product.name == config.productName) { + if (product.name == config.whitesource.productName) { foundMetaProduct = product break } @@ -45,15 +45,15 @@ class WhitesourceOrgAdminRepository implements Serializable { def createProduct() { def requestBody = [ requestType: "createProduct", - orgToken: config.orgToken, - productName: config.productName + orgToken: config.whitesource.orgToken, + productName: config.whitesource.productName ] def parsedResponse = issueHttpRequest(requestBody) def metaInfo = parsedResponse def groups = [] def users = [] - config.emailAddressesOfInitialProductAdmins.each { + config.whitesource.emailAddressesOfInitialProductAdmins.each { email -> users.add(["email": email]) } @@ -80,27 +80,27 @@ class WhitesourceOrgAdminRepository implements Serializable { @NonCPS protected def httpWhitesource(requestBody) { - requestBody["userKey"] = config.orgAdminUserKey + requestBody["userKey"] = config.whitesource.orgAdminUserKey def serializedBody = new JsonUtils().jsonToString(requestBody) def params = [ - url : config.serviceUrl, + url : config.whitesource.serviceUrl, httpMode : 'POST', acceptType : 'APPLICATION_JSON', contentType: 'APPLICATION_JSON', requestBody: serializedBody, - quiet : !config.verbose, - timeout : config.timeout + quiet : !config.whitesource.verbose, + timeout : config.whitesource.timeout ] if (script.env.HTTP_PROXY) params["httpProxy"] = script.env.HTTP_PROXY - if (config.verbose) + if (config.whitesource.verbose) script.echo "Sending http request with parameters ${params}" def response = script.httpRequest(params) - if (config.verbose) + if (config.whitesource.verbose) script.echo "Received response ${response}" return response diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy index 6882f075f..af8210524 100644 --- a/src/com/sap/piper/integration/WhitesourceRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -12,13 +12,13 @@ class WhitesourceRepository implements Serializable { this.script = script this.config = config - if(!config.serviceUrl) - script.error "Parameter 'serviceUrl' must be provided as part of the configuration." + if(!config.whitesource.serviceUrl) + script.error "Parameter 'whitesource.serviceUrl' must be provided as part of the configuration." } List fetchVulnerabilities(whitesourceProjectsMetaInformation) { def fetchedVulnerabilities = [] - if (config.projectNames) { + if (config.whitesource.projectNames) { for (int i = 0; i < whitesourceProjectsMetaInformation.size(); i++) { def metaInfo = whitesourceProjectsMetaInformation[i] @@ -35,7 +35,7 @@ class WhitesourceRepository implements Serializable { def requestBody = [ requestType : "getProductAlertsByType", alertType : "SECURITY_VULNERABILITY", - productToken: config.productToken, + productToken: config.whitesource.productToken, ] def response = fetchWhitesourceResource(requestBody) @@ -99,17 +99,17 @@ class WhitesourceRepository implements Serializable { List fetchProjectsMetaInfo() { def projectsMetaInfo = [] - if(config.projectNames){ + if(config.whitesource.projectNames){ def requestBody = [ requestType: "getProductProjectVitals", - productToken: config.productToken + productToken: config.whitesource.productToken ] def response = fetchWhitesourceResource(requestBody) if(response?.projectVitals) { projectsMetaInfo.addAll(findProjectsMeta(response.projectVitals)) } else { - script.error "[WhiteSource] Could not fetch any projects for product '${config.productName}' from backend, response was ${response}" + script.error "[WhiteSource] Could not fetch any projects for product '${config.whitesource.productName}' from backend, response was ${response}" } } return projectsMetaInfo @@ -117,8 +117,8 @@ class WhitesourceRepository implements Serializable { List findProjectsMeta(projectVitals) { def matchedProjects = [] - for (int i = 0; i < config.projectNames?.size(); i++) { - def requestedProjectName = config.projectNames[i].trim() + for (int i = 0; i < config.whitesource.projectNames?.size(); i++) { + def requestedProjectName = config.whitesource.projectNames[i].trim() def matchedProjectInfo = null for (int j = 0; j < projectVitals.size(); j++) { @@ -132,7 +132,7 @@ class WhitesourceRepository implements Serializable { if (matchedProjectInfo != null) { matchedProjects.add(matchedProjectInfo) } else { - script.error "[WhiteSource] Could not fetch/find requested project '${requestedProjectName}' for product '${config.productName}'" + script.error "[WhiteSource] Could not fetch/find requested project '${requestedProjectName}' for product '${config.whitesource.productName}'" } } @@ -142,7 +142,7 @@ class WhitesourceRepository implements Serializable { void fetchReportForProduct(reportName) { def requestContent = [ requestType: "getProductRiskReport", - productToken: config.productToken + productToken: config.whitesource.productToken ] fetchFileFromWhiteSource(reportName, requestContent) @@ -152,7 +152,7 @@ class WhitesourceRepository implements Serializable { def requestContent = [ requestType: "getProductAlertsByType", alertType: "REJECTED_BY_POLICY_RESOURCE", - productToken: config.productToken + productToken: config.whitesource.productToken ] def parsedResponse = fetchWhitesourceResource(requestContent) @@ -175,24 +175,24 @@ class WhitesourceRepository implements Serializable { handleAdditionalRequestParameters(requestBody) def serializedBody = new JsonUtils().getPrettyJsonString(requestBody) def params = [ - url : config.serviceUrl, + url : config.whitesource.serviceUrl, httpMode : 'POST', acceptType : 'APPLICATION_JSON', contentType: 'APPLICATION_JSON', requestBody: serializedBody, - quiet : !config.verbose, - timeout : config.timeout + quiet : !config.whitesource.verbose, + timeout : config.whitesource.timeout ] if (script.env.HTTP_PROXY) params["httpProxy"] = script.env.HTTP_PROXY - if(config.verbose) + if(config.whitesource.verbose) script.echo "Sending http request with parameters ${params}" def response = script.httpRequest(params) - if(config.verbose) + if(config.whitesource.verbose) script.echo "Received response ${response}" return response @@ -203,15 +203,15 @@ class WhitesourceRepository implements Serializable { handleAdditionalRequestParameters(params) def serializedContent = new JsonUtils().jsonToString(params) - if(config.verbose) + if(config.whitesource.verbose) script.echo "Sending curl request with parameters ${params}" - script.sh "${config.verbose ? '' : '#!/bin/sh -e\n'}curl -o ${fileName} -X POST ${config.serviceUrl} -H 'Content-Type: application/json' -d \'${serializedContent}\'" + script.sh "${config.whitesource.verbose ? '' : '#!/bin/sh -e\n'}curl -o ${fileName} -X POST ${config.whitesource.serviceUrl} -H 'Content-Type: application/json' -d \'${serializedContent}\'" } @NonCPS protected void handleAdditionalRequestParameters(params) { - if(config.userKey) - params["userKey"] = config.userKey + if(config.whitesource.userKey) + params["userKey"] = config.whitesource.userKey } } diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index 0e5269539..b78f4ada3 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -26,10 +26,12 @@ class WhitesourceExecuteScanTest extends BasePiperTest { private JenkinsWriteFileRule writeFileRule = new JenkinsWriteFileRule(this) private JenkinsStepRule stepRule = new JenkinsStepRule(this) private JenkinsErrorRule errorRule = new JenkinsErrorRule(this) + private JenkinsEnvironmentRule environmentRule = new JenkinsEnvironmentRule(this) @Rule public RuleChain ruleChain = Rules .getCommonRules(this) + .around(environmentRule) .around(new JenkinsReadYamlRule(this)) .around(thrown) .around(dockerExecuteRule) @@ -103,10 +105,10 @@ class WhitesourceExecuteScanTest extends BasePiperTest { return [].toArray() }) - whitesourceOrgAdminRepositoryStub = new WhitesourceOrgAdminRepository(nullScript, [serviceUrl: "http://some.host.whitesource.com/api/"]) + whitesourceOrgAdminRepositoryStub = new WhitesourceOrgAdminRepository(nullScript, [whitesource: [serviceUrl: "http://some.host.whitesource.com/api/"]]) LibraryLoadingTestExecutionListener.prepareObjectInterceptors(whitesourceOrgAdminRepositoryStub) - whitesourceStub = new WhitesourceRepository(nullScript, [serviceUrl: "http://some.host.whitesource.com/api/"]) + whitesourceStub = new WhitesourceRepository(nullScript, [whitesource: [serviceUrl: "http://some.host.whitesource.com/api/"]]) LibraryLoadingTestExecutionListener.prepareObjectInterceptors(whitesourceStub) helper.registerAllowedMethod("fetchProductMetaInfo", [], { @@ -527,11 +529,13 @@ class WhitesourceExecuteScanTest extends BasePiperTest { whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, descriptorUtilsStub : descriptorUtilsStub, scanType : 'mta', - productName : 'SHC - Piper', + whitesource: [ + productName : 'SHC - Piper', + orgToken : 'b39d1328-52e2-42e3-98f0-932709daf3f0' + ], buildDescriptorExcludeList : ["maven2${File.separator}pom.xml".toString(), "npm2${File.separator}package.json".toString()], reporting : true, - juStabUtils : utils, - orgToken : 'b39d1328-52e2-42e3-98f0-932709daf3f0' + juStabUtils : utils ]) assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) @@ -542,19 +546,26 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(parallelMap, hasKey('Whitesource - pip')) assertThat(parallelMap.keySet(), hasSize(4)) - assertThat(whitesourceCalls, hasItem(allOf( - hasEntry('scanType', 'maven'), - hasEntry('buildDescriptorFile', "maven1${File.separator}pom.xml".toString()) - ))) - assertThat(whitesourceCalls, hasItem(allOf( - hasEntry('scanType', 'npm'), - hasEntry('buildDescriptorFile', "npm1${File.separator}package.json".toString()) - ))) - assertThat(whitesourceCalls, hasItem(allOf( - hasEntry('scanType', 'pip'), - hasEntry('projectNames', ["com.sap.maven.test-java - 1.2.3", "com.sap.node.test-node - 1.2.3", "test-python - 1.2.3"]), - hasEntry('buildDescriptorFile', "pip${File.separator}setup.py".toString()) - ))) + assertThat(whitesourceCalls, + contains( + allOf( + hasEntry('scanType', 'maven'), + hasEntry('buildDescriptorFile', "maven1${File.separator}pom.xml".toString()) + ), + allOf( + hasEntry('scanType', 'npm'), + hasEntry('buildDescriptorFile', "npm1${File.separator}package.json".toString()) + ), + allOf( + hasEntry('scanType', 'pip'), + hasEntry('buildDescriptorFile', "pip${File.separator}setup.py".toString()) + ) + ) + ) + + assertThat(whitesourceCalls[0]['whitesource']['projectNames'], contains("com.sap.maven.test-java - 1.2.3", "com.sap.node.test-node - 1.2.3", "test-python - 1.2.3")) + assertThat(whitesourceCalls[1]['whitesource']['projectNames'], contains("com.sap.maven.test-java - 1.2.3", "com.sap.node.test-node - 1.2.3", "test-python - 1.2.3")) + assertThat(whitesourceCalls[2]['whitesource']['projectNames'], contains("com.sap.maven.test-java - 1.2.3", "com.sap.node.test-node - 1.2.3", "test-python - 1.2.3")) } @Test @@ -669,36 +680,36 @@ class WhitesourceExecuteScanTest extends BasePiperTest { @Test void testNPMStatusCheckScanException() { thrown.expect(AbortException.class) - stepRule.step.checkStatus(-1 & 0xFF, [licensingVulnerabilities: true]) + stepRule.step.checkStatus(-1 & 0xFF, [whitesource:[licensingVulnerabilities: true]]) } @Test void testNPMStatusCheckPolicyViolation() { thrown.expect(AbortException.class) - stepRule.step.checkStatus(-2 & 0xFF, [licensingVulnerabilities: true]) + stepRule.step.checkStatus(-2 & 0xFF, [whitesource:[licensingVulnerabilities: true]]) } @Test void testNPMStatusCheckNoPolicyViolation() { - stepRule.step.checkStatus(-2 & 0xFF, [licensingVulnerabilities: false]) + stepRule.step.checkStatus(-2 & 0xFF, [whitesource:[licensingVulnerabilities: false]]) } @Test void testNPMStatusCheckClientException() { thrown.expect(AbortException.class) - stepRule.step.checkStatus(-3 & 0xFF, [licensingVulnerabilities: true]) + stepRule.step.checkStatus(-3 & 0xFF, [whitesource:[licensingVulnerabilities: true]]) } @Test void testNPMStatusCheckConnectionException() { thrown.expect(AbortException.class) - stepRule.step.checkStatus(-4 & 0xFF, [licensingVulnerabilities: true]) + stepRule.step.checkStatus(-4 & 0xFF, [whitesource:[licensingVulnerabilities: true]]) } @Test void testNPMStatusCheckServerException() { thrown.expect(AbortException.class) - stepRule.step.checkStatus(-3 & 0xFF, [licensingVulnerabilities: true]) + stepRule.step.checkStatus(-3 & 0xFF, [whitesource:[licensingVulnerabilities: true]]) } @Test @@ -736,8 +747,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { @Test void testFetchViolationCountProjectNotZero() { - def config = [projectNames: ["piper-java-cc - 0.0.1", "pipeline-test - 0.0.1"]] - + def config = [whitesource: [projectNames: ["piper-java-cc - 0.0.1", "pipeline-test - 0.0.1"]]] def projectTokens = [[token:"abc-project-token"],[token:"def-project-token"]] helper.registerAllowedMethod('fetchProjectsMetaInfo', [], { return projectTokens }) helper.registerAllowedMethod('fetchProjectLicenseAlerts', [String], { projectToken -> @@ -976,7 +986,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { void testCheckStatus_0() { def error = false try { - stepRule.step.checkStatus(0, [licensingVulnerabilities: true]) + stepRule.step.checkStatus(0, [whitesource:[licensingVulnerabilities: true]]) } catch (e) { error = true } @@ -987,7 +997,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { void testCheckStatus_255() { def error = false try { - stepRule.step.checkStatus(255, [licensingVulnerabilities: true]) + stepRule.step.checkStatus(255, [whitesource:[licensingVulnerabilities: true]]) } catch (e) { error = true assertThat(e.getMessage(), is("[whitesourceExecuteScan] The scan resulted in an error")) @@ -999,7 +1009,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { void testCheckStatus_254() { def error = false try { - stepRule.step.checkStatus(254, [licensingVulnerabilities: true]) + stepRule.step.checkStatus(254, [whitesource:[licensingVulnerabilities: true]]) } catch (e) { error = true assertThat(e.getMessage(), is("[whitesourceExecuteScan] Whitesource found one or multiple policy violations")) @@ -1011,7 +1021,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { void testCheckStatus_253() { def error = false try { - stepRule.step.checkStatus(253, [licensingVulnerabilities: true]) + stepRule.step.checkStatus(253, [whitesource:[licensingVulnerabilities: true]]) } catch (e) { error = true assertThat(e.getMessage(), is("[whitesourceExecuteScan] The local scan client failed to execute the scan")) @@ -1023,7 +1033,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { void testCheckStatus_252() { def error = false try { - stepRule.step.checkStatus(252, [licensingVulnerabilities: true]) + stepRule.step.checkStatus(252, [whitesource:[licensingVulnerabilities: true]]) } catch (e) { error = true assertThat(e.getMessage(), is("[whitesourceExecuteScan] There was a failure in the connection to the WhiteSource servers")) @@ -1035,7 +1045,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { void testCheckStatus_251() { def error = false try { - stepRule.step.checkStatus(251, [licensingVulnerabilities: true]) + stepRule.step.checkStatus(251, [whitesource:[licensingVulnerabilities: true]]) } catch (e) { error = true assertThat(e.getMessage(), is("[whitesourceExecuteScan] The server failed to analyze the scan")) @@ -1047,7 +1057,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { void testCheckStatus_250() { def error = false try { - stepRule.step.checkStatus(250, [licensingVulnerabilities: true]) + stepRule.step.checkStatus(250, [whitesource:[licensingVulnerabilities: true]]) } catch (e) { error = true assertThat(e.getMessage(), is("[whitesourceExecuteScan] Pre-step failure")) @@ -1059,7 +1069,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { void testCheckStatus_127() { def error = false try { - stepRule.step.checkStatus(127, [licensingVulnerabilities: true]) + stepRule.step.checkStatus(127, [whitesource:[licensingVulnerabilities: true]]) } catch (e) { error = true assertThat(e.getMessage(), is("[whitesourceExecuteScan] Whitesource scan failed with unknown error code '127'")) @@ -1071,7 +1081,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { void testCheckStatus_vulnerability() { def error = false try { - stepRule.step.checkStatus(0, [licensingVulnerabilities: false, securityVulnerabilities: true, severeVulnerabilities: 5, cvssSeverityLimit: 7]) + stepRule.step.checkStatus(0, [whitesource:[licensingVulnerabilities: false, securityVulnerabilities: true, severeVulnerabilities: 5, cvssSeverityLimit: 7]]) } catch (e) { error = true assertThat(e.getMessage(), is("[whitesourceExecuteScan] 5 Open Source Software Security vulnerabilities with CVSS score greater or equal 7 detected. - ")) diff --git a/test/groovy/com/sap/piper/ConfigurationHelperTest.groovy b/test/groovy/com/sap/piper/ConfigurationHelperTest.groovy index 33b9b6c2c..8be56c64c 100644 --- a/test/groovy/com/sap/piper/ConfigurationHelperTest.groovy +++ b/test/groovy/com/sap/piper/ConfigurationHelperTest.groovy @@ -166,17 +166,6 @@ class ConfigurationHelperTest { Assert.assertThat(configuration.deep.deeper.newStructure.new2, is('oldValue2')) } - @Test - void testHandleCompatibilityFlatten() { - def configuration = ConfigurationHelper.newInstance(mockScript) - .mixin([old1: [deeper: 'oldValue1'], old2: [deeper: 'oldValue2'], test: 'testValue'], null, [new1: 'old1.deeper', new2: 'old2.deeper']) - .use() - - Assert.assertThat(configuration.size(), is(5)) - Assert.assertThat(configuration.new1, is('oldValue1')) - Assert.assertThat(configuration.new2, is('oldValue2')) - } - @Test void testHandleCompatibilityNewAvailable() { def configuration = ConfigurationHelper.newInstance(mockScript, [old1: 'oldValue1', newStructure: [new1: 'newValue1'], test: 'testValue']) @@ -207,6 +196,17 @@ class ConfigurationHelperTest { Assert.assertThat(configuration.newStructure.new1, is(null)) } + @Test + void testHandleCompatibilityPremigratedValues() { + def configuration = ConfigurationHelper.newInstance(mockScript, [old1: null, test: 'testValue']) + .mixin([someValueToMigrate: 'testValue2'], null, [someValueToMigrateSecondTime: 'someValueToMigrate', newStructure: [new1: 'old1', new2: 'someValueToMigrateSecondTime']]) + .use() + + Assert.assertThat(configuration.size(), is(4)) + Assert.assertThat(configuration.newStructure.new1, is(null)) + Assert.assertThat(configuration.newStructure.new2, is('testValue2')) + } + @Test public void testWithMandoryParameterReturnDefaultFailureMessage() { diff --git a/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy b/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy index 9d7374af0..8f7138d5f 100644 --- a/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy +++ b/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy @@ -29,7 +29,7 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { @Test void testExtendConfigurationFileUnifiedAgentPip() { - WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") + WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', configFilePath: './config', whitesource: [serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000']], "./") assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=name")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) @@ -39,7 +39,7 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { @Test void testExtendConfigurationFileUnifiedAgentVerbose() { - WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', verbose: true, configFilePath: './config', orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], "./") + WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', configFilePath: './config', whitesource: [serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000', verbose: true]], "./") assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=name")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) @@ -47,5 +47,24 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("python.resolveDependencies=true")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("log.level=debug")) } + + @Test + void testExtendConfigurationFileUnifiedAgentEnforcement() { + def p = new Properties() + p.putAll(['python.resolveDependencies': 'false', 'python.ignoreSourceFiles': 'false', 'python.ignorePipInstallErrors': 'true','python.installVirtualenv': 'false']) + helper.registerAllowedMethod('readProperties', [Map], {return p}) + + WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', configFilePath: './config', whitesource: [serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000', verbose: true]], "./") + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=name")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("python.resolveDependencies=true")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("log.level=debug")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("python.resolveDependencies=true")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("python.ignoreSourceFiles=true")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("python.ignorePipInstallErrors=true")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("python.installVirtualenv=false")) + } } diff --git a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy index 1f4c56e19..63cd4877e 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy @@ -35,7 +35,7 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { @Before void init() throws Exception { - repository = new WhitesourceOrgAdminRepository(nullScript, [serviceUrl: "http://some.host.whitesource.com/api/", verbose: true]) + repository = new WhitesourceOrgAdminRepository(nullScript, [whitesource: [serviceUrl: "http://some.host.whitesource.com/api/"], verbose: true]) LibraryLoadingTestExecutionListener.prepareObjectInterceptors(repository) } @@ -53,14 +53,14 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { } catch (e) { errorCaught = true assertThat(e, isA(AbortException.class)) - assertThat(e.getMessage(), is("Parameter 'serviceUrl' must be provided as part of the configuration.")) + assertThat(e.getMessage(), is("Parameter 'whitesource.serviceUrl' must be provided as part of the configuration.")) } assertThat(errorCaught, is(true)) } @Test void testAccessor() { - new WhitesourceOrgAdminRepository(nullScript, [whitesourceAccessor: "com.sap.piper.integration.WhitesourceRepository", serviceUrl: "http://test.com"]) + new WhitesourceOrgAdminRepository(nullScript, [whitesourceAccessor: "com.sap.piper.integration.WhitesourceRepository", whitesource: [serviceUrl: "http://test.com"]]) } @Test @@ -83,7 +83,7 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { ] ] - repository.config['productName'] = "Correct Name Cloud" + repository.config.putAll([whitesource: [productName: "Correct Name Cloud"]]) def result = repository.findProductMeta(whitesourceMetaResponse) diff --git a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy index db2ea3f23..8f7fca218 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy @@ -39,7 +39,7 @@ class WhitesourceRepositoryTest extends BasePiperTest { void init() throws Exception { nullScript.env['HTTP_PROXY'] = "http://proxy.wdf.sap.corp:8080" - repository = new WhitesourceRepository(nullScript, [serviceUrl: "http://some.host.whitesource.com/api/"]) + repository = new WhitesourceRepository(nullScript, [whitesource: [serviceUrl: "http://some.host.whitesource.com/api/"]]) LibraryLoadingTestExecutionListener.prepareObjectInterceptors(repository) } @@ -57,7 +57,7 @@ class WhitesourceRepositoryTest extends BasePiperTest { } catch (e) { errorCaught = true assertThat(e, isA(AbortException.class)) - assertThat(e.getMessage(), is("Parameter 'serviceUrl' must be provided as part of the configuration.")) + assertThat(e.getMessage(), is("Parameter 'whitesource.serviceUrl' must be provided as part of the configuration.")) } assertThat(errorCaught, is(true)) } @@ -126,7 +126,7 @@ class WhitesourceRepositoryTest extends BasePiperTest { exception.expectMessage("Correct Project Name") - repository.config['projectNames'] = ["Correct Project Name"] + repository.config.putAll([whitesource : [ projectNames: ["Correct Project Name"]]]) repository.findProjectsMeta(whitesourceMetaResponse.projectVitals) } diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 332bee554..0087b4896 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -15,6 +15,10 @@ import static com.sap.piper.Prerequisites.checkScript @Field String STEP_NAME = getClass().getName() @Field Set GENERAL_CONFIG_KEYS = [ + /** + * Wrapper object to bundle any of the other configuration settings on general and stage level. + */ + 'whitesource', /** * Jenkins credentials ID referring to the organization admin's token. */ @@ -148,21 +152,39 @@ import static com.sap.piper.Prerequisites.checkScript @Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS @Field Map CONFIG_KEY_COMPATIBILITY = [ - whitesource : [ - productName : 'whitesourceProductName', - productToken : 'whitesourceProductToken', - projectNames : 'whitesourceProjectNames', - userTokenCredentialsId : 'whitesourceUserTokenCredentialsId' - ], - orgAdminUserTokenCredentialsId : 'whitesource.orgAdminUserTokenCredentialsId', - orgToken : 'whitesource.orgToken', - productName : 'whitesource.productName', - productVersion : 'whitesource.productVersion', - productToken : 'whitesource.productToken', - projectNames : 'whitesource.projectNames', - scanType : 'whitesource.scanType', - serviceUrl : 'whitesource.serviceUrl', - userTokenCredentialsId : 'whitesource.userTokenCredentialsId' + productName : 'whitesourceProductName', + productToken : 'whitesourceProductToken', + projectNames : 'whitesourceProjectNames', + userTokenCredentialsId : 'whitesourceUserTokenCredentialsId', + serviceUrl : 'whitesourceServiceUrl', + agentDownloadUrl : 'fileAgentDownloadUrl', + agentParameters : 'fileAgentParameters', + whitesource : [ + orgAdminUserTokenCredentialsId : 'orgAdminUserTokenCredentialsId', + orgToken : 'orgToken', + productName : 'productName', + productToken : 'productToken', + projectNames : 'projectNames', + serviceUrl : 'serviceUrl', + userTokenCredentialsId : 'userTokenCredentialsId', + verbose : 'verbose', + agentDownloadUrl : 'agentDownloadUrl', + agentFileName : 'agentFileName', + agentParameters : 'agentParameters', + buildDescriptorExcludeList : 'buildDescriptorExcludeList', + buildDescriptorFile : 'buildDescriptorFile', + createProductFromPipeline : 'createProductFromPipeline', + emailAddressesOfInitialProductAdmins : 'emailAddressesOfInitialProductAdmins', + jreDownloadUrl : 'jreDownloadUrl', + licensingVulnerabilities : 'licensingVulnerabilities', + parallelLimit : 'parallelLimit', + reporting : 'reporting', + securityVulnerabilities : 'securityVulnerabilities', + cvssSeverityLimit : 'cvssSeverityLimit', + timeout : 'timeout', + vulnerabilityReportFileName : 'vulnerabilityReportFileName', + vulnerabilityReportTitle : 'vulnerabilityReportTitle' + ] ] /** @@ -199,16 +221,17 @@ void call(Map parameters = [:]) { .dependingOn('scanType').mixin('dockerImage') .dependingOn('scanType').mixin('dockerWorkspace') .dependingOn('scanType').mixin('stashContent') - .withMandatoryProperty('serviceUrl') - .withMandatoryProperty('orgToken') - .withMandatoryProperty('userTokenCredentialsId') - .withMandatoryProperty('productName') + .withMandatoryProperty('whitesource/serviceUrl') + .withMandatoryProperty('whitesource/orgToken') + .withMandatoryProperty('whitesource/userTokenCredentialsId') + .withMandatoryProperty('whitesource/productName') .use() - config.cvssSeverityLimit = config.cvssSeverityLimit == null ? -1 : Integer.valueOf(config.cvssSeverityLimit) + config.whitesource.cvssSeverityLimit = config.whitesource.cvssSeverityLimit == null ? -1 : Integer.valueOf(config.whitesource.cvssSeverityLimit) config.stashContent = utils.unstashAll(config.stashContent) - config.projectNames = (config.projectNames instanceof List) ? config.projectNames : config.projectNames?.tokenize(',') - parameters.projectNames = config.projectNames + config.whitesource['projectNames'] = (config.whitesource['projectNames'] instanceof List) ? config.whitesource['projectNames'] : config.whitesource['projectNames']?.tokenize(',') + parameters.whitesource = parameters.whitesource ?: [:] + parameters.whitesource['projectNames'] = config.whitesource['projectNames'] script.commonPipelineEnvironment.setInfluxStepData('whitesource', false) @@ -223,7 +246,7 @@ void call(Map parameters = [:]) { def whitesourceRepository = parameters.whitesourceRepositoryStub ?: new WhitesourceRepository(this, config) def whitesourceOrgAdminRepository = parameters.whitesourceOrgAdminRepositoryStub ?: new WhitesourceOrgAdminRepository(this, config) - if(config.orgAdminUserTokenCredentialsId) { + if(config.whitesource.orgAdminUserTokenCredentialsId) { statusCode = triggerWhitesourceScanWithOrgAdminUserKey(script, config, utils, descriptorUtils, parameters, whitesourceRepository, whitesourceOrgAdminRepository) } else { statusCode = triggerWhitesourceScanWithUserKey(script, config, utils, descriptorUtils, parameters, whitesourceRepository, whitesourceOrgAdminRepository) @@ -236,34 +259,34 @@ void call(Map parameters = [:]) { private def triggerWhitesourceScanWithOrgAdminUserKey(script, config, utils, descriptorUtils, parameters, repository, orgAdminRepository) { withCredentials ([script.string( - credentialsId: config.orgAdminUserTokenCredentialsId, + credentialsId: config.whitesource.orgAdminUserTokenCredentialsId, variable: 'orgAdminUserKey' )]) { - config.orgAdminUserKey = orgAdminUserKey + config.whitesource.orgAdminUserKey = orgAdminUserKey triggerWhitesourceScanWithUserKey(script, config, utils, descriptorUtils, parameters, repository, orgAdminRepository) } } private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorUtils, parameters, repository, orgAdminRepository) { withCredentials ([string( - credentialsId: config.userTokenCredentialsId, + credentialsId: config.whitesource.userTokenCredentialsId, variable: 'userKey' )]) { - config.userKey = userKey + config.whitesource.userKey = userKey def statusCode = 1 - echo "Triggering Whitesource scan on product '${config.productName}'${config.productToken ? ' with token \'' + config.productToken + '\'' : ''} using product admin credentials with ID '${config.userTokenCredentialsId}'${config.orgAdminUserTokenCredentialsId ? ' and organization admin credentials with ID \'' + config.orgAdminUserTokenCredentialsId + '\'' : ''}" + echo "Triggering Whitesource scan on product '${config.whitesource.productName}'${config.whitesource.productToken ? ' with token \'' + config.whitesource.productToken + '\'' : ''} using product admin credentials with ID '${config.whitesource.userTokenCredentialsId}'${config.whitesource.orgAdminUserTokenCredentialsId ? ' and organization admin credentials with ID \'' + config.whitesource.orgAdminUserTokenCredentialsId + '\'' : ''}" - if (!config.productToken) { + if (!config.whitesource.productToken) { def metaInfo = orgAdminRepository.fetchProductMetaInfo() def key = "token" if((null == metaInfo || !metaInfo[key]) && config.createProductFromPipeline) { metaInfo = orgAdminRepository.createProduct() key = "productToken" } else if(null == metaInfo || !metaInfo[key]) { - error "[WhiteSource] Could not fetch/find requested product '${config.productName}' and automatic creation has been disabled" + error "[WhiteSource] Could not fetch/find requested product '${config.whitesource.productName}' and automatic creation has been disabled" } echo "Meta Info: ${metaInfo}" - config.productToken = metaInfo[key] + config.whitesource.productToken = metaInfo[key] } switch (config.scanType) { @@ -285,7 +308,7 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU this, mtaParameters, config.buildDescriptorExcludeList, 'Whitesource', 'setup.py', 'pip' ) { options -> whitesourceExecuteScan(options) }) // execute scan jobs - if (config.parallelLimit > 0 && config.parallelLimit < scanJobs.keySet().size()) { + if (config.whitesource.parallelLimit > 0 && config.whitesource.parallelLimit < scanJobs.keySet().size()) { // block wise def scanJobsAll = scanJobs scanJobs = [failFast: false] @@ -293,7 +316,7 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU def index = i - 1 def key = scanJobsAll.keySet()[index] scanJobs[key] = scanJobsAll[key] - if (i % config.parallelLimit == 0 || i == scanJobsAll.keySet().size()) { + if (i % config.whitesource.parallelLimit == 0 || i == scanJobsAll.keySet().size()) { parallel scanJobs scanJobs = [failFast: false] } @@ -311,53 +334,53 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU switch (config.scanType) { case 'npm': gav = descriptorUtils.getNpmGAV(config.buildDescriptorFile) - config.projectName = gav.group + "." + gav.artifact - config.productVersion = gav.version + config.whitesource.projectName = gav.group + "." + gav.artifact + config.whitesource.productVersion = gav.version break case 'sbt': gav = descriptorUtils.getSbtGAV(config.buildDescriptorFile) - config.projectName = gav.group + "." + gav.artifact - config.productVersion = gav.version + config.whitesource.projectName = gav.group + "." + gav.artifact + config.whitesource.productVersion = gav.version break case 'pip': gav = descriptorUtils.getPipGAV(config.buildDescriptorFile) - config.projectName = gav.artifact - config.productVersion = gav.version + config.whitesource.projectName = gav.artifact + config.whitesource.productVersion = gav.version break default: gav = descriptorUtils.getMavenGAV(config.buildDescriptorFile) - config.projectName = gav.group + "." + gav.artifact - config.productVersion = gav.version + config.whitesource.projectName = gav.group + "." + gav.artifact + config.whitesource.productVersion = gav.version break } - config.projectNames.add("${config.projectName} - ${config.productVersion}".toString()) + config.whitesource['projectNames'].add("${config.whitesource.projectName} - ${config.whitesource.productVersion}".toString()) WhitesourceConfigurationHelper.extendUAConfigurationFile(script, utils, config, path) dockerExecute(script: script, dockerImage: config.dockerImage, dockerWorkspace: config.dockerWorkspace, stashContent: config.stashContent) { - if (config.agentDownloadUrl) { - def agentDownloadUrl = new GStringTemplateEngine().createTemplate(config.agentDownloadUrl).make([config: config]).toString() + if (config.whitesource.agentDownloadUrl) { + def agentDownloadUrl = new GStringTemplateEngine().createTemplate(config.whitesource.agentDownloadUrl).make([config: config]).toString() //if agentDownloadUrl empty, rely on dockerImage to contain unifiedAgent correctly set up and available - sh "curl ${script.env.HTTP_PROXY ? '--proxy ' + script.env.HTTP_PROXY + ' ' : ''}--location --output ${config.agentFileName} ${agentDownloadUrl}".toString() + sh "curl ${script.env.HTTP_PROXY ? '--proxy ' + script.env.HTTP_PROXY + ' ' : ''}--location --output ${config.whitesource.agentFileName} ${agentDownloadUrl}".toString() } def javaCmd = 'java' - if (config.jreDownloadUrl) { + if (config.whitesource.jreDownloadUrl) { //if jreDownloadUrl empty, rely on dockerImage to contain java correctly set up and available on the path - sh "curl ${script.env.HTTP_PROXY ? '--proxy ' + script.env.HTTP_PROXY + ' ' : ''}--location --output jvm.tar.gz ${config.jreDownloadUrl} && tar --strip-components=1 -xzf jvm.tar.gz".toString() + sh "curl ${script.env.HTTP_PROXY ? '--proxy ' + script.env.HTTP_PROXY + ' ' : ''}--location --output jvm.tar.gz ${config.whitesource.jreDownloadUrl} && tar --strip-components=1 -xzf jvm.tar.gz".toString() javaCmd = './bin/java' } - def options = ["-jar ${config.agentFileName} -c \'${config.configFilePath}\'"] - if (config.orgToken) options.push("-apiKey '${config.orgToken}'") - if (config.userKey) options.push("-userKey '${config.userKey}'") - if (config.productName) options.push("-product '${config.productName}'") + def options = ["-jar ${config.whitesource.agentFileName} -c \'${config.configFilePath}\'"] + if (config.whitesource.orgToken) options.push("-apiKey '${config.whitesource.orgToken}'") + if (config.whitesource.userKey) options.push("-userKey '${config.whitesource.userKey}'") + if (config.whitesource.productName) options.push("-product '${config.whitesource.productName}'") - statusCode = sh(script: "${javaCmd} ${options.join(' ')} ${config.agentParameters}", returnStatus: true) + statusCode = sh(script: "${javaCmd} ${options.join(' ')} ${config.whitesource.agentParameters}", returnStatus: true) - if (config.agentDownloadUrl) { - sh "rm -f ${config.agentFileName}" + if (config.whitesource.agentDownloadUrl) { + sh "rm -f ${config.whitesource.agentFileName}" } - if (config.jreDownloadUrl) { + if (config.whitesource.jreDownloadUrl) { sh "rm -rf ./bin ./conf ./legal ./lib ./man" } @@ -381,18 +404,18 @@ void analyseWhitesourceResults(Map config, WhitesourceRepository repository) { archiveArtifacts artifacts: pdfName echo "A summary of the Whitesource findings was stored as artifact under the name ${pdfName}" - if(config.licensingVulnerabilities) { + if(config.whitesource.licensingVulnerabilities) { def violationCount = fetchViolationCount(config, repository) checkViolationStatus(violationCount) } - if (config.securityVulnerabilities) - config.severeVulnerabilities = checkSecurityViolations(config, repository) + if (config.whitesource.securityVulnerabilities) + config.whitesource.severeVulnerabilities = checkSecurityViolations(config, repository) } int fetchViolationCount(Map config, WhitesourceRepository repository) { int violationCount = 0 - if (config.projectNames) { + if (config.whitesource?.projectNames) { def projectsMeta = repository.fetchProjectsMetaInfo() for (int i = 0; i < projectsMeta.size(); i++) { def project = projectsMeta[i] @@ -415,22 +438,22 @@ void checkViolationStatus(int violationCount) { } int checkSecurityViolations(Map config, WhitesourceRepository repository) { - def whitesourceProjectsMetaInformation = repository.fetchProjectsMetaInfo() - def whitesourceVulnerabilities = repository.fetchVulnerabilities(whitesourceProjectsMetaInformation) + def projectsMetaInformation = repository.fetchProjectsMetaInfo() + def vulnerabilities = repository.fetchVulnerabilities(projectsMetaInformation) def severeVulnerabilities = 0 - whitesourceVulnerabilities.each { + vulnerabilities.each { item -> - if ((item.vulnerability.score >= config.cvssSeverityLimit || item.vulnerability.cvss3_score >= config.cvssSeverityLimit) && config.cvssSeverityLimit >= 0) + if ((item.vulnerability.score >= config.whitesource.cvssSeverityLimit || item.vulnerability.cvss3_score >= config.whitesource.cvssSeverityLimit) && config.whitesource.cvssSeverityLimit >= 0) severeVulnerabilities++ } - writeFile(file: "${config.vulnerabilityReportFileName}.json", text: new JsonUtils().getPrettyJsonString(whitesourceVulnerabilities)) - writeFile(file: "${config.vulnerabilityReportFileName}.html", text: getReportHtml(config, whitesourceVulnerabilities, severeVulnerabilities)) + writeFile(file: "${config.vulnerabilityReportFileName}.json", text: new JsonUtils().getPrettyJsonString(vulnerabilities)) + writeFile(file: "${config.vulnerabilityReportFileName}.html", text: getReportHtml(config, vulnerabilities, severeVulnerabilities)) archiveArtifacts(artifacts: "${config.vulnerabilityReportFileName}.*") - if (whitesourceVulnerabilities.size() - severeVulnerabilities > 0) - echo "[${STEP_NAME}] WARNING: ${whitesourceVulnerabilities.size() - severeVulnerabilities} Open Source Software Security vulnerabilities with CVSS score below ${config.cvssSeverityLimit} detected." - if (whitesourceVulnerabilities.size() == 0) + if (vulnerabilities.size() - severeVulnerabilities > 0) + echo "[${STEP_NAME}] WARNING: ${vulnerabilities.size() - severeVulnerabilities} Open Source Software Security vulnerabilities with CVSS score below ${config.whitesource.cvssSeverityLimit} detected." + if (vulnerabilities.size() == 0) echo "[${STEP_NAME}] No Open Source Software Security vulnerabilities detected." return severeVulnerabilities @@ -439,9 +462,9 @@ int checkSecurityViolations(Map config, WhitesourceRepository repository) { // ExitCodes: https://whitesource.atlassian.net/wiki/spaces/WD/pages/34209870/NPM+Plugin#NPMPlugin-ExitCode void checkStatus(int statusCode, config) { def errorMessage = "" - if(config.securityVulnerabilities && config.severeVulnerabilities > 0) - errorMessage += "${config.severeVulnerabilities} Open Source Software Security vulnerabilities with CVSS score greater or equal ${config.cvssSeverityLimit} detected. - " - if (config.licensingVulnerabilities) + if(config.whitesource.securityVulnerabilities && config.whitesource.severeVulnerabilities > 0) + errorMessage += "${config.whitesource.severeVulnerabilities} Open Source Software Security vulnerabilities with CVSS score greater or equal ${config.whitesource.cvssSeverityLimit} detected. - " + if (config.whitesource.licensingVulnerabilities) switch (statusCode) { case 0: break @@ -505,12 +528,12 @@ def getReportHtml(config, vulnerabilityList, numSevereVulns) { return SimpleTemplateEngine.newInstance().createTemplate(libraryResource('com.sap.piper/templates/whitesourceVulnerabilities.html')).make( [ now : now, - reportTitle : config.vulnerabilityReportTitle, + reportTitle : config.whitesource.vulnerabilityReportTitle, style : config.style, totalSevereVulnerabilities : numSevereVulns, totalVulnerabilities : vulnerabilityList.size(), vulnerabilityTable : vulnerabilityTable, - whitesourceProductName : config.productName, - whitesourceProjectNames : config.projectNames + whitesourceProductName : config.whitesource.productName, + whitesourceProjectNames : config.whitesource.projectNames ]).toString() } From e37b714c9977fadbab9765c509dd15372bc18101 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 15 Mar 2019 13:11:34 +0100 Subject: [PATCH 067/130] Improve coverage --- src/com/sap/piper/ConfigurationHelper.groovy | 33 ++++--- .../WhitesourceConfigurationHelper.groovy | 12 +-- .../WhitesourceOrgAdminRepository.groovy | 8 +- .../integration/WhitesourceRepository.groovy | 12 +-- .../sap/piper/ConfigurationHelperTest.groovy | 97 ++++++++++++++++++- .../WhiteSourceConfigurationHelperTest.groovy | 54 +++++++++-- .../WhitesourceOrgAdminRepositoryTest.groovy | 24 ++--- .../WhitesourceRepositoryTest.groovy | 61 ++++++------ vars/whitesourceExecuteScan.groovy | 8 +- 9 files changed, 225 insertions(+), 84 deletions(-) diff --git a/src/com/sap/piper/ConfigurationHelper.groovy b/src/com/sap/piper/ConfigurationHelper.groovy index a5439990b..83a92e636 100644 --- a/src/com/sap/piper/ConfigurationHelper.groovy +++ b/src/com/sap/piper/ConfigurationHelper.groovy @@ -5,6 +5,8 @@ import com.cloudbees.groovy.cps.NonCPS @API class ConfigurationHelper implements Serializable { + def static SEPARATOR = '/' + static ConfigurationHelper newInstance(Script step, Map config = [:]) { new ConfigurationHelper(step, config) } @@ -88,9 +90,16 @@ class ConfigurationHelper implements Serializable { Map dependingOn(dependentKey){ return [ mixin: {key -> + def parts = tokenizeKey(key) + def targetMap = config + if(parts.size() > 1) { + key = parts.last() + parts.remove(key) + targetMap = getConfigPropertyNested(config, (parts as Iterable).join(SEPARATOR)) + } def dependentValue = config[dependentKey] - if(config[key] == null && dependentValue && config[dependentValue]) - config[key] = config[dependentValue][key] + if(targetMap[key] == null && dependentValue && config[dependentValue]) + targetMap[key] = config[dependentValue][key] return this } ] @@ -127,26 +136,28 @@ class ConfigurationHelper implements Serializable { /* private */ static getConfigPropertyNested(Map config, key) { - def separator = '/' + List parts = tokenizeKey(key) - // reason for cast to CharSequence: String#tokenize(./.) causes a deprecation warning. - List parts = (key in String) ? (key as CharSequence).tokenize(separator) : ([key] as List) + if (config[parts.head()] != null) { - if(config[parts.head()] != null) { - - if(config[parts.head()] in Map && ! parts.tail().isEmpty()) { - return getConfigPropertyNested(config[parts.head()], (parts.tail() as Iterable).join(separator)) + if (config[parts.head()] in Map && !parts.tail().isEmpty()) { + return getConfigPropertyNested(config[parts.head()], (parts.tail() as Iterable).join(SEPARATOR)) } if (config[parts.head()].class == String) { return (config[parts.head()] as String).trim() } } - return config[parts.head()] } - private void existsMandatoryProperty(key, errorMessage) { + /* private */ static tokenizeKey(String key) { + // reason for cast to CharSequence: String#tokenize(./.) causes a deprecation warning. + List parts = (key in String) ? (key as CharSequence).tokenize(SEPARATOR) : ([key] as List) + return parts + } + + private void existsMandatoryProperty(key, errorMessage) { def paramValue = getConfigPropertyNested(config, key) diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index 410ff0926..8bd9876a5 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -8,8 +8,8 @@ class WhitesourceConfigurationHelper implements Serializable { def mapping = [] def parsingClosure = { fileReadPath -> return script.readProperties (file: fileReadPath) } def serializationClosure = { configuration -> serializeUAConfig(configuration) } - def inputFile = config.configFilePath.replaceFirst('\\./', '') - def suffix = utils.generateSha1(config.configFilePath) + def inputFile = config.whitesource.configFilePath.replaceFirst('\\./', '') + def suffix = utils.generateSha1(config.whitesource.configFilePath) def targetFile = "${inputFile}.${suffix}" if(config.whitesource.productName.startsWith('DIST - ')) { mapping += [ @@ -22,7 +22,7 @@ class WhitesourceConfigurationHelper implements Serializable { [name: 'forceCheckAllDependencies', value: true, force: true] ] } - if(config.whitesource.verbose) + if(config.verbose) mapping += [name: 'log.level', value: 'debug', force: true] mapping += [ @@ -99,7 +99,7 @@ class WhitesourceConfigurationHelper implements Serializable { def outputFilePath = "${path}${targetFile}" def moduleSpecificFile = parsingClosure(inputFilePath) if (!moduleSpecificFile) - moduleSpecificFile = parsingClosure(config.configFilePath) + moduleSpecificFile = parsingClosure(config.whitesource.configFilePath) if (!moduleSpecificFile) moduleSpecificFile = [:] @@ -112,7 +112,7 @@ class WhitesourceConfigurationHelper implements Serializable { def output = serializationClosure(moduleSpecificFile) - if(config.whitesource.verbose) + if(config.verbose) script.echo "Writing config file ${outputFilePath} with content:\n${output}" script.writeFile file: outputFilePath, text: output if(config.stashContent && config.stashContent.size() > 0) { @@ -124,7 +124,7 @@ class WhitesourceConfigurationHelper implements Serializable { ) config.stashContent += [stashName] } - config.configFilePath = outputFilePath + config.whitesource.configFilePath = outputFilePath } @NonCPS diff --git a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy index c5d34851a..07ddf5808 100644 --- a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy @@ -12,7 +12,7 @@ class WhitesourceOrgAdminRepository implements Serializable { WhitesourceOrgAdminRepository(Script script, Map config) { this.script = script this.config = config - if(!this.config.whitesource.serviceUrl && !this.config.whitesourceAccessor) + if(!this.config.whitesource?.serviceUrl && !this.config.whitesourceAccessor) script.error "Parameter 'whitesource.serviceUrl' must be provided as part of the configuration." if(this.config.whitesourceAccessor instanceof String) { def clazz = this.class.classLoader.loadClass(this.config.whitesourceAccessor) @@ -88,19 +88,19 @@ class WhitesourceOrgAdminRepository implements Serializable { acceptType : 'APPLICATION_JSON', contentType: 'APPLICATION_JSON', requestBody: serializedBody, - quiet : !config.whitesource.verbose, + quiet : !config.verbose, timeout : config.whitesource.timeout ] if (script.env.HTTP_PROXY) params["httpProxy"] = script.env.HTTP_PROXY - if (config.whitesource.verbose) + if (config.verbose) script.echo "Sending http request with parameters ${params}" def response = script.httpRequest(params) - if (config.whitesource.verbose) + if (config.verbose) script.echo "Received response ${response}" return response diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy index af8210524..3119d6736 100644 --- a/src/com/sap/piper/integration/WhitesourceRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -12,7 +12,7 @@ class WhitesourceRepository implements Serializable { this.script = script this.config = config - if(!config.whitesource.serviceUrl) + if(!config?.whitesource?.serviceUrl) script.error "Parameter 'whitesource.serviceUrl' must be provided as part of the configuration." } @@ -180,19 +180,19 @@ class WhitesourceRepository implements Serializable { acceptType : 'APPLICATION_JSON', contentType: 'APPLICATION_JSON', requestBody: serializedBody, - quiet : !config.whitesource.verbose, + quiet : !config.verbose, timeout : config.whitesource.timeout ] if (script.env.HTTP_PROXY) params["httpProxy"] = script.env.HTTP_PROXY - if(config.whitesource.verbose) + if(config.verbose) script.echo "Sending http request with parameters ${params}" def response = script.httpRequest(params) - if(config.whitesource.verbose) + if(config.verbose) script.echo "Received response ${response}" return response @@ -203,10 +203,10 @@ class WhitesourceRepository implements Serializable { handleAdditionalRequestParameters(params) def serializedContent = new JsonUtils().jsonToString(params) - if(config.whitesource.verbose) + if(config.verbose) script.echo "Sending curl request with parameters ${params}" - script.sh "${config.whitesource.verbose ? '' : '#!/bin/sh -e\n'}curl -o ${fileName} -X POST ${config.whitesource.serviceUrl} -H 'Content-Type: application/json' -d \'${serializedContent}\'" + script.sh "${config.verbose ? '' : '#!/bin/sh -e\n'}curl -o ${fileName} -X POST ${config.whitesource.serviceUrl} -H 'Content-Type: application/json' -d \'${serializedContent}\'" } @NonCPS diff --git a/test/groovy/com/sap/piper/ConfigurationHelperTest.groovy b/test/groovy/com/sap/piper/ConfigurationHelperTest.groovy index 8be56c64c..50f064e2a 100644 --- a/test/groovy/com/sap/piper/ConfigurationHelperTest.groovy +++ b/test/groovy/com/sap/piper/ConfigurationHelperTest.groovy @@ -1,12 +1,8 @@ package com.sap.piper -import groovy.test.GroovyAssert - import static org.hamcrest.Matchers.* -import static org.junit.Assert.assertEquals import static org.junit.Assert.assertThat -import org.hamcrest.Matchers import org.junit.Assert import org.junit.Rule import org.junit.Test @@ -16,6 +12,8 @@ class ConfigurationHelperTest { Script mockScript = new Script() { + def prepareDefaultValues() {} + def run() { // it never runs throw new UnsupportedOperationException() @@ -92,6 +90,97 @@ class ConfigurationHelperTest { Assert.assertThat(config, not(hasKey('property3'))) } + @Test + void testConfigurationHelperLoadingStepDefaults() { + Set filter = ['property2'] + Map config = ConfigurationHelper.newInstance(mockScript, [property1: '27']) + .loadStepDefaults() + .mixinGeneralConfig([configuration:[general: ['general': 'test', 'oldGeneral': 'test2']]], null, [general2: 'oldGeneral']) + .mixinStageConfig([configuration:[stages:[testStage:['stage': 'test', 'oldStage': 'test2']]]], 'testStage', null, [stage2: 'oldStage']) + .mixinStepConfig([configuration:[steps:[mock: [step: 'test', 'oldStep': 'test2']]]], null, [step2: 'oldStep']) + .mixin([property1: '41', property2: '28', property3: '29'], filter) + .use() + // asserts + Assert.assertThat(config, not(hasEntry('property1', '27'))) + Assert.assertThat(config, hasEntry('property2', '28')) + Assert.assertThat(config, hasEntry('general', 'test')) + Assert.assertThat(config, hasEntry('general2', 'test2')) + Assert.assertThat(config, hasEntry('stage', 'test')) + Assert.assertThat(config, hasEntry('stage2', 'test2')) + Assert.assertThat(config, hasEntry('step', 'test')) + Assert.assertThat(config, hasEntry('step2', 'test2')) + Assert.assertThat(config, not(hasKey('property3'))) + } + + @Test + void testConfigurationHelperAddIfEmpty() { + Map config = ConfigurationHelper.newInstance(mockScript, [:]) + .mixin([property1: '41', property2: '28', property3: '29', property4: '']) + .addIfEmpty('property3', '30') + .addIfEmpty('property4', '40') + .addIfEmpty('property5', '50') + .use() + // asserts + Assert.assertThat(config, hasEntry('property1', '41')) + Assert.assertThat(config, hasEntry('property2', '28')) + Assert.assertThat(config, hasEntry('property3', '29')) + Assert.assertThat(config, hasEntry('property4', '40')) + } + + @Test + void testConfigurationHelperAddIfNull() { + Map config = ConfigurationHelper.newInstance(mockScript, [:]) + .mixin([property1: '29', property2: '', property3: null]) + .addIfNull('property1', '30') + .addIfNull('property2', '30') + .addIfNull('property3', '30') + .addIfNull('property4', '30') + .use() + // asserts + Assert.assertThat(config, hasEntry('property1', '29')) + Assert.assertThat(config, hasEntry('property2', '')) + Assert.assertThat(config, hasEntry('property3', '30')) + Assert.assertThat(config, hasEntry('property4', '30')) + } + + @Test + void testConfigurationHelperDependingOn() { + Map config = ConfigurationHelper.newInstance(mockScript, [:]) + .mixin([deep: [deeper: 'test'], scanType: 'maven', maven: [path: 'test2']]) + .dependingOn('scanType').mixin('deep/path') + .use() + // asserts + Assert.assertThat(config, hasKey('deep')) + Assert.assertThat(config.deep, allOf(hasEntry('deeper', 'test'), hasEntry('path', 'test2'))) + Assert.assertThat(config, hasEntry('scanType', 'maven')) + Assert.assertThat(config, hasKey('maven')) + Assert.assertThat(config.maven, hasEntry('path', 'test2')) + } + + @Test + void testConfigurationHelperWithPropertyInValues() { + ConfigurationHelper.newInstance(mockScript, [:]) + .mixin([test: 'allowed']) + .withPropertyInValues('test', ['allowed', 'allowed2'] as Set) + .use() + } + + @Test + void testConfigurationHelperWithPropertyInValuesException() { + def errorCaught = false + try { + ConfigurationHelper.newInstance(mockScript, [:]) + .mixin([test: 'disallowed']) + .withPropertyInValues('test', ['allowed', 'allowed2'] as Set) + .use() + } catch (e) { + errorCaught = true + assertThat(e, isA(IllegalArgumentException)) + assertThat(e.getMessage(), is('Invalid test = \'disallowed\'. Valid \'test\' values are: [allowed, allowed2].')) + } + assertThat(errorCaught, is(true)) + } + @Test void testConfigurationLoaderWithBooleanValue() { Map config = ConfigurationHelper.newInstance(mockScript, [property1: '27']) diff --git a/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy b/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy index 8f7138d5f..d01fe6efd 100644 --- a/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy +++ b/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy @@ -10,6 +10,7 @@ import util.JenkinsWriteFileRule import util.Rules import static org.hamcrest.Matchers.containsString +import static org.hamcrest.Matchers.hasItem import static org.junit.Assert.assertThat class WhiteSourceConfigurationHelperTest extends BasePiperTest { @@ -28,10 +29,46 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { } @Test - void testExtendConfigurationFileUnifiedAgentPip() { - WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', configFilePath: './config', whitesource: [serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000']], "./") + void testExtendConfigurationFileUnifiedAgentMaven() { + WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'none', whitesource: [configFilePath: './config',serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'DIST - name1', productToken: '1234', userKey: '0000']], "./") assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=name")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + } + + @Test + void testExtendConfigurationFileUnifiedAgentNpm() { + WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'npm', whitesource: [configFilePath: './config',serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'DIST - name1', productToken: '1234', userKey: '0000']], "./") + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + } + + @Test + void testExtendConfigurationFileUnifiedAgentSbt() { + WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'sbt', whitesource: [configFilePath: './config',serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'DIST - name1', productToken: '1234', userKey: '0000']], "./") + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + } + + @Test + void testExtendConfigurationFileUnifiedAgentDlang() { + WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'dlang', whitesource: [configFilePath: './config',serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'DIST - name1', productToken: '1234', userKey: '0000']], "./") + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + } + + @Test + void testExtendConfigurationFileUnifiedAgentPip() { + WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', whitesource: [configFilePath: './config',serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'DIST - name1', productToken: '1234', userKey: '0000']], "./") + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("python.resolveDependencies=true")) @@ -39,13 +76,16 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { @Test void testExtendConfigurationFileUnifiedAgentVerbose() { - WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', configFilePath: './config', whitesource: [serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000', verbose: true]], "./") + def config = [scanType: 'golang', whitesource: [configFilePath: './config', serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'SHC - name2', productToken: '1234', userKey: '0000'], stashContent: ['some', 'stashes'], verbose: true] + WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, config, "./") assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=name")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=SHC - name2")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("python.resolveDependencies=true")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("go.resolveDependencies=true")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("log.level=debug")) + + assertThat(config.stashContent, hasItem(containsString('modified whitesource config '))) } @Test @@ -54,7 +94,7 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { p.putAll(['python.resolveDependencies': 'false', 'python.ignoreSourceFiles': 'false', 'python.ignorePipInstallErrors': 'true','python.installVirtualenv': 'false']) helper.registerAllowedMethod('readProperties', [Map], {return p}) - WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', configFilePath: './config', whitesource: [serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000', verbose: true]], "./") + WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', whitesource: [configFilePath: './config', serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], verbose: true], "./") assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=name")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) diff --git a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy index 63cd4877e..cdc4b2651 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy @@ -49,7 +49,7 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { void testMissingConfig() { def errorCaught = false try { - new WhitesourceOrgAdminRepository(null, [:]) + new WhitesourceOrgAdminRepository(nullScript, [:]) } catch (e) { errorCaught = true assertThat(e, isA(AbortException.class)) @@ -95,7 +95,7 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { @Test void testHttpWhitesourceInternalCallUserKey() { - def config = [ serviceUrl: "http://some.host.whitesource.com/api/", verbose: false, orgAdminUserKey: "4711"] + def config = [whitesource: [ serviceUrl: "http://some.host.whitesource.com/api/", orgAdminUserKey: "4711"], verbose: false] repository.config.putAll(config) def requestBody = ["someJson" : [ "someObject" : "abcdef" ]] @@ -121,7 +121,7 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { @Test void testHttpWhitesourceInternalCallUserKeyVerboseProxy() { - def config = [ serviceUrl: "http://some.host.whitesource.com/api/", verbose: true, orgAdminUserKey: "4711"] + def config = [whitesource: [ serviceUrl: "http://some.host.whitesource.com/api/", orgAdminUserKey: "4711"], verbose: true] nullScript.env['HTTP_PROXY'] = "http://test.sap.com:8080" repository.config.putAll(config) def requestBody = ["someJson" : [ "someObject" : "abcdef" ]] @@ -153,12 +153,14 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { @Test void testCreateProduct() { def config = [ - serviceUrl: "http://some.host.whitesource.com/api/", - verbose: false, - orgAdminUserKey: "4711", - orgToken: "abcd1234", - productName: "testProduct", - emailAddressesOfInitialProductAdmins: ['some@somewhere.com', 'some2@somewhere.com'] + whitesource: [ + serviceUrl: "http://some.host.whitesource.com/api/", + verbose: false, + orgAdminUserKey: "4711", + orgToken: "abcd1234", + productName: "testProduct", + emailAddressesOfInitialProductAdmins: ['some@somewhere.com', 'some2@somewhere.com'] + ] ] repository.config.putAll(config) def requestBody1 = [ @@ -213,7 +215,7 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { @Test void testIssueHttpRequestError() { - def config = [ serviceUrl: "http://some.host.whitesource.com/api/", verbose: false, orgAdminUserKey: "4711"] + def config = [whitesource: [ serviceUrl: "http://some.host.whitesource.com/api/", orgAdminUserKey: "4711"], verbose: false] repository.config.putAll(config) def requestBody = ["someJson" : [ "someObject" : "abcdef" ]] @@ -248,7 +250,7 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { @Test void testFetchProductMetaInfo() { - def config = [ serviceUrl: "http://some.host.whitesource.com/api/", verbose: true, orgAdminUserKey: "4711", orgToken: "12345", productName: "testProduct"] + def config = [whitesource: [ serviceUrl: "http://some.host.whitesource.com/api/", orgAdminUserKey: "4711", orgToken: "12345", productName: "testProduct"], verbose: true] nullScript.env['HTTP_PROXY'] = "http://test.sap.com:8080" repository.config.putAll(config) diff --git a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy index 8f7fca218..73de10265 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy @@ -53,7 +53,7 @@ class WhitesourceRepositoryTest extends BasePiperTest { void testMissingConfig() { def errorCaught = false try { - new WhitesourceRepository(null, [:]) + new WhitesourceRepository(nullScript, [:]) } catch (e) { errorCaught = true assertThat(e, isA(AbortException.class)) @@ -81,8 +81,9 @@ class WhitesourceRepositoryTest extends BasePiperTest { ] ] - repository.config['productName'] = "Correct Name Cloud" - repository.config['projectNames'] = ["Correct Project Name", "Correct Project Name2"] + repository.config.whitesource = [:] + repository.config.whitesource['productName'] = "Correct Name Cloud" + repository.config.whitesource['projectNames'] = ["Correct Project Name", "Correct Project Name2"] def result = repository.findProjectsMeta(whitesourceMetaResponse.projectVitals) @@ -103,8 +104,6 @@ class WhitesourceRepositoryTest extends BasePiperTest { @Test void testResolveProjectsMetaFailNotFound() { - - def whitesourceMetaResponse = [ projectVitals: [ [ @@ -126,7 +125,7 @@ class WhitesourceRepositoryTest extends BasePiperTest { exception.expectMessage("Correct Project Name") - repository.config.putAll([whitesource : [ projectNames: ["Correct Project Name"]]]) + repository.config.putAll([whitesource : [projectNames: ["Correct Project Name"]]]) repository.findProjectsMeta(whitesourceMetaResponse.projectVitals) } @@ -245,7 +244,7 @@ class WhitesourceRepositoryTest extends BasePiperTest { @Test void testHttpWhitesourceExternalCallNoUserKey() { - def config = [ whitesourceServiceUrl: "https://saas.whitesource.com/api", verbose: true] + def config = [whitesource: [serviceUrl: "https://saas.whitesource.com/api"], verbose: true] repository.config.putAll(config) def requestBody = "{ \"someJson\" : { \"someObject\" : \"abcdef\" } }" @@ -258,7 +257,7 @@ class WhitesourceRepositoryTest extends BasePiperTest { assertThat(requestParams, is( [ - url : config.whitesourceServiceUrl, + url : config.whitesource.serviceUrl, httpMode : 'POST', acceptType : 'APPLICATION_JSON', contentType: 'APPLICATION_JSON', @@ -271,7 +270,7 @@ class WhitesourceRepositoryTest extends BasePiperTest { @Test void testHttpWhitesourceExternalCallUserKey() { - def config = [ serviceUrl: "https://saas.whitesource.com/api", verbose: true, userKey: "4711"] + def config = [whitesource: [ serviceUrl: "https://saas.whitesource.com/api", userKey: "4711"], verbose: true] def requestBody = "{ \"someJson\" : { \"someObject\" : \"abcdef\" } }" def requestParams @@ -283,7 +282,7 @@ class WhitesourceRepositoryTest extends BasePiperTest { assertThat(requestParams, is( [ - url : config.whitesourceServiceUrl, + url : config.whitesource.serviceUrl, httpMode : 'POST', acceptType : 'APPLICATION_JSON', contentType: 'APPLICATION_JSON', @@ -297,7 +296,7 @@ class WhitesourceRepositoryTest extends BasePiperTest { @Test void testHttpWhitesourceInternalCallUserKey() { - def config = [ whitesourceServiceUrl: "http://mo-323123123.sap.corp/some", verbose: false, userKey: "4711"] + def config = [whitesource: [serviceUrl: "http://mo-323123123.sap.corp/some", userKey: "4711"], verbose: false] def requestBody = "{ \"someJson\" : { \"someObject\" : \"abcdef\" } }" def requestParams @@ -309,7 +308,7 @@ class WhitesourceRepositoryTest extends BasePiperTest { assertThat(requestParams, is( [ - url : config.whitesourceServiceUrl, + url : config.whitesource.serviceUrl, httpMode : 'POST', acceptType : 'APPLICATION_JSON', contentType: 'APPLICATION_JSON', @@ -340,7 +339,7 @@ class WhitesourceRepositoryTest extends BasePiperTest { @Test void testFetchReportForProduct() { - repository.config.putAll([ whitesourceServiceUrl: "http://mo-323123123.sap.corp/some", verbose: true, productToken: "4711"]) + repository.config.putAll([whitesource: [serviceUrl: "http://mo-323123123.sap.corp/some", productToken: "4711"], verbose: true]) def command helper.registerAllowedMethod('sh', [String], { cmd -> command = cmd @@ -360,7 +359,7 @@ curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type @Test void testFetchProductLicenseAlerts() { - def config = [ serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711", productToken: "8547"] + def config = [whitesource: [serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711", productToken: "8547"]] nullScript.env['HTTP_PROXY'] = "http://test.sap.com:8080" repository.config.putAll(config) @@ -380,13 +379,13 @@ curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type assertThat(requestParams, is( [ - url : config.serviceUrl, + url : config.whitesource.serviceUrl, httpMode : 'POST', acceptType : 'APPLICATION_JSON', contentType: 'APPLICATION_JSON', requestBody: requestBody, quiet : false, - userKey : config.userKey, + userKey : config.whitesource.userKey, httpProxy : "http://test.sap.com:8080" ] )) @@ -395,7 +394,7 @@ curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type @Test void testFetchProjectLicenseAlerts() { def projectToken = "8547" - def config = [ serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711"] + def config = [whitesource: [serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711"]] repository.config.putAll(config) def requestBody = [ @@ -414,13 +413,13 @@ curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type assertThat(requestParams, is( [ - url : config.serviceUrl, + url : config.whitesource.serviceUrl, httpMode : 'POST', acceptType : 'APPLICATION_JSON', contentType: 'APPLICATION_JSON', requestBody: requestBody, quiet : false, - userKey : config.userKey, + userKey : config.whitesource.userKey, httpProxy : "http://test.sap.com:8080" ] )) @@ -428,7 +427,7 @@ curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type @Test void testFetchProjectsMetaInfo() { - def config = [ serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711", productToken: '8475', projectNames: ['testProject1', 'testProject2']] + def config = [whitesource: [serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711", productToken: '8475', projectNames: ['testProject1', 'testProject2']]] nullScript.env['HTTP_PROXY'] = "http://test.sap.com:8080" repository.config.putAll(config) @@ -447,13 +446,13 @@ curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type assertThat(requestParams, is( [ - url : config.serviceUrl, + url : config.whitesource.serviceUrl, httpMode : 'POST', acceptType : 'APPLICATION_JSON', contentType: 'APPLICATION_JSON', requestBody: requestBody, quiet : false, - userKey : config.userKey, + userKey : config.whitesource.userKey, httpProxy : "http://test.sap.com:8080" ] )) @@ -463,7 +462,7 @@ curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type @Test void testFetchProjectsMetaInfoError() { - def config = [ serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711", productName: 'kjdkjkhd', productToken: '8475', projectNames: ['testProject1', 'testProject2']] + def config = [whitesource: [serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711", productName: 'kjdkjkhd', productToken: '8475', projectNames: ['testProject1', 'testProject2']]] repository.config.putAll(config) def requestParams @@ -486,7 +485,7 @@ curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type @Test void testFetchVulnerabilitiesOnProjects() { - def config = [ serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711", productToken: '8475', projectNames: ['testProject1', 'testProject2']] + def config = [whitesource: [serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711", productToken: '8475', projectNames: ['testProject1', 'testProject2']]] nullScript.env['HTTP_PROXY'] = "http://test.sap.com:8080" repository.config.putAll(config) @@ -512,26 +511,26 @@ curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type assertThat(requestParams[0], is( [ - url : config.serviceUrl, + url : config.whitesource.serviceUrl, httpMode : 'POST', acceptType : 'APPLICATION_JSON', contentType: 'APPLICATION_JSON', requestBody: requestBody1, quiet : false, - userKey : config.userKey, + userKey : config.whitesource.userKey, httpProxy : "http://test.sap.com:8080" ] )) assertThat(requestParams[1], is( [ - url : config.serviceUrl, + url : config.whitesource.serviceUrl, httpMode : 'POST', acceptType : 'APPLICATION_JSON', contentType: 'APPLICATION_JSON', requestBody: requestBody2, quiet : false, - userKey : config.userKey, + userKey : config.whitesource.userKey, httpProxy : "http://test.sap.com:8080" ] )) @@ -541,7 +540,7 @@ curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type @Test void testFetchVulnerabilitiesOnProduct() { - def config = [ serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711", productToken: '8475', productName : 'testProduct'] + def config = [whitesource: [serviceUrl: "http://some.host.whitesource.com/api/", userKey: "4711", productToken: '8475', productName : 'testProduct']] nullScript.env['HTTP_PROXY'] = "http://test.sap.com:8080" repository.config.putAll(config) @@ -561,13 +560,13 @@ curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type assertThat(requestParams[0], is( [ - url : config.serviceUrl, + url : config.whitesource.serviceUrl, httpMode : 'POST', acceptType : 'APPLICATION_JSON', contentType: 'APPLICATION_JSON', requestBody: requestBody, quiet : false, - userKey : config.userKey, + userKey : config.whitesource.userKey, httpProxy : "http://test.sap.com:8080" ] )) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 0087b4896..2d574a463 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -166,8 +166,8 @@ import static com.sap.piper.Prerequisites.checkScript productToken : 'productToken', projectNames : 'projectNames', serviceUrl : 'serviceUrl', + configFilePath : 'configFilePath', userTokenCredentialsId : 'userTokenCredentialsId', - verbose : 'verbose', agentDownloadUrl : 'agentDownloadUrl', agentFileName : 'agentFileName', agentParameters : 'agentParameters', @@ -217,10 +217,10 @@ void call(Map parameters = [:]) { ]) .mixin(parameters, PARAMETER_KEYS, CONFIG_KEY_COMPATIBILITY) .dependingOn('scanType').mixin('buildDescriptorFile') - .dependingOn('scanType').mixin('configFilePath') .dependingOn('scanType').mixin('dockerImage') .dependingOn('scanType').mixin('dockerWorkspace') .dependingOn('scanType').mixin('stashContent') + .dependingOn('scanType').mixin('whitesource/configFilePath') .withMandatoryProperty('whitesource/serviceUrl') .withMandatoryProperty('whitesource/orgToken') .withMandatoryProperty('whitesource/userTokenCredentialsId') @@ -279,7 +279,7 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU if (!config.whitesource.productToken) { def metaInfo = orgAdminRepository.fetchProductMetaInfo() def key = "token" - if((null == metaInfo || !metaInfo[key]) && config.createProductFromPipeline) { + if((null == metaInfo || !metaInfo[key]) && config.whitesource.createProductFromPipeline) { metaInfo = orgAdminRepository.createProduct() key = "productToken" } else if(null == metaInfo || !metaInfo[key]) { @@ -369,7 +369,7 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU javaCmd = './bin/java' } - def options = ["-jar ${config.whitesource.agentFileName} -c \'${config.configFilePath}\'"] + def options = ["-jar ${config.whitesource.agentFileName} -c \'${config.whitesource.configFilePath}\'"] if (config.whitesource.orgToken) options.push("-apiKey '${config.whitesource.orgToken}'") if (config.whitesource.userKey) options.push("-userKey '${config.whitesource.userKey}'") if (config.whitesource.productName) options.push("-product '${config.whitesource.productName}'") From 1a04394a00e4d7c6bc15dbbd56159ec393382b32 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 15 Mar 2019 13:21:56 +0100 Subject: [PATCH 068/130] Ensure report severity limit is based on configuration --- .../com.sap.piper/templates/whitesourceVulnerabilities.html | 2 +- vars/whitesourceExecuteScan.groovy | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/resources/com.sap.piper/templates/whitesourceVulnerabilities.html b/resources/com.sap.piper/templates/whitesourceVulnerabilities.html index a5fea28e9..f2fe8f7dc 100644 --- a/resources/com.sap.piper/templates/whitesourceVulnerabilities.html +++ b/resources/com.sap.piper/templates/whitesourceVulnerabilities.html @@ -14,7 +14,7 @@

total number of vulnerabilities: ${totalVulnerabilities}
- total number of high/critical vulnerabilities with CVSS score >= 7.0: ${totalSevereVulnerabilities} + total number of high/critical vulnerabilities with CVSS score >= ${cvssSeverityLimit}: ${totalSevereVulnerabilities}

Snapshot taken:${now} GMT

diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 2d574a463..7a8f13076 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -530,6 +530,7 @@ def getReportHtml(config, vulnerabilityList, numSevereVulns) { now : now, reportTitle : config.whitesource.vulnerabilityReportTitle, style : config.style, + cvssSeverityLimit : config.whitesource.cvssSeverityLimit, totalSevereVulnerabilities : numSevereVulns, totalVulnerabilities : vulnerabilityList.size(), vulnerabilityTable : vulnerabilityTable, From 3d81e09ab3f43cbeb1bd7c402e9429206dc82a8d Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 15 Mar 2019 13:32:02 +0100 Subject: [PATCH 069/130] Add warning --- .../piper/WhitesourceConfigurationHelper.groovy | 2 ++ .../WhiteSourceConfigurationHelperTest.groovy | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index 8bd9876a5..7e94c0f52 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -38,6 +38,8 @@ class WhitesourceConfigurationHelper implements Serializable { [name: 'ignoreSourceFiles', value: true, force: true], [name: 'resolveAllDependencies', value: false, force: true] ] + if(!['pip', 'golang'].contains(config.scanType)) + script.echo "[Whitesource] Configuration for scanType: '${config.scanType}' is not yet hardened, please do a quality assessment of your scan results." switch (config.scanType) { case 'npm': mapping += [ diff --git a/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy b/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy index d01fe6efd..2ebd5e630 100644 --- a/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy +++ b/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy @@ -5,23 +5,27 @@ import org.junit.Rule import org.junit.Test import org.junit.rules.RuleChain import util.BasePiperTest +import util.JenkinsLoggingRule import util.JenkinsReadFileRule import util.JenkinsWriteFileRule import util.Rules import static org.hamcrest.Matchers.containsString import static org.hamcrest.Matchers.hasItem +import static org.hamcrest.Matchers.not import static org.junit.Assert.assertThat class WhiteSourceConfigurationHelperTest extends BasePiperTest { JenkinsReadFileRule jrfr = new JenkinsReadFileRule(this, 'test/resources/utilsTest/') JenkinsWriteFileRule jwfr = new JenkinsWriteFileRule(this) + JenkinsLoggingRule jlr = new JenkinsLoggingRule(this) @Rule public RuleChain ruleChain = Rules .getCommonRules(this) .around(jrfr) .around(jwfr) + .around(jlr) @Before void init() { @@ -35,6 +39,8 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + + assertThat(jlr.log, containsString("[Whitesource] Configuration for scanType: 'none' is not yet hardened, please do a quality assessment of your scan results.")) } @Test @@ -44,6 +50,8 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + + assertThat(jlr.log, containsString("[Whitesource] Configuration for scanType: 'npm' is not yet hardened, please do a quality assessment of your scan results.")) } @Test @@ -53,6 +61,9 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + + assertThat(jlr.log, containsString("[Whitesource] Configuration for scanType: 'sbt' is not yet hardened, please do a quality assessment of your scan results.")) + } @Test @@ -62,6 +73,8 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + + assertThat(jlr.log, containsString("[Whitesource] Configuration for scanType: 'dlang' is not yet hardened, please do a quality assessment of your scan results.")) } @Test @@ -72,6 +85,8 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("python.resolveDependencies=true")) + + assertThat(jlr.log, not(containsString("[Whitesource] Configuration for scanType: 'pip' is not yet hardened, please do a quality assessment of your scan results."))) } @Test @@ -86,6 +101,7 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("log.level=debug")) assertThat(config.stashContent, hasItem(containsString('modified whitesource config '))) + assertThat(jlr.log, not(containsString("[Warning][Whitesource] Configuration for scanType: 'golang' is not yet hardened, please do a quality assessment of your scan results."))) } @Test From be814c62aa2d128c2dab29a19a0dcb410e4fce6c Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Thu, 21 Mar 2019 13:25:22 +0100 Subject: [PATCH 070/130] Included comments refactored docu generation --- documentation/bin/createDocu.groovy | 92 +++++++++++++++---- .../docs/steps/whitesourceExecuteScan.md | 2 +- resources/default_pipeline_environment.yml | 7 ++ src/com/sap/piper/JsonUtils.groovy | 13 +-- .../WhitesourceConfigurationHelper.groovy | 4 +- .../WhitesourceOrgAdminRepository.groovy | 4 +- .../integration/WhitesourceRepository.groovy | 6 +- test/groovy/WhitesourceExecuteScanTest.groovy | 20 ++-- .../com/sap/piper/DescriptorUtilsTest.groovy | 11 +-- vars/influxWriteData.groovy | 8 +- vars/whitesourceExecuteScan.groovy | 44 +++++---- 11 files changed, 138 insertions(+), 73 deletions(-) diff --git a/documentation/bin/createDocu.groovy b/documentation/bin/createDocu.groovy index c380202c6..006d06761 100644 --- a/documentation/bin/createDocu.groovy +++ b/documentation/bin/createDocu.groovy @@ -93,7 +93,7 @@ class Helper { new GroovyClassLoader(classLoader, compilerConfig, true) .parseClass(new File(projectRoot, 'src/com/sap/piper/ConfigurationHelper.groovy')) .newInstance(script, [:]).loadStepDefaults() - } + } static getPrepareDefaultValuesStep(def gse) { @@ -123,6 +123,7 @@ class Helper { def prepareDefaultValues() { _prepareDefaultValuesStep() + } def run() { @@ -197,9 +198,10 @@ class Helper { boolean docu = false, value = false, mandatory = false, + parentObject = false, docuEnd = false - def docuLines = [], valueLines = [], mandatoryLines = [] + def docuLines = [], valueLines = [], mandatoryLines = [], parentObjectLines = [] f.eachLine { line -> @@ -220,13 +222,17 @@ class Helper { throw new RuntimeException('Cannot retrieve parameter for a comment') } + def _docu = [], _value = [], _mandatory = [], _parentObject = [] + docuLines.each { _docu << it } + valueLines.each { _value << it } + mandatoryLines.each { _mandatory << it } + parentObjectLines.each { _parentObject << it } + _parentObject << param + param = _parentObject*.trim().join('/').trim() + if(step.parameters[param].docu || step.parameters[param].value) System.err << "[WARNING] There is already some documentation for parameter '${param}. Is this parameter documented twice?'\n" - def _docu = [], _value = [], _mandatory = [] - docuLines.each { _docu << it } - valueLines.each { _value << it} - mandatoryLines.each { _mandatory << it} step.parameters[param].docu = _docu*.trim().join(' ').trim() step.parameters[param].value = _value*.trim().join(' ').trim() step.parameters[param].mandatory = _mandatory*.trim().join(' ').trim() @@ -234,6 +240,7 @@ class Helper { docuLines.clear() valueLines.clear() mandatoryLines.clear() + parentObjectLines.clear() } if( line.trim() ==~ /^\/\*\*.*/ ) { @@ -250,11 +257,19 @@ class Helper { if(_line ==~ /.*@possibleValues.*/) { mandatory = false // should be something like reset attributes value = true + parentObject = false } // some remark for mandatory e.g. some parameters are only mandatory under certain conditions if(_line ==~ /.*@mandatory.*/) { value = false // should be something like reset attributes ... mandatory = true + parentObject = false + } + // grouping config properties within a parent object for easier readability + if(_line ==~ /.*@parentConfigKey.*/) { + value = false // should be something like reset attributes ... + mandatory = false + parentObject = true } if(value) { @@ -271,7 +286,14 @@ class Helper { } } - if(! value && ! mandatory) { + if(parentObject) { + if(_line) { + _line = (_line =~ /.*@parentConfigKey\s*?(.*)/)[0][1] + parentObjectLines << _line + } + } + + if(!value && !mandatory && !parentObject) { docuLines << _line } } @@ -280,6 +302,7 @@ class Helper { docu = false value = false mandatory = false + parentObject = false docuEnd = true } } @@ -312,14 +335,33 @@ class Helper { def params = [] as Set f.eachLine { line -> - if( line ==~ /.*withMandatoryProperty.*/ ) { - def param = (line =~ /.*withMandatoryProperty\('(.*)'/)[0][1] - params << param - } + if (line ==~ /.*withMandatoryProperty.*/) { + def param = (line =~ /.*withMandatoryProperty\('(.*)'/)[0][1] + params << param + } } return params } + static getParentObjectMappings(File f) { + def mappings = [:] + def parentObjectKey = '' + f.eachLine { + line -> + if (line ==~ /.*parentConfigKey.*/ && !parentObjectKey) { + def param = (line =~ /.*parentConfigKey\s*?(.*)/)[0][1] + parentObjectKey = param.trim() + } else if (line ==~ /\s*?(.*)[,]{0,1}/ && parentObjectKey) { + def pName = retrieveParameterName(line) + if(pName) { + mappings.put(pName, parentObjectKey) + parentObjectKey = '' + } + } + } + return mappings + } + static getValue(Map config, def pPath) { def p =config[pPath.head()] if(pPath.size() == 1) return p // there is no tail @@ -530,6 +572,20 @@ def handleStep(stepName, prepareDefaultValuesStep, gse) { params.addAll(requiredParameters) + // translate parameter names according to compatibility annotations + def parentObjectMappings = Helper.getParentObjectMappings(theStep) + def compatibleParams = [] as Set + if(parentObjectMappings) { + params.each { + if (parentObjectMappings[it]) + compatibleParams.add(parentObjectMappings[it] + '/' + it) + else + compatibleParams.add(it) + } + if (compatibleParams) + params = compatibleParams + } + def step = [parameters:[:]] // @@ -562,14 +618,14 @@ def handleStep(stepName, prepareDefaultValuesStep, gse) { required: requiredParameters.contains((it as String)) && defaultValue == null ] - step.parameters.put(it, parameterProperties) + step.parameters.put(it, parameterProperties) - // The scope is only defined for the first level of a hierarchical configuration. - // If the first part is found, all nested parameters are allowed with that scope. - def firstPart = it.split('/').head() - scopedParameters.each { key, val -> - parameterProperties.put(key, val.contains(firstPart)) - } + // The scope is only defined for the first level of a hierarchical configuration. + // If the first part is found, all nested parameters are allowed with that scope. + def firstPart = it.split('/').head() + scopedParameters.each { key, val -> + parameterProperties.put(key, val.contains(firstPart)) + } } Helper.scanDocu(theStep, step) diff --git a/documentation/docs/steps/whitesourceExecuteScan.md b/documentation/docs/steps/whitesourceExecuteScan.md index f8accedf1..1d1bf77f7 100644 --- a/documentation/docs/steps/whitesourceExecuteScan.md +++ b/documentation/docs/steps/whitesourceExecuteScan.md @@ -23,5 +23,5 @@ None ## Examples ```groovy -whitesourceExecuteScan script: this, scanType: 'pip', whitesource: [ productName: 'My Whitesource Product', userTokenCredentialsId: 'companyAdminToken', orgAdminUserTokenCredentialsId: 'orgAdmiToken', orgToken: 'myWhitesourceOrganizationToken' ] +whitesourceExecuteScan script: this, scanType: 'pip', productName: 'My Whitesource Product', userTokenCredentialsId: 'companyAdminToken', orgAdminUserTokenCredentialsId: 'orgAdmiToken', orgToken: 'myWhitesourceOrganizationToken' ``` diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 228dd5d8c..cccd858d1 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -304,6 +304,13 @@ steps: stashContent: - 'buildDescriptor' - 'opensourceConfiguration' + golang: + buildDescriptorFile: './setup.py' + dockerImage: 'golang:1.12.1-stretch' + dockerWorkspace: '/home/golang' + stashContent: + - 'buildDescriptor' + - 'opensourceConfiguration' sbt: buildDescriptorFile: './build.sbt' dockerImage: 'hseeberger/scala-sbt:8u181_2.12.8_1.2.8' diff --git a/src/com/sap/piper/JsonUtils.groovy b/src/com/sap/piper/JsonUtils.groovy index cbc8e4602..840205561 100644 --- a/src/com/sap/piper/JsonUtils.groovy +++ b/src/com/sap/piper/JsonUtils.groovy @@ -1,20 +1,13 @@ package com.sap.piper import com.cloudbees.groovy.cps.NonCPS -import groovy.json.JsonBuilder -import groovy.json.JsonSlurperClassic @NonCPS -def jsonToString(content) { - return new JsonBuilder(content).toPrettyString() -} - -@NonCPS -String getPrettyJsonString(object) { +String groovyObjectToPrettyJsonString(object) { return groovy.json.JsonOutput.prettyPrint(groovy.json.JsonOutput.toJson(object)) } @NonCPS -def parseJsonSerializable(text) { - return new JsonSlurperClassic().parseText(text) +def jsonStringToGroovyObject(text) { + return new groovy.json.JsonSlurperClassic().parseText(text) } diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index 7e94c0f52..510c181df 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -16,7 +16,7 @@ class WhitesourceConfigurationHelper implements Serializable { [name: 'checkPolicies', value: false, force: true], [name: 'forceCheckAllDependencies', value: false, force: true] ] - } else if(config.whitesource.productName.startsWith('SHC - ')) { + } else { mapping += [ [name: 'checkPolicies', value: true, force: true], [name: 'forceCheckAllDependencies', value: true, force: true] @@ -39,7 +39,7 @@ class WhitesourceConfigurationHelper implements Serializable { [name: 'resolveAllDependencies', value: false, force: true] ] if(!['pip', 'golang'].contains(config.scanType)) - script.echo "[Whitesource] Configuration for scanType: '${config.scanType}' is not yet hardened, please do a quality assessment of your scan results." + script.echo "[Warning][Whitesource] Configuration for scanType: '${config.scanType}' is not yet hardened, please do a quality assessment of your scan results." switch (config.scanType) { case 'npm': mapping += [ diff --git a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy index 07ddf5808..5cb824df0 100644 --- a/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceOrgAdminRepository.groovy @@ -71,7 +71,7 @@ class WhitesourceOrgAdminRepository implements Serializable { def issueHttpRequest(requestBody) { def response = internalWhitesource ? internalWhitesource.httpWhitesource(requestBody) : httpWhitesource(requestBody) - def parsedResponse = new JsonUtils().parseJsonSerializable(response.content) + def parsedResponse = new JsonUtils().jsonStringToGroovyObject(response.content) if(parsedResponse?.errorCode){ script.error "[WhiteSource] Request failed with error message '${parsedResponse.errorMessage}' (${parsedResponse.errorCode})." } @@ -81,7 +81,7 @@ class WhitesourceOrgAdminRepository implements Serializable { @NonCPS protected def httpWhitesource(requestBody) { requestBody["userKey"] = config.whitesource.orgAdminUserKey - def serializedBody = new JsonUtils().jsonToString(requestBody) + def serializedBody = new JsonUtils().groovyObjectToPrettyJsonString(requestBody) def params = [ url : config.whitesource.serviceUrl, httpMode : 'POST', diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy index 3119d6736..5bddabf64 100644 --- a/src/com/sap/piper/integration/WhitesourceRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -49,7 +49,7 @@ class WhitesourceRepository implements Serializable { protected def fetchWhitesourceResource(Map requestBody) { final def response = httpWhitesource(requestBody) - def parsedResponse = new JsonUtils().parseJsonSerializable(response.content) + def parsedResponse = new JsonUtils().jsonStringToGroovyObject(response.content) if(parsedResponse?.errorCode){ script.error "[WhiteSource] Request failed with error message '${parsedResponse.errorMessage}' (${parsedResponse.errorCode})." @@ -173,7 +173,7 @@ class WhitesourceRepository implements Serializable { @NonCPS protected def httpWhitesource(requestBody) { handleAdditionalRequestParameters(requestBody) - def serializedBody = new JsonUtils().getPrettyJsonString(requestBody) + def serializedBody = new JsonUtils().groovyObjectToPrettyJsonString(requestBody) def params = [ url : config.whitesource.serviceUrl, httpMode : 'POST', @@ -201,7 +201,7 @@ class WhitesourceRepository implements Serializable { @NonCPS protected void fetchFileFromWhiteSource(String fileName, Map params) { handleAdditionalRequestParameters(params) - def serializedContent = new JsonUtils().jsonToString(params) + def serializedContent = new JsonUtils().groovyObjectToPrettyJsonString(params) if(config.verbose) script.echo "Sending curl request with parameters ${params}" diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index b78f4ada3..174c1c4b1 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -112,23 +112,23 @@ class WhitesourceExecuteScanTest extends BasePiperTest { LibraryLoadingTestExecutionListener.prepareObjectInterceptors(whitesourceStub) helper.registerAllowedMethod("fetchProductMetaInfo", [], { - return new JsonUtils().parseJsonSerializable("{ \"productVitals\": [{ \"id\": 59639, \"name\": \"SHC - Piper\", \"token\": \"e30132d8e8f04a4c8be6332c75a0ff0580ab326fa7534540ad326e97a74d945b\", \"creationDate\": \"2017-09-20 09:22:46 +0000\", \"lastUpdatedDate\": \"2018-09-19 09:44:40 +0000\" }]}") + return new JsonUtils().jsonStringToGroovyObject("{ \"productVitals\": [{ \"id\": 59639, \"name\": \"SHC - Piper\", \"token\": \"e30132d8e8f04a4c8be6332c75a0ff0580ab326fa7534540ad326e97a74d945b\", \"creationDate\": \"2017-09-20 09:22:46 +0000\", \"lastUpdatedDate\": \"2018-09-19 09:44:40 +0000\" }]}") }) helper.registerAllowedMethod("fetchProjectsMetaInfo", [], { - return new JsonUtils().parseJsonSerializable("{ \"projectVitals\": [{ \"id\": 261964, \"name\": \"piper-demo - 0.0.1\", \"token\": \"a2a62e5d7beb4170ad4dccfa3316b5a4cd3fadefc56c49f88fbf9400a09f7d94\", \"creationDate\": \"2017-09-21 00:28:06 +0000\", \"lastUpdatedDate\": \"2017-10-12 01:03:05 +0000\" }]}").projectVitals + return new JsonUtils().jsonStringToGroovyObject("{ \"projectVitals\": [{ \"id\": 261964, \"name\": \"piper-demo - 0.0.1\", \"token\": \"a2a62e5d7beb4170ad4dccfa3316b5a4cd3fadefc56c49f88fbf9400a09f7d94\", \"creationDate\": \"2017-09-21 00:28:06 +0000\", \"lastUpdatedDate\": \"2017-10-12 01:03:05 +0000\" }]}").projectVitals }) helper.registerAllowedMethod("fetchReportForProduct", [String], { }) helper.registerAllowedMethod( "fetchProjectLicenseAlerts", [Object.class], { - return new JsonUtils().parseJsonSerializable("{ \"alerts\": [] }").alerts + return new JsonUtils().jsonStringToGroovyObject("{ \"alerts\": [] }").alerts }) helper.registerAllowedMethod( "fetchProductLicenseAlerts", [], { - return new JsonUtils().parseJsonSerializable("{ \"alerts\": [] }").alerts + return new JsonUtils().jsonStringToGroovyObject("{ \"alerts\": [] }").alerts }) helper.registerAllowedMethod( "fetchVulnerabilities", [List], { - return new JsonUtils().parseJsonSerializable("{ \"alerts\": [] }").alerts + return new JsonUtils().jsonStringToGroovyObject("{ \"alerts\": [] }").alerts }) helper.registerAllowedMethod( "createProduct", [], { - return new JsonUtils().parseJsonSerializable("{ \"productToken\": \"e30132d8e8f04a4c8be6332c75a0ff0580ab326fa7534540ad326e97a74d945b\" }") + return new JsonUtils().jsonStringToGroovyObject("{ \"productToken\": \"e30132d8e8f04a4c8be6332c75a0ff0580ab326fa7534540ad326e97a74d945b\" }") }) helper.registerAllowedMethod( "publishHTML", [Map], {}) @@ -800,7 +800,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { return result }) helper.registerAllowedMethod("fetchVulnerabilities", [List], { - return new JsonUtils().parseJsonSerializable("{ \"alerts\": [ { \"vulnerability\": { \"name\": \"CVE-2017-15095\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 7.5, \"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15095\", \"description\": \"A deserialization flaw was discovered in the jackson-databind in versions before 2.8.10 and 2.9.1, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper. This issue extends the previous flaw CVE-2017-7525 by blacklisting more classes that could be used maliciously.\", \"topFix\": { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\"," + + return new JsonUtils().jsonStringToGroovyObject("{ \"alerts\": [ { \"vulnerability\": { \"name\": \"CVE-2017-15095\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 7.5, \"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15095\", \"description\": \"A deserialization flaw was discovered in the jackson-databind in versions before 2.8.10 and 2.9.1, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper. This issue extends the previous flaw CVE-2017-7525 by blacklisting more classes that could be used maliciously.\", \"topFix\": { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\"," + "\"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\", \"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\"," + "\"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e865a7a4464da63ded9f4b1a2328ad85c9ded78b#diff-98084d808198119d550a9211e128a16f\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-12-12\", \"message\": \"Fix #1737 (#1857)\", \"extraData\": \"key=e865a7a&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e8f043d1\"," + "\"fixResolution\": \"release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-06-30\", \"message\": \"Fix #1680\", \"extraData\": \"key=e8f043d&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" } ], \"fixResolutionText\": \"Replace or update the following files: IllegalTypesCheckTest.java, VERSION, BeanDeserializerFactory.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\", \"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\"," + @@ -882,7 +882,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { return result }) helper.registerAllowedMethod("fetchVulnerabilities", [List], { - return new JsonUtils().parseJsonSerializable("{ \"alerts\": [ { \"vulnerability\": { \"name\": \"CVE-2017-15095\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 2.1, \"cvss3_severity\": \"high\", \"cvss3_score\": 5.3, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15095\", \"description\": \"A deserialization flaw was discovered in the jackson-databind in versions before 2.8.10 and 2.9.1, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper. This issue extends the previous flaw CVE-2017-7525 by blacklisting more classes that could be used maliciously.\", \"topFix\": { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\"," + + return new JsonUtils().jsonStringToGroovyObject("{ \"alerts\": [ { \"vulnerability\": { \"name\": \"CVE-2017-15095\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 2.1, \"cvss3_severity\": \"high\", \"cvss3_score\": 5.3, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15095\", \"description\": \"A deserialization flaw was discovered in the jackson-databind in versions before 2.8.10 and 2.9.1, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper. This issue extends the previous flaw CVE-2017-7525 by blacklisting more classes that could be used maliciously.\", \"topFix\": { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\"," + "\"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\", \"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\"," + "\"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e865a7a4464da63ded9f4b1a2328ad85c9ded78b#diff-98084d808198119d550a9211e128a16f\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-12-12\", \"message\": \"Fix #1737 (#1857)\", \"extraData\": \"key=e865a7a&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\"," + "\"url\": \"https://github.com/FasterXML/jackson-databind/commit/e8f043d1\", \"fixResolution\": \"release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-06-30\", \"message\": \"Fix #1680\", \"extraData\": \"key=e8f043d&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" } ], \"fixResolutionText\": \"Replace or update the following files: IllegalTypesCheckTest.java, VERSION, BeanDeserializerFactory.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\"," + @@ -921,7 +921,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { return result }) helper.registerAllowedMethod("fetchVulnerabilities", [List], { - return new JsonUtils().parseJsonSerializable("{ \"alerts\": [ { \"vulnerability\": { \"name\": \"CVE-2017-15095\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 2.1, \"cvss3_severity\": \"high\", \"cvss3_score\": 5.3, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15095\", \"description\": \"A deserialization flaw was discovered in the jackson-databind in versions before 2.8.10 and 2.9.1, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper. This issue extends the previous flaw CVE-2017-7525 by blacklisting more classes that could be used maliciously.\", \"topFix\": { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\"," + + return new JsonUtils().jsonStringToGroovyObject("{ \"alerts\": [ { \"vulnerability\": { \"name\": \"CVE-2017-15095\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 2.1, \"cvss3_severity\": \"high\", \"cvss3_score\": 5.3, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15095\", \"description\": \"A deserialization flaw was discovered in the jackson-databind in versions before 2.8.10 and 2.9.1, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper. This issue extends the previous flaw CVE-2017-7525 by blacklisting more classes that could be used maliciously.\", \"topFix\": { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\"," + "\"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\", \"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\"," + "\"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e865a7a4464da63ded9f4b1a2328ad85c9ded78b#diff-98084d808198119d550a9211e128a16f\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-12-12\", \"message\": \"Fix #1737 (#1857)\", \"extraData\": \"key=e865a7a&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\"," + "\"url\": \"https://github.com/FasterXML/jackson-databind/commit/e8f043d1\", \"fixResolution\": \"release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-06-30\", \"message\": \"Fix #1680\", \"extraData\": \"key=e8f043d&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" } ], \"fixResolutionText\": \"Replace or update the following files: IllegalTypesCheckTest.java, VERSION, BeanDeserializerFactory.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\"," + @@ -962,7 +962,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { return result }) helper.registerAllowedMethod("fetchVulnerabilities", [Object.class], { - return new JsonUtils().parseJsonSerializable("{ \"alerts\": [] }").alerts + return new JsonUtils().jsonStringToGroovyObject("{ \"alerts\": [] }").alerts }) stepRule.step.whitesourceExecuteScan([ diff --git a/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy index 0a3159fff..1b376cb35 100644 --- a/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy +++ b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy @@ -12,7 +12,6 @@ import util.JenkinsLoggingRule import util.JenkinsSetupRule import util.LibraryLoadingTestExecutionListener -import static org.assertj.core.api.Assertions.assertThat import static org.hamcrest.Matchers.is import static org.junit.Assert.assertEquals import static org.junit.Assert.assertThat @@ -50,7 +49,7 @@ class DescriptorUtilsTest extends BasePiperTest { helper.registerAllowedMethod("readJSON", [Map.class], { searchConfig -> def packageJsonFile = new File("test/resources/DescriptorUtils/npm/${searchConfig.file}") - return new JsonUtils().parseJsonSerializable(packageJsonFile.text) + return new JsonUtils().jsonStringToGroovyObject(packageJsonFile.text) }) def gav = descriptorUtils.getNpmGAV('package2.json') @@ -66,7 +65,7 @@ class DescriptorUtilsTest extends BasePiperTest { helper.registerAllowedMethod("readJSON", [Map.class], { searchConfig -> def packageJsonFile = new File("test/resources/DescriptorUtils/npm/${searchConfig.file}") - return new JsonUtils().parseJsonSerializable(packageJsonFile.text) + return new JsonUtils().jsonStringToGroovyObject(packageJsonFile.text) }) def gav = descriptorUtils.getNpmGAV('package.json') @@ -82,7 +81,7 @@ class DescriptorUtilsTest extends BasePiperTest { helper.registerAllowedMethod("readJSON", [Map.class], { searchConfig -> def packageJsonFile = new File("test/resources/DescriptorUtils/npm/${searchConfig.file}") - return new JsonUtils().parseJsonSerializable(packageJsonFile.text) + return new JsonUtils().jsonStringToGroovyObject(packageJsonFile.text) }) def errorCaught = false @@ -102,7 +101,7 @@ class DescriptorUtilsTest extends BasePiperTest { helper.registerAllowedMethod("readJSON", [Map.class], { searchConfig -> def packageJsonFile = new File("test/resources/DescriptorUtils/sbt/${searchConfig.file}") - return new JsonUtils().parseJsonSerializable(packageJsonFile.text) + return new JsonUtils().jsonStringToGroovyObject(packageJsonFile.text) }) def gav = descriptorUtils.getSbtGAV('sbtDescriptor.json') @@ -119,7 +118,7 @@ class DescriptorUtilsTest extends BasePiperTest { helper.registerAllowedMethod("readJSON", [Map.class], { searchConfig -> def packageJsonFile = new File("test/resources/DescriptorUtils/dlang/${searchConfig.file}") - return new JsonUtils().parseJsonSerializable(packageJsonFile.text) + return new JsonUtils().jsonStringToGroovyObject(packageJsonFile.text) }) def gav = descriptorUtils.getDlangGAV('dub.json') diff --git a/vars/influxWriteData.groovy b/vars/influxWriteData.groovy index 2f51c74e2..77c1e4ec2 100644 --- a/vars/influxWriteData.groovy +++ b/vars/influxWriteData.groovy @@ -106,9 +106,9 @@ private void writeToInflux(config, script){ //write results into json file for archiving - also benefitial when no InfluxDB is available yet def jsonUtils = new JsonUtils() - writeFile file: 'jenkins_data.json', text: jsonUtils.getPrettyJsonString(config.customData) - writeFile file: 'influx_data.json', text: jsonUtils.getPrettyJsonString(config.customDataMap) - writeFile file: 'jenkins_data_tags.json', text: jsonUtils.getPrettyJsonString(config.customDataTags) - writeFile file: 'influx_data_tags.json', text: jsonUtils.getPrettyJsonString(config.customDataMapTags) + writeFile file: 'jenkins_data.json', text: jsonUtils.groovyObjectToPrettyJsonString(config.customData) + writeFile file: 'influx_data.json', text: jsonUtils.groovyObjectToPrettyJsonString(config.customDataMap) + writeFile file: 'jenkins_data_tags.json', text: jsonUtils.groovyObjectToPrettyJsonString(config.customDataTags) + writeFile file: 'influx_data_tags.json', text: jsonUtils.groovyObjectToPrettyJsonString(config.customDataMapTags) archiveArtifacts artifacts: '*data.json', allowEmptyArchive: true } diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 7a8f13076..898c78b77 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -15,47 +15,57 @@ import static com.sap.piper.Prerequisites.checkScript @Field String STEP_NAME = getClass().getName() @Field Set GENERAL_CONFIG_KEYS = [ - /** - * Wrapper object to bundle any of the other configuration settings on general and stage level. - */ 'whitesource', /** * Jenkins credentials ID referring to the organization admin's token. + * @parentConfigKey whitesource */ 'orgAdminUserTokenCredentialsId', /** * WhiteSource token identifying your organization. + * @parentConfigKey whitesource */ 'orgToken', /** * Name of the WhiteSource product to be created and used for results aggregation. + * @parentConfigKey whitesource */ 'productName', /** * Version of the WhiteSource product to be created and used for results aggregation, usually determined automatically. + * @parentConfigKey whitesource */ 'productVersion', /** * Token of the WhiteSource product to be created and used for results aggregation, usually determined automatically. + * @parentConfigKey whitesource */ 'productToken', /** * List of WhiteSource projects to be included in the assessment part of the step, usually determined automatically. + * @parentConfigKey whitesource */ 'projectNames', + /** + * URL used for downloading the Java Runtime Environment (JRE) required to run the WhiteSource Unified Agent. + * @parentConfigKey whitesource + */ + 'jreDownloadUrl', + /** + * URL to the WhiteSource server API used for communication, defaults to `https://saas.whitesourcesoftware.com/api`. + * @parentConfigKey whitesource + */ + 'serviceUrl', + /** + * Jenkins credentials ID referring to the product admin's token. + * @parentConfigKey whitesource + */ + 'userTokenCredentialsId', /** * Type of development stack used to implement the solution. * @possibleValues `maven`, `mta`, `npm`, `pip`, `sbt` */ 'scanType', - /** - * URL to the WhiteSource server API used for communication, defaults to `https://saas.whitesourcesoftware.com/api`. - */ - 'serviceUrl', - /** - * Jenkins credentials ID referring to the product admin's token. - */ - 'userTokenCredentialsId', /** * Whether verbose output should be produced. * @possibleValues `true`, `false` @@ -103,10 +113,6 @@ import static com.sap.piper.Prerequisites.checkScript * Docker workspace to be used for scanning. */ 'dockerWorkspace', - /** - * URL used for downloading the Java Runtime Environment (JRE) required to run the WhiteSource Unified Agent. - */ - 'jreDownloadUrl', /** * Whether license compliance is considered and reported as part of the assessment. * @possibleValues `true`, `false` @@ -353,7 +359,11 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU config.whitesource.productVersion = gav.version break } - config.whitesource['projectNames'].add("${config.whitesource.projectName} - ${config.whitesource.productVersion}".toString()) + + def projectName = "${config.whitesource.projectName} - ${config.whitesource.productVersion}".toString() + if(!config.whitesource['projectNames'].contains(projectName)) + config.whitesource['projectNames'].add(projectName) + WhitesourceConfigurationHelper.extendUAConfigurationFile(script, utils, config, path) dockerExecute(script: script, dockerImage: config.dockerImage, dockerWorkspace: config.dockerWorkspace, stashContent: config.stashContent) { if (config.whitesource.agentDownloadUrl) { @@ -447,7 +457,7 @@ int checkSecurityViolations(Map config, WhitesourceRepository repository) { severeVulnerabilities++ } - writeFile(file: "${config.vulnerabilityReportFileName}.json", text: new JsonUtils().getPrettyJsonString(vulnerabilities)) + writeFile(file: "${config.vulnerabilityReportFileName}.json", text: new JsonUtils().groovyObjectToPrettyJsonString(vulnerabilities)) writeFile(file: "${config.vulnerabilityReportFileName}.html", text: getReportHtml(config, vulnerabilities, severeVulnerabilities)) archiveArtifacts(artifacts: "${config.vulnerabilityReportFileName}.*") From 69f90ee7686ceeccaacead2db28fff5691ef3ef2 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Thu, 21 Mar 2019 13:34:33 +0100 Subject: [PATCH 071/130] Address sonar feedback --- src/com/sap/piper/integration/WhitesourceRepository.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy index 5bddabf64..61f5eb9c4 100644 --- a/src/com/sap/piper/integration/WhitesourceRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -81,8 +81,8 @@ class WhitesourceRepository implements Serializable { void sortVulnerabilitiesByScore(List vulnerabilities) { script.echo "${vulnerabilities.size() > 0 ? 'WARNING: ' : ''}found a total of ${vulnerabilities.size()} vulnerabilities" vulnerabilities.sort { o1, o2 -> - def cvss3score1 = o1.vulnerability.cvss3_score != 0 ? o1.vulnerability.cvss3_score : o1.vulnerability.score - def cvss3score2 = o2.vulnerability.cvss3_score != 0 ? o2.vulnerability.cvss3_score : o2.vulnerability.score + def cvss3score1 = o1.vulnerability.cvss3_score == 0 ? o1.vulnerability.score : o1.vulnerability.cvss3_score + def cvss3score2 = o2.vulnerability.cvss3_score == 0 ? o2.vulnerability.score : o2.vulnerability.cvss3_score def comparisionResult = cvss3score1 <=> cvss3score2 From 1870c6ab9c80ddfa8778d163720f056f349a28da Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Thu, 21 Mar 2019 14:06:23 +0100 Subject: [PATCH 072/130] Modify defaults --- src/com/sap/piper/WhitesourceConfigurationHelper.groovy | 4 ++-- vars/whitesourceExecuteScan.groovy | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index 510c181df..31346d094 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -86,11 +86,11 @@ class WhitesourceConfigurationHelper implements Serializable { ] break - default: - // maven + case 'maven': mapping += [ ] + break } rewriteConfiguration(script, utils, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 898c78b77..63a58ef5a 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -353,7 +353,7 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU config.whitesource.projectName = gav.artifact config.whitesource.productVersion = gav.version break - default: + case 'maven': gav = descriptorUtils.getMavenGAV(config.buildDescriptorFile) config.whitesource.projectName = gav.group + "." + gav.artifact config.whitesource.productVersion = gav.version From 8da169d2a8cb14b6818897664afb11f32b8e5b98 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 22 Mar 2019 11:55:50 +0100 Subject: [PATCH 073/130] Enable golang --- resources/default_pipeline_environment.yml | 6 +- src/com/sap/piper/DescriptorUtils.groovy | 35 ++++++++++-- test/groovy/WhitesourceExecuteScanTest.groovy | 57 +++++++++++++++++++ .../com/sap/piper/DescriptorUtilsTest.groovy | 28 +++++++-- .../WhiteSourceConfigurationHelperTest.groovy | 2 +- test/resources/DescriptorUtils/go/VERSION | 1 + vars/whitesourceExecuteScan.groovy | 7 +++ 7 files changed, 123 insertions(+), 13 deletions(-) create mode 100644 test/resources/DescriptorUtils/go/VERSION diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index cccd858d1..3652adfd4 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -305,9 +305,9 @@ steps: - 'buildDescriptor' - 'opensourceConfiguration' golang: - buildDescriptorFile: './setup.py' - dockerImage: 'golang:1.12.1-stretch' - dockerWorkspace: '/home/golang' + buildDescriptorFile: './glide.yaml' + dockerImage: 'instrumentisto/glide:0.13.1-go1.10' + dockerWorkspace: '/home/glide' stashContent: - 'buildDescriptor' - 'opensourceConfiguration' diff --git a/src/com/sap/piper/DescriptorUtils.groovy b/src/com/sap/piper/DescriptorUtils.groovy index c6976353a..979a72f41 100644 --- a/src/com/sap/piper/DescriptorUtils.groovy +++ b/src/com/sap/piper/DescriptorUtils.groovy @@ -72,7 +72,7 @@ def getSbtGAV(file = 'sbtDescriptor.json') { def getPipGAV(file = 'setup.py') { def result = [:] - def descriptor = sh(returnStdout: true, script: "cat ${file}") + def descriptor = readFile(file: file) result['group'] = '' result['packaging'] = '' @@ -81,16 +81,41 @@ def getPipGAV(file = 'setup.py') { if (result['version'] == '' || matches(method, result['version'])) { file = file.replace('setup.py', 'version.txt') - def versionString = sh(returnStdout: true, script: "cat ${file}") - if (versionString) { - result['version'] = versionString.trim() - } + result['version'] = getVersionFromFile(file) } echo "loaded ${result} from ${file}" return result } +def getGoGAV(file = './') { + def f = new File(file) + def path = f.getAbsoluteFile().getParentFile() + def result = [:] + + result['group'] = '' + result['packaging'] = '' + result['artifact'] = path.getName() + file = new File(path, 'version.txt').getAbsolutePath() + result['version'] = getVersionFromFile(file) + + if (!result['version']) { + file = new File(path, 'VERSION').getAbsolutePath() + result['version'] = getVersionFromFile(file) + } + + echo "loaded ${result} from ${file}" + return result +} + +private getVersionFromFile(file) { + def versionString = readFile(file: file) + if (versionString) { + return versionString.trim() + } + return '' +} + @NonCPS private def matches(regex, input) { def m = new Matcher(regex, input) diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index 174c1c4b1..48959ee67 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -452,6 +452,63 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=com.sap.sbt.test-scala')) } + @Test + void testGo() { + helper.registerAllowedMethod("readFile", [Map.class], { + map -> + def path = 'test/resources/DescriptorUtils/go/' + map.file.substring(map.file.lastIndexOf(File.separator) + 1, map.file.length()) + def descriptorFile = new File(path) + if(descriptorFile.exists()) + return descriptorFile.text + else + return null + }) + + helper.registerAllowedMethod("readProperties", [Map], { + def result = new Properties() + result.putAll([ + "apiKey": "b39d1328-52e2-42e3-98f0-932709daf3f0", + "productName": "SHC - Piper", + "checkPolicies": "true", + "projectName": "python-test", + "projectVersion": "1.0.0" + ]) + return result + }) + + stepRule.step.whitesourceExecuteScan([ + script : nullScript, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + descriptorUtilsStub : descriptorUtilsStub, + scanType : 'golang', + juStabUtils : utils, + productName : 'testProductName', + orgToken : 'testOrgToken', + reporting : false, + buildDescriptorFile : './myProject/glide.yaml' + ]) + + assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) + assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) + + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'instrumentisto/glide:0.13.1-go1.10')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/glide')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config d3aa80454919391024374ba46b4df082d15ab9a3'])) + + assertThat(shellRule.shell, Matchers.hasItems( + is('curl --location --output wss-unified-agent.jar https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/wss-unified-agent.jar'), + is('./bin/java -jar wss-unified-agent.jar -c \'./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProductName\'') + )) + + assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('apiKey=testOrgToken')) + assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productName=testProductName')) + assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('userKey=token-0815')) + assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1.2.3')) + assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=myProject')) + } + + @Test void testAgentNoDownloads() { helper.registerAllowedMethod("readProperties", [Map], { diff --git a/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy index 1b376cb35..01a9b814a 100644 --- a/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy +++ b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy @@ -131,9 +131,9 @@ class DescriptorUtilsTest extends BasePiperTest { @Test void testGetPipGAV() { - helper.registerAllowedMethod("sh", [Map.class], { + helper.registerAllowedMethod("readFile", [Map.class], { map -> - def descriptorFile = new File("test/resources/utilsTest/${map.script.substring(4, map.script.size())}") + def descriptorFile = new File("test/resources/utilsTest/${map.file}") return descriptorFile.text }) @@ -147,9 +147,9 @@ class DescriptorUtilsTest extends BasePiperTest { @Test void testGetPipGAVFromVersionTxt() { - helper.registerAllowedMethod("sh", [Map.class], { + helper.registerAllowedMethod("readFile", [Map.class], { map -> - def descriptorFile = new File("test/resources/DescriptorUtils/pip/${map.script.substring(4, map.script.size())}") + def descriptorFile = new File("test/resources/DescriptorUtils/pip/${map.file}") return descriptorFile.text }) @@ -208,4 +208,24 @@ class DescriptorUtilsTest extends BasePiperTest { assertEquals(gav.version, '1.2.4') assertEquals(gav.packaging, 'jar') } + + @Test + void testGetGoGAV() { + + helper.registerAllowedMethod("readFile", [Map.class], { + map -> + def path = 'test/resources/DescriptorUtils/go/' + map.file.substring(map.file.lastIndexOf(File.separator) + 1, map.file.length()) + def descriptorFile = new File(path) + if(descriptorFile.exists()) + return descriptorFile.text + else + return null + }) + + def gav = descriptorUtils.getGoGAV('./myProject/glide.yaml') + + assertEquals('', gav.group) + assertEquals('myProject', gav.artifact) + assertEquals('1.2.3', gav.version) + } } diff --git a/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy b/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy index 2ebd5e630..251998ccc 100644 --- a/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy +++ b/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy @@ -90,7 +90,7 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { } @Test - void testExtendConfigurationFileUnifiedAgentVerbose() { + void testExtendConfigurationFileUnifiedAgentGolangVerbose() { def config = [scanType: 'golang', whitesource: [configFilePath: './config', serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'SHC - name2', productToken: '1234', userKey: '0000'], stashContent: ['some', 'stashes'], verbose: true] WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, config, "./") assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) diff --git a/test/resources/DescriptorUtils/go/VERSION b/test/resources/DescriptorUtils/go/VERSION new file mode 100644 index 000000000..0495c4a88 --- /dev/null +++ b/test/resources/DescriptorUtils/go/VERSION @@ -0,0 +1 @@ +1.2.3 diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 63a58ef5a..16a4eca78 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -353,6 +353,13 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU config.whitesource.projectName = gav.artifact config.whitesource.productVersion = gav.version break + case 'golang': + gav = descriptorUtils.getGoGAV(config.buildDescriptorFile) + config.whitesource.projectName = gav.artifact + config.whitesource.productVersion = gav.version + break + case 'dlang': + break case 'maven': gav = descriptorUtils.getMavenGAV(config.buildDescriptorFile) config.whitesource.projectName = gav.group + "." + gav.artifact From c1ad3765e06f7b33ebbbc03eae04f7d626b22fd5 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 22 Mar 2019 12:06:03 +0100 Subject: [PATCH 074/130] Fix java.nio.file.NoSuchFileException --- src/com/sap/piper/DescriptorUtils.groovy | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/com/sap/piper/DescriptorUtils.groovy b/src/com/sap/piper/DescriptorUtils.groovy index 979a72f41..dc532f178 100644 --- a/src/com/sap/piper/DescriptorUtils.groovy +++ b/src/com/sap/piper/DescriptorUtils.groovy @@ -109,10 +109,12 @@ def getGoGAV(file = './') { } private getVersionFromFile(file) { - def versionString = readFile(file: file) - if (versionString) { - return versionString.trim() - } + try { + def versionString = readFile(file: file) + if (versionString) { + return versionString.trim() + } + } catch (java.nio.file.NoSuchFileException e) {/* NOP */} return '' } From a7f257dfe3e1e4ff7c1d06394c8ff6a91cf37322 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 22 Mar 2019 12:11:20 +0100 Subject: [PATCH 075/130] Fix missing reference --- vars/dockerExecuteOnKubernetes.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/dockerExecuteOnKubernetes.groovy b/vars/dockerExecuteOnKubernetes.groovy index ea21b8f85..afc3f728a 100644 --- a/vars/dockerExecuteOnKubernetes.groovy +++ b/vars/dockerExecuteOnKubernetes.groovy @@ -200,7 +200,7 @@ private String generatePodSpec(Map config) { ] podSpec.spec.securityContext = getSecurityContext(config) - return new JsonUtils().getPrettyJsonString(podSpec) + return new JsonUtils().groovyObjectToPrettyJsonString(podSpec) } From 10c7485808664d5a8b399df827ea3ba3c6ed99f4 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 22 Mar 2019 13:08:48 +0100 Subject: [PATCH 076/130] Fix project name determination for Go --- test/groovy/WhitesourceExecuteScanTest.groovy | 58 ++++++++++++++++ vars/whitesourceExecuteScan.groovy | 68 +++++++++++-------- 2 files changed, 96 insertions(+), 30 deletions(-) diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index 48959ee67..e00f951e3 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -508,6 +508,64 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=myProject')) } + @Test + void testGoDefaults() { + nullScript.commonPipelineEnvironment.setGithubOrg("testOrg") + nullScript.commonPipelineEnvironment.setGithubRepo("testRepo") + + helper.registerAllowedMethod("readFile", [Map.class], { + map -> + def path = 'test/resources/DescriptorUtils/go/' + map.file.substring(map.file.lastIndexOf(File.separator) + 1, map.file.length()) + def descriptorFile = new File(path) + if(descriptorFile.exists()) + return descriptorFile.text + else + return null + }) + + helper.registerAllowedMethod("readProperties", [Map], { + def result = new Properties() + result.putAll([ + "apiKey": "b39d1328-52e2-42e3-98f0-932709daf3f0", + "productName": "SHC - Piper", + "checkPolicies": "true", + "projectName": "python-test", + "projectVersion": "1.0.0" + ]) + return result + }) + + stepRule.step.whitesourceExecuteScan([ + script : nullScript, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub : whitesourceOrgAdminRepositoryStub, + descriptorUtilsStub : descriptorUtilsStub, + scanType : 'golang', + juStabUtils : utils, + productName : 'testProductName', + orgToken : 'testOrgToken', + reporting : false + ]) + + assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) + assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) + + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'instrumentisto/glide:0.13.1-go1.10')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/glide')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config d3aa80454919391024374ba46b4df082d15ab9a3'])) + + assertThat(shellRule.shell, Matchers.hasItems( + is('curl --location --output wss-unified-agent.jar https://github.com/whitesource/unified-agent-distribution/raw/master/standAlone/wss-unified-agent.jar'), + is('./bin/java -jar wss-unified-agent.jar -c \'./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3\' -apiKey \'testOrgToken\' -userKey \'token-0815\' -product \'testProductName\'') + )) + + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('apiKey=testOrgToken')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productName=testProductName')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('userKey=token-0815')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1.2.3')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=testOrg.testRepo')) + } + @Test void testAgentNoDownloads() { diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 16a4eca78..42e0a91eb 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -336,36 +336,7 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU break default: def path = config.buildDescriptorFile.substring(0, config.buildDescriptorFile.lastIndexOf('/') + 1) - def gav - switch (config.scanType) { - case 'npm': - gav = descriptorUtils.getNpmGAV(config.buildDescriptorFile) - config.whitesource.projectName = gav.group + "." + gav.artifact - config.whitesource.productVersion = gav.version - break - case 'sbt': - gav = descriptorUtils.getSbtGAV(config.buildDescriptorFile) - config.whitesource.projectName = gav.group + "." + gav.artifact - config.whitesource.productVersion = gav.version - break - case 'pip': - gav = descriptorUtils.getPipGAV(config.buildDescriptorFile) - config.whitesource.projectName = gav.artifact - config.whitesource.productVersion = gav.version - break - case 'golang': - gav = descriptorUtils.getGoGAV(config.buildDescriptorFile) - config.whitesource.projectName = gav.artifact - config.whitesource.productVersion = gav.version - break - case 'dlang': - break - case 'maven': - gav = descriptorUtils.getMavenGAV(config.buildDescriptorFile) - config.whitesource.projectName = gav.group + "." + gav.artifact - config.whitesource.productVersion = gav.version - break - } + resolveProjectIdentifiers(script, descriptorUtils, config) def projectName = "${config.whitesource.projectName} - ${config.whitesource.productVersion}".toString() if(!config.whitesource['projectNames'].contains(projectName)) @@ -415,6 +386,43 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU } } +private resolveProjectIdentifiers(script, descriptorUtils, config) { + if (!config.whitesource.projectName || !config.whitesource.productVersion) { + def gav + switch (config.scanType) { + case 'npm': + gav = descriptorUtils.getNpmGAV(config.buildDescriptorFile) + break + case 'sbt': + gav = descriptorUtils.getSbtGAV(config.buildDescriptorFile) + break + case 'pip': + gav = descriptorUtils.getPipGAV(config.buildDescriptorFile) + break + case 'golang': + gav = descriptorUtils.getGoGAV(config.buildDescriptorFile) + if (gav.artifact == '.') { + gav.group = script.commonPipelineEnvironment.getGithubOrg() + gav.artifact = script.commonPipelineEnvironment.getGithubRepo() + } + break + case 'dlang': + break + case 'maven': + gav = descriptorUtils.getMavenGAV(config.buildDescriptorFile) + break + } + + if(!config.whitesource.projectName) { + config.whitesource.projectName = "${gav.group?:''}${gav.group?'.':''}${gav.artifact}" + } + + if (!config.whitesource.productVersion) { + config.whitesource.productVersion = gav.version + } + } +} + void analyseWhitesourceResults(Map config, WhitesourceRepository repository) { def pdfName = "whitesource-riskReport.pdf" repository.fetchReportForProduct(pdfName) From 8470e14aed92a823a7fb8fa34b0716d992e0e9a6 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 22 Mar 2019 13:50:11 +0100 Subject: [PATCH 077/130] Fix Go image --- resources/default_pipeline_environment.yml | 2 +- test/groovy/WhitesourceExecuteScanTest.groovy | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 3f7f6a59a..831cad3c4 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -326,7 +326,7 @@ steps: - 'opensourceConfiguration' golang: buildDescriptorFile: './glide.yaml' - dockerImage: 'instrumentisto/glide:0.13.1-go1.10' + dockerImage: 'fabiorphp/golang-glide:1.11-stretch' dockerWorkspace: '/home/glide' stashContent: - 'buildDescriptor' diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index e00f951e3..64c65ae40 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -492,7 +492,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'instrumentisto/glide:0.13.1-go1.10')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'fabiorphp/golang-glide:1.11-stretch')) assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/glide')) assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config d3aa80454919391024374ba46b4df082d15ab9a3'])) @@ -550,7 +550,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'instrumentisto/glide:0.13.1-go1.10')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'fabiorphp/golang-glide:1.11-stretch')) assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/glide')) assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config d3aa80454919391024374ba46b4df082d15ab9a3'])) From 65dc3793b0b2f78b55cf4fcc5f57e531ccd5129d Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 22 Mar 2019 14:13:05 +0100 Subject: [PATCH 078/130] Fix default for Go version determination --- src/com/sap/piper/DescriptorUtils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/com/sap/piper/DescriptorUtils.groovy b/src/com/sap/piper/DescriptorUtils.groovy index dc532f178..6102cf7fc 100644 --- a/src/com/sap/piper/DescriptorUtils.groovy +++ b/src/com/sap/piper/DescriptorUtils.groovy @@ -88,7 +88,7 @@ def getPipGAV(file = 'setup.py') { return result } -def getGoGAV(file = './') { +def getGoGAV(file = './glide.yaml') { def f = new File(file) def path = f.getAbsoluteFile().getParentFile() def result = [:] From b1cf27aa23139385ffa4810f73e43aec83257caf Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 22 Mar 2019 14:41:10 +0100 Subject: [PATCH 079/130] Improve test --- .../WhitesourceConfigurationHelper.groovy | 4 +-- ...WhitesourceConfigurationHelperTest.groovy} | 34 +++++++++++++++++-- 2 files changed, 33 insertions(+), 5 deletions(-) rename test/groovy/com/sap/piper/{WhiteSourceConfigurationHelperTest.groovy => WhitesourceConfigurationHelperTest.groovy} (77%) diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index 31346d094..51973cd3a 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -23,7 +23,7 @@ class WhitesourceConfigurationHelper implements Serializable { ] } if(config.verbose) - mapping += [name: 'log.level', value: 'debug', force: true] + mapping += [name: 'log.level', value: 'debug'] mapping += [ [name: 'apiKey', value: config.whitesource.orgToken, force: true], @@ -100,7 +100,7 @@ class WhitesourceConfigurationHelper implements Serializable { def inputFilePath = "${path}${inputFile}" def outputFilePath = "${path}${targetFile}" def moduleSpecificFile = parsingClosure(inputFilePath) - if (!moduleSpecificFile) + if (!moduleSpecificFile && inputFilePath != config.whitesource.configFilePath) moduleSpecificFile = parsingClosure(config.whitesource.configFilePath) if (!moduleSpecificFile) moduleSpecificFile = [:] diff --git a/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy b/test/groovy/com/sap/piper/WhitesourceConfigurationHelperTest.groovy similarity index 77% rename from test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy rename to test/groovy/com/sap/piper/WhitesourceConfigurationHelperTest.groovy index 251998ccc..e49ed29ea 100644 --- a/test/groovy/com/sap/piper/WhiteSourceConfigurationHelperTest.groovy +++ b/test/groovy/com/sap/piper/WhitesourceConfigurationHelperTest.groovy @@ -15,7 +15,7 @@ import static org.hamcrest.Matchers.hasItem import static org.hamcrest.Matchers.not import static org.junit.Assert.assertThat -class WhiteSourceConfigurationHelperTest extends BasePiperTest { +class WhitesourceConfigurationHelperTest extends BasePiperTest { JenkinsReadFileRule jrfr = new JenkinsReadFileRule(this, 'test/resources/utilsTest/') JenkinsWriteFileRule jwfr = new JenkinsWriteFileRule(this) JenkinsLoggingRule jlr = new JenkinsLoggingRule(this) @@ -29,7 +29,35 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { @Before void init() { + def p = new Properties() + p.put("log.level", "debug") + helper.registerAllowedMethod('readProperties', [Map], {return p}) + } + + @Test + void testExtendConfigurationFileUnifiedAgentEmptyConfig() { helper.registerAllowedMethod('readProperties', [Map], {return new Properties()}) + WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'none', whitesource: [configFilePath: './config',serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'DIST - name1', productToken: '1234', userKey: '0000']], "./") + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], not(containsString("log.level=debug"))) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + + assertThat(jlr.log, containsString("[Whitesource] Configuration for scanType: 'none' is not yet hardened, please do a quality assessment of your scan results.")) + } + + @Test + void testExtendConfigurationFileUnifiedAgentConfigDeeper() { + helper.registerAllowedMethod('readProperties', [Map], { m -> if (!m.file.contains('testModule')) return new Properties() else return null }) + WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'none', whitesource: [configFilePath: './config',serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'DIST - name1', productToken: '1234', userKey: '0000']], "./testModule/") + assertThat(jwfr.files['./testModule/config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], not(containsString("log.level=debug"))) + assertThat(jwfr.files['./testModule/config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) + assertThat(jwfr.files['./testModule/config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) + assertThat(jwfr.files['./testModule/config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) + assertThat(jwfr.files['./testModule/config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + + assertThat(jlr.log, containsString("[Whitesource] Configuration for scanType: 'none' is not yet hardened, please do a quality assessment of your scan results.")) } @Test @@ -110,8 +138,8 @@ class WhiteSourceConfigurationHelperTest extends BasePiperTest { p.putAll(['python.resolveDependencies': 'false', 'python.ignoreSourceFiles': 'false', 'python.ignorePipInstallErrors': 'true','python.installVirtualenv': 'false']) helper.registerAllowedMethod('readProperties', [Map], {return p}) - WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', whitesource: [configFilePath: './config', serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'name', productToken: '1234', userKey: '0000'], verbose: true], "./") - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) + WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', whitesource: [configFilePath: './config', serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'cdfg', productName: 'name', productToken: '1234', userKey: '0000'], verbose: true], "./") + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=cdfg")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=name")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) From 9322769d4708e895c5aeee175139db5fe91ba6c2 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 22 Mar 2019 14:43:10 +0100 Subject: [PATCH 080/130] Improve reported version --- test/groovy/WhitesourceExecuteScanTest.groovy | 32 +++++++++---------- vars/whitesourceExecuteScan.groovy | 4 +-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index 64c65ae40..aa93f506b 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -238,7 +238,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('apiKey=testOrgToken')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productName=testProductName')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productToken=testProductToken')) - assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1.2.3')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=com.sap.node.test-node')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('userKey=token-0815')) } @@ -318,7 +318,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('apiKey=testOrgToken')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productName=testProductName')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('userKey=token-0815')) - assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1.2.3')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=test-python')) } @@ -364,7 +364,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('apiKey=testOrgToken')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productName=testProductName')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('userKey=token-0815')) - assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1.2.3')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=test-python')) } @@ -448,7 +448,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('apiKey=testOrgToken')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productName=testProductName')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('userKey=token-0815')) - assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1.2.3')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=com.sap.sbt.test-scala')) } @@ -504,7 +504,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('apiKey=testOrgToken')) assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productName=testProductName')) assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('userKey=token-0815')) - assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1.2.3')) + assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1')) assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=myProject')) } @@ -530,7 +530,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { "productName": "SHC - Piper", "checkPolicies": "true", "projectName": "python-test", - "projectVersion": "1.0.0" + "projectVersion": "2.0.0" ]) return result }) @@ -562,7 +562,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('apiKey=testOrgToken')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productName=testProductName')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('userKey=token-0815')) - assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1.2.3')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=testOrg.testRepo')) } @@ -576,7 +576,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { "productName": "SHC - Piper", "checkPolicies": "true", "projectName": "python-test", - "projectVersion": "1.0.0" + "projectVersion": "2.0.0" ]) return result }) @@ -606,7 +606,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { "productName": "SHC - Piper", "checkPolicies": "true", "projectName": "python-test", - "projectVersion": "1.0.0" + "projectVersion": "2.0.0" ]) return result }) @@ -678,9 +678,9 @@ class WhitesourceExecuteScanTest extends BasePiperTest { ) ) - assertThat(whitesourceCalls[0]['whitesource']['projectNames'], contains("com.sap.maven.test-java - 1.2.3", "com.sap.node.test-node - 1.2.3", "test-python - 1.2.3")) - assertThat(whitesourceCalls[1]['whitesource']['projectNames'], contains("com.sap.maven.test-java - 1.2.3", "com.sap.node.test-node - 1.2.3", "test-python - 1.2.3")) - assertThat(whitesourceCalls[2]['whitesource']['projectNames'], contains("com.sap.maven.test-java - 1.2.3", "com.sap.node.test-node - 1.2.3", "test-python - 1.2.3")) + assertThat(whitesourceCalls[0]['whitesource']['projectNames'], contains("com.sap.maven.test-java - 1", "com.sap.node.test-node - 1", "test-python - 1")) + assertThat(whitesourceCalls[1]['whitesource']['projectNames'], contains("com.sap.maven.test-java - 1", "com.sap.node.test-node - 1", "test-python - 1")) + assertThat(whitesourceCalls[2]['whitesource']['projectNames'], contains("com.sap.maven.test-java - 1", "com.sap.node.test-node - 1", "test-python - 1")) } @Test @@ -910,7 +910,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { "productName": "SHC - Piper", "checkPolicies": "true", "projectName": "python-test", - "projectVersion": "1.0.0" + "projectVersion": "2.0.0" ]) return result }) @@ -992,7 +992,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { "productName": "SHC - Piper", "checkPolicies": "true", "projectName": "pipeline-test-node", - "projectVersion": "1.0.0" + "projectVersion": "2.0.0" ]) return result }) @@ -1031,7 +1031,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { "productName": "SHC - Piper", "checkPolicies": "true", "projectName": "pipeline-test-node", - "projectVersion": "1.0.0" + "projectVersion": "2.0.0" ]) return result }) @@ -1072,7 +1072,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { "productName": "SHC - Piper", "checkPolicies": "true", "projectName": "pipeline-test-node", - "projectVersion": "1.0.0" + "projectVersion": "2.0.0" ]) return result }) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 42e0a91eb..7b8d53274 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -338,7 +338,7 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU def path = config.buildDescriptorFile.substring(0, config.buildDescriptorFile.lastIndexOf('/') + 1) resolveProjectIdentifiers(script, descriptorUtils, config) - def projectName = "${config.whitesource.projectName} - ${config.whitesource.productVersion}".toString() + def projectName = "${config.whitesource.projectName}${config.whitesource.productVersion?' - ':''}${config.whitesource.productVersion}".toString() if(!config.whitesource['projectNames'].contains(projectName)) config.whitesource['projectNames'].add(projectName) @@ -418,7 +418,7 @@ private resolveProjectIdentifiers(script, descriptorUtils, config) { } if (!config.whitesource.productVersion) { - config.whitesource.productVersion = gav.version + config.whitesource.productVersion = gav.version?.tokenize('.')?.get(0) } } } From 3ef908ff068b76718cdea7361bc30e92c89809f3 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 22 Mar 2019 14:59:51 +0100 Subject: [PATCH 081/130] Fix path to read version --- src/com/sap/piper/DescriptorUtils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/com/sap/piper/DescriptorUtils.groovy b/src/com/sap/piper/DescriptorUtils.groovy index 6102cf7fc..f35025c11 100644 --- a/src/com/sap/piper/DescriptorUtils.groovy +++ b/src/com/sap/piper/DescriptorUtils.groovy @@ -90,7 +90,7 @@ def getPipGAV(file = 'setup.py') { def getGoGAV(file = './glide.yaml') { def f = new File(file) - def path = f.getAbsoluteFile().getParentFile() + def path = f.getCanonicalFile().getParentFile() def result = [:] result['group'] = '' From c269746ed366bcc7141395e073c67bfc3725bf77 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 22 Mar 2019 15:04:19 +0100 Subject: [PATCH 082/130] Fix version determination --- vars/whitesourceExecuteScan.groovy | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 7b8d53274..eb15d3876 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -413,13 +413,12 @@ private resolveProjectIdentifiers(script, descriptorUtils, config) { break } - if(!config.whitesource.projectName) { + if(!config.whitesource.projectName) config.whitesource.projectName = "${gav.group?:''}${gav.group?'.':''}${gav.artifact}" - } - if (!config.whitesource.productVersion) { - config.whitesource.productVersion = gav.version?.tokenize('.')?.get(0) - } + def version = gav.version?.tokenize('.')?.head() + if(version && !config.whitesource.productVersion) + config.whitesource.productVersion = version } } From 50cc1cb32d5e539e03900afda45368ab84f2827b Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 25 Mar 2019 08:41:30 +0100 Subject: [PATCH 083/130] Fix version determination 2 --- vars/whitesourceExecuteScan.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index eb15d3876..b905e817f 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -416,7 +416,8 @@ private resolveProjectIdentifiers(script, descriptorUtils, config) { if(!config.whitesource.projectName) config.whitesource.projectName = "${gav.group?:''}${gav.group?'.':''}${gav.artifact}" - def version = gav.version?.tokenize('.')?.head() + def versionFragments = gav.version?.tokenize('.') + def version = versionFragments ? versionFragments.head() : null if(version && !config.whitesource.productVersion) config.whitesource.productVersion = version } From db8c86e9de44a67c075e0964cf80f6690093cc2e Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 25 Mar 2019 08:51:23 +0100 Subject: [PATCH 084/130] Fix project name determination --- vars/whitesourceExecuteScan.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index b905e817f..469ce70a1 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -401,7 +401,7 @@ private resolveProjectIdentifiers(script, descriptorUtils, config) { break case 'golang': gav = descriptorUtils.getGoGAV(config.buildDescriptorFile) - if (gav.artifact == '.') { + if (!gav.artifact) { gav.group = script.commonPipelineEnvironment.getGithubOrg() gav.artifact = script.commonPipelineEnvironment.getGithubRepo() } From d1223fb346287e6cb596563834283b2697ea6373 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 25 Mar 2019 09:01:49 +0100 Subject: [PATCH 085/130] Temporary verbosity --- src/com/sap/piper/DescriptorUtils.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/com/sap/piper/DescriptorUtils.groovy b/src/com/sap/piper/DescriptorUtils.groovy index f35025c11..eacc53dc1 100644 --- a/src/com/sap/piper/DescriptorUtils.groovy +++ b/src/com/sap/piper/DescriptorUtils.groovy @@ -114,7 +114,9 @@ private getVersionFromFile(file) { if (versionString) { return versionString.trim() } - } catch (java.nio.file.NoSuchFileException e) {/* NOP */} + } catch (java.nio.file.NoSuchFileException e) { + echo "Failed to load version string from file ${file} due to ${e}" + } return '' } From c71f4209951f94aebe6eac705efe099048c0fb39 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 25 Mar 2019 09:42:25 +0100 Subject: [PATCH 086/130] Fix version handling --- src/com/sap/piper/WhitesourceConfigurationHelper.groovy | 4 ++-- vars/whitesourceExecuteScan.groovy | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index 51973cd3a..3170e8493 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -28,9 +28,9 @@ class WhitesourceConfigurationHelper implements Serializable { mapping += [ [name: 'apiKey', value: config.whitesource.orgToken, force: true], [name: 'productName', value: config.whitesource.productName, force: true], - [name: 'productVersion', value: config.whitesource.productVersion, force: true], + [name: 'productVersion', value: config.whitesource.productVersion?:'', force: true], [name: 'projectName', value: config.whitesource.projectName, force: true], - [name: 'projectVersion', value: config.whitesource.productVersion, force: true], + [name: 'projectVersion', value: config.whitesource.productVersion?:'', force: true], [name: 'productToken', value: config.whitesource.productToken, omitIfPresent: 'projectToken', force: true], [name: 'userKey', value: config.whitesource.userKey, force: true], [name: 'forceUpdate', value: true, force: true], diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 469ce70a1..472ad9f8e 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -338,7 +338,7 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU def path = config.buildDescriptorFile.substring(0, config.buildDescriptorFile.lastIndexOf('/') + 1) resolveProjectIdentifiers(script, descriptorUtils, config) - def projectName = "${config.whitesource.projectName}${config.whitesource.productVersion?' - ':''}${config.whitesource.productVersion}".toString() + def projectName = "${config.whitesource.projectName}${config.whitesource.productVersion?' - ':''}${config.whitesource.productVersion?:''}".toString() if(!config.whitesource['projectNames'].contains(projectName)) config.whitesource['projectNames'].add(projectName) From f928878c5e7da541125734e9364cd83dd4d44a5f Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 25 Mar 2019 10:38:00 +0100 Subject: [PATCH 087/130] Fix path handling --- src/com/sap/piper/DescriptorUtils.groovy | 9 ++++----- test/groovy/WhitesourceExecuteScanTest.groovy | 6 +++--- vars/whitesourceExecuteScan.groovy | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/com/sap/piper/DescriptorUtils.groovy b/src/com/sap/piper/DescriptorUtils.groovy index eacc53dc1..86b4544ed 100644 --- a/src/com/sap/piper/DescriptorUtils.groovy +++ b/src/com/sap/piper/DescriptorUtils.groovy @@ -89,18 +89,17 @@ def getPipGAV(file = 'setup.py') { } def getGoGAV(file = './glide.yaml') { - def f = new File(file) - def path = f.getCanonicalFile().getParentFile() + def path = file.substring(0, file.lastIndexOf('/') + 1) def result = [:] result['group'] = '' result['packaging'] = '' - result['artifact'] = path.getName() - file = new File(path, 'version.txt').getAbsolutePath() + result['artifact'] = path?.replaceAll(/\./, '')?.replaceAll('/', '') + file = path + 'version.txt' result['version'] = getVersionFromFile(file) if (!result['version']) { - file = new File(path, 'VERSION').getAbsolutePath() + file = path + 'VERSION' result['version'] = getVersionFromFile(file) } diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index aa93f506b..e0f9ad569 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -456,7 +456,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { void testGo() { helper.registerAllowedMethod("readFile", [Map.class], { map -> - def path = 'test/resources/DescriptorUtils/go/' + map.file.substring(map.file.lastIndexOf(File.separator) + 1, map.file.length()) + def path = 'test/resources/DescriptorUtils/go/' + map.file.substring(map.file.lastIndexOf('/') + 1, map.file.length()) def descriptorFile = new File(path) if(descriptorFile.exists()) return descriptorFile.text @@ -471,7 +471,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { "productName": "SHC - Piper", "checkPolicies": "true", "projectName": "python-test", - "projectVersion": "1.0.0" + "projectVersion": "2.0.0" ]) return result }) @@ -515,7 +515,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { helper.registerAllowedMethod("readFile", [Map.class], { map -> - def path = 'test/resources/DescriptorUtils/go/' + map.file.substring(map.file.lastIndexOf(File.separator) + 1, map.file.length()) + def path = 'test/resources/DescriptorUtils/go/' + map.file.substring(map.file.lastIndexOf('/') + 1, map.file.length()) def descriptorFile = new File(path) if(descriptorFile.exists()) return descriptorFile.text diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 472ad9f8e..f65d08f22 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -417,7 +417,7 @@ private resolveProjectIdentifiers(script, descriptorUtils, config) { config.whitesource.projectName = "${gav.group?:''}${gav.group?'.':''}${gav.artifact}" def versionFragments = gav.version?.tokenize('.') - def version = versionFragments ? versionFragments.head() : null + def version = versionFragments.size() > 0 ? versionFragments.head() : null if(version && !config.whitesource.productVersion) config.whitesource.productVersion = version } From 6fbf659553c2b7614d9f6bab93d038fdd6b4aed4 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 25 Mar 2019 10:58:10 +0100 Subject: [PATCH 088/130] Extend build descriptor stash for glide --- resources/default_pipeline_environment.yml | 2 +- src/com/sap/piper/WhitesourceConfigurationHelper.groovy | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 831cad3c4..452eda928 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -356,7 +356,7 @@ steps: pipelineStashFilesBeforeBuild: runCheckmarx: false stashIncludes: - buildDescriptor: '**/pom.xml, **/.mvn/**, **/assembly.xml, **/.swagger-codegen-ignore, **/package.json, **/requirements.txt, **/setup.py, **/whitesource_config.py, **/mta*.y*ml, **/.npmrc, **/whitesource.*.json, **/whitesource-fs-agent.config, Dockerfile, **/VERSION, **/version.txt, **/build.sbt, **/sbtDescriptor.json, **/project/*' + buildDescriptor: '**/pom.xml, **/.mvn/**, **/assembly.xml, **/.swagger-codegen-ignore, **/package.json, **/requirements.txt, **/setup.py, **/whitesource_config.py, **/mta*.y*ml, **/.npmrc, **/whitesource.*.json, **/whitesource-fs-agent.config, Dockerfile, **/VERSION, **/version.txt, **/glide.*, **/Gopkg.*, **/build.sbt, **/sbtDescriptor.json, **/project/*' deployDescriptor: '**/manifest*.y*ml, **/*.mtaext.y*ml, **/*.mtaext, **/xs-app.json, helm/**, *.y*ml' git: '**/gitmetadata/**' opa5: '**/*.*' diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index 3170e8493..f320252ed 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -38,8 +38,11 @@ class WhitesourceConfigurationHelper implements Serializable { [name: 'ignoreSourceFiles', value: true, force: true], [name: 'resolveAllDependencies', value: false, force: true] ] + + // To be removed once all scenarios are fully supported if(!['pip', 'golang'].contains(config.scanType)) script.echo "[Warning][Whitesource] Configuration for scanType: '${config.scanType}' is not yet hardened, please do a quality assessment of your scan results." + switch (config.scanType) { case 'npm': mapping += [ @@ -74,7 +77,7 @@ class WhitesourceConfigurationHelper implements Serializable { [name: 'go.resolveDependencies', value: true, force: true], [name: 'go.ignoreSourceFiles', value: true, force: true], [name: 'go.collectDependenciesAtRuntime', value: true], - [name: 'go.dependencyManager', value: ''], + [name: 'go.dependencyManager', value: 'glide'], [name: 'includes', value: '**/*.lock'], [name: 'excludes', value: '**/*sources.jar **/*javadoc.jar'], [name: 'case.sensitive.glob', value: false], From 686de62782d13d6aa2fb30666049695d129eb392 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 25 Mar 2019 11:50:40 +0100 Subject: [PATCH 089/130] Fix includes for glide build --- src/com/sap/piper/WhitesourceConfigurationHelper.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index f320252ed..ae399f1f4 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -78,7 +78,7 @@ class WhitesourceConfigurationHelper implements Serializable { [name: 'go.ignoreSourceFiles', value: true, force: true], [name: 'go.collectDependenciesAtRuntime', value: true], [name: 'go.dependencyManager', value: 'glide'], - [name: 'includes', value: '**/*.lock'], + [name: 'includes', value: '**/glide.yaml **/glide.lock'], [name: 'excludes', value: '**/*sources.jar **/*javadoc.jar'], [name: 'case.sensitive.glob', value: false], [name: 'followSymbolicLinks', value: true] From 7de9718e13c41722ad9d006c34030fd4b72dc1b3 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 25 Mar 2019 12:50:00 +0100 Subject: [PATCH 090/130] Fix interpretation of descriptor --- src/com/sap/piper/DescriptorUtils.groovy | 4 +- test/groovy/WhitesourceExecuteScanTest.groovy | 42 +++++++++++++++++-- test/resources/DescriptorUtils/go/glide.yaml | 7 ++++ vars/whitesourceExecuteScan.groovy | 4 -- 4 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 test/resources/DescriptorUtils/go/glide.yaml diff --git a/src/com/sap/piper/DescriptorUtils.groovy b/src/com/sap/piper/DescriptorUtils.groovy index 86b4544ed..624f5ae64 100644 --- a/src/com/sap/piper/DescriptorUtils.groovy +++ b/src/com/sap/piper/DescriptorUtils.groovy @@ -89,12 +89,14 @@ def getPipGAV(file = 'setup.py') { } def getGoGAV(file = './glide.yaml') { + def descriptor = readYaml(file: file) def path = file.substring(0, file.lastIndexOf('/') + 1) + def module = path?.replaceAll(/\./, '')?.replaceAll('/', '') def result = [:] result['group'] = '' result['packaging'] = '' - result['artifact'] = path?.replaceAll(/\./, '')?.replaceAll('/', '') + result['artifact'] = "${descriptor.package}${module?'.':''}${module?:''}" file = path + 'version.txt' result['version'] = getVersionFromFile(file) diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index e0f9ad569..671a2b0ab 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -12,6 +12,7 @@ import org.junit.Test import org.junit.rules.ExpectedException import org.junit.rules.RuleChain import org.springframework.beans.factory.annotation.Autowired +import org.yaml.snakeyaml.Yaml import util.* import static org.hamcrest.Matchers.* @@ -454,6 +455,24 @@ class WhitesourceExecuteScanTest extends BasePiperTest { @Test void testGo() { + helper.registerAllowedMethod("readYaml", [Map.class], { + map -> + def result = null + if(map.file) { + def path = 'test/resources/DescriptorUtils/go/' + map.file.substring(map.file.lastIndexOf('/') + 1, map.file.length()) + def descriptorFile = new File(path) + if (descriptorFile.exists()) + result = descriptorFile.text + } else { + result = map.text + } + + if(result) + new Yaml().load(result) + else + null + }) + helper.registerAllowedMethod("readFile", [Map.class], { map -> def path = 'test/resources/DescriptorUtils/go/' + map.file.substring(map.file.lastIndexOf('/') + 1, map.file.length()) @@ -505,13 +524,28 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productName=testProductName')) assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('userKey=token-0815')) assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1')) - assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=myProject')) + assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=github.wdf.sap.corp/TestOrg/GolangTest.myProject')) } @Test void testGoDefaults() { - nullScript.commonPipelineEnvironment.setGithubOrg("testOrg") - nullScript.commonPipelineEnvironment.setGithubRepo("testRepo") + helper.registerAllowedMethod("readYaml", [Map.class], { + map -> + def result = null + if(map.file) { + def path = 'test/resources/DescriptorUtils/go/' + map.file.substring(map.file.lastIndexOf('/') + 1, map.file.length()) + def descriptorFile = new File(path) + if (descriptorFile.exists()) + result = descriptorFile.text + } else { + result = map.text + } + + if(result) + new Yaml().load(result) + else + null + }) helper.registerAllowedMethod("readFile", [Map.class], { map -> @@ -563,7 +597,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productName=testProductName')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('userKey=token-0815')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1')) - assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=testOrg.testRepo')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=github.wdf.sap.corp/TestOrg/GolangTest')) } diff --git a/test/resources/DescriptorUtils/go/glide.yaml b/test/resources/DescriptorUtils/go/glide.yaml new file mode 100644 index 000000000..300df9240 --- /dev/null +++ b/test/resources/DescriptorUtils/go/glide.yaml @@ -0,0 +1,7 @@ +package: github.wdf.sap.corp/TestOrg/GolangTest +import: +- package: github.com/julienschmidt/httprouter + version: ^1.1.0 +- package: github.com/tebeka/go2xunit + version: ^1.4.4 +- package: github.wdf.sap.corp/dtxmake-acceptance/golang-sample diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index f65d08f22..a3e52eecb 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -401,10 +401,6 @@ private resolveProjectIdentifiers(script, descriptorUtils, config) { break case 'golang': gav = descriptorUtils.getGoGAV(config.buildDescriptorFile) - if (!gav.artifact) { - gav.group = script.commonPipelineEnvironment.getGithubOrg() - gav.artifact = script.commonPipelineEnvironment.getGithubRepo() - } break case 'dlang': break From cbc81acaa53606b9c9f7aba53989eb003fc83409 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 25 Mar 2019 14:00:57 +0100 Subject: [PATCH 091/130] Refactor stash --- resources/default_pipeline_environment.yml | 8 ++++---- src/com/sap/piper/WhitesourceConfigurationHelper.groovy | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 452eda928..862b73f93 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -325,9 +325,9 @@ steps: - 'buildDescriptor' - 'opensourceConfiguration' golang: - buildDescriptorFile: './glide.yaml' - dockerImage: 'fabiorphp/golang-glide:1.11-stretch' - dockerWorkspace: '/home/glide' + buildDescriptorFile: './Gopkg.toml' + dockerImage: 'golang:1.12-stretch' + dockerWorkspace: '/home/dep' stashContent: - 'buildDescriptor' - 'opensourceConfiguration' @@ -356,7 +356,7 @@ steps: pipelineStashFilesBeforeBuild: runCheckmarx: false stashIncludes: - buildDescriptor: '**/pom.xml, **/.mvn/**, **/assembly.xml, **/.swagger-codegen-ignore, **/package.json, **/requirements.txt, **/setup.py, **/whitesource_config.py, **/mta*.y*ml, **/.npmrc, **/whitesource.*.json, **/whitesource-fs-agent.config, Dockerfile, **/VERSION, **/version.txt, **/glide.*, **/Gopkg.*, **/build.sbt, **/sbtDescriptor.json, **/project/*' + buildDescriptor: '**/pom.xml, **/.mvn/**, **/assembly.xml, **/.swagger-codegen-ignore, **/package.json, **/requirements.txt, **/setup.py, **/whitesource_config.py, **/mta*.y*ml, **/.npmrc, **/whitesource.*.json, **/whitesource-fs-agent.config, Dockerfile, **/VERSION, **/version.txt, **/Gopkg.*, **/build.sbt, **/sbtDescriptor.json, **/project/*' deployDescriptor: '**/manifest*.y*ml, **/*.mtaext.y*ml, **/*.mtaext, **/xs-app.json, helm/**, *.y*ml' git: '**/gitmetadata/**' opa5: '**/*.*' diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index ae399f1f4..3b671b3cd 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -77,8 +77,8 @@ class WhitesourceConfigurationHelper implements Serializable { [name: 'go.resolveDependencies', value: true, force: true], [name: 'go.ignoreSourceFiles', value: true, force: true], [name: 'go.collectDependenciesAtRuntime', value: true], - [name: 'go.dependencyManager', value: 'glide'], - [name: 'includes', value: '**/glide.yaml **/glide.lock'], + [name: 'go.dependencyManager', value: ''], + [name: 'includes', value: '**/*.lock'], [name: 'excludes', value: '**/*sources.jar **/*javadoc.jar'], [name: 'case.sensitive.glob', value: false], [name: 'followSymbolicLinks', value: true] From b305039f27c7103ff76df7b1b44c986cc7ac1d5b Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 25 Mar 2019 14:01:49 +0100 Subject: [PATCH 092/130] Fix descriptor default name --- src/com/sap/piper/DescriptorUtils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/com/sap/piper/DescriptorUtils.groovy b/src/com/sap/piper/DescriptorUtils.groovy index 624f5ae64..815033842 100644 --- a/src/com/sap/piper/DescriptorUtils.groovy +++ b/src/com/sap/piper/DescriptorUtils.groovy @@ -88,7 +88,7 @@ def getPipGAV(file = 'setup.py') { return result } -def getGoGAV(file = './glide.yaml') { +def getGoGAV(file = './Gopkg.toml') { def descriptor = readYaml(file: file) def path = file.substring(0, file.lastIndexOf('/') + 1) def module = path?.replaceAll(/\./, '')?.replaceAll('/', '') From 82d70677fbe03c210b6e8e10670d6977a2f713e4 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 25 Mar 2019 14:32:36 +0100 Subject: [PATCH 093/130] Refactor to dep as dependency manager --- src/com/sap/piper/DescriptorUtils.groovy | 6 +-- test/groovy/WhitesourceExecuteScanTest.groovy | 48 ++++--------------- .../com/sap/piper/DescriptorUtilsTest.groovy | 6 +-- vars/whitesourceExecuteScan.groovy | 2 +- 4 files changed, 15 insertions(+), 47 deletions(-) diff --git a/src/com/sap/piper/DescriptorUtils.groovy b/src/com/sap/piper/DescriptorUtils.groovy index 815033842..223bf1c2a 100644 --- a/src/com/sap/piper/DescriptorUtils.groovy +++ b/src/com/sap/piper/DescriptorUtils.groovy @@ -88,15 +88,15 @@ def getPipGAV(file = 'setup.py') { return result } -def getGoGAV(file = './Gopkg.toml') { - def descriptor = readYaml(file: file) +def getGoGAV(file = './Gopkg.toml', URI repoUrl) { + def name = "${repoUrl.getHost()}${repoUrl.getPath()}" def path = file.substring(0, file.lastIndexOf('/') + 1) def module = path?.replaceAll(/\./, '')?.replaceAll('/', '') def result = [:] result['group'] = '' result['packaging'] = '' - result['artifact'] = "${descriptor.package}${module?'.':''}${module?:''}" + result['artifact'] = "${name}${module?'.':''}${module?:''}".toString() file = path + 'version.txt' result['version'] = getVersionFromFile(file) diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index 671a2b0ab..bf4bdecde 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -455,23 +455,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { @Test void testGo() { - helper.registerAllowedMethod("readYaml", [Map.class], { - map -> - def result = null - if(map.file) { - def path = 'test/resources/DescriptorUtils/go/' + map.file.substring(map.file.lastIndexOf('/') + 1, map.file.length()) - def descriptorFile = new File(path) - if (descriptorFile.exists()) - result = descriptorFile.text - } else { - result = map.text - } - - if(result) - new Yaml().load(result) - else - null - }) + nullScript.commonPipelineEnvironment.gitHttpsUrl = 'https://github.wdf.sap.corp/test/golang' helper.registerAllowedMethod("readFile", [Map.class], { map -> @@ -511,8 +495,8 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'fabiorphp/golang-glide:1.11-stretch')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/glide')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'golang:1.12-stretch')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/dep')) assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config d3aa80454919391024374ba46b4df082d15ab9a3'])) assertThat(shellRule.shell, Matchers.hasItems( @@ -524,28 +508,12 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productName=testProductName')) assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('userKey=token-0815')) assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1')) - assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=github.wdf.sap.corp/TestOrg/GolangTest.myProject')) + assertThat(writeFileRule.files['./myProject/wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=github.wdf.sap.corp/test/golang.myProject')) } @Test void testGoDefaults() { - helper.registerAllowedMethod("readYaml", [Map.class], { - map -> - def result = null - if(map.file) { - def path = 'test/resources/DescriptorUtils/go/' + map.file.substring(map.file.lastIndexOf('/') + 1, map.file.length()) - def descriptorFile = new File(path) - if (descriptorFile.exists()) - result = descriptorFile.text - } else { - result = map.text - } - - if(result) - new Yaml().load(result) - else - null - }) + nullScript.commonPipelineEnvironment.gitHttpsUrl = 'https://github.wdf.sap.corp/test/golang' helper.registerAllowedMethod("readFile", [Map.class], { map -> @@ -584,8 +552,8 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(loggingRule.log, containsString('Unstash content: buildDescriptor')) assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'fabiorphp/golang-glide:1.11-stretch')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/glide')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'golang:1.12-stretch')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/dep')) assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config d3aa80454919391024374ba46b4df082d15ab9a3'])) assertThat(shellRule.shell, Matchers.hasItems( @@ -597,7 +565,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productName=testProductName')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('userKey=token-0815')) assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('productVersion=1')) - assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=github.wdf.sap.corp/TestOrg/GolangTest')) + assertThat(writeFileRule.files['./wss-unified-agent.config.d3aa80454919391024374ba46b4df082d15ab9a3'], containsString('projectName=github.wdf.sap.corp/test/golang')) } diff --git a/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy index 01a9b814a..0f0c86d9e 100644 --- a/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy +++ b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy @@ -214,7 +214,7 @@ class DescriptorUtilsTest extends BasePiperTest { helper.registerAllowedMethod("readFile", [Map.class], { map -> - def path = 'test/resources/DescriptorUtils/go/' + map.file.substring(map.file.lastIndexOf(File.separator) + 1, map.file.length()) + def path = 'test/resources/DescriptorUtils/go/' + map.file.substring(map.file.lastIndexOf('/') + 1, map.file.length()) def descriptorFile = new File(path) if(descriptorFile.exists()) return descriptorFile.text @@ -222,10 +222,10 @@ class DescriptorUtilsTest extends BasePiperTest { return null }) - def gav = descriptorUtils.getGoGAV('./myProject/glide.yaml') + def gav = descriptorUtils.getGoGAV('./myProject/Gopkg.toml', new URI('https://github.wdf.sap.corp/test/golang')) assertEquals('', gav.group) - assertEquals('myProject', gav.artifact) + assertEquals('github.wdf.sap.corp/test/golang.myProject', gav.artifact) assertEquals('1.2.3', gav.version) } } diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index a3e52eecb..0a7e5fc40 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -400,7 +400,7 @@ private resolveProjectIdentifiers(script, descriptorUtils, config) { gav = descriptorUtils.getPipGAV(config.buildDescriptorFile) break case 'golang': - gav = descriptorUtils.getGoGAV(config.buildDescriptorFile) + gav = descriptorUtils.getGoGAV(config.buildDescriptorFile, new URI(script.commonPipelineEnvironment.getGitHttpsUrl())) break case 'dlang': break From 250fbccfeb0b5654ded5a13e386b4f9c975d4d5d Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 25 Mar 2019 15:30:28 +0100 Subject: [PATCH 094/130] Refactor include pattern --- src/com/sap/piper/WhitesourceConfigurationHelper.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index 3b671b3cd..9fc2942c0 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -78,7 +78,7 @@ class WhitesourceConfigurationHelper implements Serializable { [name: 'go.ignoreSourceFiles', value: true, force: true], [name: 'go.collectDependenciesAtRuntime', value: true], [name: 'go.dependencyManager', value: ''], - [name: 'includes', value: '**/*.lock'], + [name: 'includes', value: '**/Gopkg.*'], [name: 'excludes', value: '**/*sources.jar **/*javadoc.jar'], [name: 'case.sensitive.glob', value: false], [name: 'followSymbolicLinks', value: true] From 9f18272af69fca2db47df97474eff49d95a80e64 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 25 Mar 2019 15:54:06 +0100 Subject: [PATCH 095/130] Fix project name and revert include change since it showed no effect --- src/com/sap/piper/DescriptorUtils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/com/sap/piper/DescriptorUtils.groovy b/src/com/sap/piper/DescriptorUtils.groovy index 223bf1c2a..9a626cb85 100644 --- a/src/com/sap/piper/DescriptorUtils.groovy +++ b/src/com/sap/piper/DescriptorUtils.groovy @@ -89,7 +89,7 @@ def getPipGAV(file = 'setup.py') { } def getGoGAV(file = './Gopkg.toml', URI repoUrl) { - def name = "${repoUrl.getHost()}${repoUrl.getPath()}" + def name = "${repoUrl.getHost()}${repoUrl.getPath().replaceAll(/\.git/, '')}" def path = file.substring(0, file.lastIndexOf('/') + 1) def module = path?.replaceAll(/\./, '')?.replaceAll('/', '') def result = [:] From 630150d48167b6a6f1ce2a721cb5a778a7afece6 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 25 Mar 2019 16:15:57 +0100 Subject: [PATCH 096/130] Switch to dep image --- resources/default_pipeline_environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 862b73f93..da8be5883 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -326,7 +326,7 @@ steps: - 'opensourceConfiguration' golang: buildDescriptorFile: './Gopkg.toml' - dockerImage: 'golang:1.12-stretch' + dockerImage: 'instrumentisto/dep:0.5' dockerWorkspace: '/home/dep' stashContent: - 'buildDescriptor' From 514e3f7c8498670f81f1c76f32cf5d250ad5ddb3 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 25 Mar 2019 16:47:51 +0100 Subject: [PATCH 097/130] Switch working directory --- resources/default_pipeline_environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index da8be5883..22585f239 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -327,7 +327,7 @@ steps: golang: buildDescriptorFile: './Gopkg.toml' dockerImage: 'instrumentisto/dep:0.5' - dockerWorkspace: '/home/dep' + dockerWorkspace: '/go/src' stashContent: - 'buildDescriptor' - 'opensourceConfiguration' From b36ad6d8d3549f6f6e261530d167ddfec807561a Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 25 Mar 2019 17:02:46 +0100 Subject: [PATCH 098/130] Switch working directory 2 --- resources/default_pipeline_environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 22585f239..01e7119bc 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -327,7 +327,7 @@ steps: golang: buildDescriptorFile: './Gopkg.toml' dockerImage: 'instrumentisto/dep:0.5' - dockerWorkspace: '/go/src' + dockerWorkspace: '/go/src/scan' stashContent: - 'buildDescriptor' - 'opensourceConfiguration' From 56ae339d9c774f1768937e0b552b075c527d8804 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 26 Mar 2019 09:36:59 +0100 Subject: [PATCH 099/130] Switch working directory 3 --- resources/default_pipeline_environment.yml | 4 ++-- test/groovy/WhitesourceExecuteScanTest.groovy | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 01e7119bc..1801ca37e 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -326,8 +326,8 @@ steps: - 'opensourceConfiguration' golang: buildDescriptorFile: './Gopkg.toml' - dockerImage: 'instrumentisto/dep:0.5' - dockerWorkspace: '/go/src/scan' + dockerImage: 'golang:1.12-stretch' + dockerWorkspace: '/home/modules' stashContent: - 'buildDescriptor' - 'opensourceConfiguration' diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index bf4bdecde..7e79a3755 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -496,7 +496,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'golang:1.12-stretch')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/dep')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/modules')) assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config d3aa80454919391024374ba46b4df082d15ab9a3'])) assertThat(shellRule.shell, Matchers.hasItems( @@ -553,7 +553,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'golang:1.12-stretch')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/dep')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/modules')) assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config d3aa80454919391024374ba46b4df082d15ab9a3'])) assertThat(shellRule.shell, Matchers.hasItems( From a5138ed341f70bcc893da80b3ccb3ac4558e276d Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 26 Mar 2019 14:22:02 +0100 Subject: [PATCH 100/130] Extended documentation --- vars/whitesourceExecuteScan.groovy | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 0a7e5fc40..3e5e1fe11 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -194,6 +194,8 @@ import static com.sap.piper.Prerequisites.checkScript ] /** + * BETA + * * With this step [WhiteSource](https://www.whitesourcesoftware.com) security and license compliance scans can be executed and assessed. * * WhiteSource is a Software as a Service offering based on a so called unified agent that locally determines the dependency @@ -201,8 +203,16 @@ import static com.sap.piper.Prerequisites.checkScript * check and additional Free and Open Source Software Publicly Known Vulnerabilities detection. * * !!! note "Docker Images" - * The underlying Docker images are public and specific to the solution's programming language(s) and may therefore be exchanged - * to fit and suite the relevant scenario. The default Python environment used is i.e. Python 3 based. + * The underlying Docker images are public and specific to the solution's programming language(s) and therefore may have to be exchanged + * to fit to and support the relevant scenario. The default Python environment used is i.e. Python 3 based. + * + * !!! warn "Restrictions" + * Currently the step does contain hardened scan configurations for `scanType` `'pip'` and `'go'`. Other environments are still being elaborated, + * so please thoroughly check your results and do not take them for granted by default. + * Also not all environments have been thoroughly tested already therefore you might need to tweak around with the default containers used or + * create your own ones to adequately support your scenario. To do so please modify `dockerImage` and `dockerWorkspace` parameters. + * The step expects an environment containing the programming language related compiler/interpreter as well as the related build tool. For a list + * of the supported build tools per environment please refer to the [WhiteSource Unified Agent Documentation](https://whitesource.atlassian.net/wiki/spaces/WD/pages/33718339/Unified+Agent). */ @GenerateDocumentation void call(Map parameters = [:]) { From 5ae1f40983682c80eba166321a590de34668ab59 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 26 Mar 2019 15:23:58 +0100 Subject: [PATCH 101/130] Switch image --- resources/default_pipeline_environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 1801ca37e..01e7119bc 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -326,8 +326,8 @@ steps: - 'opensourceConfiguration' golang: buildDescriptorFile: './Gopkg.toml' - dockerImage: 'golang:1.12-stretch' - dockerWorkspace: '/home/modules' + dockerImage: 'instrumentisto/dep:0.5' + dockerWorkspace: '/go/src/scan' stashContent: - 'buildDescriptor' - 'opensourceConfiguration' From 8725f28ed65bc5a25cb4427af140ee8e0c8622b3 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 26 Mar 2019 16:52:57 +0100 Subject: [PATCH 102/130] Switch image, dep on the fly --- resources/default_pipeline_environment.yml | 9 +++++++-- vars/whitesourceExecuteScan.groovy | 7 ++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 01e7119bc..33642461c 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -326,11 +326,16 @@ steps: - 'opensourceConfiguration' golang: buildDescriptorFile: './Gopkg.toml' - dockerImage: 'instrumentisto/dep:0.5' - dockerWorkspace: '/go/src/scan' + dockerImage: 'golang:1.12.1-stretch' + dockerWorkspace: '/home/dep' stashContent: - 'buildDescriptor' - 'opensourceConfiguration' + additionalInstallCommand: > + curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh \ + mkdir -p /go/src/${config.whitesource.projectName} \ + ln -s $(pwd) $GOPATH/${config.whitesource.projectName} \ + cd $GOPATH/src/${config.whitesource.projectName} && dep ensure sbt: buildDescriptorFile: './build.sbt' dockerImage: 'hseeberger/scala-sbt:8u181_2.12.8_1.2.8' diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 3e5e1fe11..97c3ca8a1 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -189,7 +189,8 @@ import static com.sap.piper.Prerequisites.checkScript cvssSeverityLimit : 'cvssSeverityLimit', timeout : 'timeout', vulnerabilityReportFileName : 'vulnerabilityReportFileName', - vulnerabilityReportTitle : 'vulnerabilityReportTitle' + vulnerabilityReportTitle : 'vulnerabilityReportTitle', + additionalInstallCommand : 'additionalInstallCommand' ] ] @@ -237,6 +238,7 @@ void call(Map parameters = [:]) { .dependingOn('scanType').mixin('dockerWorkspace') .dependingOn('scanType').mixin('stashContent') .dependingOn('scanType').mixin('whitesource/configFilePath') + .dependingOn('scanType').mixin('whitesource/additionalInstallCommand') .withMandatoryProperty('whitesource/serviceUrl') .withMandatoryProperty('whitesource/orgToken') .withMandatoryProperty('whitesource/userTokenCredentialsId') @@ -367,6 +369,9 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU javaCmd = './bin/java' } + if(config.whitesource.additionalInstallCommand) + sh new GStringTemplateEngine().createTemplate(config.whitesource.additionalInstallCommand).make([config: config]).toString() + def options = ["-jar ${config.whitesource.agentFileName} -c \'${config.whitesource.configFilePath}\'"] if (config.whitesource.orgToken) options.push("-apiKey '${config.whitesource.orgToken}'") if (config.whitesource.userKey) options.push("-userKey '${config.whitesource.userKey}'") From 9184102295383a2646d64f4f829c06758596b272 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 26 Mar 2019 17:05:12 +0100 Subject: [PATCH 103/130] Fix string --- resources/default_pipeline_environment.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 33642461c..b304b8d05 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -331,11 +331,11 @@ steps: stashContent: - 'buildDescriptor' - 'opensourceConfiguration' - additionalInstallCommand: > - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh \ - mkdir -p /go/src/${config.whitesource.projectName} \ - ln -s $(pwd) $GOPATH/${config.whitesource.projectName} \ - cd $GOPATH/src/${config.whitesource.projectName} && dep ensure + additionalInstallCommand: >- + curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + && mkdir -p /go/src/${config.whitesource.projectName} + && ln -s $(pwd) $GOPATH/${config.whitesource.projectName} + && cd $GOPATH/src/${config.whitesource.projectName} && dep ensure sbt: buildDescriptorFile: './build.sbt' dockerImage: 'hseeberger/scala-sbt:8u181_2.12.8_1.2.8' From e5420b1e8ab58029ce677aaad649e18cc9f3d3d5 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 26 Mar 2019 17:07:15 +0100 Subject: [PATCH 104/130] Add parameter --- vars/whitesourceExecuteScan.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 97c3ca8a1..9436275bd 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -73,6 +73,10 @@ import static com.sap.piper.Prerequisites.checkScript 'verbose' ] @Field Set STEP_CONFIG_KEYS = GENERAL_CONFIG_KEYS + [ + /** + * Additional install command that can be used to populate the default docker image for some scenarios. + */ + 'additionalInstallCommand', /** * URL used to download the latest version of the WhiteSource Unified Agent. */ From 8adf1833956e9c95749ce4beb0ca5ced42368feb Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 26 Mar 2019 17:16:44 +0100 Subject: [PATCH 105/130] Fix quoting --- resources/default_pipeline_environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index b304b8d05..67a0f2425 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -334,8 +334,8 @@ steps: additionalInstallCommand: >- curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh && mkdir -p /go/src/${config.whitesource.projectName} - && ln -s $(pwd) $GOPATH/${config.whitesource.projectName} - && cd $GOPATH/src/${config.whitesource.projectName} && dep ensure + && ln -s \$(pwd) \$GOPATH/${config.whitesource.projectName} + && cd \$GOPATH/src/${config.whitesource.projectName} && dep ensure sbt: buildDescriptorFile: './build.sbt' dockerImage: 'hseeberger/scala-sbt:8u181_2.12.8_1.2.8' From 957bad0d4572edf345a83b53fbccc7d1f9d33a2a Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 26 Mar 2019 17:22:51 +0100 Subject: [PATCH 106/130] Fix path --- resources/default_pipeline_environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 67a0f2425..bf4a596e3 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -333,8 +333,8 @@ steps: - 'opensourceConfiguration' additionalInstallCommand: >- curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - && mkdir -p /go/src/${config.whitesource.projectName} - && ln -s \$(pwd) \$GOPATH/${config.whitesource.projectName} + && mkdir -p \$GOPATH/src/${config.whitesource.projectName} + && ln -s \$(pwd) \$GOPATH/src/${config.whitesource.projectName} && cd \$GOPATH/src/${config.whitesource.projectName} && dep ensure sbt: buildDescriptorFile: './build.sbt' From b2929a907010aa9fb3d49ea8503e4cc9999cad99 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 26 Mar 2019 17:26:09 +0100 Subject: [PATCH 107/130] Fix path 2 --- resources/default_pipeline_environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index bf4a596e3..4c04c69f6 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -333,7 +333,7 @@ steps: - 'opensourceConfiguration' additionalInstallCommand: >- curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - && mkdir -p \$GOPATH/src/${config.whitesource.projectName} + && mkdir -p \$GOPATH/src/${config.whitesource.projectName.substring(0, config.whitesource.projectName.lastIndexOf('/'))} && ln -s \$(pwd) \$GOPATH/src/${config.whitesource.projectName} && cd \$GOPATH/src/${config.whitesource.projectName} && dep ensure sbt: From b188a59ee57b930b039cc9d6e3f2fe88d5b2bb5f Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 26 Mar 2019 17:33:04 +0100 Subject: [PATCH 108/130] Fix config --- src/com/sap/piper/WhitesourceConfigurationHelper.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index 9fc2942c0..a64f6e546 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -76,9 +76,9 @@ class WhitesourceConfigurationHelper implements Serializable { mapping += [ [name: 'go.resolveDependencies', value: true, force: true], [name: 'go.ignoreSourceFiles', value: true, force: true], - [name: 'go.collectDependenciesAtRuntime', value: true], - [name: 'go.dependencyManager', value: ''], - [name: 'includes', value: '**/Gopkg.*'], + [name: 'go.collectDependenciesAtRuntime', value: false], + [name: 'go.dependencyManager', value: 'dep'], + [name: 'includes', value: '**/*.lock'], [name: 'excludes', value: '**/*sources.jar **/*javadoc.jar'], [name: 'case.sensitive.glob', value: false], [name: 'followSymbolicLinks', value: true] From 9466beec42bc9271b546eae11c80769ad7fc8cfe Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 26 Mar 2019 17:45:29 +0100 Subject: [PATCH 109/130] Fix tests --- resources/default_pipeline_environment.yml | 2 +- test/groovy/WhitesourceExecuteScanTest.groovy | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index 4c04c69f6..e00fcbe48 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -326,7 +326,7 @@ steps: - 'opensourceConfiguration' golang: buildDescriptorFile: './Gopkg.toml' - dockerImage: 'golang:1.12.1-stretch' + dockerImage: 'golang:1.12-stretch' dockerWorkspace: '/home/dep' stashContent: - 'buildDescriptor' diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index 7e79a3755..876be0642 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -144,7 +144,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { nullScript.commonPipelineEnvironment.configuration['general'] = nullScript.commonPipelineEnvironment.configuration['general'] ?: [:] nullScript.commonPipelineEnvironment.configuration['general']['whitesource'] = nullScript.commonPipelineEnvironment.configuration['general']['whitesource'] ?: [:] nullScript.commonPipelineEnvironment.configuration['general']['whitesource']['serviceUrl'] = "http://some.host.whitesource.com/api/" - nullScript.commonPipelineEnvironment.setConfigProperty('userTokenCredentialsId', 'ID-123456789') + nullScript.commonPipelineEnvironment.configuration['general']['whitesource']['userTokenCredentialsId'] = 'ID-123456789' nullScript.commonPipelineEnvironment.configuration['steps']['whitesourceExecuteScan']['userTokenCredentialsId'] = 'ID-123456789' } @@ -496,7 +496,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'golang:1.12-stretch')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/modules')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/dep')) assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config d3aa80454919391024374ba46b4df082d15ab9a3'])) assertThat(shellRule.shell, Matchers.hasItems( @@ -553,7 +553,7 @@ class WhitesourceExecuteScanTest extends BasePiperTest { assertThat(loggingRule.log, containsString('Unstash content: opensourceConfiguration')) assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerImage', 'golang:1.12-stretch')) - assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/modules')) + assertThat(dockerExecuteRule.dockerParams, hasEntry('dockerWorkspace', '/home/dep')) assertThat(dockerExecuteRule.dockerParams, hasEntry('stashContent', ['buildDescriptor', 'opensourceConfiguration', 'modified whitesource config d3aa80454919391024374ba46b4df082d15ab9a3'])) assertThat(shellRule.shell, Matchers.hasItems( From 6c85699689b32b724f36c6f934cd859286a2bfa1 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 27 Mar 2019 11:49:56 +0100 Subject: [PATCH 110/130] Fix docu template --- .../docs/steps/whitesourceExecuteScan.md | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/documentation/docs/steps/whitesourceExecuteScan.md b/documentation/docs/steps/whitesourceExecuteScan.md index 1d1bf77f7..1c92aedbd 100644 --- a/documentation/docs/steps/whitesourceExecuteScan.md +++ b/documentation/docs/steps/whitesourceExecuteScan.md @@ -1,20 +1,16 @@ -# whitesourceExecuteScan +# ${docGenStepName} -## Description - -Content here is generated from corresponding step, see `vars`. +## ${docGenDescription} ## Prerequisites -None +Your company has registered an account with WhiteSource and you have enabled the use of so called `User Keys` to manage +access to your organization in WhiteSource via dedicated privileges. Scanning your products without adequate user level +access protection imposed on the WhiteSource backend would simply allow access based on the organization token. -## Parameters +## ${docGenParameters} -Content here is generated from corresponding step, see `vars`. - -## Step configuration - -Content here is generated from corresponding step, see `vars`. +## ${docGenConfiguration} ## Exceptions @@ -23,5 +19,5 @@ None ## Examples ```groovy -whitesourceExecuteScan script: this, scanType: 'pip', productName: 'My Whitesource Product', userTokenCredentialsId: 'companyAdminToken', orgAdminUserTokenCredentialsId: 'orgAdmiToken', orgToken: 'myWhitesourceOrganizationToken' +whitesourceExecuteScan script: this, scanType: 'pip', productName: 'My Whitesource Product', userTokenCredentialsId: 'companyAdminToken', orgAdminUserTokenCredentialsId: 'orgAdminToken', orgToken: 'myWhitesourceOrganizationToken' ``` From eab1dacde654d3be7ab046efa3c18a5718a3614f Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 27 Mar 2019 12:57:57 +0100 Subject: [PATCH 111/130] Fix stashes --- resources/default_pipeline_environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index e00fcbe48..bc7074bde 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -361,11 +361,11 @@ steps: pipelineStashFilesBeforeBuild: runCheckmarx: false stashIncludes: - buildDescriptor: '**/pom.xml, **/.mvn/**, **/assembly.xml, **/.swagger-codegen-ignore, **/package.json, **/requirements.txt, **/setup.py, **/whitesource_config.py, **/mta*.y*ml, **/.npmrc, **/whitesource.*.json, **/whitesource-fs-agent.config, Dockerfile, **/VERSION, **/version.txt, **/Gopkg.*, **/build.sbt, **/sbtDescriptor.json, **/project/*' + buildDescriptor: '**/pom.xml, **/.mvn/**, **/assembly.xml, **/.swagger-codegen-ignore, **/package.json, **/requirements.txt, **/setup.py, **/mta*.y*ml, **/.npmrc, Dockerfile, **/VERSION, **/version.txt, **/Gopkg.*, **/build.sbt, **/sbtDescriptor.json, **/project/*' deployDescriptor: '**/manifest*.y*ml, **/*.mtaext.y*ml, **/*.mtaext, **/xs-app.json, helm/**, *.y*ml' git: '**/gitmetadata/**' opa5: '**/*.*' - opensourceConfiguration: '**/srcclr.yml, **/vulas-custom.properties, **/.nsprc, **/.retireignore, **/.retireignore.json, **/.snyk' + opensourceConfiguration: '**/srcclr.yml, **/vulas-custom.properties, **/.nsprc, **/.retireignore, **/.retireignore.json, **/.snyk, **/wss-unified-agent.config, **/vendor/**/*' pipelineConfigAndTests: '.pipeline/**' securityDescriptor: '**/xs-security.json' tests: '**/pom.xml, **/*.json, **/*.xml, **/src/**, **/node_modules/**, **/specs/**, **/env/**, **/*.js, **/tests/**' From ea211f598788a7f3fa34a603732cc2f67d14d6ef Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 27 Mar 2019 14:03:11 +0100 Subject: [PATCH 112/130] Add archiving of debug output to new step --- vars/whitesourceExecuteScan.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 9436275bd..6699d8f52 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -397,6 +397,9 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU break } + // archive whitesource debug files, if available + archiveArtifacts artifacts: "**/ws-l*", allowEmptyArchive: true + if (config.reporting) { analyseWhitesourceResults(config, repository) } From 6b1caf5a3cf074b207e624e719dd27b907794160 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 27 Mar 2019 14:10:29 +0100 Subject: [PATCH 113/130] Add deletion of downloaded JVM archive --- vars/whitesourceExecuteScan.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 6699d8f52..d430ab464 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -389,6 +389,7 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU if (config.whitesource.jreDownloadUrl) { sh "rm -rf ./bin ./conf ./legal ./lib ./man" + sh "rm -f jvm.tar.gz" } // archive whitesource result files From b9596aa84f43fa62f712d6b4371425e82497de9c Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 27 Mar 2019 14:11:54 +0100 Subject: [PATCH 114/130] Add deletion of downloaded JVM archive --- vars/whitesourceExecuteScan.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index d430ab464..bd89011d6 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -394,13 +394,13 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU // archive whitesource result files archiveArtifacts artifacts: "whitesource/*.*", allowEmptyArchive: true + + // archive whitesource debug files, if available + archiveArtifacts artifacts: "**/ws-l*", allowEmptyArchive: true } break } - // archive whitesource debug files, if available - archiveArtifacts artifacts: "**/ws-l*", allowEmptyArchive: true - if (config.reporting) { analyseWhitesourceResults(config, repository) } From 4ed0f8df364f322456ec6e73c973c00e7702bd97 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 27 Mar 2019 23:46:43 +0100 Subject: [PATCH 115/130] Trigger build --- vars/whitesourceExecuteScan.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index bd89011d6..827c6f9b8 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -392,7 +392,7 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU sh "rm -f jvm.tar.gz" } - // archive whitesource result files + // archive whitesource result files for UA archiveArtifacts artifacts: "whitesource/*.*", allowEmptyArchive: true // archive whitesource debug files, if available From 50b0e8e377218781131eee302c3e72b50eaafd63 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 29 Mar 2019 12:11:10 +0100 Subject: [PATCH 116/130] Use new failErrorLevel parameter --- src/com/sap/piper/WhitesourceConfigurationHelper.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index a64f6e546..0feeddab9 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -36,7 +36,8 @@ class WhitesourceConfigurationHelper implements Serializable { [name: 'forceUpdate', value: true, force: true], [name: 'offline', value: false, force: true], [name: 'ignoreSourceFiles', value: true, force: true], - [name: 'resolveAllDependencies', value: false, force: true] + [name: 'resolveAllDependencies', value: false, force: true], + [name: 'failErrorLevel', value: 'ALL', force: true] ] // To be removed once all scenarios are fully supported From c82bcc9c8681c8a7bf3ee2a20afe6d91190bcf7f Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 29 Mar 2019 13:58:41 +0100 Subject: [PATCH 117/130] Addressed feedback --- resources/default_pipeline_environment.yml | 2 +- src/com/sap/piper/DescriptorUtils.groovy | 21 ++- .../integration/WhitesourceRepository.groovy | 33 ++--- test/groovy/WhitesourceExecuteScanTest.groovy | 1 - .../com/sap/piper/DescriptorUtilsTest.groovy | 6 +- .../WhitesourceConfigurationHelperTest.groovy | 131 +++++++++++------- 6 files changed, 115 insertions(+), 79 deletions(-) diff --git a/resources/default_pipeline_environment.yml b/resources/default_pipeline_environment.yml index bc7074bde..e633a7c80 100644 --- a/resources/default_pipeline_environment.yml +++ b/resources/default_pipeline_environment.yml @@ -332,7 +332,7 @@ steps: - 'buildDescriptor' - 'opensourceConfiguration' additionalInstallCommand: >- - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + curl --fail https://raw.githubusercontent.com/golang/dep/master/install.sh | sh && mkdir -p \$GOPATH/src/${config.whitesource.projectName.substring(0, config.whitesource.projectName.lastIndexOf('/'))} && ln -s \$(pwd) \$GOPATH/src/${config.whitesource.projectName} && cd \$GOPATH/src/${config.whitesource.projectName} && dep ensure diff --git a/src/com/sap/piper/DescriptorUtils.groovy b/src/com/sap/piper/DescriptorUtils.groovy index 9a626cb85..17b3a3dd8 100644 --- a/src/com/sap/piper/DescriptorUtils.groovy +++ b/src/com/sap/piper/DescriptorUtils.groovy @@ -13,20 +13,22 @@ def version = Pattern.compile("(.*)version=['\"](.*?)['\"](.*)", Pattern.DOTALL) @Field def method = Pattern.compile("(.*)\\(\\)", Pattern.DOTALL) -def getMavenGAV(fileName) { +@NonCPS +def getMavenGAV(file = 'pom.xml') { def result = [:] - def descriptor = readMavenPom(file: fileName) + def descriptor = readMavenPom(file: file) def group = descriptor.getGroupId() def artifact = descriptor.getArtifactId() def version = descriptor.getVersion() result['packaging'] = descriptor.getPackaging() - result['group'] = (null != group && group.length() > 0) ? group : sh(returnStdout: true, script: "mvn -f ${fileName} help:evaluate -Dexpression=project.groupId | grep -Ev '(^\\s*\\[|Download|Java\\w+:)'").trim() - result['artifact'] = (null != artifact && artifact.length() > 0) ? artifact : sh(returnStdout: true, script: "mvn -f ${fileName} help:evaluate -Dexpression=project.artifactId | grep -Ev '(^\\s*\\[|Download|Java\\w+:)'").trim() - result['version'] = (null != version && version.length() > 0) ? version : sh(returnStdout: true, script: "mvn -f ${fileName} help:evaluate -Dexpression=project.version | grep ^[0-9].*").trim() - echo "loaded ${result} from ${fileName}" + result['group'] = (null != group && group.length() > 0) ? group : sh(returnStdout: true, script: "mvn -f ${file} help:evaluate -Dexpression=project.groupId | grep -Ev '(^\\s*\\[|Download|Java\\w+:)'").trim() + result['artifact'] = (null != artifact && artifact.length() > 0) ? artifact : sh(returnStdout: true, script: "mvn -f ${file} help:evaluate -Dexpression=project.artifactId | grep -Ev '(^\\s*\\[|Download|Java\\w+:)'").trim() + result['version'] = (null != version && version.length() > 0) ? version : sh(returnStdout: true, script: "mvn -f ${file} help:evaluate -Dexpression=project.version | grep ^[0-9].*").trim() + echo "loaded ${result} from ${file}" return result } +@NonCPS def getNpmGAV(file = 'package.json') { def result = [:] def descriptor = readJSON(file: file) @@ -46,6 +48,7 @@ def getNpmGAV(file = 'package.json') { return result } +@NonCPS def getDlangGAV(file = 'dub.json') { def result = [:] def descriptor = readJSON(file: file) @@ -58,6 +61,7 @@ def getDlangGAV(file = 'dub.json') { return result } +@NonCPS def getSbtGAV(file = 'sbtDescriptor.json') { def result = [:] def descriptor = readJSON(file: file) @@ -70,6 +74,7 @@ def getSbtGAV(file = 'sbtDescriptor.json') { return result } +@NonCPS def getPipGAV(file = 'setup.py') { def result = [:] def descriptor = readFile(file: file) @@ -88,7 +93,8 @@ def getPipGAV(file = 'setup.py') { return result } -def getGoGAV(file = './Gopkg.toml', URI repoUrl) { +@NonCPS +def getGoGAV(file = 'Gopkg.toml', URI repoUrl) { def name = "${repoUrl.getHost()}${repoUrl.getPath().replaceAll(/\.git/, '')}" def path = file.substring(0, file.lastIndexOf('/') + 1) def module = path?.replaceAll(/\./, '')?.replaceAll('/', '') @@ -109,6 +115,7 @@ def getGoGAV(file = './Gopkg.toml', URI repoUrl) { return result } +@NonCPS private getVersionFromFile(file) { try { def versionString = readFile(file: file) diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy index 61f5eb9c4..85b2a4adc 100644 --- a/src/com/sap/piper/integration/WhitesourceRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -20,26 +20,10 @@ class WhitesourceRepository implements Serializable { def fetchedVulnerabilities = [] if (config.whitesource.projectNames) { for (int i = 0; i < whitesourceProjectsMetaInformation.size(); i++) { - def metaInfo = whitesourceProjectsMetaInformation[i] - - def requestBody = [ - requestType : "getProjectAlertsByType", - alertType : "SECURITY_VULNERABILITY", - projectToken: metaInfo.token - ] - - def response = fetchWhitesourceResource(requestBody) - fetchedVulnerabilities.addAll(response.alerts) + fetchSecurityAlertsPerItem(whitesourceProjectsMetaInformation[i].token, "getProjectAlertsByType", fetchedVulnerabilities) } } else { - def requestBody = [ - requestType : "getProductAlertsByType", - alertType : "SECURITY_VULNERABILITY", - productToken: config.whitesource.productToken, - ] - - def response = fetchWhitesourceResource(requestBody) - fetchedVulnerabilities.addAll(response.alerts) + fetchSecurityAlertsPerItem(config.whitesource.productToken, "getProductAlertsByType", fetchedVulnerabilities) } sortVulnerabilitiesByScore(fetchedVulnerabilities) @@ -47,6 +31,17 @@ class WhitesourceRepository implements Serializable { return fetchedVulnerabilities } + private fetchSecurityAlertsPerItem(token, type, List fetchedVulnerabilities) { + def requestBody = [ + requestType : type, + alertType : "SECURITY_VULNERABILITY", + projectToken: token + ] + + def response = fetchWhitesourceResource(requestBody) + fetchedVulnerabilities.addAll(response.alerts) + } + protected def fetchWhitesourceResource(Map requestBody) { final def response = httpWhitesource(requestBody) def parsedResponse = new JsonUtils().jsonStringToGroovyObject(response.content) @@ -206,7 +201,7 @@ class WhitesourceRepository implements Serializable { if(config.verbose) script.echo "Sending curl request with parameters ${params}" - script.sh "${config.verbose ? '' : '#!/bin/sh -e\n'}curl -o ${fileName} -X POST ${config.whitesource.serviceUrl} -H 'Content-Type: application/json' -d \'${serializedContent}\'" + script.sh "${config.verbose ? '' : '#!/bin/sh -e\n'}curl --fail -o ${fileName} -X POST ${config.whitesource.serviceUrl} -H 'Content-Type: application/json' -d \'${serializedContent}\'" } @NonCPS diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index 876be0642..cef26b15f 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -12,7 +12,6 @@ import org.junit.Test import org.junit.rules.ExpectedException import org.junit.rules.RuleChain import org.springframework.beans.factory.annotation.Autowired -import org.yaml.snakeyaml.Yaml import util.* import static org.hamcrest.Matchers.* diff --git a/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy index 0f0c86d9e..3e86cc661 100644 --- a/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy +++ b/test/groovy/com/sap/piper/DescriptorUtilsTest.groovy @@ -11,6 +11,7 @@ import util.JenkinsErrorRule import util.JenkinsLoggingRule import util.JenkinsSetupRule import util.LibraryLoadingTestExecutionListener +import util.Rules import static org.hamcrest.Matchers.is import static org.junit.Assert.assertEquals @@ -29,10 +30,7 @@ class DescriptorUtilsTest extends BasePiperTest { public JenkinsLoggingRule loggingRule = new JenkinsLoggingRule(this) @Rule - public RuleChain ruleChain = - RuleChain.outerRule(setUpRule) - .around(errorRule) - .around(environmentRule) + public RuleChain ruleChain = Rules.getCommonRules(this) .around(loggingRule) DescriptorUtils descriptorUtils diff --git a/test/groovy/com/sap/piper/WhitesourceConfigurationHelperTest.groovy b/test/groovy/com/sap/piper/WhitesourceConfigurationHelperTest.groovy index e49ed29ea..416552c65 100644 --- a/test/groovy/com/sap/piper/WhitesourceConfigurationHelperTest.groovy +++ b/test/groovy/com/sap/piper/WhitesourceConfigurationHelperTest.groovy @@ -13,6 +13,7 @@ import util.Rules import static org.hamcrest.Matchers.containsString import static org.hamcrest.Matchers.hasItem import static org.hamcrest.Matchers.not +import static org.hamcrest.Matchers.allOf import static org.junit.Assert.assertThat class WhitesourceConfigurationHelperTest extends BasePiperTest { @@ -38,11 +39,15 @@ class WhitesourceConfigurationHelperTest extends BasePiperTest { void testExtendConfigurationFileUnifiedAgentEmptyConfig() { helper.registerAllowedMethod('readProperties', [Map], {return new Properties()}) WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'none', whitesource: [configFilePath: './config',serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'DIST - name1', productToken: '1234', userKey: '0000']], "./") - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], not(containsString("log.level=debug"))) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], + allOf( + not(containsString("log.level=debug")), + containsString("apiKey=abcd"), + containsString("productName=DIST - name1"), + containsString("productToken=1234"), + containsString("userKey=0000") + ) + ) assertThat(jlr.log, containsString("[Whitesource] Configuration for scanType: 'none' is not yet hardened, please do a quality assessment of your scan results.")) } @@ -51,11 +56,15 @@ class WhitesourceConfigurationHelperTest extends BasePiperTest { void testExtendConfigurationFileUnifiedAgentConfigDeeper() { helper.registerAllowedMethod('readProperties', [Map], { m -> if (!m.file.contains('testModule')) return new Properties() else return null }) WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'none', whitesource: [configFilePath: './config',serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'DIST - name1', productToken: '1234', userKey: '0000']], "./testModule/") - assertThat(jwfr.files['./testModule/config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], not(containsString("log.level=debug"))) - assertThat(jwfr.files['./testModule/config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) - assertThat(jwfr.files['./testModule/config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) - assertThat(jwfr.files['./testModule/config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) - assertThat(jwfr.files['./testModule/config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + assertThat(jwfr.files['./testModule/config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], + allOf( + not(containsString("log.level=debug")), + containsString("apiKey=abcd"), + containsString("productName=DIST - name1"), + containsString("productToken=1234"), + containsString("userKey=0000") + ) + ) assertThat(jlr.log, containsString("[Whitesource] Configuration for scanType: 'none' is not yet hardened, please do a quality assessment of your scan results.")) } @@ -63,10 +72,14 @@ class WhitesourceConfigurationHelperTest extends BasePiperTest { @Test void testExtendConfigurationFileUnifiedAgentMaven() { WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'none', whitesource: [configFilePath: './config',serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'DIST - name1', productToken: '1234', userKey: '0000']], "./") - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], + allOf( + containsString("apiKey=abcd"), + containsString("productName=DIST - name1"), + containsString("productToken=1234"), + containsString("userKey=0000") + ) + ) assertThat(jlr.log, containsString("[Whitesource] Configuration for scanType: 'none' is not yet hardened, please do a quality assessment of your scan results.")) } @@ -74,10 +87,14 @@ class WhitesourceConfigurationHelperTest extends BasePiperTest { @Test void testExtendConfigurationFileUnifiedAgentNpm() { WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'npm', whitesource: [configFilePath: './config',serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'DIST - name1', productToken: '1234', userKey: '0000']], "./") - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], + allOf( + containsString("apiKey=abcd"), + containsString("productName=DIST - name1"), + containsString("productToken=1234"), + containsString("userKey=0000") + ) + ) assertThat(jlr.log, containsString("[Whitesource] Configuration for scanType: 'npm' is not yet hardened, please do a quality assessment of your scan results.")) } @@ -85,10 +102,14 @@ class WhitesourceConfigurationHelperTest extends BasePiperTest { @Test void testExtendConfigurationFileUnifiedAgentSbt() { WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'sbt', whitesource: [configFilePath: './config',serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'DIST - name1', productToken: '1234', userKey: '0000']], "./") - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], + allOf( + containsString("apiKey=abcd"), + containsString("productName=DIST - name1"), + containsString("productToken=1234"), + containsString("userKey=0000") + ) + ) assertThat(jlr.log, containsString("[Whitesource] Configuration for scanType: 'sbt' is not yet hardened, please do a quality assessment of your scan results.")) @@ -97,10 +118,14 @@ class WhitesourceConfigurationHelperTest extends BasePiperTest { @Test void testExtendConfigurationFileUnifiedAgentDlang() { WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'dlang', whitesource: [configFilePath: './config',serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'DIST - name1', productToken: '1234', userKey: '0000']], "./") - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], + allOf( + containsString("apiKey=abcd"), + containsString("productName=DIST - name1"), + containsString("productToken=1234"), + containsString("userKey=0000") + ) + ) assertThat(jlr.log, containsString("[Whitesource] Configuration for scanType: 'dlang' is not yet hardened, please do a quality assessment of your scan results.")) } @@ -108,11 +133,15 @@ class WhitesourceConfigurationHelperTest extends BasePiperTest { @Test void testExtendConfigurationFileUnifiedAgentPip() { WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', whitesource: [configFilePath: './config',serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'DIST - name1', productToken: '1234', userKey: '0000']], "./") - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=DIST - name1")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("python.resolveDependencies=true")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], + allOf( + containsString("apiKey=abcd"), + containsString("productName=DIST - name1"), + containsString("productToken=1234"), + containsString("userKey=0000"), + containsString("python.resolveDependencies=true") + ) + ) assertThat(jlr.log, not(containsString("[Whitesource] Configuration for scanType: 'pip' is not yet hardened, please do a quality assessment of your scan results."))) } @@ -121,12 +150,16 @@ class WhitesourceConfigurationHelperTest extends BasePiperTest { void testExtendConfigurationFileUnifiedAgentGolangVerbose() { def config = [scanType: 'golang', whitesource: [configFilePath: './config', serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'abcd', productName: 'SHC - name2', productToken: '1234', userKey: '0000'], stashContent: ['some', 'stashes'], verbose: true] WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, config, "./") - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=abcd")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=SHC - name2")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("go.resolveDependencies=true")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("log.level=debug")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], + allOf( + containsString("apiKey=abcd"), + containsString("productName=SHC - name2"), + containsString("productToken=1234"), + containsString("userKey=0000"), + containsString("go.resolveDependencies=true"), + containsString("log.level=debug") + ) + ) assertThat(config.stashContent, hasItem(containsString('modified whitesource config '))) assertThat(jlr.log, not(containsString("[Warning][Whitesource] Configuration for scanType: 'golang' is not yet hardened, please do a quality assessment of your scan results."))) @@ -139,16 +172,20 @@ class WhitesourceConfigurationHelperTest extends BasePiperTest { helper.registerAllowedMethod('readProperties', [Map], {return p}) WhitesourceConfigurationHelper.extendUAConfigurationFile(nullScript, utils, [scanType: 'pip', whitesource: [configFilePath: './config', serviceUrl: "http://some.host.whitesource.com/api/", orgToken: 'cdfg', productName: 'name', productToken: '1234', userKey: '0000'], verbose: true], "./") - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("apiKey=cdfg")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productName=name")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("productToken=1234")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("userKey=0000")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("python.resolveDependencies=true")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("log.level=debug")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("python.resolveDependencies=true")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("python.ignoreSourceFiles=true")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("python.ignorePipInstallErrors=true")) - assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], containsString("python.installVirtualenv=false")) + assertThat(jwfr.files['./config.847f9aec2f93de9000d5fa4e6eaace2283ae6377'], + allOf( + containsString("apiKey=cdfg"), + containsString("productName=name"), + containsString("productToken=1234"), + containsString("userKey=0000"), + containsString("python.resolveDependencies=true"), + containsString("log.level=debug"), + containsString("python.resolveDependencies=true"), + containsString("python.ignoreSourceFiles=true"), + containsString("python.ignorePipInstallErrors=true"), + containsString("python.installVirtualenv=false") + ) + ) } } From d169bfb61926d77d8977392f8de4dae6d626d3e4 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 29 Mar 2019 14:22:42 +0100 Subject: [PATCH 118/130] Refactor file download --- .../integration/WhitesourceRepository.groovy | 10 +++++-- .../WhitesourceRepositoryTest.groovy | 30 ++++++++++++------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy index 85b2a4adc..078a879f9 100644 --- a/src/com/sap/piper/integration/WhitesourceRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -140,7 +140,8 @@ class WhitesourceRepository implements Serializable { productToken: config.whitesource.productToken ] - fetchFileFromWhiteSource(reportName, requestContent) + //fetchFileFromWhiteSource(reportName, requestContent) + httpWhitesource(requestContent, 'APPLICATION_PDF', reportName) } def fetchProductLicenseAlerts() { @@ -166,19 +167,22 @@ class WhitesourceRepository implements Serializable { } @NonCPS - protected def httpWhitesource(requestBody) { + protected def httpWhitesource(requestBody, acceptType = 'APPLICATION_JSON', outputFile = null) { handleAdditionalRequestParameters(requestBody) def serializedBody = new JsonUtils().groovyObjectToPrettyJsonString(requestBody) def params = [ url : config.whitesource.serviceUrl, httpMode : 'POST', - acceptType : 'APPLICATION_JSON', + acceptType : acceptType, contentType: 'APPLICATION_JSON', requestBody: serializedBody, quiet : !config.verbose, timeout : config.whitesource.timeout ] + if (outputFile) + params["outputFile"] = outputFile + if (script.env.HTTP_PROXY) params["httpProxy"] = script.env.HTTP_PROXY diff --git a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy index 73de10265..40f1f9621 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy @@ -329,7 +329,6 @@ class WhitesourceRepositoryTest extends BasePiperTest { exception.expectMessage("[WhiteSource] Request failed with error message 'User is not allowed to perform this action' (5001)") helper.registerAllowedMethod('httpRequest', [Map], { p -> - requestParams = p return [content: responseBody] }) @@ -339,22 +338,31 @@ class WhitesourceRepositoryTest extends BasePiperTest { @Test void testFetchReportForProduct() { - repository.config.putAll([whitesource: [serviceUrl: "http://mo-323123123.sap.corp/some", productToken: "4711"], verbose: true]) - def command - helper.registerAllowedMethod('sh', [String], { cmd -> - command = cmd + repository.config.putAll([whitesource: [serviceUrl: "http://mo-323123123.sap.corp/some", productToken: "4712", userKey: "4711"], verbose: true]) + def requestBody = "{ \"requestType\": \"getProductRiskReport\", \"productToken\": \"${repository.config.whitesource.productToken}\" }" + + def requestParams + helper.registerAllowedMethod('httpRequest', [Map], { p -> + requestParams = p }) repository.fetchReportForProduct("test.file") - assertThat(command, equals('''#!/bin/sh -e -curl -o test.file -X POST http://some.host.whitesource.com/api/ -H 'Content-Type: application/json' -d '{ - "requestType": "getProductRiskReport", - "productToken": "4711" -}''' + assertThat(requestParams, is( + [ + url : repository.config.whitesource.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_PDF', + contentType: 'APPLICATION_JSON', + requestBody: requestBody, + quiet : false, + userKey : repository.config.whitesource.userKey, + httpProxy : "http://test.sap.com:8080", + outputFile : "test.file" + ] )) - assertThat(jlr.log, containsString("Sending curl request with parameters [requestType:getProductRiskReport, productToken:4711]")) + assertThat(jlr.log, containsString("Sending http request with parameters [requestType:getProductRiskReport, productToken:4711]")) } @Test From 8698795ba6b40ee6179c7192a17dd47d3982c507 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 29 Mar 2019 14:24:46 +0100 Subject: [PATCH 119/130] Fix NonCPS --- src/com/sap/piper/DescriptorUtils.groovy | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/com/sap/piper/DescriptorUtils.groovy b/src/com/sap/piper/DescriptorUtils.groovy index 17b3a3dd8..bbd2f590d 100644 --- a/src/com/sap/piper/DescriptorUtils.groovy +++ b/src/com/sap/piper/DescriptorUtils.groovy @@ -13,7 +13,6 @@ def version = Pattern.compile("(.*)version=['\"](.*?)['\"](.*)", Pattern.DOTALL) @Field def method = Pattern.compile("(.*)\\(\\)", Pattern.DOTALL) -@NonCPS def getMavenGAV(file = 'pom.xml') { def result = [:] def descriptor = readMavenPom(file: file) @@ -28,7 +27,6 @@ def getMavenGAV(file = 'pom.xml') { return result } -@NonCPS def getNpmGAV(file = 'package.json') { def result = [:] def descriptor = readJSON(file: file) @@ -48,7 +46,6 @@ def getNpmGAV(file = 'package.json') { return result } -@NonCPS def getDlangGAV(file = 'dub.json') { def result = [:] def descriptor = readJSON(file: file) @@ -61,7 +58,6 @@ def getDlangGAV(file = 'dub.json') { return result } -@NonCPS def getSbtGAV(file = 'sbtDescriptor.json') { def result = [:] def descriptor = readJSON(file: file) @@ -74,7 +70,6 @@ def getSbtGAV(file = 'sbtDescriptor.json') { return result } -@NonCPS def getPipGAV(file = 'setup.py') { def result = [:] def descriptor = readFile(file: file) @@ -93,7 +88,6 @@ def getPipGAV(file = 'setup.py') { return result } -@NonCPS def getGoGAV(file = 'Gopkg.toml', URI repoUrl) { def name = "${repoUrl.getHost()}${repoUrl.getPath().replaceAll(/\.git/, '')}" def path = file.substring(0, file.lastIndexOf('/') + 1) @@ -115,7 +109,6 @@ def getGoGAV(file = 'Gopkg.toml', URI repoUrl) { return result } -@NonCPS private getVersionFromFile(file) { try { def versionString = readFile(file: file) From 55e3622e6b5b1fbbe276b298893a2f8ce3451c43 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 29 Mar 2019 14:38:43 +0100 Subject: [PATCH 120/130] Remove obsolete code, avoid caching of request --- .../integration/WhitesourceRepository.groovy | 24 ++++++------------- .../WhitesourceRepositoryTest.groovy | 19 ++++++++------- 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy index 078a879f9..0735c0b7d 100644 --- a/src/com/sap/piper/integration/WhitesourceRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -135,13 +135,14 @@ class WhitesourceRepository implements Serializable { } void fetchReportForProduct(reportName) { + def headers = [[name: 'Cache-Control', value: 'no-cache, no-store, must-revalidate'], [name: 'Pragma', value: 'no-cache']] def requestContent = [ requestType: "getProductRiskReport", productToken: config.whitesource.productToken ] //fetchFileFromWhiteSource(reportName, requestContent) - httpWhitesource(requestContent, 'APPLICATION_PDF', reportName) + httpWhitesource(requestContent, 'APPLICATION_PDF', headers, reportName) } def fetchProductLicenseAlerts() { @@ -167,7 +168,7 @@ class WhitesourceRepository implements Serializable { } @NonCPS - protected def httpWhitesource(requestBody, acceptType = 'APPLICATION_JSON', outputFile = null) { + protected def httpWhitesource(requestBody, acceptType = 'APPLICATION_JSON', customHeaders = null, outputFile = null) { handleAdditionalRequestParameters(requestBody) def serializedBody = new JsonUtils().groovyObjectToPrettyJsonString(requestBody) def params = [ @@ -180,11 +181,11 @@ class WhitesourceRepository implements Serializable { timeout : config.whitesource.timeout ] - if (outputFile) - params["outputFile"] = outputFile + if(customHeaders) params["customHeaders"] = customHeaders - if (script.env.HTTP_PROXY) - params["httpProxy"] = script.env.HTTP_PROXY + if (outputFile) params["outputFile"] = outputFile + + if (script.env.HTTP_PROXY) params["httpProxy"] = script.env.HTTP_PROXY if(config.verbose) script.echo "Sending http request with parameters ${params}" @@ -197,17 +198,6 @@ class WhitesourceRepository implements Serializable { return response } - @NonCPS - protected void fetchFileFromWhiteSource(String fileName, Map params) { - handleAdditionalRequestParameters(params) - def serializedContent = new JsonUtils().groovyObjectToPrettyJsonString(params) - - if(config.verbose) - script.echo "Sending curl request with parameters ${params}" - - script.sh "${config.verbose ? '' : '#!/bin/sh -e\n'}curl --fail -o ${fileName} -X POST ${config.whitesource.serviceUrl} -H 'Content-Type: application/json' -d \'${serializedContent}\'" - } - @NonCPS protected void handleAdditionalRequestParameters(params) { if(config.whitesource.userKey) diff --git a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy index 40f1f9621..0cfe9f963 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy @@ -350,15 +350,16 @@ class WhitesourceRepositoryTest extends BasePiperTest { assertThat(requestParams, is( [ - url : repository.config.whitesource.serviceUrl, - httpMode : 'POST', - acceptType : 'APPLICATION_PDF', - contentType: 'APPLICATION_JSON', - requestBody: requestBody, - quiet : false, - userKey : repository.config.whitesource.userKey, - httpProxy : "http://test.sap.com:8080", - outputFile : "test.file" + url : repository.config.whitesource.serviceUrl, + httpMode : 'POST', + acceptType : 'APPLICATION_PDF', + contentType : 'APPLICATION_JSON', + requestBody : requestBody, + quiet : false, + userKey : repository.config.whitesource.userKey, + httpProxy : "http://test.sap.com:8080", + outputFile : "test.file", + customHeaders : [[name: 'Cache-Control', value: 'no-cache, no-store, must-revalidate'], [name: 'Pragma', value: 'no-cache']] ] )) From 3191d8bb26a0b88ccfcc3dcda0d1a6b589192e9f Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Fri, 29 Mar 2019 14:45:05 +0100 Subject: [PATCH 121/130] Fix mime type for accept header --- src/com/sap/piper/integration/WhitesourceRepository.groovy | 2 +- .../com/sap/piper/integration/WhitesourceRepositoryTest.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/com/sap/piper/integration/WhitesourceRepository.groovy b/src/com/sap/piper/integration/WhitesourceRepository.groovy index 0735c0b7d..cba167794 100644 --- a/src/com/sap/piper/integration/WhitesourceRepository.groovy +++ b/src/com/sap/piper/integration/WhitesourceRepository.groovy @@ -142,7 +142,7 @@ class WhitesourceRepository implements Serializable { ] //fetchFileFromWhiteSource(reportName, requestContent) - httpWhitesource(requestContent, 'APPLICATION_PDF', headers, reportName) + httpWhitesource(requestContent, 'APPLICATION_OCTETSTREAM', headers, reportName) } def fetchProductLicenseAlerts() { diff --git a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy index 0cfe9f963..29b380503 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy @@ -352,7 +352,7 @@ class WhitesourceRepositoryTest extends BasePiperTest { [ url : repository.config.whitesource.serviceUrl, httpMode : 'POST', - acceptType : 'APPLICATION_PDF', + acceptType : 'APPLICATION_OCTETSTREAM', contentType : 'APPLICATION_JSON', requestBody : requestBody, quiet : false, From 1b7b1c5214d21a5f91324add8afc6eea9d4e6d71 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 1 Apr 2019 09:01:31 +0200 Subject: [PATCH 122/130] Refactor test to address review comments --- .../integration/WhitesourceRepositoryTest.groovy | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy index 29b380503..270cff08b 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy @@ -51,15 +51,9 @@ class WhitesourceRepositoryTest extends BasePiperTest { @Test void testMissingConfig() { - def errorCaught = false - try { - new WhitesourceRepository(nullScript, [:]) - } catch (e) { - errorCaught = true - assertThat(e, isA(AbortException.class)) - assertThat(e.getMessage(), is("Parameter 'whitesource.serviceUrl' must be provided as part of the configuration.")) - } - assertThat(errorCaught, is(true)) + exception.expect(AbortException) + exception.expectMessage("Parameter 'whitesource.serviceUrl' must be provided as part of the configuration.") + new WhitesourceRepository(nullScript, [:]) } @Test From 8d1ce13e4ef9568d757c7dc78ede9f60889b8b31 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 1 Apr 2019 09:03:56 +0200 Subject: [PATCH 123/130] Refactor test to address review comments --- .../WhitesourceOrgAdminRepositoryTest.groovy | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy index cdc4b2651..2d554abe0 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy @@ -5,6 +5,7 @@ import org.junit.After import org.junit.Before import org.junit.Rule import org.junit.Test +import org.junit.rules.ExpectedException import org.junit.rules.RuleChain import util.BasePiperTest import util.JenkinsEnvironmentRule @@ -20,6 +21,7 @@ import static org.hamcrest.Matchers.isA class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { + private ExpectedException expectedException = ExpectedException.none() private JenkinsErrorRule thrown = new JenkinsErrorRule(this) private JenkinsLoggingRule loggingRule = new JenkinsLoggingRule(this) private JenkinsEnvironmentRule environmentRule = new JenkinsEnvironmentRule(this) @@ -27,6 +29,7 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { @Rule public RuleChain ruleChain = Rules .getCommonRules(this) + .around(expectedException) .around(thrown) .around(loggingRule) .around(environmentRule) @@ -47,15 +50,9 @@ class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { @Test void testMissingConfig() { - def errorCaught = false - try { - new WhitesourceOrgAdminRepository(nullScript, [:]) - } catch (e) { - errorCaught = true - assertThat(e, isA(AbortException.class)) - assertThat(e.getMessage(), is("Parameter 'whitesource.serviceUrl' must be provided as part of the configuration.")) - } - assertThat(errorCaught, is(true)) + expectedException.expect(AbortException) + expectedException.expectMessage("Parameter 'whitesource.serviceUrl' must be provided as part of the configuration.") + new WhitesourceOrgAdminRepository(nullScript, [:]) } @Test From 4e144b80a12643670889aae81ea0b014ebcbecc0 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 1 Apr 2019 14:43:44 +0200 Subject: [PATCH 124/130] Refactor test to address last review comments --- test/groovy/WhitesourceExecuteScanTest.groovy | 4 ---- .../WhitesourceOrgAdminRepositoryTest.groovy | 4 ---- .../piper/integration/WhitesourceRepositoryTest.groovy | 10 +++------- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index cef26b15f..2544a95f1 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -25,13 +25,10 @@ class WhitesourceExecuteScanTest extends BasePiperTest { private JenkinsLoggingRule loggingRule = new JenkinsLoggingRule(this) private JenkinsWriteFileRule writeFileRule = new JenkinsWriteFileRule(this) private JenkinsStepRule stepRule = new JenkinsStepRule(this) - private JenkinsErrorRule errorRule = new JenkinsErrorRule(this) - private JenkinsEnvironmentRule environmentRule = new JenkinsEnvironmentRule(this) @Rule public RuleChain ruleChain = Rules .getCommonRules(this) - .around(environmentRule) .around(new JenkinsReadYamlRule(this)) .around(thrown) .around(dockerExecuteRule) @@ -39,7 +36,6 @@ class WhitesourceExecuteScanTest extends BasePiperTest { .around(loggingRule) .around(writeFileRule) .around(stepRule) - .around(errorRule) def whitesourceOrgAdminRepositoryStub def whitesourceStub diff --git a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy index 2d554abe0..ed1ab28c9 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceOrgAdminRepositoryTest.groovy @@ -22,17 +22,13 @@ import static org.hamcrest.Matchers.isA class WhitesourceOrgAdminRepositoryTest extends BasePiperTest { private ExpectedException expectedException = ExpectedException.none() - private JenkinsErrorRule thrown = new JenkinsErrorRule(this) private JenkinsLoggingRule loggingRule = new JenkinsLoggingRule(this) - private JenkinsEnvironmentRule environmentRule = new JenkinsEnvironmentRule(this) @Rule public RuleChain ruleChain = Rules .getCommonRules(this) .around(expectedException) - .around(thrown) .around(loggingRule) - .around(environmentRule) WhitesourceOrgAdminRepository repository diff --git a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy index 270cff08b..5f3043a0f 100644 --- a/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy +++ b/test/groovy/com/sap/piper/integration/WhitesourceRepositoryTest.groovy @@ -9,29 +9,25 @@ import org.junit.Test import org.junit.rules.ExpectedException import org.junit.rules.RuleChain import util.BasePiperTest -import util.JenkinsEnvironmentRule import util.JenkinsLoggingRule import util.LibraryLoadingTestExecutionListener import util.Rules import static org.assertj.core.api.Assertions.assertThat import static org.hamcrest.Matchers.containsString -import static org.hamcrest.Matchers.containsString import static org.hamcrest.Matchers.is import static org.hamcrest.Matchers.isA class WhitesourceRepositoryTest extends BasePiperTest { private ExpectedException exception = ExpectedException.none() - private JenkinsLoggingRule jlr = new JenkinsLoggingRule(this) - private JenkinsEnvironmentRule jer = new JenkinsEnvironmentRule(this) + private JenkinsLoggingRule loggingRule = new JenkinsLoggingRule(this) @Rule public RuleChain ruleChain = Rules .getCommonRules(this) .around(exception) - .around(jlr) - .around(jer) + .around(loggingRule) WhitesourceRepository repository @@ -357,7 +353,7 @@ class WhitesourceRepositoryTest extends BasePiperTest { ] )) - assertThat(jlr.log, containsString("Sending http request with parameters [requestType:getProductRiskReport, productToken:4711]")) + assertThat(loggingRule.log, containsString("Sending http request with parameters [requestType:getProductRiskReport, productToken:4711]")) } @Test From a43760d34aaa6c1f65c468e34066210ffec320db Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 1 Apr 2019 15:59:20 +0200 Subject: [PATCH 125/130] Use JenkinsCredentialsRule --- test/groovy/WhitesourceExecuteScanTest.groovy | 49 +-------------- .../groovy/util/JenkinsCredentialsRule.groovy | 59 +++++++++++++++---- 2 files changed, 51 insertions(+), 57 deletions(-) diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index 2544a95f1..02398c917 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -36,6 +36,9 @@ class WhitesourceExecuteScanTest extends BasePiperTest { .around(loggingRule) .around(writeFileRule) .around(stepRule) + .around(new JenkinsCredentialsRule(this) + .withCredentials('ID-123456789', 'token-0815') + .withCredentials('ID-9876543', 'token-0816')) def whitesourceOrgAdminRepositoryStub def whitesourceStub @@ -45,52 +48,6 @@ class WhitesourceExecuteScanTest extends BasePiperTest { @Before void init() { - def credentialsStore = ['ID-123456789': 'token-0815', 'ID-9876543': 'token-0816', 'ID-abcdefg': ['testUser', 'testPassword']] - def withCredentialsBindings - helper.registerAllowedMethod('string', [Map], { - m -> - withCredentialsBindings = ["${m.credentialsId}": "${m.variable}"] - return m - }) - helper.registerAllowedMethod('usernamePassword', [Map], { - m -> - withCredentialsBindings = ["${m.credentialsId}": ["${m.usernameVariable}", "${m.passwordVariable}"]] - return m - }) - helper.registerAllowedMethod('withCredentials', [List.class, Closure.class], { - l, body -> - def index = 0 - withCredentialsBindings.each { - entry -> - if(entry.value instanceof List) { - entry.value.each { - subEntry -> - def value = credentialsStore[entry.key] - getBinding().setProperty(subEntry, value[index]) - index++ - - } - } else { - getBinding().setProperty(entry.value, credentialsStore[entry.key]) - } - } - try { - body() - } finally { - withCredentialsBindings.each { - entry -> - if(entry.value instanceof List) { - entry.value.each { - subEntry -> - getBinding().setProperty(subEntry, null) - - } - } else { - getBinding().setProperty(entry.value, null) - } - } - } - }) helper.registerAllowedMethod("archiveArtifacts", [Map.class], { m -> if (m.artifacts == null) { throw new Exception('artifacts cannot be null') diff --git a/test/groovy/util/JenkinsCredentialsRule.groovy b/test/groovy/util/JenkinsCredentialsRule.groovy index dc3388c0d..6b855c843 100644 --- a/test/groovy/util/JenkinsCredentialsRule.groovy +++ b/test/groovy/util/JenkinsCredentialsRule.groovy @@ -14,6 +14,7 @@ import org.jenkinsci.plugins.credentialsbinding.impl.CredentialNotFoundException class JenkinsCredentialsRule implements TestRule { Map credentials = [:] + Map bindingTypes = [:] final BasePipelineTest testInstance @@ -26,6 +27,11 @@ class JenkinsCredentialsRule implements TestRule { return this } + JenkinsCredentialsRule withCredentials(String credentialsId, String token) { + credentials.put(credentialsId, [token: token]) + return this + } + @Override Statement apply(Statement base, Description description) { return statement(base) @@ -39,27 +45,58 @@ class JenkinsCredentialsRule implements TestRule { void evaluate() throws Throwable { testInstance.helper.registerAllowedMethod('usernamePassword', [Map.class], - { m -> if (credentials.keySet().contains(m.credentialsId)) return m; - // this is what really happens in case of an unknown credentials id, - // checked with reality using credentials plugin 2.1.18. - throw new CredentialNotFoundException( - "Could not find credentials entry with ID '${m.credentialsId}'") + { m -> + if (credentials.keySet().contains(m.credentialsId)) { bindingTypes[m.credentialsId] = 'usernamePassword'; return m } + // this is what really happens in case of an unknown credentials id, + // checked with reality using credentials plugin 2.1.18. + throw new CredentialNotFoundException( + "Could not find credentials entry with ID '${m.credentialsId}'") + }) + + testInstance.helper.registerAllowedMethod('string', [Map.class], + { m -> + if (credentials.keySet().contains(m.credentialsId)) { bindingTypes[m.credentialsId] = 'string'; return m } + // this is what really happens in case of an unknown credentials id, + // checked with reality using credentials plugin 2.1.18. + throw new CredentialNotFoundException( + "Could not find credentials entry with ID '${m.credentialsId}'") }) testInstance.helper.registerAllowedMethod('withCredentials', [List, Closure], { config, closure -> def credsId = config[0].credentialsId - def passwordVariable = config[0].passwordVariable - def usernameVariable = config[0].usernameVariable + def credentialsBindingType = bindingTypes.get(credsId) def creds = credentials.get(credsId) - binding.setProperty(usernameVariable, creds?.user) - binding.setProperty(passwordVariable, creds?.passwd) + def tokenVariable, usernameVariable, passwordVariable, prepare, destruct + if(credentialsBindingType == "usernamePassword") { + passwordVariable = config[0].passwordVariable + usernameVariable = config[0].usernameVariable + prepare = { + binding.setProperty(usernameVariable, creds?.user) + binding.setProperty(passwordVariable, creds?.passwd) + } + destruct = { + binding.setProperty(usernameVariable, null) + binding.setProperty(passwordVariable, null) + } + } else if(credentialsBindingType == "string") { + tokenVariable = config[0].variable + prepare = { + binding.setProperty(tokenVariable, creds?.token) + } + destruct = { + binding.setProperty(tokenVariable, null) + } + } else { + throw new RuntimeException("Unknown binding type") + } + + prepare() try { closure() } finally { - binding.setProperty(usernameVariable, null) - binding.setProperty(passwordVariable, null) + destruct() } }) From b618764a001d95104b127651f2b4c3b0cefbe56f Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 1 Apr 2019 16:13:48 +0200 Subject: [PATCH 126/130] Use ExpectedException --- test/groovy/WhitesourceExecuteScanTest.groovy | 153 ++++++------------ 1 file changed, 48 insertions(+), 105 deletions(-) diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index 02398c917..07b54b01e 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -997,24 +997,24 @@ class WhitesourceExecuteScanTest extends BasePiperTest { "\"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-databind\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:5,\", \"date\": \"2017-11-15\" } ] }").alerts }) - try { - stepRule.step.whitesourceExecuteScan([ - script : nullScript, - whitesourceRepositoryStub : whitesourceStub, - whitesourceOrgAdminRepositoryStub: whitesourceOrgAdminRepositoryStub, - descriptorUtilsStub : descriptorUtilsStub, - scanType : 'npm', - juStabUtils : utils, - securityVulnerabilities : true, - orgToken : 'testOrgToken', - productName : 'SHC - Piper', - projectNames : ['piper-demo - 0.0.1'], - cvssSeverityLimit : 0 - ]) - } catch (e) { - assertThat(e.getMessage(), containsString('[whitesourceExecuteScan] 1 Open Source Software Security vulnerabilities with CVSS score greater or equal 0 detected. - ')) - assertThat(writeFileRule.files['piper_whitesource_vulnerability_report.json'], not(isEmptyOrNullString())) - } + thrown.expect(AbortException) + thrown.expectMessage('[whitesourceExecuteScan] 1 Open Source Software Security vulnerabilities with CVSS score greater or equal 0 detected. - ') + + stepRule.step.whitesourceExecuteScan([ + script : nullScript, + whitesourceRepositoryStub : whitesourceStub, + whitesourceOrgAdminRepositoryStub: whitesourceOrgAdminRepositoryStub, + descriptorUtilsStub : descriptorUtilsStub, + scanType : 'npm', + juStabUtils : utils, + securityVulnerabilities : true, + orgToken : 'testOrgToken', + productName : 'SHC - Piper', + projectNames : ['piper-demo - 0.0.1'], + cvssSeverityLimit : 0 + ]) + + assertThat(writeFileRule.files['piper_whitesource_vulnerability_report.json'], not(isEmptyOrNullString())) } @Test @@ -1053,132 +1053,75 @@ class WhitesourceExecuteScanTest extends BasePiperTest { @Test void testCheckStatus_0() { - def error = false - try { - stepRule.step.checkStatus(0, [whitesource:[licensingVulnerabilities: true]]) - } catch (e) { - error = true - } - assertThat(error, is(false)) + stepRule.step.checkStatus(0, [whitesource:[licensingVulnerabilities: true]]) } @Test void testCheckStatus_255() { - def error = false - try { - stepRule.step.checkStatus(255, [whitesource:[licensingVulnerabilities: true]]) - } catch (e) { - error = true - assertThat(e.getMessage(), is("[whitesourceExecuteScan] The scan resulted in an error")) - } - assertThat(error, is(true)) + thrown.expect(AbortException) + thrown.expectMessage("[whitesourceExecuteScan] The scan resulted in an error") + stepRule.step.checkStatus(255, [whitesource:[licensingVulnerabilities: true]]) } @Test void testCheckStatus_254() { - def error = false - try { - stepRule.step.checkStatus(254, [whitesource:[licensingVulnerabilities: true]]) - } catch (e) { - error = true - assertThat(e.getMessage(), is("[whitesourceExecuteScan] Whitesource found one or multiple policy violations")) - } - assertThat(error, is(true)) + thrown.expect(AbortException) + thrown.expectMessage("[whitesourceExecuteScan] Whitesource found one or multiple policy violations") + stepRule.step.checkStatus(254, [whitesource:[licensingVulnerabilities: true]]) } @Test void testCheckStatus_253() { - def error = false - try { - stepRule.step.checkStatus(253, [whitesource:[licensingVulnerabilities: true]]) - } catch (e) { - error = true - assertThat(e.getMessage(), is("[whitesourceExecuteScan] The local scan client failed to execute the scan")) - } - assertThat(error, is(true)) + thrown.expect(AbortException) + thrown.expectMessage("[whitesourceExecuteScan] The local scan client failed to execute the scan") + stepRule.step.checkStatus(253, [whitesource:[licensingVulnerabilities: true]]) } @Test void testCheckStatus_252() { - def error = false - try { - stepRule.step.checkStatus(252, [whitesource:[licensingVulnerabilities: true]]) - } catch (e) { - error = true - assertThat(e.getMessage(), is("[whitesourceExecuteScan] There was a failure in the connection to the WhiteSource servers")) - } - assertThat(error, is(true)) + thrown.expect(AbortException) + thrown.expectMessage("[whitesourceExecuteScan] There was a failure in the connection to the WhiteSource servers") + stepRule.step.checkStatus(252, [whitesource:[licensingVulnerabilities: true]]) } @Test void testCheckStatus_251() { - def error = false - try { - stepRule.step.checkStatus(251, [whitesource:[licensingVulnerabilities: true]]) - } catch (e) { - error = true - assertThat(e.getMessage(), is("[whitesourceExecuteScan] The server failed to analyze the scan")) - } - assertThat(error, is(true)) + thrown.expect(AbortException) + thrown.expectMessage("[whitesourceExecuteScan] The server failed to analyze the scan") + stepRule.step.checkStatus(251, [whitesource:[licensingVulnerabilities: true]]) } @Test void testCheckStatus_250() { - def error = false - try { - stepRule.step.checkStatus(250, [whitesource:[licensingVulnerabilities: true]]) - } catch (e) { - error = true - assertThat(e.getMessage(), is("[whitesourceExecuteScan] Pre-step failure")) - } - assertThat(error, is(true)) + thrown.expect(AbortException) + thrown.expectMessage("[whitesourceExecuteScan] Pre-step failure") + stepRule.step.checkStatus(250, [whitesource:[licensingVulnerabilities: true]]) } @Test void testCheckStatus_127() { - def error = false - try { - stepRule.step.checkStatus(127, [whitesource:[licensingVulnerabilities: true]]) - } catch (e) { - error = true - assertThat(e.getMessage(), is("[whitesourceExecuteScan] Whitesource scan failed with unknown error code '127'")) - } - assertThat(error, is(true)) + thrown.expect(AbortException) + thrown.expectMessage("[whitesourceExecuteScan] Whitesource scan failed with unknown error code '127'") + stepRule.step.checkStatus(127, [whitesource:[licensingVulnerabilities: true]]) } @Test void testCheckStatus_vulnerability() { - def error = false - try { - stepRule.step.checkStatus(0, [whitesource:[licensingVulnerabilities: false, securityVulnerabilities: true, severeVulnerabilities: 5, cvssSeverityLimit: 7]]) - } catch (e) { - error = true - assertThat(e.getMessage(), is("[whitesourceExecuteScan] 5 Open Source Software Security vulnerabilities with CVSS score greater or equal 7 detected. - ")) - } - assertThat(error, is(true)) + thrown.expect(AbortException) + thrown.expectMessage("[whitesourceExecuteScan] 5 Open Source Software Security vulnerabilities with CVSS score greater or equal 7 detected. - ") + stepRule.step.checkStatus(0, [whitesource:[licensingVulnerabilities: false, securityVulnerabilities: true, severeVulnerabilities: 5, cvssSeverityLimit: 7]]) } @Test void testCheckViolationStatus_0() { - def error = false - try { - stepRule.step.checkViolationStatus(0) - } catch (e) { - error = true - } - assertThat(error, is(false)) - assertThat(loggingRule.log, containsString("[whitesourceExecuteScan] No policy violations found")) + stepRule.step.checkViolationStatus(0) + assertThat(loggingRule.log, containsString ("[whitesourceExecuteScan] No policy violations found")) } @Test void testCheckViolationStatus_5() { - def error = false - try { - stepRule.step.checkViolationStatus(5) - } catch (e) { - error = true - assertThat(e.getMessage(), is("[whitesourceExecuteScan] Whitesource found 5 policy violations for your product")) - } - assertThat(error, is(true)) + thrown.expect(AbortException) + thrown.expectMessage("[whitesourceExecuteScan] Whitesource found 5 policy violations for your product") + stepRule.step.checkViolationStatus(5) } } From 7f4f0eef000df506c395c500e8545dfe1fbf5cc7 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 1 Apr 2019 16:22:49 +0200 Subject: [PATCH 127/130] JSON strings into tripple quotes --- test/groovy/WhitesourceExecuteScanTest.groovy | 81 +++++-------------- 1 file changed, 22 insertions(+), 59 deletions(-) diff --git a/test/groovy/WhitesourceExecuteScanTest.groovy b/test/groovy/WhitesourceExecuteScanTest.groovy index 07b54b01e..9329197c7 100644 --- a/test/groovy/WhitesourceExecuteScanTest.groovy +++ b/test/groovy/WhitesourceExecuteScanTest.groovy @@ -869,55 +869,18 @@ class WhitesourceExecuteScanTest extends BasePiperTest { return result }) helper.registerAllowedMethod("fetchVulnerabilities", [List], { - return new JsonUtils().jsonStringToGroovyObject("{ \"alerts\": [ { \"vulnerability\": { \"name\": \"CVE-2017-15095\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 7.5, \"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15095\", \"description\": \"A deserialization flaw was discovered in the jackson-databind in versions before 2.8.10 and 2.9.1, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper. This issue extends the previous flaw CVE-2017-7525 by blacklisting more classes that could be used maliciously.\", \"topFix\": { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\"," + - "\"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\", \"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\"," + - "\"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e865a7a4464da63ded9f4b1a2328ad85c9ded78b#diff-98084d808198119d550a9211e128a16f\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-12-12\", \"message\": \"Fix #1737 (#1857)\", \"extraData\": \"key=e865a7a&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e8f043d1\"," + - "\"fixResolution\": \"release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-06-30\", \"message\": \"Fix #1680\", \"extraData\": \"key=e8f043d&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" } ], \"fixResolutionText\": \"Replace or update the following files: IllegalTypesCheckTest.java, VERSION, BeanDeserializerFactory.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\", \"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\"," + - "\"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-databind\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:5,\", \"date\": \"2017-11-15\" }, { \"vulnerability\": { \"name\": \"CVE-2017-17485\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 7.5, \"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-01-10\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-17485\"," + - "\"description\": \"FasterXML jackson-databind through 2.8.10 and 2.9.x through 2.9.3 allows unauthenticated remote code execution because of an incomplete fix for the CVE-2017-7525 deserialization flaw. This is exploitable by sending maliciously crafted JSON input to the readValue method of the ObjectMapper, bypassing a blacklist that is ineffective if the Spring libraries are available in the classpath.\", \"topFix\": { \"vulnerability\": \"CVE-2017-17485\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/bb45fb16709018842f858f1a6e1118676aaa34bd#diff-727a6e8db3603b95f185697108af6c48\", \"fixResolution\": \"src/test/java/org/springframework/jacksontest/AbstractApplicationContext.java,src/test/java/org/springframework/jacksontest/AbstractPointcutAdvisor.java,src/test/java/org/springframework/jacksontest/BogusApplicationContext.java,src/main/java/com/fasterxml/jackson/databind/jsontype/impl/SubTypeValidator.java,src/test/java/org/springframework/jacksontest/BogusPointcutAdvisor.java,src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java\"," + - "\"date\": \"2017-12-19\", \"message\": \"Fix issues with earlier fix for #1855\", \"extraData\": \"key=bb45fb1&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-17485\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/bb45fb16709018842f858f1a6e1118676aaa34bd#diff-727a6e8db3603b95f185697108af6c48\", \"fixResolution\": \"src/test/java/org/springframework/jacksontest/AbstractApplicationContext.java,src/test/java/org/springframework/jacksontest/AbstractPointcutAdvisor.java,src/test/java/org/springframework/jacksontest/BogusApplicationContext.java,src/main/java/com/fasterxml/jackson/databind/jsontype/impl/SubTypeValidator.java,src/test/java/org/springframework/jacksontest/BogusPointcutAdvisor.java,src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java\", \"date\": \"2017-12-19\", \"message\": \"Fix issues with earlier fix for #1855\"," + - "\"extraData\": \"key=bb45fb1&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-17485\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/bb45fb16709018842f858f1a6e1118676aaa34bd\", \"fixResolution\": \"src/test/java/org/springframework/jacksontest/AbstractApplicationContext.java,src/test/java/org/springframework/jacksontest/AbstractPointcutAdvisor.java,src/test/java/org/springframework/jacksontest/BogusApplicationContext.java,src/main/java/com/fasterxml/jackson/databind/jsontype/impl/SubTypeValidator.java,src/test/java/org/springframework/jacksontest/BogusPointcutAdvisor.java,src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java\", \"date\": \"2017-12-19\", \"message\": \"Fix issues with earlier fix for #1855\", \"extraData\": \"key=bb45fb1&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" } ]," + - "\"fixResolutionText\": \"Replace or update the following files: AbstractApplicationContext.java, AbstractPointcutAdvisor.java, BogusApplicationContext.java, SubTypeValidator.java, BogusPointcutAdvisor.java, IllegalTypesCheckTest.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\", \"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-databind\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\"," + - "\"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:5,\", \"date\": \"2017-11-15\" }, { \"vulnerability\": { \"name\": \"CVE-2017-7525\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 7.5, \"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-7525\", \"description\": \"A deserialization flaw was discovered in the jackson-databind, versions before 2.6.7.1, 2.7.9.1 and 2.8.9, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper.\", \"topFix\": { \"vulnerability\": \"CVE-2017-7525\"," + - "\"type\": \"UPGRADE_VERSION\", \"origin\": \"BUGZILLA\", \"url\": \"https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2017-7525\", \"fixResolution\": \"jackson-databind 2.8.9,jackson-databind 2.9.0\", \"message\": \"CVE-2017-7525 jackson-databind: Deserialization vulnerability via readValue method of ObjectMapper\", \"extraData\": \"key=1462702&assignee=Red Hat Product Security\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-7525\", \"type\": \"UPGRADE_VERSION\", \"origin\": \"BUGZILLA\", \"url\": \"https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2017-7525\", \"fixResolution\": \"jackson-databind 2.8.9,jackson-databind 2.9.0\", \"message\": \"CVE-2017-7525 jackson-databind: Deserialization vulnerability via readValue method of ObjectMapper\", \"extraData\": \"key=1462702&assignee=Red Hat Product Security\" }, { \"vulnerability\": \"CVE-2017-7525\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459cedcf079c6106ae7da2ac562bc32dcabe1#diff-98084d808198119d550a9211e128a16f\"," + - "\"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\", \"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-7525\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459cedcf079c6106ae7da2ac562bc32dcabe1\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\"," + - "\"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-7525\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/6ce32ffd18facac6abdbbf559c817b47fcb622c1\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.7(.10)\", \"extraData\": \"key=6ce32ff&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" } ], \"fixResolutionText\": \"Upgrade to version jackson-databind 2.8.9, jackson-databind 2.9.0 or greater\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\"," + - "\"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\", \"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-databind\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }," + - "\"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:5,\", \"date\": \"2017-11-15\" }, { \"vulnerability\": { \"name\": \"CVE-2018-5968\", \"type\": \"CVE\", \"severity\": \"medium\", \"score\": 5.1, \"cvss3_severity\": \"high\", \"cvss3_score\": 8.1, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-01-22\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-5968\", \"description\": \"FasterXML jackson-databind through 2.8.11 and 2.9.x through 2.9.3 allows unauthenticated remote code execution because of an incomplete fix for the CVE-2017-7525 and CVE-2017-17485 deserialization flaws. This is exploitable via two different gadgets that bypass a blacklist.\", \"topFix\": { \"vulnerability\": \"CVE-2018-5968\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\"," + - "\"url\": \"https://github.com/FasterXML/jackson-databind/commit/038b471e2efde2e8f96b4e0be958d3e5a1ff1d05\", \"fixResolution\": \"src/main/java/com/fasterxml/jackson/databind/jsontype/impl/SubTypeValidator.java,release-notes/VERSION\", \"date\": \"2018-01-22\", \"message\": \"Fix #1899\", \"extraData\": \"key=038b471&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [], \"fixResolutionText\": \"Replace or update the following files: SubTypeValidator.java, VERSION\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\"," + - "\"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-databind\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:5,\", \"date\": \"2017-11-15\" }, { \"vulnerability\": { \"name\": \"CVE-2018-7489\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 7.5," + - "\"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-26\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-7489\", \"description\": \"FasterXML jackson-databind before 2.8.11.1 and 2.9.x before 2.9.5 allows unauthenticated remote code execution because of an incomplete fix for the CVE-2017-7525 deserialization flaw. This is exploitable by sending maliciously crafted JSON input to the readValue method of the ObjectMapper, bypassing a blacklist that is ineffective if the c3p0 libraries are available in the classpath.\", \"topFix\": { \"vulnerability\": \"CVE-2018-7489\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/6799f8f10cc78e9af6d443ed6982d00a13f2e7d2\", \"fixResolution\": \"src/main/java/com/fasterxml/jackson/databind/jsontype/impl/SubTypeValidator.java,src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,src/test/java/com/mchange/v2/c3p0/jacksontest/ComboPooledDataSource.java,release-notes/VERSION\"," + - "\"date\": \"2018-02-11\", \"message\": \"Fix #1931\", \"extraData\": \"key=6799f8f&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [], \"fixResolutionText\": \"Replace or update the following files: SubTypeValidator.java, IllegalTypesCheckTest.java, ComboPooledDataSource.java, VERSION\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\", \"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\"," + - "\"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-databind\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:5,\", \"date\": \"2017-11-15\" }, { \"vulnerability\": { \"name\": \"CVE-2016-3720\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 7.5, \"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2016-06-10\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-3720\"," + - "\"description\": \"XML external entity (XXE) vulnerability in XmlMapper in the Data format extension for Jackson (aka jackson-dataformat-xml) allows attackers to have unspecified impact via unknown vectors.\", \"allFixes\": [], \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"9c7e7805-0ba0-480a-8647-ed0a457b4a88\", \"keyId\": 441802, \"filename\": \"jackson-dataformat-xml-2.4.2.jar\", \"name\": \"Jackson-dataformat-XML\", \"groupId\": \"com.fasterxml.jackson.dataformat\", \"artifactId\": \"jackson-dataformat-xml\", \"version\": \"2.4.2\", \"sha1\": \"02f2d96f68b2d3475452d95dde7a3fbee225f6ae\", \"type\": \"Java\", \"references\": { \"url\": \"http://wiki.fasterxml.com/JacksonExtensionXmlDataBinding\", \"issueUrl\": \"https://github.com/FasterXML/jackson-dataformat-xml/issues\", \"pomUrl\": \"http://maven.ibiblio.org/maven2/com/fasterxml/jackson/dataformat/jackson-dataformat-xml/2.4.2/jackson-dataformat-xml-2.4.2.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-dataformat-xml\" }," + - "\"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:1,\", \"date\": \"2017-10-26\" }, { \"vulnerability\": { \"name\": \"CVE-2017-5929\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 7.5, \"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2017-03-13\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-5929\", \"description\": \"QOS.ch Logback before 1.2.0 has a serialization vulnerability affecting the SocketServer and ServerSocketReceiver components.\", \"topFix\": { \"vulnerability\": \"CVE-2017-5929\"," + - "\"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/qos-ch/logback/commit/f46044b805bca91efe5fd6afe52257cd02f775f8\", \"fixResolution\": \"logback-classic/src/main/java/ch/qos/logback/classic/net/server/LogbackClassicSerializationHelper.java,logback-core/src/test/java/ch/qos/logback/core/net/Innocent.java,logback-classic/src/main/java/ch/qos/logback/classic/net/SimpleSocketServer.java,logback-core/src/main/java/ch/qos/logback/core/net/HardenedObjectInputStream.java,logback-classic/src/test/java/ch/qos/logback/classic/LoggerSerializationTest.java,logback-core/src/test/java/ch/qos/logback/core/net/HardenedObjectInputStreamTest.java\", \"date\": \"2017-02-07\", \"message\": \"harden serialization\", \"extraData\": \"key=f46044b&committerName=ceki&committerUrl=https://github.com/ceki&committerAvatar=https://avatars1.githubusercontent.com/u/115476?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-5929\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\"," + - "\"url\": \"https://github.com/qos-ch/logback/commit/f46044b805bca91efe5fd6afe52257cd02f775f8\", \"fixResolution\": \"logback-classic/src/main/java/ch/qos/logback/classic/net/server/LogbackClassicSerializationHelper.java,logback-core/src/test/java/ch/qos/logback/core/net/Innocent.java,logback-classic/src/main/java/ch/qos/logback/classic/net/SimpleSocketServer.java,logback-core/src/main/java/ch/qos/logback/core/net/HardenedObjectInputStream.java,logback-classic/src/test/java/ch/qos/logback/classic/LoggerSerializationTest.java,logback-core/src/test/java/ch/qos/logback/core/net/HardenedObjectInputStreamTest.java\", \"date\": \"2017-02-07\", \"message\": \"harden serialization\", \"extraData\": \"key=f46044b&committerName=ceki&committerUrl=https://github.com/ceki&committerAvatar=https://avatars1.githubusercontent.com/u/115476?v=4\" }, { \"vulnerability\": \"CVE-2017-5929\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/victims/victims-cve-db/commit/94745e07d4b10a58d98ffc3916d9711fa407b018\"," + - "\"fixResolution\": \"database/java/2017/5929.yaml\", \"date\": \"2017-03-15\", \"message\": \"Added CVE-2017-5929 per issue #76\", \"extraData\": \"key=94745e0&committerName=cplvic&committerUrl=https://github.com/cplvic&committerAvatar=https://avatars0.githubusercontent.com/u/11528385?v=4\" } ], \"fixResolutionText\": \"Replace or update the following files: LogbackClassicSerializationHelper.java, Innocent.java, SimpleSocketServer.java, HardenedObjectInputStream.java, LoggerSerializationTest.java, HardenedObjectInputStreamTest.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"1259a151-8d7c-4921-b293-3b4b8708ac5e\", \"keyId\": 754707, \"filename\": \"logback-classic-1.1.3.jar\", \"name\": \"Logback Classic Module\", \"groupId\": \"ch.qos.logback\", \"artifactId\": \"logback-classic\", \"version\": \"1.1.3\", \"sha1\": \"d90276fff414f06cb375f2057f6778cd63c6082f\", \"type\": \"Java\", \"references\": { \"url\": \"http://logback.qos.ch/logback-classic\"," + - "\"pomUrl\": \"http://maven.ibiblio.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom\", \"scmUrl\": \"https://github.com/ceki/logback/logback-classic\" }, \"licenses\": [ { \"name\": \"LGPL 2.1\", \"url\": \"http://opensource.org/licenses/lgpl-2.1\", \"profileInfo\": { \"copyrightRiskScore\": \"FIVE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"PARTIAL\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } }, { \"name\": \"Eclipse 1.0\", \"url\": \"http://opensource.org/licenses/eclipse-1.0.php\", \"profileInfo\": { \"copyrightRiskScore\": \"SIX\", \"patentRiskScore\": \"FOUR\", \"copyleft\": \"PARTIAL\", \"linking\": \"VIRAL\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": true, \"description\": \"High:1,\", \"date\": \"2017-10-26\" }, { \"vulnerability\": { \"name\": \"CVE-2017-5929\", \"type\": \"CVE\", \"severity\": \"high\"," + - "\"score\": 7.5, \"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2017-03-13\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-5929\", \"description\": \"QOS.ch Logback before 1.2.0 has a serialization vulnerability affecting the SocketServer and ServerSocketReceiver components.\", \"topFix\": { \"vulnerability\": \"CVE-2017-5929\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/qos-ch/logback/commit/f46044b805bca91efe5fd6afe52257cd02f775f8\", \"fixResolution\": \"logback-classic/src/main/java/ch/qos/logback/classic/net/server/LogbackClassicSerializationHelper.java,logback-core/src/test/java/ch/qos/logback/core/net/Innocent.java,logback-classic/src/main/java/ch/qos/logback/classic/net/SimpleSocketServer.java,logback-core/src/main/java/ch/qos/logback/core/net/HardenedObjectInputStream.java,logback-classic/src/test/java/ch/qos/logback/classic/LoggerSerializationTest.java,logback-core/src/test/java/ch/qos/logback/core/net/HardenedObjectInputStreamTest.java\", \"date\": \"2017-02-07\"," + - "\"message\": \"harden serialization\", \"extraData\": \"key=f46044b&committerName=ceki&committerUrl=https://github.com/ceki&committerAvatar=https://avatars1.githubusercontent.com/u/115476?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-5929\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/qos-ch/logback/commit/f46044b805bca91efe5fd6afe52257cd02f775f8\", \"fixResolution\": \"logback-classic/src/main/java/ch/qos/logback/classic/net/server/LogbackClassicSerializationHelper.java,logback-core/src/test/java/ch/qos/logback/core/net/Innocent.java,logback-classic/src/main/java/ch/qos/logback/classic/net/SimpleSocketServer.java,logback-core/src/main/java/ch/qos/logback/core/net/HardenedObjectInputStream.java,logback-classic/src/test/java/ch/qos/logback/classic/LoggerSerializationTest.java,logback-core/src/test/java/ch/qos/logback/core/net/HardenedObjectInputStreamTest.java\", \"date\": \"2017-02-07\", \"message\": \"harden serialization\", \"extraData\": \"key=f46044b&committerName=ceki&committerUrl=https://github.com/ceki&committerAvatar=https://avatars1.githubusercontent.com/u/115476?v=4\" }," + - "{ \"vulnerability\": \"CVE-2017-5929\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/victims/victims-cve-db/commit/94745e07d4b10a58d98ffc3916d9711fa407b018\", \"fixResolution\": \"database/java/2017/5929.yaml\", \"date\": \"2017-03-15\", \"message\": \"Added CVE-2017-5929 per issue #76\", \"extraData\": \"key=94745e0&committerName=cplvic&committerUrl=https://github.com/cplvic&committerAvatar=https://avatars0.githubusercontent.com/u/11528385?v=4\" } ], \"fixResolutionText\": \"Replace or update the following files: LogbackClassicSerializationHelper.java, Innocent.java, SimpleSocketServer.java, HardenedObjectInputStream.java, LoggerSerializationTest.java, HardenedObjectInputStreamTest.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"d2b236f1-ac7d-4413-92fa-848cc7c3d3e5\", \"keyId\": 754708, \"filename\": \"logback-core-1.1.3.jar\", \"name\": \"Logback Core Module\", \"groupId\": \"ch.qos.logback\", \"artifactId\": \"logback-core\", \"version\": \"1.1.3\", \"sha1\": \"e3c02049f2dbbc764681b40094ecf0dcbc99b157\", \"type\": \"Java\"," + - "\"references\": { \"url\": \"http://logback.qos.ch/logback-core\", \"pomUrl\": \"http://maven.ibiblio.org/maven2/ch/qos/logback/logback-core/1.1.3/logback-core-1.1.3.pom\", \"scmUrl\": \"https://github.com/ceki/logback/logback-core\" }, \"licenses\": [ { \"name\": \"Eclipse 1.0\", \"url\": \"http://opensource.org/licenses/eclipse-1.0.php\", \"profileInfo\": { \"copyrightRiskScore\": \"SIX\", \"patentRiskScore\": \"FOUR\", \"copyleft\": \"PARTIAL\", \"linking\": \"VIRAL\", \"royaltyFree\": \"CONDITIONAL\" } }, { \"name\": \"LGPL 2.1\", \"url\": \"http://opensource.org/licenses/lgpl-2.1\", \"profileInfo\": { \"copyrightRiskScore\": \"FIVE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"PARTIAL\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:1,\", \"date\": \"2017-10-26\" }, { \"vulnerability\": { \"name\": \"CVE-2012-4529\", \"type\": \"CVE\", \"severity\": \"medium\", \"score\": 4.3, \"cvss3_score\": 0," + - "\"publishDate\": \"2013-10-28\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2012-4529\", \"description\": \"The org.apache.catalina.connector.Response.encodeURL method in Red Hat JBoss Web 7.1.x and earlier, when the tracking mode is set to COOKIE, sends the jsessionid in the URL of the first response of a session, which allows remote attackers to obtain the session id (1) via a man-in-the-middle attack or (2) by reading a log.\", \"allFixes\": [], \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"aa3a10d7-4e2c-46fe-bbf9-3c2d06e43b02\", \"keyId\": 24276785, \"filename\": \"tomcat-embed-core-7.0.78.jar\", \"name\": \"tomcat-embed-core\", \"groupId\": \"org.apache.tomcat.embed\", \"artifactId\": \"tomcat-embed-core\", \"version\": \"7.0.78\", \"sha1\": \"ddb63d615ec3944b4394aed6dc825cd0cbb16b21\", \"type\": \"Java\", \"references\": { \"url\": \"http://tomcat.apache.org/\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/org/apache/tomcat/embed/tomcat-embed-core/7.0.78/tomcat-embed-core-7.0.78.pom\" }, \"licenses\": [ { \"name\": \"Apache 2.0\"," + - "\"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": true, \"description\": \"High:3,Medium:2,\", \"date\": \"2017-10-26\" }, { \"vulnerability\": { \"name\": \"CVE-2017-12615\", \"type\": \"CVE\", \"severity\": \"medium\", \"score\": 6.8, \"cvss3_severity\": \"high\", \"cvss3_score\": 8.1, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2017-09-19\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-12615\", \"description\": \"When running Apache Tomcat 7.0.0 to 7.0.79 on Windows with HTTP PUTs enabled (e.g. via setting the readonly initialisation parameter of the Default to false) it was possible to upload a JSP file to the server via a specially crafted request. This JSP could then be requested and any code it contained would be executed by the server.\"," + - "\"topFix\": { \"vulnerability\": \"CVE-2017-12615\", \"type\": \"UPGRADE_VERSION\", \"origin\": \"SECURITY_TRACKER\", \"url\": \"http://www.securitytracker.com/id/1039392\", \"fixResolution\": \"The vendor has issued a fix (7.0.81).\\n\\nThe vendor advisory is available at:\\n\\nhttps://tomcat.apache.org/security-7.html#Fixed_in_Apache_Tomcat_7.0.81\", \"date\": \"2017-12-31\", \"message\": \"Apache Tomcat on Windows HTTP PUT Request Processing Flaw Lets Remote Users Execute Arbitrary Code on the Target System\", \"extraData\": \"key=1039392\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-12615\", \"type\": \"UPGRADE_VERSION\", \"origin\": \"SECURITY_TRACKER\", \"url\": \"http://www.securitytracker.com/id/1039392\", \"fixResolution\": \"The vendor has issued a fix (7.0.81).\\n\\nThe vendor advisory is available at:\\n\\nhttps://tomcat.apache.org/security-7.html#Fixed_in_Apache_Tomcat_7.0.81\", \"date\": \"2017-12-31\", \"message\": \"Apache Tomcat on Windows HTTP PUT Request Processing Flaw Lets Remote Users Execute Arbitrary Code on the Target System\", \"extraData\": \"key=1039392\" }, { \"vulnerability\": \"CVE-2017-12615\", \"type\": \"CHANGE_FILES\"," + - "\"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat70/commit/07dc0ea2745f0afab6415f22b16a29f1c6de5727\", \"fixResolution\": \"java/org/apache/naming/resources/VirtualDirContext.java,webapps/docs/changelog.xml,java/org/apache/naming/resources/FileDirContext.java\", \"date\": \"2017-08-10\", \"message\": \"Correct regression in r1804604 that broke WebDAV.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc7.0.x/trunk@1804729 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=07dc0ea&committerName=markt-asf&committerUrl=https://github.com/markt-asf&committerAvatar=https://avatars3.githubusercontent.com/u/4690029?v=4\" } ], \"fixResolutionText\": \"The vendor has issued a fix (7.0.81).\\n\\nThe vendor advisory is available at:\\n\\nhttps://tomcat.apache.org/security-7.html#Fixed_in_Apache_Tomcat_7.0.81\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"aa3a10d7-4e2c-46fe-bbf9-3c2d06e43b02\", \"keyId\": 24276785, \"filename\": \"tomcat-embed-core-7.0.78.jar\", \"name\": \"tomcat-embed-core\", \"groupId\": \"org.apache.tomcat.embed\", \"artifactId\": \"tomcat-embed-core\"," + - "\"version\": \"7.0.78\", \"sha1\": \"ddb63d615ec3944b4394aed6dc825cd0cbb16b21\", \"type\": \"Java\", \"references\": { \"url\": \"http://tomcat.apache.org/\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/org/apache/tomcat/embed/tomcat-embed-core/7.0.78/tomcat-embed-core-7.0.78.pom\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": true, \"description\": \"High:3,Medium:2,\", \"date\": \"2017-10-26\" }, { \"vulnerability\": { \"name\": \"CVE-2017-12616\", \"type\": \"CVE\", \"severity\": \"medium\", \"score\": 5, \"cvss3_severity\": \"high\", \"cvss3_score\": 7.5, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N\", \"publishDate\": \"2017-09-19\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-12616\"," + - "\"description\": \"When using a VirtualDirContext with Apache Tomcat 7.0.0 to 7.0.80 it was possible to bypass security constraints and/or view the source code of JSPs for resources served by the VirtualDirContext using a specially crafted request.\", \"topFix\": { \"vulnerability\": \"CVE-2017-12616\", \"type\": \"UPGRADE_VERSION\", \"origin\": \"SECURITY_TRACKER\", \"url\": \"http://www.securitytracker.com/id/1039393\", \"fixResolution\": \"The vendor has issued a fix (7.0.81).\\n\\nThe vendor advisory is available at:\\n\\nhttps://tomcat.apache.org/security-7.html#Fixed_in_Apache_Tomcat_7.0.81\", \"date\": \"2017-12-31\", \"message\": \"Apache Tomcat VirtualDirContext Flaw Lets Remote Users View JSP Source Code for the Affected Resource\", \"extraData\": \"key=1039393\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-12616\", \"type\": \"UPGRADE_VERSION\", \"origin\": \"SECURITY_TRACKER\", \"url\": \"http://www.securitytracker.com/id/1039393\", \"fixResolution\": \"The vendor has issued a fix (7.0.81).\\n\\nThe vendor advisory is available at:\\n\\nhttps://tomcat.apache.org/security-7.html#Fixed_in_Apache_Tomcat_7.0.81\", \"date\": \"2017-12-31\"," + - "\"message\": \"Apache Tomcat VirtualDirContext Flaw Lets Remote Users View JSP Source Code for the Affected Resource\", \"extraData\": \"key=1039393\" }, { \"vulnerability\": \"CVE-2017-12616\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat70/commit/07dc0ea2745f0afab6415f22b16a29f1c6de5727\", \"fixResolution\": \"java/org/apache/naming/resources/VirtualDirContext.java,webapps/docs/changelog.xml,java/org/apache/naming/resources/FileDirContext.java\", \"date\": \"2017-08-10\", \"message\": \"Correct regression in r1804604 that broke WebDAV.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc7.0.x/trunk@1804729 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=07dc0ea&committerName=markt-asf&committerUrl=https://github.com/markt-asf&committerAvatar=https://avatars3.githubusercontent.com/u/4690029?v=4\" } ], \"fixResolutionText\": \"The vendor has issued a fix (7.0.81).\\n\\nThe vendor advisory is available at:\\n\\nhttps://tomcat.apache.org/security-7.html#Fixed_in_Apache_Tomcat_7.0.81\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\"," + - "\"library\": { \"keyUuid\": \"aa3a10d7-4e2c-46fe-bbf9-3c2d06e43b02\", \"keyId\": 24276785, \"filename\": \"tomcat-embed-core-7.0.78.jar\", \"name\": \"tomcat-embed-core\", \"groupId\": \"org.apache.tomcat.embed\", \"artifactId\": \"tomcat-embed-core\", \"version\": \"7.0.78\", \"sha1\": \"ddb63d615ec3944b4394aed6dc825cd0cbb16b21\", \"type\": \"Java\", \"references\": { \"url\": \"http://tomcat.apache.org/\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/org/apache/tomcat/embed/tomcat-embed-core/7.0.78/tomcat-embed-core-7.0.78.pom\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": true, \"description\": \"High:3,Medium:2,\", \"date\": \"2017-10-26\" }, { \"vulnerability\": { \"name\": \"CVE-2017-7674\", \"type\": \"CVE\", \"severity\": \"medium\", \"score\": 4.3," + - "\"cvss3_severity\": \"medium\", \"cvss3_score\": 4.3, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:L/A:N\", \"publishDate\": \"2017-08-11\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-7674\", \"description\": \"The CORS Filter in Apache Tomcat 9.0.0.M1 to 9.0.0.M21, 8.5.0 to 8.5.15, 8.0.0.RC1 to 8.0.44 and 7.0.41 to 7.0.78 did not add an HTTP Vary header indicating that the response varies depending on Origin. This permitted client and server side cache poisoning in some circumstances.\", \"topFix\": { \"vulnerability\": \"CVE-2017-7674\", \"type\": \"UPGRADE_VERSION\", \"origin\": \"BUGZILLA\", \"url\": \"https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2017-7674\", \"fixResolution\": \"tomcat 7.0.79,tomcat 8.0.45,tomcat 8.5.16\", \"message\": \"CVE-2017-7674 tomcat: Vary header not added by CORS filter leading to cache poisoning\", \"extraData\": \"key=1480618&assignee=Red Hat Product Security\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-7674\", \"type\": \"UPGRADE_VERSION\", \"origin\": \"BUGZILLA\", \"url\": \"https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2017-7674\"," + - "\"fixResolution\": \"tomcat 7.0.79,tomcat 8.0.45,tomcat 8.5.16\", \"message\": \"CVE-2017-7674 tomcat: Vary header not added by CORS filter leading to cache poisoning\", \"extraData\": \"key=1480618&assignee=Red Hat Product Security\" }, { \"vulnerability\": \"CVE-2017-7674\", \"type\": \"UPGRADE_VERSION\", \"origin\": \"BUGZILLA\", \"url\": \"https://bugzilla.redhat.com/show_bug.cgi?id=1480618\", \"fixResolution\": \"tomcat 7.0.79,tomcat 8.0.45,tomcat 8.5.16\", \"message\": \"CVE-2017-7674 tomcat: Vary header not added by CORS filter leading to cache poisoning\", \"extraData\": \"key=1480618&assignee=Red Hat Product Security\" }, { \"vulnerability\": \"CVE-2017-7674\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat/commit/b94478d45b7e1fc06134a785571f78772fa30fed\", \"fixResolution\": \"java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml\", \"date\": \"2017-05-22\", \"message\": \"BZ61101: CORS filter should set Vary header in response. Submitted by Rick Riemer.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/trunk@1795813 13f79535-47bb-0310-9956-ffa450edef68\"," + - "\"extraData\": \"key=b94478d&committerName=rmaucher&committerUrl=https://github.com/rmaucher&committerAvatar=https://avatars2.githubusercontent.com/u/324250?v=4\" }, { \"vulnerability\": \"CVE-2017-7674\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat85/commit/9044c1672bbe4b2cf4c55028cc8b977cc62650e7\", \"fixResolution\": \"java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml\", \"date\": \"2017-05-22\", \"message\": \"BZ61101: CORS filter should set Vary header in response. Submitted by Rick Riemer.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc8.5.x/trunk@1795814 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=9044c16&committerName=rmaucher&committerUrl=https://github.com/rmaucher&committerAvatar=https://avatars2.githubusercontent.com/u/324250?v=4\" }, { \"vulnerability\": \"CVE-2017-7674\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat70/commit/52382ebfbce20a98b01cd9d37184a12703987a5a\", \"fixResolution\": \"java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml\"," + - "\"date\": \"2017-05-22\", \"message\": \"BZ61101: CORS filter should set Vary header in response. Submitted by Rick Riemer.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc7.0.x/trunk@1795816 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=52382eb&committerName=rmaucher&committerUrl=https://github.com/rmaucher&committerAvatar=https://avatars2.githubusercontent.com/u/324250?v=4\" }, { \"vulnerability\": \"CVE-2017-7674\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat80/commit/f52c242d92d4563dd1226dcc993ec37370ba9ce3\", \"fixResolution\": \"java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml\", \"date\": \"2017-05-22\", \"message\": \"BZ61101: CORS filter should set Vary header in response. Submitted by Rick Riemer.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc8.0.x/trunk@1795815 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=f52c242&committerName=rmaucher&committerUrl=https://github.com/rmaucher&committerAvatar=https://avatars2.githubusercontent.com/u/324250?v=4\" } ], \"fixResolutionText\": \"Upgrade to version tomcat 7.0.79, tomcat 8.0.45, tomcat 8.5.16 or greater\"," + - "\"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"aa3a10d7-4e2c-46fe-bbf9-3c2d06e43b02\", \"keyId\": 24276785, \"filename\": \"tomcat-embed-core-7.0.78.jar\", \"name\": \"tomcat-embed-core\", \"groupId\": \"org.apache.tomcat.embed\", \"artifactId\": \"tomcat-embed-core\", \"version\": \"7.0.78\", \"sha1\": \"ddb63d615ec3944b4394aed6dc825cd0cbb16b21\", \"type\": \"Java\", \"references\": { \"url\": \"http://tomcat.apache.org/\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/org/apache/tomcat/embed/tomcat-embed-core/7.0.78/tomcat-embed-core-7.0.78.pom\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": true, \"description\": \"High:3,Medium:2,\", \"date\": \"2017-10-26\" }, { \"vulnerability\": { \"name\": \"CVE-2018-8014\"," + - "\"type\": \"CVE\", \"severity\": \"high\", \"score\": 7.5, \"cvss3_severity\": \"high\", \"cvss3_score\": 9.8, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-05-16\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-8014\", \"description\": \"The defaults settings for the CORS filter provided in Apache Tomcat 9.0.0.M1 to 9.0.8, 8.5.0 to 8.5.31, 8.0.0.RC1 to 8.0.52, 7.0.41 to 7.0.88 are insecure and enable 'supportsCredentials' for all origins. It is expected that users of the CORS filter will have configured it appropriately for their environment rather than using it in the default configuration. Therefore, it is expected that most users will not be impacted by this issue.\", \"topFix\": { \"vulnerability\": \"CVE-2018-8014\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat70/commit/5877390a9605f56d9bd6859a54ccbfb16374a78b\", \"fixResolution\": \"java/org/apache/catalina/filters/LocalStrings.properties,test/org/apache/catalina/filters/TestCorsFilter.java,java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml,test/org/apache/catalina/filters/TesterFilterConfigs.java\"," + - "\"date\": \"2018-05-16\", \"message\": \"Fix https://bz.apache.org/bugzilla/show_bug.cgi?id=62343\\nMake CORS filter defaults more secure.\\nThis is the fix for CVE-2018-8014.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc7.0.x/trunk@1831730 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=5877390&committerName=markt-asf&committerUrl=https://github.com/markt-asf&committerAvatar=https://avatars3.githubusercontent.com/u/4690029?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2018-8014\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat70/commit/5877390a9605f56d9bd6859a54ccbfb16374a78b\", \"fixResolution\": \"java/org/apache/catalina/filters/LocalStrings.properties,test/org/apache/catalina/filters/TestCorsFilter.java,java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml,test/org/apache/catalina/filters/TesterFilterConfigs.java\", \"date\": \"2018-05-16\", \"message\": \"Fix https://bz.apache.org/bugzilla/show_bug.cgi?id=62343\\nMake CORS filter defaults more secure.\\nThis is the fix for CVE-2018-8014.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc7.0.x/trunk@1831730 13f79535-47bb-0310-9956-ffa450edef68\"," + - "\"extraData\": \"key=5877390&committerName=markt-asf&committerUrl=https://github.com/markt-asf&committerAvatar=https://avatars3.githubusercontent.com/u/4690029?v=4\" }, { \"vulnerability\": \"CVE-2018-8014\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat80/commit/2c9d8433bd3247a2856d4b2555447108758e813e#diff-32f241c95d21b1b224601e52f83af334\", \"fixResolution\": \"java/org/apache/catalina/filters/LocalStrings.properties,test/org/apache/catalina/filters/TestCorsFilter.java,java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml,test/org/apache/catalina/filters/TesterFilterConfigs.java\", \"date\": \"2018-05-16\", \"message\": \"Fix https://bz.apache.org/bugzilla/show_bug.cgi?id=62343\\nMake CORS filter defaults more secure.\\nThis is the fix for CVE-2018-8014.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc8.0.x/trunk@1831729 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=2c9d843&committerName=markt-asf&committerUrl=https://github.com/markt-asf&committerAvatar=https://avatars3.githubusercontent.com/u/4690029?v=4\" }, { \"vulnerability\": \"CVE-2018-8014\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\"," + - "\"url\": \"https://github.com/apache/tomcat/commit/d83a76732e6804739b81d8b2056365307637b42d\", \"fixResolution\": \"java/org/apache/catalina/filters/LocalStrings.properties,test/org/apache/catalina/filters/TestCorsFilter.java,java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml,test/org/apache/catalina/filters/TesterFilterConfigs.java\", \"date\": \"2018-05-16\", \"message\": \"Fix https://bz.apache.org/bugzilla/show_bug.cgi?id=62343\\nMake CORS filter defaults more secure.\\nThis is the fix for CVE-2018-8014.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/trunk@1831726 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=d83a767&committerName=markt-asf&committerUrl=https://github.com/markt-asf&committerAvatar=https://avatars3.githubusercontent.com/u/4690029?v=4\" }, { \"vulnerability\": \"CVE-2018-8014\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/apache/tomcat85/commit/60f596a21fd6041335a3a1a4015d4512439cecb5\", \"fixResolution\": \"java/org/apache/catalina/filters/LocalStrings.properties,test/org/apache/catalina/filters/TestCorsFilter.java,java/org/apache/catalina/filters/CorsFilter.java,webapps/docs/changelog.xml,test/org/apache/catalina/filters/TesterFilterConfigs.java\"," + - "\"date\": \"2018-05-16\", \"message\": \"Fix https://bz.apache.org/bugzilla/show_bug.cgi?id=62343\\nMake CORS filter defaults more secure.\\nThis is the fix for CVE-2018-8014.\\n\\ngit-svn-id: https://svn.apache.org/repos/asf/tomcat/tc8.5.x/trunk@1831728 13f79535-47bb-0310-9956-ffa450edef68\", \"extraData\": \"key=60f596a&committerName=markt-asf&committerUrl=https://github.com/markt-asf&committerAvatar=https://avatars3.githubusercontent.com/u/4690029?v=4\" } ], \"fixResolutionText\": \"Replace or update the following files: LocalStrings.properties, TestCorsFilter.java, CorsFilter.java, changelog.xml, TesterFilterConfigs.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"aa3a10d7-4e2c-46fe-bbf9-3c2d06e43b02\", \"keyId\": 24276785, \"filename\": \"tomcat-embed-core-7.0.78.jar\", \"name\": \"tomcat-embed-core\", \"groupId\": \"org.apache.tomcat.embed\", \"artifactId\": \"tomcat-embed-core\", \"version\": \"7.0.78\", \"sha1\": \"ddb63d615ec3944b4394aed6dc825cd0cbb16b21\", \"type\": \"Java\", \"references\": { \"url\": \"http://tomcat.apache.org/\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/org/apache/tomcat/embed/tomcat-embed-core/7.0.78/tomcat-embed-core-7.0.78.pom\" }," + - "\"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": true, \"description\": \"High:3,Medium:2,\", \"date\": \"2017-10-26\" } ] }").alerts + return new JsonUtils().jsonStringToGroovyObject("""{"alerts":[{"vulnerability":{"name":"CVE-2017-15095","type":"CVE","severity":"high","score":7.5,"cvss3_severity":"high","cvss3_score":9.8,"scoreMetadataVector":"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H","publishDate":"2018-02-06","url":"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15095", + "description":"A deserialization flaw was discovered in the jackson-databind in versions before 2.8.10 and 2.9.1, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper. This issue extends the previous flaw CVE-2017-7525 by blacklisting more classes that could be used maliciously.", + "topFix":{"vulnerability":"CVE-2017-15095","type":"CHANGE_FILES","origin":"GITHUB_COMMIT","url":"https://github.com/FasterXML/jackson-databind/commit/60d459ce","fixResolution":"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java","date":"2017-04-13", + "message":"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8","extraData":"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4"},"allFixes":[{"vulnerability":"CVE-2017-15095","type":"CHANGE_FILES","origin":"GITHUB_COMMIT", + "url":"https://github.com/FasterXML/jackson-databind/commit/60d459ce","fixResolution":"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java","date":"2017-04-13","message":"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8", + "extraData":"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4"},{"vulnerability":"CVE-2017-15095","type":"CHANGE_FILES","origin":"GITHUB_COMMIT","url":"https://github.com/FasterXML/jackson-databind/commit/e865a7a4464da63ded9f4b1a2328ad85c9ded78b#diff-98084d808198119d550a9211e128a16f", + "fixResolution":"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java","date":"2017-12-12","message":"Fix #1737 (#1857)","extraData":"key=e865a7a&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4"}, + {"vulnerability":"CVE-2017-15095","type":"CHANGE_FILES","origin":"GITHUB_COMMIT","url":"https://github.com/FasterXML/jackson-databind/commit/e8f043d1","fixResolution":"release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java","date":"2017-06-30","message":"Fix #1680", + "extraData":"key=e8f043d&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4"}],"fixResolutionText":"Replace or update the following files: IllegalTypesCheckTest.java, VERSION, BeanDeserializerFactory.java","references":[]}, + "type":"SECURITY_VULNERABILITY","level":"MAJOR","library":{"keyUuid":"13f7802e-8aa1-4303-a5db-1d0c85e871a9","keyId":23410061,"filename":"jackson-databind-2.8.8.jar","name":"jackson-databind","groupId":"com.fasterxml.jackson.core","artifactId":"jackson-databind","version":"2.8.8","sha1":"bf88c7b27e95cbadce4e7c316a56c3efffda8026", + "type":"Java","references":{"url":"http://github.com/FasterXML/jackson","issueUrl":"https://github.com/FasterXML/jackson-databind/issues","pomUrl":"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom","scmUrl":"http://github.com/FasterXML/jackson-databind"}, + "licenses":[{"name":"Apache 2.0","url":"http://apache.org/licenses/LICENSE-2.0","profileInfo":{"copyrightRiskScore":"THREE","patentRiskScore":"ONE","copyleft":"NO","linking":"DYNAMIC","royaltyFree":"CONDITIONAL"}}]},"project":"pipeline-test - 0.0.1","projectId":302194,"projectToken":"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7","directDependency":false,"description":"High:5,","date":"2017-11-15"}]}""").alerts }) thrown.expect(AbortException.class) @@ -951,11 +914,11 @@ class WhitesourceExecuteScanTest extends BasePiperTest { return result }) helper.registerAllowedMethod("fetchVulnerabilities", [List], { - return new JsonUtils().jsonStringToGroovyObject("{ \"alerts\": [ { \"vulnerability\": { \"name\": \"CVE-2017-15095\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 2.1, \"cvss3_severity\": \"high\", \"cvss3_score\": 5.3, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15095\", \"description\": \"A deserialization flaw was discovered in the jackson-databind in versions before 2.8.10 and 2.9.1, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper. This issue extends the previous flaw CVE-2017-7525 by blacklisting more classes that could be used maliciously.\", \"topFix\": { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\"," + - "\"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\", \"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\"," + - "\"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e865a7a4464da63ded9f4b1a2328ad85c9ded78b#diff-98084d808198119d550a9211e128a16f\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-12-12\", \"message\": \"Fix #1737 (#1857)\", \"extraData\": \"key=e865a7a&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\"," + - "\"url\": \"https://github.com/FasterXML/jackson-databind/commit/e8f043d1\", \"fixResolution\": \"release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-06-30\", \"message\": \"Fix #1680\", \"extraData\": \"key=e8f043d&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" } ], \"fixResolutionText\": \"Replace or update the following files: IllegalTypesCheckTest.java, VERSION, BeanDeserializerFactory.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\"," + - "\"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-databind\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:5,\", \"date\": \"2017-11-15\" } ] }").alerts + return new JsonUtils().jsonStringToGroovyObject("""{ \"alerts\": [ { \"vulnerability\": { \"name\": \"CVE-2017-15095\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 2.1, \"cvss3_severity\": \"high\", \"cvss3_score\": 5.3, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15095\", \"description\": \"A deserialization flaw was discovered in the jackson-databind in versions before 2.8.10 and 2.9.1, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper. This issue extends the previous flaw CVE-2017-7525 by blacklisting more classes that could be used maliciously.\", \"topFix\": { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\", + \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\", \"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\", + \"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e865a7a4464da63ded9f4b1a2328ad85c9ded78b#diff-98084d808198119d550a9211e128a16f\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-12-12\", \"message\": \"Fix #1737 (#1857)\", \"extraData\": \"key=e865a7a&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", + \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e8f043d1\", \"fixResolution\": \"release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-06-30\", \"message\": \"Fix #1680\", \"extraData\": \"key=e8f043d&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" } ], \"fixResolutionText\": \"Replace or update the following files: IllegalTypesCheckTest.java, VERSION, BeanDeserializerFactory.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\", + \"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-databind\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:5,\", \"date\": \"2017-11-15\" } ] }""").alerts }) stepRule.step.whitesourceExecuteScan([ @@ -990,11 +953,11 @@ class WhitesourceExecuteScanTest extends BasePiperTest { return result }) helper.registerAllowedMethod("fetchVulnerabilities", [List], { - return new JsonUtils().jsonStringToGroovyObject("{ \"alerts\": [ { \"vulnerability\": { \"name\": \"CVE-2017-15095\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 2.1, \"cvss3_severity\": \"high\", \"cvss3_score\": 5.3, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15095\", \"description\": \"A deserialization flaw was discovered in the jackson-databind in versions before 2.8.10 and 2.9.1, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper. This issue extends the previous flaw CVE-2017-7525 by blacklisting more classes that could be used maliciously.\", \"topFix\": { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\"," + - "\"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\", \"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\"," + - "\"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e865a7a4464da63ded9f4b1a2328ad85c9ded78b#diff-98084d808198119d550a9211e128a16f\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-12-12\", \"message\": \"Fix #1737 (#1857)\", \"extraData\": \"key=e865a7a&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\"," + - "\"url\": \"https://github.com/FasterXML/jackson-databind/commit/e8f043d1\", \"fixResolution\": \"release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-06-30\", \"message\": \"Fix #1680\", \"extraData\": \"key=e8f043d&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" } ], \"fixResolutionText\": \"Replace or update the following files: IllegalTypesCheckTest.java, VERSION, BeanDeserializerFactory.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\"," + - "\"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-databind\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:5,\", \"date\": \"2017-11-15\" } ] }").alerts + return new JsonUtils().jsonStringToGroovyObject("""{ \"alerts\": [ { \"vulnerability\": { \"name\": \"CVE-2017-15095\", \"type\": \"CVE\", \"severity\": \"high\", \"score\": 2.1, \"cvss3_severity\": \"high\", \"cvss3_score\": 5.3, \"scoreMetadataVector\": \"CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"publishDate\": \"2018-02-06\", \"url\": \"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15095\", \"description\": \"A deserialization flaw was discovered in the jackson-databind in versions before 2.8.10 and 2.9.1, which could allow an unauthenticated user to perform code execution by sending the maliciously crafted input to the readValue method of the ObjectMapper. This issue extends the previous flaw CVE-2017-7525 by blacklisting more classes that could be used maliciously.\", \"topFix\": { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\", + \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\", \"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, \"allFixes\": [ { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/60d459ce\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-04-13\", \"message\": \"Fix #1599 for 2.8.9\\n\\nMerge branch '2.7' into 2.8\", + \"extraData\": \"key=60d459c&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e865a7a4464da63ded9f4b1a2328ad85c9ded78b#diff-98084d808198119d550a9211e128a16f\", \"fixResolution\": \"src/test/java/com/fasterxml/jackson/databind/interop/IllegalTypesCheckTest.java,release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-12-12\", \"message\": \"Fix #1737 (#1857)\", \"extraData\": \"key=e865a7a&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" }, { \"vulnerability\": \"CVE-2017-15095\", \"type\": \"CHANGE_FILES\", \"origin\": \"GITHUB_COMMIT\", + \"url\": \"https://github.com/FasterXML/jackson-databind/commit/e8f043d1\", \"fixResolution\": \"release-notes/VERSION,src/main/java/com/fasterxml/jackson/databind/deser/BeanDeserializerFactory.java\", \"date\": \"2017-06-30\", \"message\": \"Fix #1680\", \"extraData\": \"key=e8f043d&committerName=cowtowncoder&committerUrl=https://github.com/cowtowncoder&committerAvatar=https://avatars0.githubusercontent.com/u/55065?v=4\" } ], \"fixResolutionText\": \"Replace or update the following files: IllegalTypesCheckTest.java, VERSION, BeanDeserializerFactory.java\", \"references\": [] }, \"type\": \"SECURITY_VULNERABILITY\", \"level\": \"MAJOR\", \"library\": { \"keyUuid\": \"13f7802e-8aa1-4303-a5db-1d0c85e871a9\", \"keyId\": 23410061, \"filename\": \"jackson-databind-2.8.8.jar\", \"name\": \"jackson-databind\", \"groupId\": \"com.fasterxml.jackson.core\", \"artifactId\": \"jackson-databind\", \"version\": \"2.8.8\", \"sha1\": \"bf88c7b27e95cbadce4e7c316a56c3efffda8026\", + \"type\": \"Java\", \"references\": { \"url\": \"http://github.com/FasterXML/jackson\", \"issueUrl\": \"https://github.com/FasterXML/jackson-databind/issues\", \"pomUrl\": \"http://repo.jfrog.org/artifactory/list/repo1/com/fasterxml/jackson/core/jackson-databind/2.8.8/jackson-databind-2.8.8.pom\", \"scmUrl\": \"http://github.com/FasterXML/jackson-databind\" }, \"licenses\": [ { \"name\": \"Apache 2.0\", \"url\": \"http://apache.org/licenses/LICENSE-2.0\", \"profileInfo\": { \"copyrightRiskScore\": \"THREE\", \"patentRiskScore\": \"ONE\", \"copyleft\": \"NO\", \"linking\": \"DYNAMIC\", \"royaltyFree\": \"CONDITIONAL\" } } ] }, \"project\": \"pipeline-test - 0.0.1\", \"projectId\": 302194, \"projectToken\": \"1b8fdc36cb6949f482d0fd936a39dab69d6b34f43fff4dda8a9241f2c6e536c7\", \"directDependency\": false, \"description\": \"High:5,\", \"date\": \"2017-11-15\" } ] }""").alerts }) thrown.expect(AbortException) From 8fc6eb521987a143beb65e89bd0a4208e9ff384a Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Wed, 3 Apr 2019 12:31:40 +0200 Subject: [PATCH 128/130] Fix time displayed in report --- .../com.sap.piper/templates/whitesourceVulnerabilities.html | 2 +- vars/whitesourceExecuteScan.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/com.sap.piper/templates/whitesourceVulnerabilities.html b/resources/com.sap.piper/templates/whitesourceVulnerabilities.html index f2fe8f7dc..816eb7c96 100644 --- a/resources/com.sap.piper/templates/whitesourceVulnerabilities.html +++ b/resources/com.sap.piper/templates/whitesourceVulnerabilities.html @@ -17,7 +17,7 @@ total number of high/critical vulnerabilities with CVSS score >= ${cvssSeverityLimit}: ${totalSevereVulnerabilities} -

Snapshot taken:${now} GMT

+

Snapshot taken:${now}

Entry #DateCVECVSS ScoreCVSS VersionProjectLibrary file nameLibrary group IDLibrary artifact IDLibrary versionDescriptionTop fixEntry #DateCVECVSS ScoreCVSS VersionProjectLibrary file nameLibrary group IDLibrary artifact IDLibrary versionDescriptionTop fix
${i + 1} ${item.date} ${item.vulnerability.name}${score}${score} ${item.vulnerability.cvss3_score > 0 ? 'v3' : 'v2'} ${item.project} ${item.library.filename}
diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index 827c6f9b8..ec0c4d92c 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -539,7 +539,7 @@ void checkStatus(int statusCode, config) { } def getReportHtml(config, vulnerabilityList, numSevereVulns) { - def now = new Date().format('MMM dd, yyyy - HH:mm:ss') + def now = new Date().format('MMM dd, yyyy - HH:mm:ss z', TimeZone.getTimeZone('UTC')) def vulnerabilityTable = '' if (vulnerabilityList.size() == 0) { vulnerabilityTable += ''' From 097ee4179bea9d38ed47a2dd66d4f1035f3f7a87 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Thu, 4 Apr 2019 16:05:26 +0200 Subject: [PATCH 129/130] Chris' final comments + Scala config --- .../WhitesourceConfigurationHelper.groovy | 40 ++++++------------- vars/whitesourceExecuteScan.groovy | 14 +++---- 2 files changed, 20 insertions(+), 34 deletions(-) diff --git a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy index 0feeddab9..306575892 100644 --- a/src/com/sap/piper/WhitesourceConfigurationHelper.groovy +++ b/src/com/sap/piper/WhitesourceConfigurationHelper.groovy @@ -37,19 +37,12 @@ class WhitesourceConfigurationHelper implements Serializable { [name: 'offline', value: false, force: true], [name: 'ignoreSourceFiles', value: true, force: true], [name: 'resolveAllDependencies', value: false, force: true], - [name: 'failErrorLevel', value: 'ALL', force: true] + [name: 'failErrorLevel', value: 'ALL', force: true], + [name: 'case.sensitive.glob', value: false], + [name: 'followSymbolicLinks', value: true] ] - // To be removed once all scenarios are fully supported - if(!['pip', 'golang'].contains(config.scanType)) - script.echo "[Warning][Whitesource] Configuration for scanType: '${config.scanType}' is not yet hardened, please do a quality assessment of your scan results." - switch (config.scanType) { - case 'npm': - mapping += [ - - ] - break case 'pip': mapping += [ [name: 'python.resolveDependencies', value: true, force: true], @@ -63,14 +56,17 @@ class WhitesourceConfigurationHelper implements Serializable { [name: 'python.pipenvDevDependencies', value: true], [name: 'python.IgnorePipenvInstallErrors', value: false], [name: 'includes', value: '**/*.py **/*.txt'], - [name: 'excludes', value: '**/*sources.jar **/*javadoc.jar'], - [name: 'case.sensitive.glob', value: false], - [name: 'followSymbolicLinks', value: true] + [name: 'excludes', value: '**/*sources.jar **/*javadoc.jar'] ] break case 'sbt': mapping += [ - + [name: 'sbt.resolveDependencies', value: true, force: true], + [name: 'sbt.ignoreSourceFiles', value: true, force: true], + [name: 'sbt.aggregateModules', value: false, force: true], + [name: 'sbt.runPreStep', value: true], + [name: 'includes', value: '**/*.jar'], + [name: 'excludes', value: '**/*sources.jar **/*javadoc.jar'] ] break case 'golang': @@ -80,21 +76,11 @@ class WhitesourceConfigurationHelper implements Serializable { [name: 'go.collectDependenciesAtRuntime', value: false], [name: 'go.dependencyManager', value: 'dep'], [name: 'includes', value: '**/*.lock'], - [name: 'excludes', value: '**/*sources.jar **/*javadoc.jar'], - [name: 'case.sensitive.glob', value: false], - [name: 'followSymbolicLinks', value: true] - ] - break - case 'dlang': - mapping += [ - - ] - break - case 'maven': - mapping += [ - + [name: 'excludes', value: '**/*sources.jar **/*javadoc.jar'] ] break + default: + script.echo "[Warning][Whitesource] Configuration for scanType: '${config.scanType}' is not yet hardened, please do a quality assessment of your scan results." } rewriteConfiguration(script, utils, config, mapping, suffix, path, inputFile, targetFile, parsingClosure, serializationClosure) diff --git a/vars/whitesourceExecuteScan.groovy b/vars/whitesourceExecuteScan.groovy index ec0c4d92c..f5bdc5142 100644 --- a/vars/whitesourceExecuteScan.groovy +++ b/vars/whitesourceExecuteScan.groovy @@ -74,9 +74,9 @@ import static com.sap.piper.Prerequisites.checkScript ] @Field Set STEP_CONFIG_KEYS = GENERAL_CONFIG_KEYS + [ /** - * Additional install command that can be used to populate the default docker image for some scenarios. + * Install command that can be used to populate the default docker image for some scenarios. */ - 'additionalInstallCommand', + 'installCommand', /** * URL used to download the latest version of the WhiteSource Unified Agent. */ @@ -194,7 +194,7 @@ import static com.sap.piper.Prerequisites.checkScript timeout : 'timeout', vulnerabilityReportFileName : 'vulnerabilityReportFileName', vulnerabilityReportTitle : 'vulnerabilityReportTitle', - additionalInstallCommand : 'additionalInstallCommand' + installCommand : 'installCommand' ] ] @@ -242,14 +242,14 @@ void call(Map parameters = [:]) { .dependingOn('scanType').mixin('dockerWorkspace') .dependingOn('scanType').mixin('stashContent') .dependingOn('scanType').mixin('whitesource/configFilePath') - .dependingOn('scanType').mixin('whitesource/additionalInstallCommand') + .dependingOn('scanType').mixin('whitesource/installCommand') .withMandatoryProperty('whitesource/serviceUrl') .withMandatoryProperty('whitesource/orgToken') .withMandatoryProperty('whitesource/userTokenCredentialsId') .withMandatoryProperty('whitesource/productName') .use() - config.whitesource.cvssSeverityLimit = config.whitesource.cvssSeverityLimit == null ? -1 : Integer.valueOf(config.whitesource.cvssSeverityLimit) + config.whitesource.cvssSeverityLimit = config.whitesource.cvssSeverityLimit == null ?: Integer.valueOf(config.whitesource.cvssSeverityLimit) config.stashContent = utils.unstashAll(config.stashContent) config.whitesource['projectNames'] = (config.whitesource['projectNames'] instanceof List) ? config.whitesource['projectNames'] : config.whitesource['projectNames']?.tokenize(',') parameters.whitesource = parameters.whitesource ?: [:] @@ -373,8 +373,8 @@ private def triggerWhitesourceScanWithUserKey(script, config, utils, descriptorU javaCmd = './bin/java' } - if(config.whitesource.additionalInstallCommand) - sh new GStringTemplateEngine().createTemplate(config.whitesource.additionalInstallCommand).make([config: config]).toString() + if(config.whitesource.installCommand) + sh new GStringTemplateEngine().createTemplate(config.whitesource.installCommand).make([config: config]).toString() def options = ["-jar ${config.whitesource.agentFileName} -c \'${config.whitesource.configFilePath}\'"] if (config.whitesource.orgToken) options.push("-apiKey '${config.whitesource.orgToken}'") From bccec4259acc041226042b89c903b3e787b4ac7d Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Thu, 4 Apr 2019 21:20:25 +0200 Subject: [PATCH 130/130] Fix scala test --- .../sap/piper/WhitesourceConfigurationHelperTest.groovy | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/groovy/com/sap/piper/WhitesourceConfigurationHelperTest.groovy b/test/groovy/com/sap/piper/WhitesourceConfigurationHelperTest.groovy index 416552c65..92497f7f8 100644 --- a/test/groovy/com/sap/piper/WhitesourceConfigurationHelperTest.groovy +++ b/test/groovy/com/sap/piper/WhitesourceConfigurationHelperTest.groovy @@ -107,12 +107,11 @@ class WhitesourceConfigurationHelperTest extends BasePiperTest { containsString("apiKey=abcd"), containsString("productName=DIST - name1"), containsString("productToken=1234"), - containsString("userKey=0000") + containsString("userKey=0000"), + containsString("sbt.resolveDependencies=true"), + containsString("log.level=debug") ) ) - - assertThat(jlr.log, containsString("[Whitesource] Configuration for scanType: 'sbt' is not yet hardened, please do a quality assessment of your scan results.")) - } @Test
Entry #