update stashing behavior (#628)

* update stashing behavior close #619 * add test
2025-03-03 15:02:35 +02:00 · 2019-04-02 13:13:25 +02:00 · 2019-04-02 13:13:25 +02:00 · 24563db155
commit 24563db155
parent 78400cad9b
6 changed files with 46 additions and 145 deletions
--- a/resources/default_pipeline_environment.yml
+++ b/resources/default_pipeline_environment.yml
@ -284,7 +284,6 @@ steps:
    sendMail: true
    timeoutInSeconds: 900
  pipelineStashFilesAfterBuild:
-    runOpaTests: false
    stashIncludes:
      checkmarx: '**/*.js, **/*.scala, **/*.py, **/*.go, **/*.xml, **/*.html'
      classFiles: '**/target/classes/**/*.class, **/target/test-classes/**/*.class'
@ -293,12 +292,12 @@ steps:
      checkmarx: '**/*.mockserver.js, node_modules/**/*.js'
      classFiles: ''
      sonar: ''
+    noDefaultExludes: []
  pipelineStashFilesBeforeBuild:
-    runCheckmarx: false
    stashIncludes:
      buildDescriptor: '**/pom.xml, **/.mvn/**, **/assembly.xml, **/.swagger-codegen-ignore, **/package.json, **/requirements.txt, **/setup.py, **/whitesource_config.py, **/mta*.y*ml, **/.npmrc, **/whitesource.*.json, **/whitesource-fs-agent.config, Dockerfile, **/VERSION, **/version.txt, **/build.sbt, **/sbtDescriptor.json, **/project/*'
      deployDescriptor: '**/manifest*.y*ml, **/*.mtaext.y*ml, **/*.mtaext, **/xs-app.json, helm/**, *.y*ml'
-      git: '**/gitmetadata/**'
+      git: '.git/**'
      opa5: '**/*.*'
      opensourceConfiguration: '**/srcclr.yml, **/vulas-custom.properties, **/.nsprc, **/.retireignore, **/.retireignore.json, **/.snyk'
      pipelineConfigAndTests: '.pipeline/**'
@ -313,6 +312,8 @@ steps:
      pipelineConfigAndTests: ''
      securityDescriptor: ''
      tests: ''
+    noDefaultExludes:
+      - 'git'
  seleniumExecuteTests:
    buildTool: 'npm'
    containerPortMappings:
--- a/src/com/sap/piper/Utils.groovy
+++ b/src/com/sap/piper/Utils.groovy
@ -21,9 +21,19 @@ def getMandatoryParameter(Map map, paramName, defaultValue = null) {

 }

-def stash(name, include = '**/*.*', exclude = '') {
-    echo "Stash content: ${name} (include: ${include}, exclude: ${exclude})"
-    steps.stash name: name, includes: include, excludes: exclude
+def stash(name, include = '**/*.*', exclude = '', useDefaultExcludes = true) {
+    echo "Stash content: ${name} (include: ${include}, exclude: ${exclude}, useDefaultExcludes: ${useDefaultExcludes})"
+
+    Map stashParams = [
+        name: name,
+        includes: include,
+        excludes: exclude
+    ]
+    //only set the optional parameter if default excludes should not be applied
+    if (!useDefaultExcludes) {
+        stashParams.useDefaultExcludes = useDefaultExcludes
+    }
+    steps.stash stashParams
 }

 def stashList(script, List stashes) {
@ -46,9 +56,9 @@ def stashList(script, List stashes) {
    }
 }

-def stashWithMessage(name, msg, include = '**/*.*', exclude = '') {
+def stashWithMessage(name, msg, include = '**/*.*', exclude = '', useDefaultExcludes = true) {
    try {
-        stash(name, include, exclude)
+        stash(name, include, exclude, useDefaultExcludes)
    } catch (e) {
        echo msg + name + " (${e.getMessage()})"
    }
--- a/test/groovy/PipelineStashFilesAfterBuildTest.groovy
+++ b/test/groovy/PipelineStashFilesAfterBuildTest.groovy
@ -22,22 +22,6 @@ class PipelineStashFilesAfterBuildTest extends BasePiperTest {

    @Test
    void testStashAfterBuild() {
-        helper.registerAllowedMethod("fileExists", [String.class], {
-            searchTerm ->
-                return false
-        })
-        stepRule.step.pipelineStashFilesAfterBuild(
-            script: nullScript,
-            juStabUtils: utils
-        )
-        // asserts
-        assertFalse(loggingRule.log.contains('Stash content: checkmarx'))
-        assertThat(loggingRule.log, containsString('Stash content: classFiles'))
-        assertThat(loggingRule.log, containsString('Stash content: sonar'))
-    }
-
-    @Test
-    void testStashAfterBuildWithCheckmarx() {
        helper.registerAllowedMethod("fileExists", [String.class], {
            searchTerm ->
                return true
@ -52,21 +36,4 @@ class PipelineStashFilesAfterBuildTest extends BasePiperTest {
        assertThat(loggingRule.log, containsString('Stash content: classFiles'))
        assertThat(loggingRule.log, containsString('Stash content: sonar'))
    }
-
-    @Test
-    void testStashAfterBuildWithCheckmarxConfig() {
-        helper.registerAllowedMethod("fileExists", [String.class], {
-            searchTerm ->
-                return true
-        })
-        stepRule.step.pipelineStashFilesAfterBuild(
-            script: [commonPipelineEnvironment: [configuration: [steps: [executeCheckmarxScan: [checkmarxProject: 'TestProject']]]]],
-            juStabUtils: utils,
-        )
-        // asserts
-        assertThat(loggingRule.log, containsString('Stash content: checkmarx'))
-        assertThat(loggingRule.log, containsString('Stash content: classFiles'))
-        assertThat(loggingRule.log, containsString('Stash content: sonar'))
-    }
-
 }
--- a/test/groovy/PipelineStashFilesBeforeBuildTest.groovy
+++ b/test/groovy/PipelineStashFilesBeforeBuildTest.groovy
@ -22,27 +22,7 @@ class PipelineStashFilesBeforeBuildTest extends BasePiperTest {
        .around(stepRule)

    @Test
-    void testStashBeforeBuildNoOpa() {
-
-        stepRule.step.pipelineStashFilesBeforeBuild(script: nullScript, juStabUtils: utils)
-
-        // asserts
-        assertEquals('mkdir -p gitmetadata', shellRule.shell[0])
-        assertEquals('cp -rf .git/* gitmetadata', shellRule.shell[1])
-        assertEquals('chmod -R u+w gitmetadata', shellRule.shell[2])
-
-        assertThat(loggingRule.log, containsString('Stash content: buildDescriptor'))
-        assertThat(loggingRule.log, containsString('Stash content: deployDescriptor'))
-        assertThat(loggingRule.log, containsString('Stash content: git'))
-        assertFalse(loggingRule.log.contains('Stash content: opa5'))
-        assertThat(loggingRule.log, containsString('Stash content: opensourceConfiguration'))
-        assertThat(loggingRule.log, containsString('Stash content: pipelineConfigAndTests'))
-        assertThat(loggingRule.log, containsString('Stash content: securityDescriptor'))
-        assertThat(loggingRule.log, containsString('Stash content: tests'))
-    }
-
-    @Test
-    void testStashBeforeBuildOpa() {
+    void testStashBeforeBuild() {

        stepRule.step.pipelineStashFilesBeforeBuild(script: nullScript, juStabUtils: utils, runOpaTests: true)

@ -56,4 +36,21 @@ class PipelineStashFilesBeforeBuildTest extends BasePiperTest {
        assertThat(loggingRule.log, containsString('Stash content: securityDescriptor'))
        assertThat(loggingRule.log, containsString('Stash content: tests'))
    }
+
+    @Test
+    void testStashBeforeBuildCustomConfig() {
+
+        stepRule.step.pipelineStashFilesBeforeBuild(script: nullScript, juStabUtils: utils, runOpaTests: true, stashIncludes: ['myStash': '**.myTest'])
+
+        // asserts
+        assertThat(loggingRule.log, containsString('Stash content: buildDescriptor'))
+        assertThat(loggingRule.log, containsString('Stash content: deployDescriptor'))
+        assertThat(loggingRule.log, containsString('Stash content: git'))
+        assertThat(loggingRule.log, containsString('Stash content: opa5'))
+        assertThat(loggingRule.log, containsString('Stash content: opensourceConfiguration'))
+        assertThat(loggingRule.log, containsString('Stash content: pipelineConfigAndTests'))
+        assertThat(loggingRule.log, containsString('Stash content: securityDescriptor'))
+        assertThat(loggingRule.log, containsString('Stash content: tests'))
+        assertThat(loggingRule.log, containsString('Stash content: myStash'))
+    }
 }
--- a/vars/pipelineStashFilesAfterBuild.groovy
+++ b/vars/pipelineStashFilesAfterBuild.groovy
@ -5,7 +5,7 @@ import com.sap.piper.ConfigurationHelper
 import groovy.transform.Field

@Field String STEP_NAME = getClass().getName()
-@Field Set STEP_CONFIG_KEYS = ['runCheckmarx', 'stashIncludes', 'stashExcludes']
+@Field Set STEP_CONFIG_KEYS = ['noDefaultExludes', 'stashIncludes', 'stashExcludes']
@Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS

 void call(Map parameters = [:]) {
@ -28,9 +28,6 @@ void call(Map parameters = [:]) {
            .mixinGeneralConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS)
            .mixinStepConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS)
            .mixinStageConfig(script.commonPipelineEnvironment, parameters.stageName?:env.STAGE_NAME, STEP_CONFIG_KEYS)
-            .mixin([
-                runCheckmarx: (script.commonPipelineEnvironment.configuration?.steps?.executeCheckmarxScan?.checkmarxProject != null && script.commonPipelineEnvironment.configuration.steps.executeCheckmarxScan.checkmarxProject.length()>0)
-            ])
            .mixin(parameters, PARAMETER_KEYS)
            .use()

@ -40,27 +37,9 @@ void call(Map parameters = [:]) {
            stepParam1: parameters?.script == null
        ], config)

-        // store files to be checked with checkmarx
-        if (config.runCheckmarx) {
-            utils.stash(
-                'checkmarx',
-                config.stashIncludes.checkmarx,
-                config.stashExcludes.checkmarx
-            )
+        config.stashIncludes.each {stashKey, stashIncludes ->
+            def useDefaultExcludes = !config.noDefaultExludes.contains(stashKey)
+            utils.stashWithMessage(stashKey, "[${STEP_NAME}] no files detected for stash '${stashKey}': ", stashIncludes, config.stashExcludes[stashKey]?:'', useDefaultExcludes)
        }
-
-        utils.stashWithMessage(
-            'classFiles',
-            "[${STEP_NAME}] Failed to stash class files.",
-            config.stashIncludes.classFiles,
-            config.stashExcludes.classFiles
-        )
-
-        utils.stashWithMessage(
-            'sonar',
-            "[${STEP_NAME}] Failed to stash sonar files.",
-            config.stashIncludes.sonar,
-            config.stashExcludes.sonar
-        )
    }
 }
--- a/vars/pipelineStashFilesBeforeBuild.groovy
+++ b/vars/pipelineStashFilesBeforeBuild.groovy
@ -5,14 +5,14 @@ import com.sap.piper.ConfigurationHelper
 import groovy.transform.Field

@Field String STEP_NAME = getClass().getName()
-@Field Set STEP_CONFIG_KEYS = ['runOpaTests', 'stashIncludes', 'stashExcludes']
+@Field Set STEP_CONFIG_KEYS = ['noDefaultExludes', 'stashIncludes', 'stashExcludes']
@Field Set PARAMETER_KEYS = STEP_CONFIG_KEYS

 void call(Map parameters = [:]) {

    handlePipelineStepErrors (stepName: STEP_NAME, stepParameters: parameters, stepNameDoc: 'stashFiles') {

-        def utils = parameters.juStabUtils
+        Utils utils = parameters.juStabUtils
        if (utils == null) {
            utils = new Utils()
        }
@ -21,9 +21,6 @@ void call(Map parameters = [:]) {
        if (script == null)
            script = this

-        //additional includes via passing e.g. stashIncludes: [opa5: '**/*.include']
-        //additional excludes via passing e.g. stashExcludes: [opa5: '**/*.exclude']
-
        Map config = ConfigurationHelper.newInstance(this)
            .loadStepDefaults()
            .mixinGeneralConfig(script.commonPipelineEnvironment, STEP_CONFIG_KEYS)
@ -38,59 +35,9 @@ void call(Map parameters = [:]) {
            stepParam1: parameters?.script == null
        ], config)

-        if (config.runOpaTests){
-            utils.stash('opa5', config.stashIncludes?.get('opa5')?config.stashIncludes.opa5:'**/*.*', config.stashExcludes?.get('opa5')?config.stashExcludes.opa5:'')
+        config.stashIncludes.each {stashKey, stashIncludes ->
+            def useDefaultExcludes = !config.noDefaultExludes.contains(stashKey)
+            utils.stashWithMessage(stashKey, "[${STEP_NAME}] no files detected for stash '${stashKey}': ", stashIncludes, config.stashExcludes[stashKey]?:'', useDefaultExcludes)
        }
-
-        //store build descriptor files depending on technology, e.g. pom.xml, package.json
-        utils.stash(
-            'buildDescriptor',
-            config.stashIncludes.buildDescriptor,
-            config.stashExcludes.buildDescriptor
-        )
-        //store deployment descriptor files depending on technology, e.g. *.mtaext.yml
-        utils.stashWithMessage(
-            'deployDescriptor',
-            "[${STEP_NAME}] no deployment descriptor files provided: ",
-            config.stashIncludes.deployDescriptor,
-            config.stashExcludes.deployDescriptor
-        )
-        //store git metadata for SourceClear agent
-        sh "mkdir -p gitmetadata"
-        sh "cp -rf .git/* gitmetadata"
-        sh "chmod -R u+w gitmetadata"
-        utils.stashWithMessage(
-            'git',
-            "[${STEP_NAME}] no git repo files detected: ",
-            config.stashIncludes.git,
-            config.stashExcludes.git
-        )
-        //store nsp & retire exclusion file for future use
-        utils.stashWithMessage(
-            'opensourceConfiguration',
-            "[${STEP_NAME}] no opensourceConfiguration files provided: ",
-            config.stashIncludes.get('opensourceConfiguration'),
-            config.stashExcludes.get('opensourceConfiguration')
-        )
-        //store pipeline configuration including additional groovy test scripts for future use
-        utils.stashWithMessage(
-            'pipelineConfigAndTests',
-            "[${STEP_NAME}] no pipeline configuration and test files found: ",
-            config.stashIncludes.pipelineConfigAndTests,
-            config.stashExcludes.pipelineConfigAndTests
-        )
-        utils.stashWithMessage(
-            'securityDescriptor',
-            "[${STEP_NAME}] no security descriptor found: ",
-            config.stashIncludes.securityDescriptor,
-            config.stashExcludes.securityDescriptor
-        )
-        //store files required for tests, e.g. Gauge, SUT, ...
-        utils.stashWithMessage(
-            'tests',
-            "[${STEP_NAME}] no files for tests provided: ",
-            config.stashIncludes.tests,
-            config.stashExcludes.tests
-        )
    }
 }