fix(whitesourceExecuteScan) properly handle output resources (#2266)

* fix(whitesourceExecuteScan) properly handle output resources

* fix merge issues

* add required aliases

* update generation

* fix reading custom and container environment parameters from cpe

Co-authored-by: Stephan Aßmus <stephan.assmus@sap.com>

cmd/whitesourceExecuteScan.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/SAP/jenkins-library/pkg/npm"
 	"os"
 	"path/filepath"
 	"sort"
@@ -10,6 +9,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/SAP/jenkins-library/pkg/npm"
+
 	"github.com/360EntSecGroup-Skylar/excelize/v2"
 	"github.com/SAP/jenkins-library/pkg/command"
 	piperhttp "github.com/SAP/jenkins-library/pkg/http"
@@ -107,18 +108,17 @@ func newWhitesourceScan(config *ScanOptions) *ws.Scan {
 	}
 }
 
-func whitesourceExecuteScan(config ScanOptions, _ *telemetry.CustomData) {
+func whitesourceExecuteScan(config ScanOptions, _ *telemetry.CustomData, commonPipelineEnvironment *whitesourceExecuteScanCommonPipelineEnvironment) {
 	utils := newWhitesourceUtils(&config)
 	scan := newWhitesourceScan(&config)
-	sys := ws.NewSystem(config.ServiceURL, config.OrgToken, config.UserToken,
-		time.Duration(config.Timeout)*time.Second)
-	err := runWhitesourceExecuteScan(&config, scan, utils, sys)
+	sys := ws.NewSystem(config.ServiceURL, config.OrgToken, config.UserToken, time.Duration(config.Timeout)*time.Second)
+	err := runWhitesourceExecuteScan(&config, scan, utils, sys, commonPipelineEnvironment)
 	if err != nil {
 		log.Entry().WithError(err).Fatal("step execution failed")
 	}
 }
 
-func runWhitesourceExecuteScan(config *ScanOptions, scan *ws.Scan, utils whitesourceUtils, sys whitesource) error {
+func runWhitesourceExecuteScan(config *ScanOptions, scan *ws.Scan, utils whitesourceUtils, sys whitesource, commonPipelineEnvironment *whitesourceExecuteScanCommonPipelineEnvironment) error {
 	if err := resolveProjectIdentifiers(config, scan, utils, sys); err != nil {
 		return fmt.Errorf("failed to resolve project identifiers: %w", err)
 	}
@@ -135,14 +135,14 @@ func runWhitesourceExecuteScan(config *ScanOptions, scan *ws.Scan, utils whiteso
 			return fmt.Errorf("failed to aggregate version wide vulnerabilities: %w", err)
 		}
 	} else {
-		if err := runWhitesourceScan(config, scan, utils, sys); err != nil {
+		if err := runWhitesourceScan(config, scan, utils, sys, commonPipelineEnvironment); err != nil {
 			return fmt.Errorf("failed to execute WhiteSource scan: %w", err)
 		}
 	}
 	return nil
 }
 
-func runWhitesourceScan(config *ScanOptions, scan *ws.Scan, utils whitesourceUtils, sys whitesource) error {
+func runWhitesourceScan(config *ScanOptions, scan *ws.Scan, utils whitesourceUtils, sys whitesource, commonPipelineEnvironment *whitesourceExecuteScanCommonPipelineEnvironment) error {
 	// Start the scan
 	if err := executeScan(config, scan, utils); err != nil {
 		return err
@@ -165,9 +165,7 @@ func runWhitesourceScan(config *ScanOptions, scan *ws.Scan, utils whitesourceUti
 		return err
 	}
 
-	if err := persistScannedProjects(config, scan, utils); err != nil {
-		return fmt.Errorf("failed to persist scanned WhiteSource project names: %w", err)
-	}
+	persistScannedProjects(config, scan, commonPipelineEnvironment)
 
 	return nil
 }
@@ -656,8 +654,8 @@ func newLibraryCSVReport(libraries map[string][]ws.Library, config *ScanOptions,
 
 // persistScannedProjects writes all actually scanned WhiteSource project names as comma separated
 // string into the Common Pipeline Environment, from where it can be used by sub-sequent steps.
-func persistScannedProjects(config *ScanOptions, scan *ws.Scan, utils whitesourceUtils) error {
-	var projectNames []string
+func persistScannedProjects(config *ScanOptions, scan *ws.Scan, commonPipelineEnvironment *whitesourceExecuteScanCommonPipelineEnvironment) {
+	projectNames := []string{}
 	if config.ProjectName != "" {
 		projectNames = []string{config.ProjectName + " - " + config.ProductVersion}
 	} else {
@@ -668,14 +666,5 @@ func persistScannedProjects(config *ScanOptions, scan *ws.Scan, utils whitesourc
 		// as the order in which we travers map keys is not deterministic.
 		sort.Strings(projectNames)
 	}
-	resourceDir := filepath.Join(".pipeline", "commonPipelineEnvironment", "custom")
-	if err := utils.MkdirAll(resourceDir, 0755); err != nil {
-		return err
-	}
-	fileContents := strings.Join(projectNames, ",")
-	resource := filepath.Join(resourceDir, "whitesourceProjectNames")
-	if err := utils.FileWrite(resource, []byte(fileContents), 0644); err != nil {
-		return err
-	}
-	return nil
+	commonPipelineEnvironment.custom.whitesourceProjectNames = projectNames
 }

cmd/whitesourceExecuteScan_generated.go

@@ -5,10 +5,12 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/SAP/jenkins-library/pkg/config"
 	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/SAP/jenkins-library/pkg/piperenv"
 	"github.com/SAP/jenkins-library/pkg/telemetry"
 	"github.com/spf13/cobra"
 )
@@ -54,6 +56,34 @@ type whitesourceExecuteScanOptions struct {
 	DefaultNpmRegistry                   string   `json:"defaultNpmRegistry,omitempty"`
 }
 
+type whitesourceExecuteScanCommonPipelineEnvironment struct {
+	custom struct {
+		whitesourceProjectNames []string
+	}
+}
+
+func (p *whitesourceExecuteScanCommonPipelineEnvironment) persist(path, resourceName string) {
+	content := []struct {
+		category string
+		name     string
+		value    interface{}
+	}{
+		{category: "custom", name: "whitesourceProjectNames", value: p.custom.whitesourceProjectNames},
+	}
+
+	errCount := 0
+	for _, param := range content {
+		err := piperenv.SetResourceParameter(path, resourceName, filepath.Join(param.category, param.name), param.value)
+		if err != nil {
+			log.Entry().WithError(err).Error("Error persisting piper environment.")
+			errCount++
+		}
+	}
+	if errCount > 0 {
+		log.Entry().Fatal("failed to persist Piper environment")
+	}
+}
+
 // WhitesourceExecuteScanCommand BETA
 func WhitesourceExecuteScanCommand() *cobra.Command {
 	const STEP_NAME = "whitesourceExecuteScan"
@@ -61,6 +91,7 @@ func WhitesourceExecuteScanCommand() *cobra.Command {
 	metadata := whitesourceExecuteScanMetadata()
 	var stepConfig whitesourceExecuteScanOptions
 	var startTime time.Time
+	var commonPipelineEnvironment whitesourceExecuteScanCommonPipelineEnvironment
 
 	var createWhitesourceExecuteScanCmd = &cobra.Command{
 		Use:   STEP_NAME,
@@ -109,6 +140,7 @@ check and additional Free and Open Source Software Publicly Known Vulnerabilitie
 			telemetryData.ErrorCode = "1"
 			handler := func() {
 				config.RemoveVaultSecretFiles()
+				commonPipelineEnvironment.persist(GeneralConfig.EnvRootPath, "commonPipelineEnvironment")
 				telemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds())
 				telemetryData.ErrorCategory = log.GetErrorCategory().String()
 				telemetry.Send(&telemetryData)
@@ -116,7 +148,7 @@ check and additional Free and Open Source Software Publicly Known Vulnerabilitie
 			log.DeferExitHandler(handler)
 			defer handler()
 			telemetry.Initialize(GeneralConfig.NoTelemetry, STEP_NAME)
-			whitesourceExecuteScan(stepConfig, &telemetryData)
+			whitesourceExecuteScan(stepConfig, &telemetryData, &commonPipelineEnvironment)
 			telemetryData.ErrorCode = "0"
 			log.Entry().Info("SUCCESS")
 		},
@@ -384,7 +416,7 @@ func whitesourceExecuteScanMetadata() config.StepData {
 						Scope:       []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"},
 						Type:        "string",
 						Mandatory:   true,
-						Aliases:     []config.Alias{},
+						Aliases:     []config.Alias{{Name: "whitesourceProductName"}},
 					},
 					{
 						Name:        "projectName",

cmd/whitesourceExecuteScan_test.go

@@ -1,13 +1,14 @@
 package cmd
 
 import (
+	"path/filepath"
+	"testing"
+	"time"
+
 	"github.com/SAP/jenkins-library/pkg/mock"
 	"github.com/SAP/jenkins-library/pkg/versioning"
 	ws "github.com/SAP/jenkins-library/pkg/whitesource"
 	"github.com/stretchr/testify/assert"
-	"path/filepath"
-	"testing"
-	"time"
 )
 
 type whitesourceCoordinatesMock struct {
@@ -194,63 +195,49 @@ func TestBlockUntilProjectIsUpdated(t *testing.T) {
 }
 
 func TestPersistScannedProjects(t *testing.T) {
-	resource := filepath.Join(".pipeline", "commonPipelineEnvironment", "custom", "whitesourceProjectNames")
-
 	t.Parallel()
 	t.Run("write 1 scanned projects", func(t *testing.T) {
 		// init
+		cpe := whitesourceExecuteScanCommonPipelineEnvironment{}
 		config := &ScanOptions{ProductVersion: "1"}
-		utils := newWhitesourceUtilsMock()
 		scan := newWhitesourceScan(config)
 		_ = scan.AppendScannedProject("project")
 		// test
-		err := persistScannedProjects(config, scan, utils)
+		persistScannedProjects(config, scan, &cpe)
 		// assert
-		if assert.NoError(t, err) && assert.True(t, utils.HasWrittenFile(resource)) {
-			contents, _ := utils.FileRead(resource)
-			assert.Equal(t, "project - 1", string(contents))
-		}
+		assert.Equal(t, []string{"project - 1"}, cpe.custom.whitesourceProjectNames)
 	})
 	t.Run("write 2 scanned projects", func(t *testing.T) {
 		// init
+		cpe := whitesourceExecuteScanCommonPipelineEnvironment{}
 		config := &ScanOptions{ProductVersion: "1"}
-		utils := newWhitesourceUtilsMock()
 		scan := newWhitesourceScan(config)
 		_ = scan.AppendScannedProject("project-app")
 		_ = scan.AppendScannedProject("project-db")
 		// test
-		err := persistScannedProjects(config, scan, utils)
+		persistScannedProjects(config, scan, &cpe)
 		// assert
-		if assert.NoError(t, err) && assert.True(t, utils.HasWrittenFile(resource)) {
-			contents, _ := utils.FileRead(resource)
-			assert.Equal(t, "project-app - 1,project-db - 1", string(contents))
-		}
+		assert.Equal(t, []string{"project-app - 1", "project-db - 1"}, cpe.custom.whitesourceProjectNames)
 	})
 	t.Run("write no projects", func(t *testing.T) {
 		// init
+		cpe := whitesourceExecuteScanCommonPipelineEnvironment{}
 		config := &ScanOptions{ProductVersion: "1"}
-		utils := newWhitesourceUtilsMock()
 		scan := newWhitesourceScan(config)
 		// test
-		err := persistScannedProjects(config, scan, utils)
+		persistScannedProjects(config, scan, &cpe)
 		// assert
-		if assert.NoError(t, err) && assert.True(t, utils.HasWrittenFile(resource)) {
-			contents, _ := utils.FileRead(resource)
-			assert.Equal(t, "", string(contents))
-		}
+		assert.Equal(t, []string{}, cpe.custom.whitesourceProjectNames)
 	})
 	t.Run("write aggregated project", func(t *testing.T) {
 		// init
+		cpe := whitesourceExecuteScanCommonPipelineEnvironment{}
 		config := &ScanOptions{ProjectName: "project", ProductVersion: "1"}
-		utils := newWhitesourceUtilsMock()
 		scan := newWhitesourceScan(config)
 		// test
-		err := persistScannedProjects(config, scan, utils)
+		persistScannedProjects(config, scan, &cpe)
 		// assert
-		if assert.NoError(t, err) && assert.True(t, utils.HasWrittenFile(resource)) {
-			contents, _ := utils.FileRead(resource)
-			assert.Equal(t, "project - 1", string(contents))
-		}
+		assert.Equal(t, []string{"project - 1"}, cpe.custom.whitesourceProjectNames)
 	})
 }
 

pkg/generator/helper/helper.go

@@ -495,7 +495,7 @@ func getOutputResourceDetails(stepData *config.StepData) ([]map[string]string, e
 						envResource.Categories = append(envResource.Categories, category)
 					}
 				}
-				envParam := PiperEnvironmentParameter{Category: category, Name: name}
+				envParam := PiperEnvironmentParameter{Category: category, Name: name, Type: fmt.Sprint(param["type"])}
 				envResource.Parameters = append(envResource.Parameters, envParam)
 			}
 			def, err := envResource.StructString()
@@ -594,7 +594,7 @@ func longName(long string) string {
 	return l
 }
 
-func influxType(fieldType string) string {
+func resourceFieldType(fieldType string) string {
 	//TODO: clarify why fields are initialized with <nil> and tags are initialized with ''
 	if len(fieldType) == 0 || fieldType == "<nil>" {
 		return "string"

pkg/generator/helper/helper_test.go

@@ -31,6 +31,8 @@ spec:
           - name: artifactVersion
           - name: git/commitId
           - name: git/branch
+          - name: custom/customList
+            type: "[]string"
       - name: influxTest
         type: influx
         params:

pkg/generator/helper/resources.go

@@ -19,19 +19,20 @@ type PiperEnvironmentResource struct {
 type PiperEnvironmentParameter struct {
 	Category string
 	Name     string
+	Type     string
 }
 
 const piperEnvStructTemplate = `type {{ .StepName }}{{ .Name | title}} struct {
 	{{- range $notused, $param := .Parameters }}
 	{{- if not $param.Category}}
-	{{ $param.Name | golangName }} string
+	{{ $param.Name | golangName }} {{ $param.Type | resourceFieldType }}
 	{{- end }}
 	{{- end }}
 	{{- range $notused, $category := .Categories }}
 	{{ $category }} struct {
 		{{- range $notused, $param := $.Parameters }}
 		{{- if eq $category $param.Category }}
-		{{ $param.Name | golangName }} string
+		{{ $param.Name | golangName }} {{ $param.Type | resourceFieldType }}
 		{{- end }}
 		{{- end }}
 	}
@@ -66,16 +67,17 @@ func (p *{{ .StepName }}{{ .Name | title}}) persist(path, resourceName string) {
 	}
 }`
 
-// StructName returns the name of the influx resource struct
+// StructName returns the name of the environment resource struct
 func (p *PiperEnvironmentResource) StructName() string {
 	return fmt.Sprintf("%v%v", p.StepName, strings.Title(p.Name))
 }
 
-// StructString returns the golang coding for the struct definition of the InfluxResource
+// StructString returns the golang coding for the struct definition of the environment resource
 func (p *PiperEnvironmentResource) StructString() (string, error) {
 	funcMap := template.FuncMap{
-		"title":      strings.Title,
-		"golangName": golangName,
+		"title":             strings.Title,
+		"golangName":        golangName,
+		"resourceFieldType": resourceFieldType,
 	}
 
 	tmpl, err := template.New("resources").Funcs(funcMap).Parse(piperEnvStructTemplate)
@@ -125,12 +127,12 @@ const influxStructTemplate = `type {{ .StepName }}{{ .Name | title}} struct {
 	{{ $measurement.Name }} struct {
 		fields struct {
 			{{- range $notused, $field := $measurement.Fields }}
-			{{ $field.Name | golangName }} {{ $field.Type | influxType }}
+			{{ $field.Name | golangName }} {{ $field.Type | resourceFieldType }}
 			{{- end }}
 		}
 		tags struct {
 			{{- range $notused, $tag := $measurement.Tags }}
-			{{ $tag.Name | golangName }} {{ $tag.Type | influxType }}
+			{{ $tag.Name | golangName }} {{ $tag.Type | resourceFieldType }}
 			{{- end }}
 		}
 	}
@@ -170,9 +172,9 @@ func (i *{{ .StepName }}{{ .Name | title}}) persist(path, resourceName string) {
 // StructString returns the golang coding for the struct definition of the InfluxResource
 func (i *InfluxResource) StructString() (string, error) {
 	funcMap := template.FuncMap{
-		"title":      strings.Title,
-		"golangName": golangName,
-		"influxType": influxType,
+		"title":             strings.Title,
+		"golangName":        golangName,
+		"resourceFieldType": resourceFieldType,
 	}
 
 	tmpl, err := template.New("resources").Funcs(funcMap).Parse(influxStructTemplate)

pkg/generator/helper/testdata/TestProcessMetaFiles/custom_step_code_generated.golden

@@ -29,6 +29,9 @@ type testStepCommonPipelineEnvironment struct {
 		commitID string
 		branch string
 	}
+	custom struct {
+		customList []string
+	}
 }
 
 func (p *testStepCommonPipelineEnvironment) persist(path, resourceName string) {
@@ -40,6 +43,7 @@ func (p *testStepCommonPipelineEnvironment) persist(path, resourceName string) {
 		{category: "", name: "artifactVersion", value: p.artifactVersion},
 		{category: "git", name: "commitId", value: p.git.commitID},
 		{category: "git", name: "branch", value: p.git.branch},
+		{category: "custom", name: "customList", value: p.custom.customList},
 	}
 
 	errCount := 0

pkg/generator/helper/testdata/TestProcessMetaFiles/step_code_generated.golden

@@ -28,6 +28,9 @@ type testStepCommonPipelineEnvironment struct {
 		commitID string
 		branch string
 	}
+	custom struct {
+		customList []string
+	}
 }
 
 func (p *testStepCommonPipelineEnvironment) persist(path, resourceName string) {
@@ -39,6 +42,7 @@ func (p *testStepCommonPipelineEnvironment) persist(path, resourceName string) {
 		{category: "", name: "artifactVersion", value: p.artifactVersion},
 		{category: "git", name: "commitId", value: p.git.commitID},
 		{category: "git", name: "branch", value: p.git.branch},
+		{category: "custom", name: "customList", value: p.custom.customList},
 	}
 
 	errCount := 0

resources/metadata/whitesource.yaml

@@ -21,9 +21,13 @@ spec:
   inputs:
     secrets:
       - name: userTokenCredentialsId
+        aliases:
+          - name: whitesourceUserTokenCredentialsId
         description: Jenkins 'Secret text' credentials ID containing Whitesource user token.
         type: jenkins
       - name: orgAdminUserTokenCredentialsId
+        aliases:
+          - name: whitesourceOrgAdminUserTokenCredentialsId
         description: Jenkins 'Secret text' credentials ID containing Whitesource org admin token.
         type: jenkins
     params:
@@ -231,6 +235,8 @@ spec:
           - STAGES
           - STEPS
       - name: productName
+        aliases:
+          - name: whitesourceProductName
         type: string
         description: "Name of the WhiteSource product used for results aggregation.
           This parameter is mandatory if the parameter `createProductFromPipeline` is set to `true`
@@ -495,6 +501,13 @@ spec:
             params:
               - name: scanType
                 value: dub
+  outputs:
+    resources:
+      - name: commonPipelineEnvironment
+        type: piperEnvironment
+        params:
+          - name: custom/whitesourceProjectNames
+            type: "[]string"
   containers:
     - image: maven:3.5-jdk-8
       workingDir: /home/java

vars/commonPipelineEnvironment.groovy

@@ -236,7 +236,7 @@ class commonPipelineEnvironment implements Serializable {
             def param = fileName.split('/')[fileName.split('\\/').size()-1]
             if (param.endsWith(".json")){
                 param = param.replace(".json","")
-                valueMap[param] = script.readJSON(test: fileContent)
+                valueMap[param] = script.readJSON(text: fileContent)
             }else{
                 valueMap[param] = fileContent
             }
@@ -249,7 +249,7 @@ class commonPipelineEnvironment implements Serializable {
             def param = fileName.split('/')[fileName.split('\\/').size()-1]
             if (param.endsWith(".json")){
                 param = param.replace(".json","")
-                containerProperties[param] = script.readJSON(test: fileContent)
+                containerProperties[param] = script.readJSON(text: fileContent)
             }else{
                 containerProperties[param] = fileContent
             }