pkg/toolrecord: log json marshalling errors; fix issues in whitesource and blackduck (#3049)

* pkg/toolrecord: log json marshalling errors

* toolrecord package: add internal check to ensure that generated files
are not empty

* cmd/whitesourceExecuteScan.go : rework createToolRecordWhitesource to
include all scanned projects

* pkg/toolrecord: new helper function to override default display values

* cmd/whitesourceExecuteScan: improve toolrecord file

* cmd/detectExecuteScan.go fix toolrecord file creation #3

* pkg/toolrecord: log json marshalling errors

* toolrecord package: add internal check to ensure that generated files
are not empty

* cmd/whitesourceExecuteScan.go : rework createToolRecordWhitesource to
include all scanned projects

* pkg/toolrecord: new helper function to override default display values

* cmd/whitesourceExecuteScan: improve toolrecord file

* fix merge conflict

cmd/detectExecuteScan.go

@@ -113,12 +113,6 @@ func detectExecuteScan(config detectExecuteScanOptions, _ *telemetry.CustomData,
 	}
 
 	influx.step_data.fields.detect = true
-	// create Toolrecord file
-	toolRecordFileName, err := createToolRecordDetect("./", config)
-	if err != nil {
-		// do not fail until the framework is well established
-		log.Entry().Warning("TR_DETECT: Failed to create toolrecord file "+toolRecordFileName, err)
-	}
 }
 
 func runDetect(config detectExecuteScanOptions, utils detectUtils, influx *detectExecuteScanInflux) error {
@@ -393,6 +387,12 @@ func postScanChecksAndReporting(config detectExecuteScanOptions, influx *detectE
 	if policyJsonErr != nil {
 		return errors.Wrapf(policyJsonErr, "failed to write IP policy violations json file")
 	}
+	// create Toolrecord file
+	toolRecordFileName, err := createToolRecordDetect("./", config, sys)
+	if err != nil {
+		// do not fail until the framework is well established
+		log.Entry().Warning("TR_DETECT: Failed to create toolrecord file "+toolRecordFileName, err)
+	}
 	return nil
 }
 
@@ -671,18 +671,28 @@ func isMajorVulnerability(v bd.Vulnerability) bool {
 	}
 }
 
-// create toolrecord file for detect
-//
-//
-func createToolRecordDetect(workspace string, config detectExecuteScanOptions) (string, error) {
+// create toolrecord file for detectExecute
+func createToolRecordDetect(workspace string, config detectExecuteScanOptions, sys *blackduckSystem) (string, error) {
 	record := toolrecord.New(workspace, "detectExecute", config.ServerURL)
-
-	projectId := ""  // todo needs more research; according to synopsis documentation
-	productURL := "" // relevant ids can be found in the logfile
-	err := record.AddKeyData("project",
+	project, err := sys.Client.GetProject(config.ProjectName)
+	if err != nil {
+		return "", fmt.Errorf("TR_DETECT: GetProject failed %v", err)
+	}
+	metadata := project.Metadata
+	projectURL := metadata.Href
+	if projectURL == "" {
+		return "", fmt.Errorf("TR_DETECT: no project URL")
+	}
+	// project UUID comes as last part of the URL
+	parts := strings.Split(projectURL, "/")
+	projectId := parts[len(parts)-1]
+	if projectId == "" {
+		return "", fmt.Errorf("TR_DETECT: no project id in %v", projectURL)
+	}
+	err = record.AddKeyData("project",
 		projectId,
 		config.ProjectName,
-		productURL)
+		projectURL)
 	if err != nil {
 		return "", err
 	}

cmd/whitesourceExecuteScan.go

@@ -926,9 +926,6 @@ func persistScannedProjects(config *ScanOptions, scan *ws.Scan, commonPipelineEn
 
 // create toolrecord file for whitesource
 //
-// Limitation: as the toolrecord file is designed to point to one scan result this generate a pointer
-// to the product only, and not to the scanned projects
-//
 func createToolRecordWhitesource(workspace string, config *whitesourceExecuteScanOptions, scan *ws.Scan) (string, error) {
 	record := toolrecord.New(workspace, "whitesource", config.ServiceURL)
 	productURL := config.ServiceURL + "/Wss/WSS.html#!product;token=" + config.ProductToken
@@ -939,8 +936,31 @@ func createToolRecordWhitesource(workspace string, config *whitesourceExecuteSca
 	if err != nil {
 		return "", err
 	}
-	record.AddContext("scannedProjects", scan.ScannedProjectNames)
-	record.AddContext("configuredProject", config.ProjectName+" - "+config.Version)
+	max_idx := 0
+	for idx, project := range scan.ScannedProjects() {
+		max_idx = idx
+		name := project.Name
+		token := project.Token
+		projectURL := ""
+		if token != "" {
+			projectURL = config.ServiceURL + "/Wss/WSS.html#!project;token=" + token
+		} else {
+			// token is empty, provide a dummy to have an indication
+			token = "unknown"
+		}
+		err = record.AddKeyData("project",
+			token,
+			name,
+			projectURL)
+		if err != nil {
+			return "", err
+		}
+	}
+	// set overall display data to product if there
+	// is more than one project
+	if max_idx > 1 {
+		record.SetOverallDisplayData(config.ProductName, productURL)
+	}
 	err = record.Persist()
 	if err != nil {
 		return "", err

pkg/toolrecord/toolrecord_main.go

@@ -132,10 +132,23 @@ func (tr *Toolrecord) Persist() error {
 	tr.DisplayName = displayName
 	tr.DisplayURL = displayURL
 
-	file, _ := json.Marshal(tr)
+	file, err := json.Marshal(tr)
+	if err != nil {
+		return fmt.Errorf("TR_PERSIST: %v", err)
+	}
+	// no json generated ?
+	if len(file) == 0 {
+		return fmt.Errorf("TR_PERSIST: empty json content")
+	}
 	err = ioutil.WriteFile(tr.GetFileName(), file, 0644)
 	if err != nil {
 		return fmt.Errorf("TR_PERSIST: %v", err)
 	}
 	return nil
 }
+
+// Override the default generation for DisplayName & DisplayURL
+func (tr *Toolrecord) SetOverallDisplayData(newName, newURL string) {
+	tr.DisplayName = newName
+	tr.DisplayURL = newURL
+}

pkg/toolrecord/toolrecord_test.go

@@ -30,6 +30,12 @@ func TestToolRecord(t *testing.T) {
 			}{"goes", 42},
 		}
 		tr.AddContext("DemoContext", context)
+		context2 := "a string"
+		tr.AddContext("Context2", context2)
+		var context3 [2]string
+		context3[0] = "c3_1"
+		context3[1] = "c3_2"
+		tr.AddContext("Context3", context3)
 		err := tr.Persist()
 		assert.Nil(t, err, "internal error %s")
 		assert.FileExists(t, tr.GetFileName(), "toolrecord not persisted %s")