GCS Upload fixes (#3387)

* Fixed argument type in persist function * Fixed gcp upload to be usable in internal piper * Fixed import of packages * Updated logs Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2024-12-12 10:55:20 +02:00 · 2022-01-19 15:16:57 +03:00 · 2022-01-19 15:16:57 +03:00 · c93f1f861d
commit c93f1f861d
parent c2ebdfd9ec
7 changed files with 60 additions and 40 deletions
--- a/cmd/sonarExecuteScan_generated.go
+++ b/cmd/sonarExecuteScan_generated.go
@ -6,6 +6,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"reflect"
+	"strings"
 	"time"

 	"github.com/SAP/jenkins-library/pkg/config"
@ -17,8 +19,6 @@ import (
 	"github.com/SAP/jenkins-library/pkg/validation"
 	"github.com/bmatcuk/doublestar"
 	"github.com/spf13/cobra"
-	"reflect"
-	"strings"
 )

 type sonarExecuteScanOptions struct {
@ -54,17 +54,18 @@ type sonarExecuteScanOptions struct {
 type sonarExecuteScanReports struct {
 }

-func (p *sonarExecuteScanReports) persist(stepConfig sonarExecuteScanOptions) {
-	if GeneralConfig.GCSBucketId == "" {
+func (p *sonarExecuteScanReports) persist(stepConfig sonarExecuteScanOptions, gcpJsonKeyFilePath string, gcsBucketId string, gcsFolderPath string, gcsSubFolder string) {
+	if gcsBucketId == "" {
 		log.Entry().Info("persisting reports to GCS is disabled, because gcsBucketId is empty")
 		return
 	}
+	log.Entry().Info("Uploading reports to Google Cloud Storage...")
 	content := []gcs.ReportOutputParam{
 		{FilePattern: "**/sonarscan.json", ParamRef: "", StepResultType: "sonarqube"},
 		{FilePattern: "**/sonarscan-result.json", ParamRef: "", StepResultType: "sonarqube"},
 	}
 	envVars := []gcs.EnvVar{
-		{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: GeneralConfig.GCPJsonKeyFilePath, Modified: false},
+		{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: gcpJsonKeyFilePath, Modified: false},
 	}
 	gcsClient, err := gcs.NewClient(gcs.WithEnvVars(envVars))
 	if err != nil {
@ -81,7 +82,7 @@ func (p *sonarExecuteScanReports) persist(stepConfig sonarExecuteScanOptions) {
 			inputParameters[paramName[0]] = paramValue
 		}
 	}
-	if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, GeneralConfig.GCSFolderPath, GeneralConfig.GCSBucketId, GeneralConfig.GCSSubFolder, doublestar.Glob, os.Stat); err != nil {
+	if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, gcsFolderPath, gcsBucketId, gcsSubFolder, doublestar.Glob, os.Stat); err != nil {
 		log.Entry().Errorf("failed to persist reports: %v", err)
 	}
 }
@ -197,7 +198,7 @@ func SonarExecuteScanCommand() *cobra.Command {
 			stepTelemetryData := telemetry.CustomData{}
 			stepTelemetryData.ErrorCode = "1"
 			handler := func() {
-				reports.persist(stepConfig)
+				reports.persist(stepConfig, GeneralConfig.GCPJsonKeyFilePath, GeneralConfig.GCSBucketId, GeneralConfig.GCSFolderPath, GeneralConfig.GCSSubFolder)
 				influx.persist(GeneralConfig.EnvRootPath, "influx")
 				config.RemoveVaultSecretFiles()
 				stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds())
--- a/pkg/gcs/gcsClient.go
+++ b/pkg/gcs/gcsClient.go
@ -99,7 +99,7 @@ func NewClient(opts ...gcsOption) (*gcsClient, error) {
 // UploadFile uploads a file into a google cloud storage bucket
 func (g *gcsClient) UploadFile(bucketID string, sourcePath string, targetPath string) error {
 	target := g.client.Bucket(bucketID).Object(targetPath).NewWriter(g.context)
-	log.Entry().Debugf("uploading %v to %v\n", sourcePath, targetPath)
+	log.Entry().Debugf("uploading %v to %v", sourcePath, targetPath)
 	sourceFile, err := g.openFile(sourcePath)
 	if err != nil {
 		return errors.Wrapf(err, "could not open source file: %v", err)
--- a/pkg/generator/helper/helper.go
+++ b/pkg/generator/helper/helper.go
@ -42,10 +42,25 @@ const stepGoTemplate = `// Code generated by piper's step-generator. DO NOT EDIT

 package cmd

+{{ $reportsOutputExists := false -}}
+{{ $influxOutputExists := false -}}
+{{ $piperEnvironmentOutputExists := false -}}
+{{ if .OutputResources -}}
+	{{ range $notused, $oRes := .OutputResources -}}
+		{{ if eq (index $oRes "type") "reports" -}}{{ $reportsOutputExists = true -}}{{ end -}}
+		{{ if eq (index $oRes "type") "influx" -}}{{ $influxOutputExists = true -}}{{ end -}}
+		{{ if eq (index $oRes "type") "piperEnvironment" -}}{{ $piperEnvironmentOutputExists = true -}}{{ end -}}
+	{{ end -}}
+{{ end -}}
+
 import (
 	"fmt"
 	"os"
-	{{ if .OutputResources -}}
+	{{ if $reportsOutputExists -}}
+	"reflect"
+	"strings"
+	{{ end -}}
+	{{ if or $influxOutputExists $piperEnvironmentOutputExists -}}
 	"path/filepath"
 	{{ end -}}
 	"time"
@ -55,17 +70,11 @@ import (
 	{{ end -}}
 	"github.com/SAP/jenkins-library/pkg/config"
 	"github.com/SAP/jenkins-library/pkg/log"
-	{{ if .OutputResources -}}
-	{{ $reportsExist := false -}}
-	{{ range $notused, $oRes := .OutputResources -}}
-	{{ if eq (index $oRes "type") "reports" -}}{{ $reportsExist = true -}}{{ end -}}
-	{{ end -}}
-	{{ if $reportsExist -}}
+	{{ if $reportsOutputExists -}}
 	"github.com/bmatcuk/doublestar"
 	"github.com/SAP/jenkins-library/pkg/gcs"
-	"reflect"
-	"strings"
 	{{ end -}}
+	{{ if or $influxOutputExists $piperEnvironmentOutputExists -}}
 	"github.com/SAP/jenkins-library/pkg/piperenv"
 	{{ end -}}
 	"github.com/SAP/jenkins-library/pkg/telemetry"
@ -157,7 +166,13 @@ func {{.CobraCmdFuncName}}() *cobra.Command {
 			stepTelemetryData.ErrorCode = "1"
 			handler := func() {
 				{{- range $notused, $oRes := .OutputResources }}
-				{{ index $oRes "name" }}.persist({{ if eq (index $oRes "type") "reports" -}}stepConfig{{- else -}}
+				{{ index $oRes "name" }}.persist(
+				{{- if eq (index $oRes "type") "reports" -}}stepConfig, 
+					{{- if $.ExportPrefix}}{{ $.ExportPrefix }}.{{end}}GeneralConfig.GCPJsonKeyFilePath, 
+					{{- if $.ExportPrefix}}{{ $.ExportPrefix }}.{{end}}GeneralConfig.GCSBucketId, 
+					{{- if $.ExportPrefix}}{{ $.ExportPrefix }}.{{end}}GeneralConfig.GCSFolderPath, 
+					{{- if $.ExportPrefix}}{{ $.ExportPrefix }}.{{end}}GeneralConfig.GCSSubFolder
+				{{- else -}}
 					{{if $.ExportPrefix}}{{ $.ExportPrefix }}.{{end}}GeneralConfig.EnvRootPath, {{ index $oRes "name" | quote }}{{- end -}}
 				){{- end }}
 				config.RemoveVaultSecretFiles()
--- a/pkg/generator/helper/resources.go
+++ b/pkg/generator/helper/resources.go
@ -213,18 +213,19 @@ type ReportsParameter struct {
 const reportsStructTemplate = `type {{ .StepName }}{{ .Name | title}} struct {
 }

-func (p *{{ .StepName }}{{ .Name | title}}) persist(stepConfig sonarExecuteScanOptions) {
-	if GeneralConfig.GCSBucketId == "" {
+func (p *{{ .StepName }}{{ .Name | title}}) persist(stepConfig {{ .StepName }}Options, gcpJsonKeyFilePath string, gcsBucketId string, gcsFolderPath string, gcsSubFolder string) {
+	if gcsBucketId == "" {
 		log.Entry().Info("persisting reports to GCS is disabled, because gcsBucketId is empty")
 		return
 	}
+	log.Entry().Info("Uploading reports to Google Cloud Storage...")
 	content := []gcs.ReportOutputParam{
 		{{- range $notused, $param := .Parameters }}
 		{FilePattern: "{{ $param.FilePattern }}", ParamRef: "{{ $param.ParamRef }}", StepResultType: "{{ $param.Type }}"},
 		{{- end }}
 	}
 	envVars := []gcs.EnvVar{
-		{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: GeneralConfig.GCPJsonKeyFilePath, Modified: false},
+		{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: gcpJsonKeyFilePath, Modified: false},
 	}
 	gcsClient, err := gcs.NewClient(gcs.WithEnvVars(envVars))
 	if err != nil {
@ -241,7 +242,7 @@ func (p *{{ .StepName }}{{ .Name | title}}) persist(stepConfig sonarExecuteScanO
 			inputParameters[paramName[0]] = paramValue
 		}
 	}
-	if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, GeneralConfig.GCSFolderPath, GeneralConfig.GCSBucketId, GeneralConfig.GCSSubFolder, doublestar.Glob, os.Stat); err != nil {
+	if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, gcsFolderPath, gcsBucketId, gcsSubFolder, doublestar.Glob, os.Stat); err != nil {
 		log.Entry().Errorf("failed to persist reports: %v", err)
 	}
 }`
--- a/pkg/generator/helper/resources_test.go
+++ b/pkg/generator/helper/resources_test.go
@ -119,18 +119,19 @@ func TestReportsResource_StructString(t *testing.T) {
 			expected: `type testStepReports struct {
 }

-func (p *testStepReports) persist(stepConfig sonarExecuteScanOptions) {
-	if GeneralConfig.GCSBucketId == "" {
+func (p *testStepReports) persist(stepConfig testStepOptions, gcpJsonKeyFilePath string, gcsBucketId string, gcsFolderPath string, gcsSubFolder string) {
+	if gcsBucketId == "" {
 		log.Entry().Info("persisting reports to GCS is disabled, because gcsBucketId is empty")
 		return
 	}
+	log.Entry().Info("Uploading reports to Google Cloud Storage...")
 	content := []gcs.ReportOutputParam{
 		{FilePattern: "pattern1", ParamRef: "", StepResultType: "general"},
 		{FilePattern: "pattern2", ParamRef: "", StepResultType: ""},
 		{FilePattern: "", ParamRef: "testParam", StepResultType: ""},
 	}
 	envVars := []gcs.EnvVar{
-		{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: GeneralConfig.GCPJsonKeyFilePath, Modified: false},
+		{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: gcpJsonKeyFilePath, Modified: false},
 	}
 	gcsClient, err := gcs.NewClient(gcs.WithEnvVars(envVars))
 	if err != nil {
@ -147,7 +148,7 @@ func (p *testStepReports) persist(stepConfig sonarExecuteScanOptions) {
 			inputParameters[paramName[0]] = paramValue
 		}
 	}
-	if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, GeneralConfig.GCSFolderPath, GeneralConfig.GCSBucketId, GeneralConfig.GCSSubFolder, doublestar.Glob, os.Stat); err != nil {
+	if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, gcsFolderPath, gcsBucketId, gcsSubFolder, doublestar.Glob, os.Stat); err != nil {
 		log.Entry().Errorf("failed to persist reports: %v", err)
 	}
 }`,
--- a/pkg/generator/helper/testdata/TestProcessMetaFiles/custom_step_code_generated.golden
+++ b/pkg/generator/helper/testdata/TestProcessMetaFiles/custom_step_code_generated.golden
@ -5,6 +5,8 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"reflect"
+	"strings"
 	"path/filepath"
 	"time"

@ -13,8 +15,6 @@ import (
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/bmatcuk/doublestar"
 	"github.com/SAP/jenkins-library/pkg/gcs"
-	"reflect"
-	"strings"
 	"github.com/SAP/jenkins-library/pkg/piperenv"
 	"github.com/SAP/jenkins-library/pkg/telemetry"
 	"github.com/SAP/jenkins-library/pkg/splunk"
@ -33,17 +33,18 @@ type testStepOptions struct {
 type testStepReports struct {
 }

-func (p *testStepReports) persist(stepConfig sonarExecuteScanOptions) {
-	if GeneralConfig.GCSBucketId == "" {
+func (p *testStepReports) persist(stepConfig testStepOptions, gcpJsonKeyFilePath string, gcsBucketId string, gcsFolderPath string, gcsSubFolder string) {
+	if gcsBucketId == "" {
 		log.Entry().Info("persisting reports to GCS is disabled, because gcsBucketId is empty")
 		return
 	}
+	log.Entry().Info("Uploading reports to Google Cloud Storage...")
 	content := []gcs.ReportOutputParam{
 		{FilePattern: "test-report_*.json", ParamRef: "", StepResultType: ""},
 		{FilePattern: "report1", ParamRef: "", StepResultType: "general"},
 	}
 	envVars := []gcs.EnvVar{
-		{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: GeneralConfig.GCPJsonKeyFilePath, Modified: false},
+		{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: gcpJsonKeyFilePath, Modified: false},
 	}
 	gcsClient, err := gcs.NewClient(gcs.WithEnvVars(envVars))
 	if err != nil {
@ -60,7 +61,7 @@ func (p *testStepReports) persist(stepConfig sonarExecuteScanOptions) {
 			inputParameters[paramName[0]] = paramValue
 		}
 	}
-	if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, GeneralConfig.GCSFolderPath, GeneralConfig.GCSBucketId, GeneralConfig.GCSSubFolder, doublestar.Glob, os.Stat); err != nil {
+	if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, gcsFolderPath, gcsBucketId, gcsSubFolder, doublestar.Glob, os.Stat); err != nil {
 		log.Entry().Errorf("failed to persist reports: %v", err)
 	}
 }
@ -200,7 +201,7 @@ func TestStepCommand() *cobra.Command {
 			stepTelemetryData := telemetry.CustomData{}
 			stepTelemetryData.ErrorCode = "1"
 			handler := func() {
-				reports.persist(stepConfig)
+				reports.persist(stepConfig,piperOsCmd.GeneralConfig.GCPJsonKeyFilePath,piperOsCmd.GeneralConfig.GCSBucketId,piperOsCmd.GeneralConfig.GCSFolderPath,piperOsCmd.GeneralConfig.GCSSubFolder)
 				commonPipelineEnvironment.persist(piperOsCmd.GeneralConfig.EnvRootPath, "commonPipelineEnvironment")
 				influxTest.persist(piperOsCmd.GeneralConfig.EnvRootPath, "influxTest")
 				config.RemoveVaultSecretFiles()
--- a/pkg/generator/helper/testdata/TestProcessMetaFiles/step_code_generated.golden
+++ b/pkg/generator/helper/testdata/TestProcessMetaFiles/step_code_generated.golden
@ -5,6 +5,8 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"reflect"
+	"strings"
 	"path/filepath"
 	"time"

@ -12,8 +14,6 @@ import (
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/bmatcuk/doublestar"
 	"github.com/SAP/jenkins-library/pkg/gcs"
-	"reflect"
-	"strings"
 	"github.com/SAP/jenkins-library/pkg/piperenv"
 	"github.com/SAP/jenkins-library/pkg/telemetry"
 	"github.com/SAP/jenkins-library/pkg/splunk"
@ -32,17 +32,18 @@ type testStepOptions struct {
 type testStepReports struct {
 }

-func (p *testStepReports) persist(stepConfig sonarExecuteScanOptions) {
-	if GeneralConfig.GCSBucketId == "" {
+func (p *testStepReports) persist(stepConfig testStepOptions, gcpJsonKeyFilePath string, gcsBucketId string, gcsFolderPath string, gcsSubFolder string) {
+	if gcsBucketId == "" {
 		log.Entry().Info("persisting reports to GCS is disabled, because gcsBucketId is empty")
 		return
 	}
+	log.Entry().Info("Uploading reports to Google Cloud Storage...")
 	content := []gcs.ReportOutputParam{
 		{FilePattern: "test-report_*.json", ParamRef: "", StepResultType: ""},
 		{FilePattern: "report1", ParamRef: "", StepResultType: "general"},
 	}
 	envVars := []gcs.EnvVar{
-		{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: GeneralConfig.GCPJsonKeyFilePath, Modified: false},
+		{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: gcpJsonKeyFilePath, Modified: false},
 	}
 	gcsClient, err := gcs.NewClient(gcs.WithEnvVars(envVars))
 	if err != nil {
@ -59,7 +60,7 @@ func (p *testStepReports) persist(stepConfig sonarExecuteScanOptions) {
 			inputParameters[paramName[0]] = paramValue
 		}
 	}
-	if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, GeneralConfig.GCSFolderPath, GeneralConfig.GCSBucketId, GeneralConfig.GCSSubFolder, doublestar.Glob, os.Stat); err != nil {
+	if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, gcsFolderPath, gcsBucketId, gcsSubFolder, doublestar.Glob, os.Stat); err != nil {
 		log.Entry().Errorf("failed to persist reports: %v", err)
 	}
 }
@ -199,7 +200,7 @@ func TestStepCommand() *cobra.Command {
 			stepTelemetryData := telemetry.CustomData{}
 			stepTelemetryData.ErrorCode = "1"
 			handler := func() {
-				reports.persist(stepConfig)
+				reports.persist(stepConfig,GeneralConfig.GCPJsonKeyFilePath,GeneralConfig.GCSBucketId,GeneralConfig.GCSFolderPath,GeneralConfig.GCSSubFolder)
 				commonPipelineEnvironment.persist(GeneralConfig.EnvRootPath, "commonPipelineEnvironment")
 				influxTest.persist(GeneralConfig.EnvRootPath, "influxTest")
 				config.RemoveVaultSecretFiles()