From cd243ee54277e794f95a4eef17a15946654347f4 Mon Sep 17 00:00:00 2001 From: Siarhei Pazdniakou Date: Wed, 15 Dec 2021 17:07:47 +0300 Subject: [PATCH] feat(gcs): allow upload to gcs from steps (#3034) * Upload reports to Google Cloud Storage bucket * Added tests. Made fixes * Update step generation. GCS client was moved to GeneralConfig * Code was refactored * Fixed issues * Fixed issues * Code correction due to PR comments * Improved gcs client and integration tests * Integrated gcp config. Updated step metadata * Fixed issues. Added tests * Added cpe, vault, aliases resolving for reporting parameters * Added tests * Uncommented DeferExitHandler. Removed useless comments * fixed cloning of config * Added comments for exported functions. Removed unused mock * minor fix * Implemented setting of report name via paramRef * some refactoring. Writing tests * Update pkg/config/reporting.go * Update cmd/sonarExecuteScan_generated.go * Apply suggestions from code review * Update pkg/config/reporting.go * Update pkg/config/reporting.go * fixed removing valut secret files * Update pkg/config/reporting.go * restore order * restore order * Apply suggestions from code review * go generate * fixed tests * Update resources/metadata/sonarExecuteScan.yaml * Update resources.go * Fixed tests. Code was regenerated * changed somewhere gcp to gcs. Fixed one test * move gcsSubFolder to input parameters * fixed removing valut secret files * minor fix in integration tests * fix integration tests Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> Co-authored-by: Sven Merk <33895725+nevskrem@users.noreply.github.com> --- cmd/abapAddonAssemblyKitCheckCVs_generated.go | 2 +- cmd/abapAddonAssemblyKitCheckPV_generated.go | 2 +- ...AssemblyKitCreateTargetVector_generated.go | 2 +- ...onAssemblyKitRegisterPackages_generated.go | 2 +- ...donAssemblyKitReleasePackages_generated.go | 2 +- ...ssemblyKitReserveNextPackages_generated.go | 2 +- ...bapEnvironmentAssembleConfirm_generated.go | 2 +- ...apEnvironmentAssemblePackages_generated.go | 2 +- cmd/abapEnvironmentBuild_generated.go | 2 +- cmd/artifactPrepareVersion_generated.go | 2 +- cmd/batsExecuteTests_generated.go | 2 +- cmd/checkIfStepActive.go | 4 +- cmd/checkmarxExecuteScan_generated.go | 2 +- cmd/cloudFoundryDeploy_generated.go | 2 +- cmd/cnbBuild_generated.go | 2 +- cmd/detectExecuteScan_generated.go | 2 +- cmd/fortifyExecuteScan_generated.go | 2 +- cmd/gaugeExecuteTests_generated.go | 2 +- cmd/getConfig.go | 11 +- ...tegrationArtifactGetMplStatus_generated.go | 2 +- ...ionArtifactGetServiceEndpoint_generated.go | 2 +- cmd/isChangeInDevelopment_generated.go | 4 +- cmd/kanikoExecute_generated.go | 2 +- cmd/mavenBuild_generated.go | 2 +- cmd/mtaBuild_generated.go | 2 +- cmd/newmanExecute_generated.go | 2 +- cmd/npmExecuteScripts_generated.go | 2 +- cmd/piper.go | 37 ++- cmd/protecodeExecuteScan_generated.go | 2 +- cmd/sonarExecuteScan_generated.go | 47 +++- cmd/terraformExecute_generated.go | 4 +- cmd/transportRequestDocIDFromGit_generated.go | 2 +- cmd/transportRequestReqIDFromGit_generated.go | 2 +- cmd/transportRequestUploadCTS_generated.go | 2 +- cmd/transportRequestUploadRFC_generated.go | 2 +- cmd/transportRequestUploadSOLMAN_generated.go | 2 +- cmd/whitesourceExecuteScan_generated.go | 4 +- cmd/xsDeploy_generated.go | 2 +- go.mod | 2 + go.sum | 4 + integration/integration_gcs_test.go | 180 ++++++++++++++ .../sample-bucket/dir/test_file2.yaml | 3 + .../sample-bucket/test_file.txt | 1 + .../upload-bucket/placeholder | 0 pkg/config/config.go | 40 +++- pkg/config/config_test.go | 114 ++++++++- pkg/config/reporting.go | 81 +++++++ pkg/config/reporting_test.go | 203 ++++++++++++++++ pkg/config/run.go | 10 +- pkg/config/run_test.go | 2 +- pkg/documentation/generator/main.go | 4 +- pkg/gcs/gcsClient.go | 220 ++++++++++++++++++ pkg/gcs/gcsClient_test.go | 49 ++++ pkg/gcs/mocks/client.go | 75 ++++++ pkg/gcs/reporting.go | 56 +++++ pkg/gcs/reporting_test.go | 168 +++++++++++++ pkg/gcs/targetFolderHandler.go | 8 + pkg/gcs/targetFolderHandler_test.go | 28 +++ pkg/generator/helper/helper.go | 48 +++- pkg/generator/helper/helper_test.go | 9 +- pkg/generator/helper/resources.go | 73 ++++++ pkg/generator/helper/resources_test.go | 68 +++++- .../custom_step_code_generated.golden | 49 +++- .../step_code_generated.golden | 49 +++- resources/metadata/sonarExecuteScan.yaml | 7 + 65 files changed, 1645 insertions(+), 79 deletions(-) create mode 100644 integration/integration_gcs_test.go create mode 100644 integration/testdata/TestGCSIntegration/sample-bucket/dir/test_file2.yaml create mode 100644 integration/testdata/TestGCSIntegration/sample-bucket/test_file.txt create mode 100644 integration/testdata/TestGCSIntegration/upload-bucket/placeholder create mode 100644 pkg/config/reporting.go create mode 100644 pkg/config/reporting_test.go create mode 100644 pkg/gcs/gcsClient.go create mode 100644 pkg/gcs/gcsClient_test.go create mode 100644 pkg/gcs/mocks/client.go create mode 100644 pkg/gcs/reporting.go create mode 100644 pkg/gcs/reporting_test.go create mode 100644 pkg/gcs/targetFolderHandler.go create mode 100644 pkg/gcs/targetFolderHandler_test.go diff --git a/cmd/abapAddonAssemblyKitCheckCVs_generated.go b/cmd/abapAddonAssemblyKitCheckCVs_generated.go index b1c724db1..03becfe30 100644 --- a/cmd/abapAddonAssemblyKitCheckCVs_generated.go +++ b/cmd/abapAddonAssemblyKitCheckCVs_generated.go @@ -117,8 +117,8 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { - config.RemoveVaultSecretFiles() commonPipelineEnvironment.persist(GeneralConfig.EnvRootPath, "commonPipelineEnvironment") + config.RemoveVaultSecretFiles() stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds()) stepTelemetryData.ErrorCategory = log.GetErrorCategory().String() stepTelemetryData.PiperCommitHash = GitCommit diff --git a/cmd/abapAddonAssemblyKitCheckPV_generated.go b/cmd/abapAddonAssemblyKitCheckPV_generated.go index 26683fe44..e4b36ac28 100644 --- a/cmd/abapAddonAssemblyKitCheckPV_generated.go +++ b/cmd/abapAddonAssemblyKitCheckPV_generated.go @@ -117,8 +117,8 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { - config.RemoveVaultSecretFiles() commonPipelineEnvironment.persist(GeneralConfig.EnvRootPath, "commonPipelineEnvironment") + config.RemoveVaultSecretFiles() stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds()) stepTelemetryData.ErrorCategory = log.GetErrorCategory().String() stepTelemetryData.PiperCommitHash = GitCommit diff --git a/cmd/abapAddonAssemblyKitCreateTargetVector_generated.go b/cmd/abapAddonAssemblyKitCreateTargetVector_generated.go index e949c8306..3539582c2 100644 --- a/cmd/abapAddonAssemblyKitCreateTargetVector_generated.go +++ b/cmd/abapAddonAssemblyKitCreateTargetVector_generated.go @@ -117,8 +117,8 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { - config.RemoveVaultSecretFiles() commonPipelineEnvironment.persist(GeneralConfig.EnvRootPath, "commonPipelineEnvironment") + config.RemoveVaultSecretFiles() stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds()) stepTelemetryData.ErrorCategory = log.GetErrorCategory().String() stepTelemetryData.PiperCommitHash = GitCommit diff --git a/cmd/abapAddonAssemblyKitRegisterPackages_generated.go b/cmd/abapAddonAssemblyKitRegisterPackages_generated.go index d6c066c7b..133dccc32 100644 --- a/cmd/abapAddonAssemblyKitRegisterPackages_generated.go +++ b/cmd/abapAddonAssemblyKitRegisterPackages_generated.go @@ -118,8 +118,8 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { - config.RemoveVaultSecretFiles() commonPipelineEnvironment.persist(GeneralConfig.EnvRootPath, "commonPipelineEnvironment") + config.RemoveVaultSecretFiles() stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds()) stepTelemetryData.ErrorCategory = log.GetErrorCategory().String() stepTelemetryData.PiperCommitHash = GitCommit diff --git a/cmd/abapAddonAssemblyKitReleasePackages_generated.go b/cmd/abapAddonAssemblyKitReleasePackages_generated.go index e38339d68..d4a391ae5 100644 --- a/cmd/abapAddonAssemblyKitReleasePackages_generated.go +++ b/cmd/abapAddonAssemblyKitReleasePackages_generated.go @@ -116,8 +116,8 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { - config.RemoveVaultSecretFiles() commonPipelineEnvironment.persist(GeneralConfig.EnvRootPath, "commonPipelineEnvironment") + config.RemoveVaultSecretFiles() stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds()) stepTelemetryData.ErrorCategory = log.GetErrorCategory().String() stepTelemetryData.PiperCommitHash = GitCommit diff --git a/cmd/abapAddonAssemblyKitReserveNextPackages_generated.go b/cmd/abapAddonAssemblyKitReserveNextPackages_generated.go index f8383286d..87777928d 100644 --- a/cmd/abapAddonAssemblyKitReserveNextPackages_generated.go +++ b/cmd/abapAddonAssemblyKitReserveNextPackages_generated.go @@ -122,8 +122,8 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { - config.RemoveVaultSecretFiles() commonPipelineEnvironment.persist(GeneralConfig.EnvRootPath, "commonPipelineEnvironment") + config.RemoveVaultSecretFiles() stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds()) stepTelemetryData.ErrorCategory = log.GetErrorCategory().String() stepTelemetryData.PiperCommitHash = GitCommit diff --git a/cmd/abapEnvironmentAssembleConfirm_generated.go b/cmd/abapEnvironmentAssembleConfirm_generated.go index 67514f446..7c33609f9 100644 --- a/cmd/abapEnvironmentAssembleConfirm_generated.go +++ b/cmd/abapEnvironmentAssembleConfirm_generated.go @@ -119,8 +119,8 @@ func AbapEnvironmentAssembleConfirmCommand() *cobra.Command { stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { - config.RemoveVaultSecretFiles() commonPipelineEnvironment.persist(GeneralConfig.EnvRootPath, "commonPipelineEnvironment") + config.RemoveVaultSecretFiles() stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds()) stepTelemetryData.ErrorCategory = log.GetErrorCategory().String() stepTelemetryData.PiperCommitHash = GitCommit diff --git a/cmd/abapEnvironmentAssemblePackages_generated.go b/cmd/abapEnvironmentAssemblePackages_generated.go index b7da4c5c8..a5cce3e52 100644 --- a/cmd/abapEnvironmentAssemblePackages_generated.go +++ b/cmd/abapEnvironmentAssemblePackages_generated.go @@ -121,8 +121,8 @@ Platform ABAP Environment system and saves the corresponding [SAR archive](https stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { - config.RemoveVaultSecretFiles() commonPipelineEnvironment.persist(GeneralConfig.EnvRootPath, "commonPipelineEnvironment") + config.RemoveVaultSecretFiles() stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds()) stepTelemetryData.ErrorCategory = log.GetErrorCategory().String() stepTelemetryData.PiperCommitHash = GitCommit diff --git a/cmd/abapEnvironmentBuild_generated.go b/cmd/abapEnvironmentBuild_generated.go index a2bfe5148..0fb3d86ae 100644 --- a/cmd/abapEnvironmentBuild_generated.go +++ b/cmd/abapEnvironmentBuild_generated.go @@ -130,8 +130,8 @@ func AbapEnvironmentBuildCommand() *cobra.Command { stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { - config.RemoveVaultSecretFiles() commonPipelineEnvironment.persist(GeneralConfig.EnvRootPath, "commonPipelineEnvironment") + config.RemoveVaultSecretFiles() stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds()) stepTelemetryData.ErrorCategory = log.GetErrorCategory().String() stepTelemetryData.PiperCommitHash = GitCommit diff --git a/cmd/artifactPrepareVersion_generated.go b/cmd/artifactPrepareVersion_generated.go index 7f0d6e3fc..e6037d83a 100644 --- a/cmd/artifactPrepareVersion_generated.go +++ b/cmd/artifactPrepareVersion_generated.go @@ -204,8 +204,8 @@ Define ` + "`" + `buildTool: custom` + "`" + `, ` + "`" + `filePath: steps (-> general -> steps ...) for _, def := range c.defaults.Defaults { @@ -192,6 +197,13 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri stepConfig.mixIn(def.Steps[stepName], filters.Steps) stepConfig.mixIn(def.Stages[stageName], filters.Steps) stepConfig.mixinVaultConfig(parameters, def.General, def.Steps[stepName], def.Stages[stageName]) + reportingConfig, err := cloneConfig(&def) + if err != nil { + return StepConfig{}, err + } + reportingConfig.ApplyAliasConfig(ReportingParameters.Parameters, []StepSecrets{}, ReportingParameters.getStepFilters(), stageName, stepName, []Alias{}) + stepConfig.mixinReportingConfig(reportingConfig.General, reportingConfig.Steps[stepName], reportingConfig.Stages[stageName]) + stepConfig.mixInHookConfig(def.Hooks) } @@ -234,6 +246,14 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri } stepConfig.mixinVaultConfig(parameters, c.General, c.Steps[stepName], c.Stages[stageName]) + + reportingConfig, err := cloneConfig(c) + if err != nil { + return StepConfig{}, err + } + reportingConfig.ApplyAliasConfig(ReportingParameters.Parameters, []StepSecrets{}, ReportingParameters.getStepFilters(), stageName, stepName, []Alias{}) + stepConfig.mixinReportingConfig(reportingConfig.General, reportingConfig.Steps[stepName], reportingConfig.Stages[stageName]) + // check whether vault should be skipped if skip, ok := stepConfig.Config["skipVault"].(bool); !ok || !skip { // fetch secrets from vault @@ -243,7 +263,7 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri } if vaultClient != nil { defer vaultClient.MustRevokeToken() - resolveAllVaultReferences(&stepConfig, vaultClient, parameters) + resolveAllVaultReferences(&stepConfig, vaultClient, append(parameters, ReportingParameters.Parameters...)) resolveVaultTestCredentials(&stepConfig, vaultClient) } } @@ -312,7 +332,7 @@ func (c *Config) GetStageConfig(paramJSON string, configuration io.ReadCloser, d Parameters: acceptedParams, Env: []string{}, } - return c.GetStepConfig(map[string]interface{}{}, paramJSON, configuration, defaults, ignoreCustomDefaults, filters, []StepParameters{}, []StepSecrets{}, map[string]interface{}{}, stageName, "", []Alias{}) + return c.GetStepConfig(map[string]interface{}{}, paramJSON, configuration, defaults, ignoreCustomDefaults, filters, StepData{}, map[string]interface{}{}, stageName, "") } // GetJSON returns JSON representation of an object @@ -479,3 +499,17 @@ func sliceContains(slice []string, find string) bool { } return false } + +func cloneConfig(config *Config) (*Config, error) { + configJSON, err := json.Marshal(config) + if err != nil { + return nil, err + } + + clone := &Config{} + if err = json.Unmarshal(configJSON, &clone); err != nil { + return nil, err + } + + return clone, nil +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 573a7edab..a4c46fbe5 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -165,7 +165,19 @@ steps: }, } - stepMeta := StepData{Spec: StepSpec{Inputs: StepInputs{Parameters: parameterMetadata, Secrets: secretMetadata}}} + stepAliases := []Alias{{Name: "stepAlias"}} + + stepMeta := StepData{ + Spec: StepSpec{ + Inputs: StepInputs{ + Parameters: parameterMetadata, + Secrets: secretMetadata, + }, + }, + Metadata: StepMetadata{ + Aliases: stepAliases, + }, + } dir, err := ioutil.TempDir("", "") if err != nil { @@ -177,8 +189,7 @@ steps: piperenv.SetParameter(filepath.Join(dir, "commonPipelineEnvironment"), "test_pe1", "pe1_val") - stepAliases := []Alias{{Name: "stepAlias"}} - stepConfig, err := c.GetStepConfig(flags, paramJSON, myConfig, defaults, false, filters, parameterMetadata, secretMetadata, stepMeta.GetResourceParameters(dir, "commonPipelineEnvironment"), "stage1", "step1", stepAliases) + stepConfig, err := c.GetStepConfig(flags, paramJSON, myConfig, defaults, false, filters, stepMeta, stepMeta.GetResourceParameters(dir, "commonPipelineEnvironment"), "stage1", "step1") assert.Equal(t, nil, err, "error occurred but none expected") @@ -246,7 +257,7 @@ steps: // clean up tmp dir defer os.RemoveAll(dir) - stepConfig, err := c.GetStepConfig(map[string]interface{}{}, "", myConfig, defaults, false, filters, []StepParameters{}, []StepSecrets{}, stepMeta.GetResourceParameters(dir, "commonPipelineEnvironment"), "stage1", "step1", []Alias{}) + stepConfig, err := c.GetStepConfig(map[string]interface{}{}, "", myConfig, defaults, false, filters, stepMeta, stepMeta.GetResourceParameters(dir, "commonPipelineEnvironment"), "stage1", "step1") assert.Equal(t, nil, err, "error occurred but none expected") @@ -259,7 +270,7 @@ steps: c.openFile = customDefaultsOpenFileMock - stepConfig, err := c.GetStepConfig(nil, "", ioutil.NopCloser(strings.NewReader(testConfDefaults)), nil, false, StepFilters{General: []string{"p0"}}, []StepParameters{}, nil, nil, "stage1", "step1", []Alias{}) + stepConfig, err := c.GetStepConfig(nil, "", ioutil.NopCloser(strings.NewReader(testConfDefaults)), nil, false, StepFilters{General: []string{"p0"}}, StepData{}, nil, "stage1", "step1") assert.NoError(t, err, "Error occurred but no error expected") assert.Equal(t, "p0_custom_default", stepConfig.Config["p0"]) @@ -273,7 +284,7 @@ steps: c.openFile = customDefaultsOpenFileMock - stepConfig, err := c.GetStepConfig(nil, "", ioutil.NopCloser(strings.NewReader(testConfDefaults)), nil, true, StepFilters{General: []string{"p0"}}, []StepParameters{}, nil, nil, "stage1", "step1", []Alias{}) + stepConfig, err := c.GetStepConfig(nil, "", ioutil.NopCloser(strings.NewReader(testConfDefaults)), nil, true, StepFilters{General: []string{"p0"}}, StepData{}, nil, "stage1", "step1") assert.NoError(t, err, "Error occurred but no error expected") assert.Equal(t, nil, stepConfig.Config["p0"]) @@ -285,9 +296,16 @@ steps: var c Config stepParams := []StepParameters{{Name: "p0", Scope: []string{"GENERAL"}, Type: "string", Default: "p0_step_default", Aliases: []Alias{{Name: "p0_alias"}}}} + metadata := StepData{ + Spec: StepSpec{ + Inputs: StepInputs{ + Parameters: stepParams, + }, + }, + } testConf := "general:\n p1: p1_conf" - stepConfig, err := c.GetStepConfig(nil, "", ioutil.NopCloser(strings.NewReader(testConf)), nil, false, StepFilters{General: []string{"p0", "p1"}}, stepParams, nil, nil, "stage1", "step1", []Alias{}) + stepConfig, err := c.GetStepConfig(nil, "", ioutil.NopCloser(strings.NewReader(testConf)), nil, false, StepFilters{General: []string{"p0", "p1"}}, metadata, nil, "stage1", "step1") assert.NoError(t, err, "Error occurred but no error expected") assert.Equal(t, "p0_step_default", stepConfig.Config["p0"]) @@ -300,9 +318,16 @@ steps: stepParams := []StepParameters{ {Name: "p0", Scope: []string{"GENERAL"}, Type: "bool", Aliases: []Alias{}}, {Name: "p1", Scope: []string{"GENERAL"}, Type: "string", Aliases: []Alias{{Name: "p0/subParam"}}}} + metadata := StepData{ + Spec: StepSpec{ + Inputs: StepInputs{ + Parameters: stepParams, + }, + }, + } testConf := "general:\n p0: true" - stepConfig, err := c.GetStepConfig(nil, "", ioutil.NopCloser(strings.NewReader(testConf)), nil, false, StepFilters{General: []string{"p0", "p1"}}, stepParams, nil, nil, "stage1", "step1", []Alias{{}}) + stepConfig, err := c.GetStepConfig(nil, "", ioutil.NopCloser(strings.NewReader(testConf)), nil, false, StepFilters{General: []string{"p0", "p1"}}, metadata, nil, "stage1", "step1") assert.NoError(t, err, "Error occurred but no error expected") assert.Equal(t, true, stepConfig.Config["p0"]) @@ -314,10 +339,17 @@ steps: secrets := []StepSecrets{ {Name: "p0", Type: "string", Aliases: []Alias{{Name: "p1/subParam"}}}} + metadata := StepData{ + Spec: StepSpec{ + Inputs: StepInputs{ + Secrets: secrets, + }, + }, + } testConf := "" paramJSON := "{\"p1\":{\"subParam\":\"p1_value\"}}" - stepConfig, err := c.GetStepConfig(nil, paramJSON, ioutil.NopCloser(strings.NewReader(testConf)), nil, true, StepFilters{Parameters: []string{"p0"}}, nil, secrets, nil, "stage1", "step1", []Alias{{}}) + stepConfig, err := c.GetStepConfig(nil, paramJSON, ioutil.NopCloser(strings.NewReader(testConf)), nil, true, StepFilters{Parameters: []string{"p0"}}, metadata, nil, "stage1", "step1") assert.NoError(t, err, "Error occurred but no error expected") assert.Equal(t, "p1_value", stepConfig.Config["p0"]) @@ -326,7 +358,7 @@ steps: t.Run("Failure case config", func(t *testing.T) { var c Config myConfig := ioutil.NopCloser(strings.NewReader("invalid config")) - _, err := c.GetStepConfig(nil, "", myConfig, nil, false, StepFilters{}, []StepParameters{}, nil, nil, "stage1", "step1", []Alias{}) + _, err := c.GetStepConfig(nil, "", myConfig, nil, false, StepFilters{}, StepData{}, nil, "stage1", "step1") assert.EqualError(t, err, "failed to parse custom pipeline configuration: format of configuration is invalid \"invalid config\": error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type config.Config", "default error expected") }) @@ -334,10 +366,47 @@ steps: var c Config myConfig := ioutil.NopCloser(strings.NewReader("")) myDefaults := []io.ReadCloser{ioutil.NopCloser(strings.NewReader("invalid defaults"))} - _, err := c.GetStepConfig(nil, "", myConfig, myDefaults, false, StepFilters{}, []StepParameters{}, nil, nil, "stage1", "step1", []Alias{}) + _, err := c.GetStepConfig(nil, "", myConfig, myDefaults, false, StepFilters{}, StepData{}, nil, "stage1", "step1") assert.EqualError(t, err, "failed to read default configuration: error unmarshalling \"invalid defaults\": error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type config.Config", "default error expected") }) + t.Run("Test reporting parameters with aliases and cpe resources", func(t *testing.T) { + var c Config + testConfig := ioutil.NopCloser(strings.NewReader(`general: + gcpJsonKeyFilePath: gcpJsonKeyFilePath_value +steps: + step1: + jsonKeyFilePath: gcpJsonKeyFilePath_from_alias`)) + testDefaults := []io.ReadCloser{ioutil.NopCloser(strings.NewReader(`general: + pipelineId: gcsBucketId_from_alias +steps: + step1: + gcsBucketId: gcsBucketId_value`))} + dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal("Failed to create temporary directory") + } + + // clean up tmp dir + defer os.RemoveAll(dir) + cpeDir := filepath.Join(dir, "commonPipelineEnvironment/custom") + err = os.MkdirAll(cpeDir, 0700) + if err != nil { + t.Fatal("Failed to create sub directory") + } + + err = ioutil.WriteFile(filepath.Join(cpeDir, "gcsFolderPath.json"), []byte("\"value_from_cpe\""), 0700) + assert.NoError(t, err) + + stepMeta := StepData{Spec: StepSpec{Inputs: StepInputs{Parameters: []StepParameters{}}}} + stepConfig, err := c.GetStepConfig(nil, "", testConfig, testDefaults, false, StepFilters{General: []string{"p0", "p1"}}, stepMeta, ReportingParameters.GetResourceParameters(dir, "commonPipelineEnvironment"), "stage1", "step1") + + assert.NoError(t, err, "Error occurred but no error expected") + assert.Equal(t, "gcpJsonKeyFilePath_from_alias", stepConfig.Config["gcpJsonKeyFilePath"]) + assert.Equal(t, "gcsBucketId_value", stepConfig.Config["gcsBucketId"]) + assert.Equal(t, "value_from_cpe", stepConfig.Config["gcsFolderPath"]) + }) + //ToDo: test merging of env and parameters/flags } @@ -910,3 +979,26 @@ func TestMixInStepDefaults(t *testing.T) { assert.Equal(t, test.expected, test.stepConfig.Config, test.name) } } + +func TestCloneConfig(t *testing.T) { + testConfig := &Config{ + General: map[string]interface{}{ + "p0": "p0_general", + }, + Stages: map[string]map[string]interface{}{ + "stage1": { + "p1": "p1_stage", + }, + }, + Steps: map[string]map[string]interface{}{ + "step1": { + "p2": "p2_step", + }, + }, + } + clone, err := cloneConfig(testConfig) + assert.NoError(t, err) + assert.Equal(t, testConfig, clone) + testConfig.General["p0"] = "new_value" + assert.NotEqual(t, testConfig.General, clone.General) +} diff --git a/pkg/config/reporting.go b/pkg/config/reporting.go new file mode 100644 index 000000000..de3255cb7 --- /dev/null +++ b/pkg/config/reporting.go @@ -0,0 +1,81 @@ +package config + +// ReportingParams holds reporting parameters +type ReportingParams struct { + Parameters []StepParameters +} + +// ReportingParameters is a global variable with values of reporting parameters +var ReportingParameters = ReportingParams{ + Parameters: []StepParameters{ + { + Name: "gcpJsonKeyFilePath", + Aliases: []Alias{{Name: "jsonKeyFilePath"}}, + ResourceRef: []ResourceReference{ + { + Name: "gcpFileVaultSecretName", + Type: "vaultSecretFile", + Default: "gcp", + }, + }, + }, + { + Name: "gcsFolderPath", + ResourceRef: []ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "custom/gcsFolderPath", + }, + }, + }, + { + Name: "gcsBucketId", + Aliases: []Alias{{Name: "pipelineId"}}, + }, + { + Name: "gcsSubFolder", + }, + }, +} + +// GetResourceParameters retrieves reporting parameters from a named pipeline resource with a defined path +func (r ReportingParams) GetResourceParameters(path, name string) map[string]interface{} { + resourceParams := map[string]interface{}{} + + for _, param := range r.Parameters { + for _, res := range param.ResourceRef { + if res.Name == name { + if val := getParameterValue(path, res, param); val != nil { + resourceParams[param.Name] = val + } + } + } + } + return resourceParams +} + +func (r ReportingParams) getStepFilters() StepFilters { + var filters StepFilters + reportingFilter := r.getReportingFilter() + filters.All = append(filters.All, reportingFilter...) + filters.General = append(filters.General, reportingFilter...) + filters.Steps = append(filters.Steps, reportingFilter...) + filters.Stages = append(filters.Stages, reportingFilter...) + return filters +} + +func (r ReportingParams) getReportingFilter() []string { + var reportingFilter []string + for _, param := range r.Parameters { + reportingFilter = append(reportingFilter, param.Name) + } + reportingFilter = append(reportingFilter, getFilterForResourceReferences(r.Parameters)...) + return reportingFilter +} + +func (s *StepConfig) mixinReportingConfig(configs ...map[string]interface{}) { + reportingFilter := ReportingParameters.getReportingFilter() + for _, config := range configs { + s.mixIn(config, reportingFilter) + } +} diff --git a/pkg/config/reporting_test.go b/pkg/config/reporting_test.go new file mode 100644 index 000000000..59fc052e8 --- /dev/null +++ b/pkg/config/reporting_test.go @@ -0,0 +1,203 @@ +package config + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMixinReportingConfig(t *testing.T) { + gcpJsonKeyFilePath := "path/key.json" + gcsFolderPath := "test/folder/path" + gcsBucketID := "testBucketId" + config := StepConfig{ + Config: map[string]interface{}{}, + HookConfig: nil, + } + general := map[string]interface{}{ + "gcpJsonKeyFilePath": gcpJsonKeyFilePath, + "gcsFolderPath": gcsFolderPath, + "gcsBucketId": "generalBucketId", + } + steps := map[string]interface{}{ + "gcsBucketId": gcsBucketID, + "unknownConfig": "test", + } + + config.mixinReportingConfig(nil, general, steps) + + assert.Contains(t, config.Config, "gcpJsonKeyFilePath") + assert.Equal(t, gcpJsonKeyFilePath, config.Config["gcpJsonKeyFilePath"]) + assert.Contains(t, config.Config, "gcsFolderPath") + assert.Equal(t, gcsFolderPath, config.Config["gcsFolderPath"]) + assert.Contains(t, config.Config, "gcsBucketId") + assert.Equal(t, gcsBucketID, config.Config["gcsBucketId"]) + assert.NotContains(t, config.Config, "unknownConfig") +} + +func TestReportingParams_GetResourceParameters(t *testing.T) { + tt := []struct { + in ReportingParams + expected map[string]interface{} + }{ + { + in: ReportingParams{Parameters: []StepParameters{}}, + expected: map[string]interface{}{}, + }, + { + in: ReportingParams{Parameters: []StepParameters{ + {Name: "param1"}, + {Name: "param2"}, + }}, + expected: map[string]interface{}{}, + }, + { + in: ReportingParams{Parameters: []StepParameters{ + {Name: "param1", ResourceRef: []ResourceReference{}}, + {Name: "param2", ResourceRef: []ResourceReference{}}, + }}, + expected: map[string]interface{}{}, + }, + { + in: ReportingParams{Parameters: []StepParameters{ + {Name: "param1", ResourceRef: []ResourceReference{{Name: "notAvailable", Param: "envparam1"}}}, + {Name: "param2", ResourceRef: []ResourceReference{{Name: "commonPipelineEnvironment", Param: "envparam2"}}, Type: "string"}, + }}, + expected: map[string]interface{}{"param2": "val2"}, + }, + { + in: ReportingParams{Parameters: []StepParameters{ + {Name: "param2", ResourceRef: []ResourceReference{{Name: "commonPipelineEnvironment", Param: "envparam2"}}, Type: "string"}, + {Name: "param3", ResourceRef: []ResourceReference{{Name: "commonPipelineEnvironment", Param: "jsonList"}}, Type: "[]string"}, + }}, + expected: map[string]interface{}{"param2": "val2", "param3": []interface{}{"value1", "value2"}}, + }, + { + in: ReportingParams{Parameters: []StepParameters{ + {Name: "param4", ResourceRef: []ResourceReference{{Name: "commonPipelineEnvironment", Param: "jsonKeyValue"}}, Type: "map[string]interface{}"}, + }}, + expected: map[string]interface{}{"param4": map[string]interface{}{"key": "value"}}, + }, + { + in: ReportingParams{Parameters: []StepParameters{ + {Name: "param1", ResourceRef: []ResourceReference{{Name: "commonPipelineEnvironment", Param: "envparam1"}}, Type: "noString"}, + {Name: "param4", ResourceRef: []ResourceReference{{Name: "commonPipelineEnvironment", Param: "jsonKeyValueString"}}, Type: "string"}, + }}, + expected: map[string]interface{}{"param4": "{\"key\":\"valueString\"}"}, + }, + } + + dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal("Failed to create temporary directory") + } + // clean up tmp dir + defer os.RemoveAll(dir) + + cpeDir := filepath.Join(dir, "commonPipelineEnvironment") + err = os.MkdirAll(cpeDir, 0700) + if err != nil { + t.Fatal("Failed to create sub directory") + } + + ioutil.WriteFile(filepath.Join(cpeDir, "envparam1"), []byte("val1"), 0700) + ioutil.WriteFile(filepath.Join(cpeDir, "envparam2"), []byte("val2"), 0700) + ioutil.WriteFile(filepath.Join(cpeDir, "jsonList.json"), []byte("[\"value1\",\"value2\"]"), 0700) + ioutil.WriteFile(filepath.Join(cpeDir, "jsonKeyValue.json"), []byte("{\"key\":\"value\"}"), 0700) + ioutil.WriteFile(filepath.Join(cpeDir, "jsonKeyValueString"), []byte("{\"key\":\"valueString\"}"), 0700) + + for run, test := range tt { + t.Run(fmt.Sprintf("Run %v", run), func(t *testing.T) { + actual := test.in.GetResourceParameters(dir, "commonPipelineEnvironment") + assert.Equal(t, test.expected, actual) + }) + } +} + +func TestReportingParams_GetGetStepFilters(t *testing.T) { + tt := []struct { + in ReportingParams + expected StepFilters + }{ + { + in: ReportingParams{Parameters: []StepParameters{}}, + expected: StepFilters{}, + }, + { + in: ReportingParams{Parameters: []StepParameters{ + {Name: "param1"}, + {Name: "param2"}, + }}, + expected: StepFilters{ + All: []string{"param1", "param2"}, + General: []string{"param1", "param2"}, + Steps: []string{"param1", "param2"}, + Stages: []string{"param1", "param2"}, + }, + }, + { + in: ReportingParams{Parameters: []StepParameters{ + {Name: "param1"}, + {Name: "param2"}, + {Name: "param3"}, + {Name: "param4"}, + {Name: "param5"}, + {Name: "param6"}, + }}, + expected: StepFilters{ + All: []string{"param1", "param2", "param3", "param4", "param5", "param6"}, + General: []string{"param1", "param2", "param3", "param4", "param5", "param6"}, + Steps: []string{"param1", "param2", "param3", "param4", "param5", "param6"}, + Stages: []string{"param1", "param2", "param3", "param4", "param5", "param6"}, + }, + }, + } + + for run, test := range tt { + t.Run(fmt.Sprintf("Run %v", run), func(t *testing.T) { + actual := test.in.getStepFilters() + assert.Equal(t, test.expected, actual) + }) + } +} + +func TestReportingParams_GetReportingFilter(t *testing.T) { + tt := []struct { + in ReportingParams + expected []string + }{ + { + in: ReportingParams{Parameters: []StepParameters{}}, + expected: nil, + }, + { + in: ReportingParams{Parameters: []StepParameters{ + {Name: "param1"}, + {Name: "param2"}, + }}, + expected: []string{"param1", "param2"}, + }, + { + in: ReportingParams{Parameters: []StepParameters{ + {Name: "param1"}, + {Name: "param2"}, + {Name: "param3"}, + {Name: "param4"}, + {Name: "param5"}, + {Name: "param6"}, + }}, + expected: []string{"param1", "param2", "param3", "param4", "param5", "param6"}, + }, + } + + for run, test := range tt { + t.Run(fmt.Sprintf("Run %v", run), func(t *testing.T) { + actual := test.in.getReportingFilter() + assert.Equal(t, test.expected, actual) + }) + } +} diff --git a/pkg/config/run.go b/pkg/config/run.go index 2bb3c8cc7..68c9149f1 100644 --- a/pkg/config/run.go +++ b/pkg/config/run.go @@ -127,8 +127,14 @@ func (r *RunConfig) getStepConfig(config *Config, stageName, stepName string, fi // not considered releavant for pipeline yaml syntax resolution paramJSON := "" - return config.GetStepConfig(flagValues, paramJSON, nil, nil, false, filters[stepName], parameters[stepName], secrets[stepName], - envParameters, stageName, stepName, stepAliases[stepName]) + stepMeta := StepData{ + Spec: StepSpec{ + Inputs: StepInputs{Parameters: parameters[stepName], Secrets: secrets[stepName]}, + }, + Metadata: StepMetadata{Aliases: stepAliases[stepName]}, + } + + return config.GetStepConfig(flagValues, paramJSON, nil, nil, false, filters[stepName], stepMeta, envParameters, stageName, stepName) } func (r *RunConfig) loadConditions() error { diff --git a/pkg/config/run_test.go b/pkg/config/run_test.go index be3fe1c90..9f22ae60f 100644 --- a/pkg/config/run_test.go +++ b/pkg/config/run_test.go @@ -188,7 +188,7 @@ stages: filter := StepFilters{All: []string{}, General: []string{}, Stages: []string{}, Steps: []string{}, Env: []string{}} projectConfig := Config{} _, err := projectConfig.GetStepConfig(map[string]interface{}{}, "", tt.customConfig, - []io.ReadCloser{}, false, filter, nil, nil, nil, "", "", []Alias{}) + []io.ReadCloser{}, false, filter, StepData{}, nil, "", "") assert.NoError(t, err) err = runConfig.InitRunConfig(&projectConfig, nil, nil, nil, nil, initRunConfigGlobMock, nil) if tt.wantErr { diff --git a/pkg/documentation/generator/main.go b/pkg/documentation/generator/main.go index 09ac9a924..5e0cb19a1 100644 --- a/pkg/documentation/generator/main.go +++ b/pkg/documentation/generator/main.go @@ -48,12 +48,10 @@ func readStepConfiguration(stepMetadata config.StepData, customDefaultFiles []st defaultFiles, false, filters, - stepMetadata.Spec.Inputs.Parameters, - stepMetadata.Spec.Inputs.Secrets, + stepMetadata, map[string]interface{}{}, "", stepMetadata.Metadata.Name, - stepMetadata.Metadata.Aliases, ) checkError(err) return stepConfiguration diff --git a/pkg/gcs/gcsClient.go b/pkg/gcs/gcsClient.go new file mode 100644 index 000000000..dbf8f63fd --- /dev/null +++ b/pkg/gcs/gcsClient.go @@ -0,0 +1,220 @@ +package gcs + +import ( + "context" + "io" + "os" + "path/filepath" + + "cloud.google.com/go/storage" + "github.com/SAP/jenkins-library/pkg/log" + "github.com/pkg/errors" + "google.golang.org/api/iterator" + "google.golang.org/api/option" +) + +// Client is an interface to mock gcsClient +type Client interface { + UploadFile(bucketID string, sourcePath string, targetPath string) error + DownloadFile(bucketID string, sourcePath string, targetPath string) error + ListFiles(bucketID string) ([]string, error) + Close() error +} + +// gcsClient provides functions to interact with google cloud storage API +type gcsClient struct { + context context.Context + envVars []EnvVar + client storage.Client + clientOptions []option.ClientOption + openFile func(name string) (io.ReadCloser, error) + createFile func(name string) (io.WriteCloser, error) +} + +// EnvVar defines an environment variable incl. information about a potential modification to the variable +type EnvVar struct { + Name string + Value string + Modified bool +} + +type gcsOption func(*gcsClient) + +// WithEnvVars initializes env variables in gcsClient +func WithEnvVars(envVars []EnvVar) gcsOption { + return func(g *gcsClient) { + g.envVars = envVars + } +} + +// WithEnvVars initializes the openFile function in gcsClient +func WithOpenFileFunction(openFile func(name string) (io.ReadCloser, error)) gcsOption { + return func(g *gcsClient) { + g.openFile = openFile + } +} + +// WithEnvVars initializes the createFile function in gcsClient +func WithCreateFileFunction(createFile func(name string) (io.WriteCloser, error)) gcsOption { + return func(g *gcsClient) { + g.createFile = createFile + } +} + +// WithEnvVars initializes the Google Cloud Storage client options +func WithClientOptions(opts ...option.ClientOption) gcsOption { + return func(g *gcsClient) { + g.clientOptions = append(g.clientOptions, opts...) + } +} + +// Init intitializes the google cloud storage client +func NewClient(opts ...gcsOption) (*gcsClient, error) { + var ( + defaultOpenFile = openFileFromFS + defaultCreateFile = createFileOnFS + ) + + ctx := context.Background() + gcsClient := &gcsClient{ + context: ctx, + openFile: defaultOpenFile, + createFile: defaultCreateFile, + } + + // options handling + for _, opt := range opts { + opt(gcsClient) + } + + gcsClient.prepareEnv() + client, err := storage.NewClient(ctx, gcsClient.clientOptions...) + if err != nil { + return nil, errors.Wrapf(err, "bucket connection failed: %v", err) + } + gcsClient.client = *client + return gcsClient, nil +} + +// UploadFile uploads a file into a google cloud storage bucket +func (g *gcsClient) UploadFile(bucketID string, sourcePath string, targetPath string) error { + target := g.client.Bucket(bucketID).Object(targetPath).NewWriter(g.context) + log.Entry().Debugf("uploading %v to %v\n", sourcePath, targetPath) + sourceFile, err := g.openFile(sourcePath) + if err != nil { + return errors.Wrapf(err, "could not open source file: %v", err) + } + defer sourceFile.Close() + + if err := g.copy(sourceFile, target); err != nil { + return errors.Wrapf(err, "upload failed: %v", err) + } + + if err := target.Close(); err != nil { + return errors.Wrapf(err, "closing bucket failed: %v", err) + } + return nil +} + +// DownloadFile downloads a file from a google cloud storage bucket +func (g *gcsClient) DownloadFile(bucketID string, sourcePath string, targetPath string) error { + log.Entry().Debugf("downloading %v to %v\n", sourcePath, targetPath) + gcsReader, err := g.client.Bucket(bucketID).Object(sourcePath).NewReader(g.context) + if err != nil { + return errors.Wrapf(err, "could not open source file from a google cloud storage bucket: %v", err) + } + + targetWriter, err := g.createFile(targetPath) + if err != nil { + return errors.Wrapf(err, "could not create target file: %v", err) + } + defer targetWriter.Close() + + if err := g.copy(gcsReader, targetWriter); err != nil { + return errors.Wrapf(err, "download failed: %v", err) + } + if err := gcsReader.Close(); err != nil { + return errors.Wrapf(err, "closing bucket failed: %v", err) + } + return nil +} + +// ListFiles lists all files in certain GCS bucket +func (g *gcsClient) ListFiles(bucketID string) ([]string, error) { + fileNames := []string{} + it := g.client.Bucket(bucketID).Objects(g.context, nil) + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, err + } + fileNames = append(fileNames, attrs.Name) + } + return fileNames, nil +} + +// Close closes the client and removes previously set environment variables +func (g *gcsClient) Close() error { + if err := g.client.Close(); err != nil { + return err + } + if err := g.cleanupEnv(); err != nil { + return err + } + return nil +} + +func (g *gcsClient) copy(source io.Reader, target io.Writer) error { + if _, err := io.Copy(target, source); err != nil { + return err + } + return nil +} + +// prepareEnv sets required environment variables in case they are not set yet +func (g *gcsClient) prepareEnv() { + for key, env := range g.envVars { + g.envVars[key].Modified = setenvIfEmpty(env.Name, env.Value) + } +} + +// cleanupEnv removes environment variables set by prepareEnv +func (g *gcsClient) cleanupEnv() error { + for _, env := range g.envVars { + if err := removeEnvIfPreviouslySet(env.Name, env.Modified); err != nil { + return err + } + } + return nil +} + +func setenvIfEmpty(env, val string) bool { + if len(os.Getenv(env)) == 0 { + os.Setenv(env, val) + return true + } + return false +} + +func removeEnvIfPreviouslySet(env string, previouslySet bool) error { + if previouslySet { + if err := os.Setenv(env, ""); err != nil { + return err + } + } + return nil +} + +func openFileFromFS(name string) (io.ReadCloser, error) { + return os.Open(name) +} + +func createFileOnFS(name string) (io.WriteCloser, error) { + if err := os.MkdirAll(filepath.Dir(name), os.ModePerm); err != nil { + return nil, err + } + return os.Create(name) +} diff --git a/pkg/gcs/gcsClient_test.go b/pkg/gcs/gcsClient_test.go new file mode 100644 index 000000000..0ad5e6d0c --- /dev/null +++ b/pkg/gcs/gcsClient_test.go @@ -0,0 +1,49 @@ +package gcs + +import ( + "os" + "testing" +) + +func Test_gcsClient_prepareEnv(t *testing.T) { + os.Setenv("TESTVAR1", "test1") + + gcsClient := gcsClient{envVars: []EnvVar{ + {Name: "TESTVAR1", Value: "test1_new"}, + {Name: "TESTVAR2", Value: "test2_new"}, + }} + + gcsClient.prepareEnv() + + if gcsClient.envVars[0].Modified { + t.Errorf("%v - expected '%v' was '%v'", gcsClient.envVars[0].Name, false, gcsClient.envVars[0].Modified) + } + if !gcsClient.envVars[1].Modified { + t.Errorf("%v - expected '%v' was '%v'", gcsClient.envVars[1].Name, true, gcsClient.envVars[1].Modified) + } + + os.Setenv("TESTVAR1", "") + os.Setenv("TESTVAR2", "") +} + +func TestCleanupEnv(t *testing.T) { + os.Setenv("TESTVAR1", "test1") + os.Setenv("TESTVAR2", "test2") + + gcsClient := gcsClient{envVars: []EnvVar{ + {Name: "TESTVAR1", Modified: false}, + {Name: "TESTVAR2", Modified: true}, + }} + + gcsClient.cleanupEnv() + + if os.Getenv("TESTVAR1") != "test1" { + t.Errorf("%v - expected '%v' was '%v'", gcsClient.envVars[0].Name, "test1", os.Getenv("TESTVAR1")) + } + if len(os.Getenv("TESTVAR2")) > 0 { + t.Errorf("%v - expected '%v' was '%v'", gcsClient.envVars[1].Name, "", os.Getenv("TESTVAR2")) + } + + os.Setenv("TESTVAR1", "") + os.Setenv("TESTVAR2", "") +} diff --git a/pkg/gcs/mocks/client.go b/pkg/gcs/mocks/client.go new file mode 100644 index 000000000..4b18cfd2f --- /dev/null +++ b/pkg/gcs/mocks/client.go @@ -0,0 +1,75 @@ +// Code generated by mockery v2.7.4. DO NOT EDIT. + +package mocks + +import mock "github.com/stretchr/testify/mock" + +// Client is an autogenerated mock type for the Client type +type Client struct { + mock.Mock +} + +// Close provides a mock function with given fields: +func (_m *Client) Close() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DownloadFile provides a mock function with given fields: bucketID, sourcePath, targetPath +func (_m *Client) DownloadFile(bucketID string, sourcePath string, targetPath string) error { + ret := _m.Called(bucketID, sourcePath, targetPath) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, string) error); ok { + r0 = rf(bucketID, sourcePath, targetPath) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ListFiles provides a mock function with given fields: bucketID +func (_m *Client) ListFiles(bucketID string) ([]string, error) { + ret := _m.Called(bucketID) + + var r0 []string + if rf, ok := ret.Get(0).(func(string) []string); ok { + r0 = rf(bucketID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(bucketID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// UploadFile provides a mock function with given fields: bucketID, sourcePath, targetPath +func (_m *Client) UploadFile(bucketID string, sourcePath string, targetPath string) error { + ret := _m.Called(bucketID, sourcePath, targetPath) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, string) error); ok { + r0 = rf(bucketID, sourcePath, targetPath) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/pkg/gcs/reporting.go b/pkg/gcs/reporting.go new file mode 100644 index 000000000..84c9a2304 --- /dev/null +++ b/pkg/gcs/reporting.go @@ -0,0 +1,56 @@ +package gcs + +import ( + "fmt" + "os" + "path/filepath" +) + +type ReportOutputParam struct { + FilePattern string + ParamRef string + StepResultType string +} + +type Task struct { + SourcePath string + TargetPath string +} + +func PersistReportsToGCS(gcsClient Client, outputParams []ReportOutputParam, inputParams map[string]string, gcsFolderPath string, gcsBucketID string, gcsSubFolder string, searchFilesFunc func(string) ([]string, error), fileInfo func(string) (os.FileInfo, error)) error { + tasks := []Task{} + for _, param := range outputParams { + targetFolder := GetTargetFolder(gcsFolderPath, param.StepResultType, gcsSubFolder) + if param.ParamRef != "" { + paramValue, ok := inputParams[param.ParamRef] + if !ok { + return fmt.Errorf("there is no such input parameter as %s", param.ParamRef) + } + if paramValue == "" { + return fmt.Errorf("the value of the parameter %s must not be empty", param.ParamRef) + } + tasks = append(tasks, Task{SourcePath: paramValue, TargetPath: filepath.Join(targetFolder, paramValue)}) + } else { + foundFiles, err := searchFilesFunc(param.FilePattern) + if err != nil { + return fmt.Errorf("failed to persist reports: %v", err) + } + for _, sourcePath := range foundFiles { + fileInfo, err := fileInfo(sourcePath) + if err != nil { + return fmt.Errorf("failed to persist reports: %v", err) + } + if fileInfo.IsDir() { + continue + } + tasks = append(tasks, Task{SourcePath: sourcePath, TargetPath: filepath.Join(targetFolder, sourcePath)}) + } + } + } + for _, task := range tasks { + if err := gcsClient.UploadFile(gcsBucketID, task.SourcePath, task.TargetPath); err != nil { + return fmt.Errorf("failed to persist reports: %v", err) + } + } + return nil +} diff --git a/pkg/gcs/reporting_test.go b/pkg/gcs/reporting_test.go new file mode 100644 index 000000000..0941c8493 --- /dev/null +++ b/pkg/gcs/reporting_test.go @@ -0,0 +1,168 @@ +package gcs + +import ( + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/SAP/jenkins-library/pkg/gcs/mocks" + "github.com/bmatcuk/doublestar" + "github.com/stretchr/testify/assert" +) + +type testFileInfo struct { + path string +} + +func (t testFileInfo) Name() string { + return "" +} + +func (t testFileInfo) Size() int64 { + return 0 +} + +func (t testFileInfo) Mode() os.FileMode { + return os.FileMode(0) +} + +func (t testFileInfo) ModTime() time.Time { + return time.Time{} +} + +func (t testFileInfo) IsDir() bool { + if strings.HasSuffix(t.path, "test2") { + return true + } + return false +} + +func (t testFileInfo) Sys() interface{} { + return nil +} + +type testStepConfig struct { + FirstParameter string + SecondParameter int + ThirdParameter string + FourthParameter bool +} + +func TestPersistReportsToGCS(t *testing.T) { + var testCases = []struct { + testName string + gcsFolderPath string + gcsSubFolder string + outputParams []ReportOutputParam + expected []Task + detectedFiles []string + uploadFileErr error + expectedError error + }{ + { + testName: "success case", + gcsFolderPath: "test/folder/path", + gcsSubFolder: "sub/folder", + outputParams: []ReportOutputParam{ + {FilePattern: "*.json", ParamRef: "", StepResultType: "general"}, + {FilePattern: "*/test*", ParamRef: "", StepResultType: ""}, + {FilePattern: "*.json", ParamRef: "firstParameter", StepResultType: "general"}, + {FilePattern: "", ParamRef: "secondParameter", StepResultType: "general"}, + {FilePattern: "", ParamRef: "thirdParameter", StepResultType: ""}, + }, + expected: []Task{ + {SourcePath: "asdf.json", TargetPath: "test/folder/path/general/sub/folder/asdf.json"}, + {SourcePath: "folder/test1", TargetPath: "test/folder/path/sub/folder/folder/test1"}, + {SourcePath: "testFolder/test3", TargetPath: "test/folder/path/sub/folder/testFolder/test3"}, + {SourcePath: "report1.json", TargetPath: "test/folder/path/general/sub/folder/report1.json"}, + {SourcePath: "test-report.json", TargetPath: "test/folder/path/general/sub/folder/test-report.json"}, + {SourcePath: "test-report2.json", TargetPath: "test/folder/path/sub/folder/test-report2.json"}, + }, + detectedFiles: []string{"asdf.json", "someFolder/someFile", "folder/test1", "folder1/test2", "testFolder/test3"}, + uploadFileErr: nil, + expectedError: nil, + }, + { + testName: "failed upload to GCS", + gcsFolderPath: "test/folder/path", + gcsSubFolder: "", + outputParams: []ReportOutputParam{ + {FilePattern: "*.json", ParamRef: "", StepResultType: "general"}, + }, + expected: []Task{ + {SourcePath: "asdf.json", TargetPath: "test/folder/path/general/asdf.json"}, + }, + detectedFiles: []string{"asdf.json", "someFolder/someFile", "folder/test1", "folder1/test2", "testFolder/test3"}, + uploadFileErr: errors.New("upload failed"), + expectedError: errors.New("failed to persist reports: upload failed"), + }, + { + testName: "failed - input parameter does not exist", + gcsFolderPath: "test/folder/path", + gcsSubFolder: "", + outputParams: []ReportOutputParam{ + {FilePattern: "", ParamRef: "missingParameter", StepResultType: "general"}, + }, + expected: []Task{}, + detectedFiles: []string{"asdf.json", "someFolder/someFile", "folder/test1", "folder1/test2", "testFolder/test3"}, + uploadFileErr: nil, + expectedError: errors.New("there is no such input parameter as missingParameter"), + }, + { + testName: "failed - input parameter is empty", + gcsFolderPath: "test/folder/path", + outputParams: []ReportOutputParam{ + {FilePattern: "", ParamRef: "emptyParameter", StepResultType: "general"}, + }, + expected: []Task{}, + detectedFiles: []string{"asdf.json", "someFolder/someFile", "folder/test1", "folder1/test2", "testFolder/test3"}, + uploadFileErr: nil, + expectedError: errors.New("the value of the parameter emptyParameter must not be empty"), + }, + } + for _, tt := range testCases { + t.Run(tt.testName, func(t *testing.T) { + inputParameters := map[string]string{ + "firstParameter": "report1.json", + "secondParameter": "test-report.json", + "thirdParameter": "test-report2.json", + "emptyParameter": "", + } + gcsBucketID := "testBucketID" + mockedClient := &mocks.Client{} + + for _, expectation := range tt.expected { + mockedClient.Mock.On("UploadFile", gcsBucketID, expectation.SourcePath, expectation.TargetPath).Return( + func(pipelineId string, sourcePath string, targetPath string) error { return tt.uploadFileErr }, + ).Once() + } + + searchFn := func(path string) ([]string, error) { + matchedFiles := []string{} + for _, value := range tt.detectedFiles { + match, _ := doublestar.Match(path, value) + if match { + matchedFiles = append(matchedFiles, value) + } + } + return matchedFiles, nil + } + + fileInfoFn := func(name string) (os.FileInfo, error) { + return testFileInfo{name}, nil + } + + err := PersistReportsToGCS(mockedClient, tt.outputParams, inputParameters, tt.gcsFolderPath, gcsBucketID, tt.gcsSubFolder, searchFn, fileInfoFn) + if tt.expectedError == nil { + assert.NoError(t, err) + } else { + assert.Equal(t, tt.expectedError.Error(), err.Error()) + } + + mockedClient.Mock.AssertNumberOfCalls(t, "UploadFile", len(tt.expected)) + mockedClient.Mock.AssertExpectations(t) + }) + } +} diff --git a/pkg/gcs/targetFolderHandler.go b/pkg/gcs/targetFolderHandler.go new file mode 100644 index 000000000..5377554ab --- /dev/null +++ b/pkg/gcs/targetFolderHandler.go @@ -0,0 +1,8 @@ +package gcs + +import "path" + +// GetTargetFolder calculates the target folder in GCS bucket +func GetTargetFolder(folderPath string, stepResultType string, subFolder string) string { + return path.Join(folderPath, stepResultType, subFolder) +} diff --git a/pkg/gcs/targetFolderHandler_test.go b/pkg/gcs/targetFolderHandler_test.go new file mode 100644 index 000000000..d386d4232 --- /dev/null +++ b/pkg/gcs/targetFolderHandler_test.go @@ -0,0 +1,28 @@ +package gcs + +import ( + "fmt" + "testing" +) + +func TestGetTargetFolder(t *testing.T) { + tests := []struct { + folderPath string + stepResultType string + subFolder string + expected string + }{ + {folderPath: "folder/path/", stepResultType: "general", subFolder: "sub/folder", expected: "folder/path/general/sub/folder"}, + {folderPath: "folder/path/", subFolder: "sub/folder", expected: "folder/path/sub/folder"}, + {folderPath: "folder/path/", stepResultType: "general", expected: "folder/path/general"}, + {folderPath: "folder1", stepResultType: "general", subFolder: "folder2/", expected: "folder1/general/folder2"}} + + for key, tt := range tests { + t.Run(fmt.Sprintf("Row %v", key+1), func(t *testing.T) { + actualTargetFolder := GetTargetFolder(tt.folderPath, tt.stepResultType, tt.subFolder) + if actualTargetFolder != tt.expected { + t.Errorf("Expected '%v' was '%v'", tt.expected, actualTargetFolder) + } + }) + } +} diff --git a/pkg/generator/helper/helper.go b/pkg/generator/helper/helper.go index ce9e17f23..658e41781 100644 --- a/pkg/generator/helper/helper.go +++ b/pkg/generator/helper/helper.go @@ -56,6 +56,16 @@ import ( "github.com/SAP/jenkins-library/pkg/config" "github.com/SAP/jenkins-library/pkg/log" {{ if .OutputResources -}} + {{ $reportsExist := false -}} + {{ range $notused, $oRes := .OutputResources -}} + {{ if eq (index $oRes "type") "reports" -}}{{ $reportsExist = true -}}{{ end -}} + {{ end -}} + {{ if $reportsExist -}} + "github.com/bmatcuk/doublestar" + "github.com/SAP/jenkins-library/pkg/gcs" + "reflect" + "strings" + {{ end -}} "github.com/SAP/jenkins-library/pkg/piperenv" {{ end -}} "github.com/SAP/jenkins-library/pkg/telemetry" @@ -146,9 +156,11 @@ func {{.CobraCmdFuncName}}() *cobra.Command { stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { - config.RemoveVaultSecretFiles() {{- range $notused, $oRes := .OutputResources }} - {{ index $oRes "name" }}.persist({{if $.ExportPrefix}}{{ $.ExportPrefix }}.{{end}}GeneralConfig.EnvRootPath, {{ index $oRes "name" | quote }}){{ end }} + {{ index $oRes "name" }}.persist({{ if eq (index $oRes "type") "reports" -}}stepConfig{{- else -}} + {{if $.ExportPrefix}}{{ $.ExportPrefix }}.{{end}}GeneralConfig.EnvRootPath, {{ index $oRes "name" | quote }}{{- end -}} + ){{- end }} + config.RemoveVaultSecretFiles() stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds()) stepTelemetryData.ErrorCategory = log.GetErrorCategory().String() stepTelemetryData.PiperCommitHash = {{if .ExportPrefix}}{{ .ExportPrefix }}.{{end}}GitCommit @@ -168,7 +180,7 @@ func {{.CobraCmdFuncName}}() *cobra.Command { {{if .ExportPrefix}}{{ .ExportPrefix }}.{{end}}GeneralConfig.HookConfig.SplunkConfig.Index, {{if .ExportPrefix}}{{ .ExportPrefix }}.{{end}}GeneralConfig.HookConfig.SplunkConfig.SendLogs) } - {{.StepName}}(stepConfig, &stepTelemetryData{{ range $notused, $oRes := .OutputResources}}, &{{ index $oRes "name" }}{{ end }}) + {{.StepName}}(stepConfig, &stepTelemetryData{{ range $notused, $oRes := .OutputResources}}{{ if ne (index $oRes "type") "reports" }}, &{{ index $oRes "name" }}{{ end }}{{ end }}) stepTelemetryData.ErrorCode = "0" log.Entry().Info("SUCCESS") }, @@ -287,6 +299,9 @@ func {{ .StepName }}Metadata() config.StepData { {{ if $p.name}}"name": {{ $p.name | quote }},{{ end -}} {{ if $p.fields}}"fields": []map[string]string{ {{- range $j, $f := $p.fields}} {"name": {{ $f.name | quote }}}, {{end -}} },{{ end -}} {{ if $p.tags}}"tags": []map[string]string{ {{- range $j, $t := $p.tags}} {"name": {{ $t.name | quote }}}, {{end -}} },{{ end -}} + {{ if $p.filePattern}}"filePattern": {{ $p.filePattern | quote }},{{ end -}} + {{ if $p.type}}"type": {{ $p.type | quote }},{{ end -}} + {{ if $p.subFolder}}"subFolder": {{ $p.subFolder | quote }},{{ end -}} {{ if $p }}}, {{- end -}} {{ end }} {{ if $res.Parameters -}} }, {{- end }} @@ -638,6 +653,7 @@ func getOutputResourceDetails(stepData *config.StepData) ([]map[string]string, e for _, res := range stepData.Spec.Outputs.Resources { currentResource := map[string]string{} currentResource["name"] = res.Name + currentResource["type"] = res.Type switch res.Type { case "piperEnvironment": @@ -695,6 +711,27 @@ func getOutputResourceDetails(stepData *config.StepData) ([]map[string]string, e currentResource["def"] = def currentResource["objectname"] = influxResource.StructName() outputResources = append(outputResources, currentResource) + case "reports": + var reportsResource ReportsResource + reportsResource.Name = res.Name + reportsResource.StepName = stepData.Metadata.Name + for _, param := range res.Parameters { + filePattern, _ := param["filePattern"].(string) + paramRef, _ := param["paramRef"].(string) + if filePattern == "" && paramRef == "" { + return outputResources, errors.New("both filePattern and paramRef cannot be empty at the same time") + } + stepResultType, _ := param["type"].(string) + reportsParam := ReportsParameter{FilePattern: filePattern, ParamRef: paramRef, Type: stepResultType} + reportsResource.Parameters = append(reportsResource.Parameters, reportsParam) + } + def, err := reportsResource.StructString() + if err != nil { + return outputResources, err + } + currentResource["def"] = def + currentResource["objectname"] = reportsResource.StructName() + outputResources = append(outputResources, currentResource) } } @@ -725,7 +762,7 @@ func isCLIParam(myType string) bool { func stepTemplate(myStepInfo stepInfo, templateName, goTemplate string) []byte { funcMap := sprig.HermeticTxtFuncMap() funcMap["flagType"] = flagType - funcMap["golangName"] = golangNameTitle + funcMap["golangName"] = GolangNameTitle funcMap["title"] = strings.Title funcMap["longName"] = longName funcMap["uniqueName"] = mustUniqName @@ -778,7 +815,8 @@ func golangName(name string) string { return properName } -func golangNameTitle(name string) string { +// GolangNameTitle returns name in title case with abbriviations in capital (API, URL, ID, JSON, TLS) +func GolangNameTitle(name string) string { return strings.Title(golangName(name)) } diff --git a/pkg/generator/helper/helper_test.go b/pkg/generator/helper/helper_test.go index 9ceb89421..3c8fc6fe0 100644 --- a/pkg/generator/helper/helper_test.go +++ b/pkg/generator/helper/helper_test.go @@ -25,6 +25,13 @@ func configOpenFileMock(name string) (io.ReadCloser, error) { spec: outputs: resources: + - name: reports + type: reports + params: + - filePattern: "test-report_*.json" + subFolder: "sonarExecuteScan" + - filePattern: "report1" + type: general - name: commonPipelineEnvironment type: piperEnvironment params: @@ -289,7 +296,7 @@ func TestGolangNameTitle(t *testing.T) { } for k, v := range tt { - assert.Equal(t, v.expected, golangNameTitle(v.input), fmt.Sprintf("wrong golang name for run %v", k)) + assert.Equal(t, v.expected, GolangNameTitle(v.input), fmt.Sprintf("wrong golang name for run %v", k)) } } diff --git a/pkg/generator/helper/resources.go b/pkg/generator/helper/resources.go index 8b4dcfd92..ef8ed4186 100644 --- a/pkg/generator/helper/resources.go +++ b/pkg/generator/helper/resources.go @@ -195,3 +195,76 @@ func (i *InfluxResource) StructString() (string, error) { func (i *InfluxResource) StructName() string { return fmt.Sprintf("%v%v", i.StepName, strings.Title(i.Name)) } + +// PiperEnvironmentResource defines a piper environement resource which stores data across multiple pipeline steps +type ReportsResource struct { + Name string + StepName string + Parameters []ReportsParameter +} + +// PiperEnvironmentParameter defines a parameter within the Piper environment +type ReportsParameter struct { + FilePattern string + ParamRef string + Type string +} + +const reportsStructTemplate = `type {{ .StepName }}{{ .Name | title}} struct { +} + +func (p *{{ .StepName }}{{ .Name | title}}) persist(stepConfig sonarExecuteScanOptions) { + content := []gcs.ReportOutputParam{ + {{- range $notused, $param := .Parameters }} + {FilePattern: "{{ $param.FilePattern }}", ParamRef: "{{ $param.ParamRef }}", StepResultType: "{{ $param.Type }}"}, + {{- end }} + } + envVars := []gcs.EnvVar{ + {Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: GeneralConfig.GCPJsonKeyFilePath, Modified: false}, + } + gcsClient, err := gcs.NewClient(gcs.WithEnvVars(envVars)) + if err != nil { + log.Entry().Errorf("creation of GCS client failed: %v", err) + } + defer gcsClient.Close() + structVal := reflect.ValueOf(&stepConfig).Elem() + inputParameters := map[string]string{} + for i := 0; i < structVal.NumField(); i++ { + field := structVal.Type().Field(i) + if field.Type.String() == "string" { + paramName := strings.Split(field.Tag.Get("json"), ",") + paramValue, _ := structVal.Field(i).Interface().(string) + inputParameters[paramName[0]] = paramValue + } + } + if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, GeneralConfig.GCSFolderPath, GeneralConfig.GCSBucketId, GeneralConfig.GCSSubFolder, doublestar.Glob, os.Stat); err != nil { + log.Entry().Errorf("failed to persist reports: %v", err) + } +}` + +// StructName returns the name of the environment resource struct +func (p *ReportsResource) StructName() string { + return fmt.Sprintf("%v%v", p.StepName, strings.Title(p.Name)) +} + +// StructString returns the golang coding for the struct definition of the environment resource +func (p *ReportsResource) StructString() (string, error) { + funcMap := template.FuncMap{ + "title": strings.Title, + "golangName": golangName, + "resourceFieldType": resourceFieldType, + } + + tmpl, err := template.New("resources").Funcs(funcMap).Parse(reportsStructTemplate) + if err != nil { + return "", err + } + + var generatedCode bytes.Buffer + err = tmpl.Execute(&generatedCode, &p) + if err != nil { + return "", err + } + + return string(generatedCode.String()), nil +} diff --git a/pkg/generator/helper/resources_test.go b/pkg/generator/helper/resources_test.go index d0e4f14f4..a09e089ad 100644 --- a/pkg/generator/helper/resources_test.go +++ b/pkg/generator/helper/resources_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestStructString(t *testing.T) { +func TestInfluxResource_StructString(t *testing.T) { tt := []struct { in InfluxResource expected string @@ -93,3 +93,69 @@ func (i *TestStepTestInflux) persist(path, resourceName string) { } } + +func TestReportsResource_StructString(t *testing.T) { + tt := []struct { + in ReportsResource + expected string + }{ + { + in: ReportsResource{ + Name: "reports", + StepName: "testStep", + Parameters: []ReportsParameter{ + { + FilePattern: "pattern1", + Type: "general", + }, + { + FilePattern: "pattern2", + }, + { + ParamRef: "testParam", + }, + }, + }, + expected: `type testStepReports struct { +} + +func (p *testStepReports) persist(stepConfig sonarExecuteScanOptions) { + content := []gcs.ReportOutputParam{ + {FilePattern: "pattern1", ParamRef: "", StepResultType: "general"}, + {FilePattern: "pattern2", ParamRef: "", StepResultType: ""}, + {FilePattern: "", ParamRef: "testParam", StepResultType: ""}, + } + envVars := []gcs.EnvVar{ + {Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: GeneralConfig.GCPJsonKeyFilePath, Modified: false}, + } + gcsClient, err := gcs.NewClient(gcs.WithEnvVars(envVars)) + if err != nil { + log.Entry().Errorf("creation of GCS client failed: %v", err) + } + defer gcsClient.Close() + structVal := reflect.ValueOf(&stepConfig).Elem() + inputParameters := map[string]string{} + for i := 0; i < structVal.NumField(); i++ { + field := structVal.Type().Field(i) + if field.Type.String() == "string" { + paramName := strings.Split(field.Tag.Get("json"), ",") + paramValue, _ := structVal.Field(i).Interface().(string) + inputParameters[paramName[0]] = paramValue + } + } + if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, GeneralConfig.GCSFolderPath, GeneralConfig.GCSBucketId, GeneralConfig.GCSSubFolder, doublestar.Glob, os.Stat); err != nil { + log.Entry().Errorf("failed to persist reports: %v", err) + } +}`, + }, + } + + for run, test := range tt { + t.Run(fmt.Sprintf("Run %v", run), func(t *testing.T) { + got, err := test.in.StructString() + assert.NoError(t, err) + assert.Equal(t, test.expected, got) + }) + + } +} diff --git a/pkg/generator/helper/testdata/TestProcessMetaFiles/custom_step_code_generated.golden b/pkg/generator/helper/testdata/TestProcessMetaFiles/custom_step_code_generated.golden index 8d588f54a..5efbbf928 100644 --- a/pkg/generator/helper/testdata/TestProcessMetaFiles/custom_step_code_generated.golden +++ b/pkg/generator/helper/testdata/TestProcessMetaFiles/custom_step_code_generated.golden @@ -11,6 +11,10 @@ import ( piperOsCmd "github.com/SAP/jenkins-library/cmd" "github.com/SAP/jenkins-library/pkg/config" "github.com/SAP/jenkins-library/pkg/log" + "github.com/bmatcuk/doublestar" + "github.com/SAP/jenkins-library/pkg/gcs" + "reflect" + "strings" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/SAP/jenkins-library/pkg/telemetry" "github.com/SAP/jenkins-library/pkg/splunk" @@ -26,6 +30,37 @@ type testStepOptions struct { } +type testStepReports struct { +} + +func (p *testStepReports) persist(stepConfig sonarExecuteScanOptions) { + content := []gcs.ReportOutputParam{ + {FilePattern: "test-report_*.json", ParamRef: "", StepResultType: ""}, + {FilePattern: "report1", ParamRef: "", StepResultType: "general"}, + } + envVars := []gcs.EnvVar{ + {Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: GeneralConfig.GCPJsonKeyFilePath, Modified: false}, + } + gcsClient, err := gcs.NewClient(gcs.WithEnvVars(envVars)) + if err != nil { + log.Entry().Errorf("creation of GCS client failed: %v", err) + } + defer gcsClient.Close() + structVal := reflect.ValueOf(&stepConfig).Elem() + inputParameters := map[string]string{} + for i := 0; i < structVal.NumField(); i++ { + field := structVal.Type().Field(i) + if field.Type.String() == "string" { + paramName := strings.Split(field.Tag.Get("json"), ",") + paramValue, _ := structVal.Field(i).Interface().(string) + inputParameters[paramName[0]] = paramValue + } + } + if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, GeneralConfig.GCSFolderPath, GeneralConfig.GCSBucketId, GeneralConfig.GCSSubFolder, doublestar.Glob, os.Stat); err != nil { + log.Entry().Errorf("failed to persist reports: %v", err) + } +} + type testStepCommonPipelineEnvironment struct { artifactVersion string git struct { @@ -107,6 +142,7 @@ func TestStepCommand() *cobra.Command { metadata := testStepMetadata() var stepConfig testStepOptions var startTime time.Time + var reports testStepReports var commonPipelineEnvironment testStepCommonPipelineEnvironment var influxTest testStepInfluxTest var logCollector *log.CollectorHook @@ -160,9 +196,10 @@ func TestStepCommand() *cobra.Command { stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { - config.RemoveVaultSecretFiles() + reports.persist(stepConfig) commonPipelineEnvironment.persist(piperOsCmd.GeneralConfig.EnvRootPath, "commonPipelineEnvironment") influxTest.persist(piperOsCmd.GeneralConfig.EnvRootPath, "influxTest") + config.RemoveVaultSecretFiles() stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds()) stepTelemetryData.ErrorCategory = log.GetErrorCategory().String() stepTelemetryData.PiperCommitHash = piperOsCmd.GitCommit @@ -256,6 +293,14 @@ func testStepMetadata() config.StepData { }, Outputs: config.StepOutputs{ Resources: []config.StepResources{ + { + Name: "reports", + Type: "reports", + Parameters: []map[string]interface{}{ + {"filePattern": "test-report_*.json","subFolder": "sonarExecuteScan",}, + {"filePattern": "report1","type": "general",}, + }, + }, { Name: "commonPipelineEnvironment", Type: "piperEnvironment", @@ -264,7 +309,7 @@ func testStepMetadata() config.StepData { {"name": "git/commitId",}, {"name": "git/headCommitId",}, {"name": "git/branch",}, - {"name": "custom/customList",}, + {"name": "custom/customList","type": "[]string",}, }, }, { diff --git a/pkg/generator/helper/testdata/TestProcessMetaFiles/step_code_generated.golden b/pkg/generator/helper/testdata/TestProcessMetaFiles/step_code_generated.golden index 047df9a86..3d0d6bc43 100644 --- a/pkg/generator/helper/testdata/TestProcessMetaFiles/step_code_generated.golden +++ b/pkg/generator/helper/testdata/TestProcessMetaFiles/step_code_generated.golden @@ -10,6 +10,10 @@ import ( "github.com/SAP/jenkins-library/pkg/config" "github.com/SAP/jenkins-library/pkg/log" + "github.com/bmatcuk/doublestar" + "github.com/SAP/jenkins-library/pkg/gcs" + "reflect" + "strings" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/SAP/jenkins-library/pkg/telemetry" "github.com/SAP/jenkins-library/pkg/splunk" @@ -25,6 +29,37 @@ type testStepOptions struct { } +type testStepReports struct { +} + +func (p *testStepReports) persist(stepConfig sonarExecuteScanOptions) { + content := []gcs.ReportOutputParam{ + {FilePattern: "test-report_*.json", ParamRef: "", StepResultType: ""}, + {FilePattern: "report1", ParamRef: "", StepResultType: "general"}, + } + envVars := []gcs.EnvVar{ + {Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: GeneralConfig.GCPJsonKeyFilePath, Modified: false}, + } + gcsClient, err := gcs.NewClient(gcs.WithEnvVars(envVars)) + if err != nil { + log.Entry().Errorf("creation of GCS client failed: %v", err) + } + defer gcsClient.Close() + structVal := reflect.ValueOf(&stepConfig).Elem() + inputParameters := map[string]string{} + for i := 0; i < structVal.NumField(); i++ { + field := structVal.Type().Field(i) + if field.Type.String() == "string" { + paramName := strings.Split(field.Tag.Get("json"), ",") + paramValue, _ := structVal.Field(i).Interface().(string) + inputParameters[paramName[0]] = paramValue + } + } + if err := gcs.PersistReportsToGCS(gcsClient, content, inputParameters, GeneralConfig.GCSFolderPath, GeneralConfig.GCSBucketId, GeneralConfig.GCSSubFolder, doublestar.Glob, os.Stat); err != nil { + log.Entry().Errorf("failed to persist reports: %v", err) + } +} + type testStepCommonPipelineEnvironment struct { artifactVersion string git struct { @@ -106,6 +141,7 @@ func TestStepCommand() *cobra.Command { metadata := testStepMetadata() var stepConfig testStepOptions var startTime time.Time + var reports testStepReports var commonPipelineEnvironment testStepCommonPipelineEnvironment var influxTest testStepInfluxTest var logCollector *log.CollectorHook @@ -159,9 +195,10 @@ func TestStepCommand() *cobra.Command { stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { - config.RemoveVaultSecretFiles() + reports.persist(stepConfig) commonPipelineEnvironment.persist(GeneralConfig.EnvRootPath, "commonPipelineEnvironment") influxTest.persist(GeneralConfig.EnvRootPath, "influxTest") + config.RemoveVaultSecretFiles() stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds()) stepTelemetryData.ErrorCategory = log.GetErrorCategory().String() stepTelemetryData.PiperCommitHash = GitCommit @@ -255,6 +292,14 @@ func testStepMetadata() config.StepData { }, Outputs: config.StepOutputs{ Resources: []config.StepResources{ + { + Name: "reports", + Type: "reports", + Parameters: []map[string]interface{}{ + {"filePattern": "test-report_*.json","subFolder": "sonarExecuteScan",}, + {"filePattern": "report1","type": "general",}, + }, + }, { Name: "commonPipelineEnvironment", Type: "piperEnvironment", @@ -263,7 +308,7 @@ func testStepMetadata() config.StepData { {"name": "git/commitId",}, {"name": "git/headCommitId",}, {"name": "git/branch",}, - {"name": "custom/customList",}, + {"name": "custom/customList","type": "[]string",}, }, }, { diff --git a/resources/metadata/sonarExecuteScan.yaml b/resources/metadata/sonarExecuteScan.yaml index 15a830e9e..ec2615f97 100644 --- a/resources/metadata/sonarExecuteScan.yaml +++ b/resources/metadata/sonarExecuteScan.yaml @@ -269,6 +269,13 @@ spec: outputs: resources: + - name: reports + type: reports + params: + - filePattern: "sonarscan.json" + type: sonarqube + - filePattern: "sonarExecuteScan_*.json" + type: sonarqube - name: influx type: influx params: