diff --git a/cmd/metadata_generated.go b/cmd/metadata_generated.go index ef031729d..2dc26c5ec 100644 --- a/cmd/metadata_generated.go +++ b/cmd/metadata_generated.go @@ -101,6 +101,7 @@ func GetAllStepMetadata() map[string]config.StepData { "shellExecute": shellExecuteMetadata(), "sonarExecuteScan": sonarExecuteScanMetadata(), "terraformExecute": terraformExecuteMetadata(), + "tmsExport": tmsExportMetadata(), "tmsUpload": tmsUploadMetadata(), "transportRequestDocIDFromGit": transportRequestDocIDFromGitMetadata(), "transportRequestReqIDFromGit": transportRequestReqIDFromGitMetadata(), diff --git a/cmd/piper.go b/cmd/piper.go index 2dafcc277..cd34b4e33 100644 --- a/cmd/piper.go +++ b/cmd/piper.go @@ -192,6 +192,7 @@ func Execute() { rootCmd.AddCommand(AnsSendEventCommand()) rootCmd.AddCommand(ApiProviderListCommand()) rootCmd.AddCommand(TmsUploadCommand()) + rootCmd.AddCommand(TmsExportCommand()) rootCmd.AddCommand(IntegrationArtifactTransportCommand()) addRootFlags(rootCmd) diff --git a/cmd/tmsExport.go b/cmd/tmsExport.go new file mode 100644 index 000000000..1def2705b --- /dev/null +++ b/cmd/tmsExport.go @@ -0,0 +1,65 @@ +package cmd + +import ( + "fmt" + + "github.com/SAP/jenkins-library/pkg/command" + "github.com/SAP/jenkins-library/pkg/log" + "github.com/SAP/jenkins-library/pkg/piperutils" + "github.com/SAP/jenkins-library/pkg/telemetry" + "github.com/SAP/jenkins-library/pkg/tms" +) + +type tmsExportUtilsBundle struct { + *command.Command + *piperutils.Files +} + +func tmsExport(exportConfig tmsExportOptions, telemetryData *telemetry.CustomData, influx *tmsExportInflux) { + utils := tms.NewTmsUtils() + config := convertExportOptions(exportConfig) + communicationInstance := tms.SetupCommunication(config) + + err := runTmsExport(exportConfig, communicationInstance, utils) + if err != nil { + log.Entry().WithError(err).Fatal("Failed to run tmsExport") + } +} + +func runTmsExport(exportConfig tmsExportOptions, communicationInstance tms.CommunicationInterface, utils tms.TmsUtils) error { + config := convertExportOptions(exportConfig) + fileId, errUploadFile := tms.UploadFile(config, communicationInstance, utils) + if errUploadFile != nil { + return errUploadFile + } + + errUploadDescriptors := tms.UploadDescriptors(config, communicationInstance, utils) + if errUploadDescriptors != nil { + return errUploadDescriptors + } + + _, errExportFileToNode := communicationInstance.ExportFileToNode(config.NodeName, fileId, config.CustomDescription, config.NamedUser) + if errExportFileToNode != nil { + log.SetErrorCategory(log.ErrorService) + return fmt.Errorf("failed to export file to node: %w", errExportFileToNode) + } + + return nil +} + +func convertExportOptions(exportConfig tmsExportOptions) tms.Options { + var config tms.Options + config.TmsServiceKey = exportConfig.TmsServiceKey + config.CustomDescription = exportConfig.CustomDescription + if config.CustomDescription == "" { + config.CustomDescription = tms.DEFAULT_TR_DESCRIPTION + } + config.NamedUser = exportConfig.NamedUser + config.NodeName = exportConfig.NodeName + config.MtaPath = exportConfig.MtaPath + config.MtaVersion = exportConfig.MtaVersion + config.NodeExtDescriptorMapping = exportConfig.NodeExtDescriptorMapping + config.Proxy = exportConfig.Proxy + config.Verbose = GeneralConfig.Verbose + return config +} diff --git a/cmd/tmsExport_generated.go b/cmd/tmsExport_generated.go new file mode 100644 index 000000000..8a5e7bf30 --- /dev/null +++ b/cmd/tmsExport_generated.go @@ -0,0 +1,300 @@ +// Code generated by piper's step-generator. DO NOT EDIT. + +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/log" + "github.com/SAP/jenkins-library/pkg/piperenv" + "github.com/SAP/jenkins-library/pkg/splunk" + "github.com/SAP/jenkins-library/pkg/telemetry" + "github.com/SAP/jenkins-library/pkg/validation" + "github.com/spf13/cobra" +) + +type tmsExportOptions struct { + TmsServiceKey string `json:"tmsServiceKey,omitempty"` + CustomDescription string `json:"customDescription,omitempty"` + NamedUser string `json:"namedUser,omitempty"` + NodeName string `json:"nodeName,omitempty"` + MtaPath string `json:"mtaPath,omitempty"` + MtaVersion string `json:"mtaVersion,omitempty"` + NodeExtDescriptorMapping map[string]interface{} `json:"nodeExtDescriptorMapping,omitempty"` + Proxy string `json:"proxy,omitempty"` +} + +type tmsExportInflux struct { + step_data struct { + fields struct { + tms bool + } + tags struct { + } + } +} + +func (i *tmsExportInflux) persist(path, resourceName string) { + measurementContent := []struct { + measurement string + valType string + name string + value interface{} + }{ + {valType: config.InfluxField, measurement: "step_data", name: "tms", value: i.step_data.fields.tms}, + } + + errCount := 0 + for _, metric := range measurementContent { + err := piperenv.SetResourceParameter(path, resourceName, filepath.Join(metric.measurement, fmt.Sprintf("%vs", metric.valType), metric.name), metric.value) + if err != nil { + log.Entry().WithError(err).Error("Error persisting influx environment.") + errCount++ + } + } + if errCount > 0 { + log.Entry().Error("failed to persist Influx environment") + } +} + +// TmsExportCommand This step allows you to export an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape. +func TmsExportCommand() *cobra.Command { + const STEP_NAME = "tmsExport" + + metadata := tmsExportMetadata() + var stepConfig tmsExportOptions + var startTime time.Time + var influx tmsExportInflux + var logCollector *log.CollectorHook + var splunkClient *splunk.Splunk + telemetryClient := &telemetry.Telemetry{} + + var createTmsExportCmd = &cobra.Command{ + Use: STEP_NAME, + Short: "This step allows you to export an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape.", + Long: `This step allows you to export an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape. The MTA file is attached to a new transport request which is added to the import queues of the follow-on transport nodes of the specified export node. + +TMS lets you manage transports between SAP Business Technology Platform accounts in Neo and Cloud Foundry, such as from DEV to TEST and PROD accounts. +For more information, see [official documentation of SAP Cloud Transport Management service](https://help.sap.com/viewer/p/TRANSPORT_MANAGEMENT_SERVICE) + +!!! note "Prerequisites" +* You have subscribed to and set up TMS, as described in [Initial Setup](https://help.sap.com/viewer/7f7160ec0d8546c6b3eab72fb5ad6fd8/Cloud/en-US/66fd7283c62f48adb23c56fb48c84a60.html), which includes the configuration of your transport landscape. +* A corresponding service key has been created, as described in [Set Up the Environment to Transport Content Archives directly in an Application](https://help.sap.com/viewer/7f7160ec0d8546c6b3eab72fb5ad6fd8/Cloud/en-US/8d9490792ed14f1bbf8a6ac08a6bca64.html). This service key (JSON) must be stored as a secret text within the Jenkins secure store or provided as value of tmsServiceKey parameter.`, + PreRunE: func(cmd *cobra.Command, _ []string) error { + startTime = time.Now() + log.SetStepName(STEP_NAME) + log.SetVerbose(GeneralConfig.Verbose) + + GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) + + path, _ := os.Getwd() + fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} + log.RegisterHook(fatalHook) + + err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + if err != nil { + log.SetErrorCategory(log.ErrorConfiguration) + return err + } + log.RegisterSecret(stepConfig.TmsServiceKey) + + if len(GeneralConfig.HookConfig.SentryConfig.Dsn) > 0 { + sentryHook := log.NewSentryHook(GeneralConfig.HookConfig.SentryConfig.Dsn, GeneralConfig.CorrelationID) + log.RegisterHook(&sentryHook) + } + + if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 { + splunkClient = &splunk.Splunk{} + logCollector = &log.CollectorHook{CorrelationID: GeneralConfig.CorrelationID} + log.RegisterHook(logCollector) + } + + if err = log.RegisterANSHookIfConfigured(GeneralConfig.CorrelationID); err != nil { + log.Entry().WithError(err).Warn("failed to set up SAP Alert Notification Service log hook") + } + + validation, err := validation.New(validation.WithJSONNamesForStructFields(), validation.WithPredefinedErrorMessages()) + if err != nil { + return err + } + if err = validation.ValidateStruct(stepConfig); err != nil { + log.SetErrorCategory(log.ErrorConfiguration) + return err + } + + return nil + }, + Run: func(_ *cobra.Command, _ []string) { + stepTelemetryData := telemetry.CustomData{} + stepTelemetryData.ErrorCode = "1" + handler := func() { + influx.persist(GeneralConfig.EnvRootPath, "influx") + config.RemoveVaultSecretFiles() + stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds()) + stepTelemetryData.ErrorCategory = log.GetErrorCategory().String() + stepTelemetryData.PiperCommitHash = GitCommit + telemetryClient.SetData(&stepTelemetryData) + telemetryClient.Send() + if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 { + splunkClient.Send(telemetryClient.GetData(), logCollector) + } + } + log.DeferExitHandler(handler) + defer handler() + telemetryClient.Initialize(GeneralConfig.NoTelemetry, STEP_NAME) + if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 { + splunkClient.Initialize(GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.SplunkConfig.Dsn, + GeneralConfig.HookConfig.SplunkConfig.Token, + GeneralConfig.HookConfig.SplunkConfig.Index, + GeneralConfig.HookConfig.SplunkConfig.SendLogs) + } + tmsExport(stepConfig, &stepTelemetryData, &influx) + stepTelemetryData.ErrorCode = "0" + log.Entry().Info("SUCCESS") + }, + } + + addTmsExportFlags(createTmsExportCmd, &stepConfig) + return createTmsExportCmd +} + +func addTmsExportFlags(cmd *cobra.Command, stepConfig *tmsExportOptions) { + cmd.Flags().StringVar(&stepConfig.TmsServiceKey, "tmsServiceKey", os.Getenv("PIPER_tmsServiceKey"), "Service key JSON string to access the SAP Cloud Transport Management service instance APIs. If not specified and if pipeline is running on Jenkins, service key, stored under ID provided with credentialsId parameter, is used.") + cmd.Flags().StringVar(&stepConfig.CustomDescription, "customDescription", os.Getenv("PIPER_customDescription"), "Can be used as the description of a transport request. Will overwrite the default, which is corresponding Git commit ID.") + cmd.Flags().StringVar(&stepConfig.NamedUser, "namedUser", `Piper-Pipeline`, "Defines the named user to execute transport request with. The default value is 'Piper-Pipeline'. If pipeline is running on Jenkins, the name of the user, who started the job, is tried to be used at first.") + cmd.Flags().StringVar(&stepConfig.NodeName, "nodeName", os.Getenv("PIPER_nodeName"), "Defines the name of the export node - starting node in TMS landscape. The transport request is added to the queues of the follow-on nodes of export node.") + cmd.Flags().StringVar(&stepConfig.MtaPath, "mtaPath", os.Getenv("PIPER_mtaPath"), "Defines the relative path to *.mtar file for the export to the SAP Cloud Transport Management service. If not specified, it will use the *.mtar file created in mtaBuild.") + cmd.Flags().StringVar(&stepConfig.MtaVersion, "mtaVersion", `*`, "Defines the version of the MTA for which the MTA extension descriptor will be used. You can use an asterisk (*) to accept any MTA version, or use a specific version compliant with SemVer 2.0, e.g. 1.0.0 (see semver.org). If the parameter is not configured, an asterisk is used.") + + cmd.Flags().StringVar(&stepConfig.Proxy, "proxy", os.Getenv("PIPER_proxy"), "Proxy URL which should be used for communication with the SAP Cloud Transport Management service backend.") + + cmd.MarkFlagRequired("tmsServiceKey") + cmd.MarkFlagRequired("nodeName") +} + +// retrieve step metadata +func tmsExportMetadata() config.StepData { + var theMetaData = config.StepData{ + Metadata: config.StepMetadata{ + Name: "tmsExport", + Aliases: []config.Alias{}, + Description: "This step allows you to export an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape.", + }, + Spec: config.StepSpec{ + Inputs: config.StepInputs{ + Secrets: []config.StepSecrets{ + {Name: "credentialsId", Description: "Jenkins 'Secret text' credentials ID containing service key for SAP Cloud Transport Management service.", Type: "jenkins"}, + }, + Resources: []config.StepResources{ + {Name: "buildResult", Type: "stash"}, + }, + Parameters: []config.StepParameters{ + { + Name: "tmsServiceKey", + ResourceRef: []config.ResourceReference{ + { + Name: "credentialsId", + Param: "tmsServiceKey", + Type: "secret", + }, + }, + Scope: []string{"PARAMETERS", "STEPS", "STAGES"}, + Type: "string", + Mandatory: true, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_tmsServiceKey"), + }, + { + Name: "customDescription", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "git/commitId", + }, + }, + Scope: []string{"PARAMETERS", "STEPS", "STAGES"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_customDescription"), + }, + { + Name: "namedUser", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STEPS", "STAGES"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: `Piper-Pipeline`, + }, + { + Name: "nodeName", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STEPS", "STAGES"}, + Type: "string", + Mandatory: true, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_nodeName"), + }, + { + Name: "mtaPath", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "mtarFilePath", + }, + }, + Scope: []string{"PARAMETERS", "STEPS", "STAGES"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_mtaPath"), + }, + { + Name: "mtaVersion", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STEPS", "STAGES"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: `*`, + }, + { + Name: "nodeExtDescriptorMapping", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STEPS", "STAGES"}, + Type: "map[string]interface{}", + Mandatory: false, + Aliases: []config.Alias{}, + }, + { + Name: "proxy", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STEPS", "STAGES"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_proxy"), + }, + }, + }, + Outputs: config.StepOutputs{ + Resources: []config.StepResources{ + { + Name: "influx", + Type: "influx", + Parameters: []map[string]interface{}{ + {"name": "step_data", "fields": []map[string]string{{"name": "tms"}}}, + }, + }, + }, + }, + }, + } + return theMetaData +} diff --git a/cmd/tmsExport_generated_test.go b/cmd/tmsExport_generated_test.go new file mode 100644 index 000000000..e52b23951 --- /dev/null +++ b/cmd/tmsExport_generated_test.go @@ -0,0 +1,17 @@ +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTmsExportCommand(t *testing.T) { + t.Parallel() + + testCmd := TmsExportCommand() + + // only high level testing performed - details are tested in step generation procedure + assert.Equal(t, "tmsExport", testCmd.Use, "command name incorrect") + +} diff --git a/cmd/tmsExport_test.go b/cmd/tmsExport_test.go new file mode 100644 index 000000000..3965fc088 --- /dev/null +++ b/cmd/tmsExport_test.go @@ -0,0 +1,148 @@ +package cmd + +import ( + "os" + "strconv" + "testing" + + "github.com/SAP/jenkins-library/pkg/mock" + "github.com/SAP/jenkins-library/pkg/tms" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +type tmsExportMockUtils struct { + *mock.ExecMockRunner + *mock.FilesMock +} + +func newTmsExportTestsUtils() tmsExportMockUtils { + utils := tmsExportMockUtils{ + ExecMockRunner: &mock.ExecMockRunner{}, + FilesMock: &mock.FilesMock{}, + } + return utils +} + +func (cim *communicationInstanceMock) ExportFileToNode(nodeName, fileId, description, namedUser string) (tms.NodeUploadResponseEntity, error) { + var nodeUploadResponseEntity tms.NodeUploadResponseEntity + if description != CUSTOM_DESCRIPTION || nodeName != NODE_NAME || fileId != strconv.FormatInt(FILE_ID, 10) || namedUser != NAMED_USER { + return nodeUploadResponseEntity, errors.New(INVALID_INPUT_MSG) + } + + if cim.isErrorOnExportFileToNode { + return nodeUploadResponseEntity, errors.New("Something went wrong on exporting file to node") + } else { + return cim.exportFileToNodeResponse, nil + } +} + +func TestRunTmsExport(t *testing.T) { + t.Parallel() + + t.Run("happy path: 1. get nodes 2. get MTA ext descriptor -> nothing obtained 3. upload MTA ext descriptor to node 4. upload file 5. export file to node", func(t *testing.T) { + t.Parallel() + + // init + nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}} + fileInfo := tms.FileInfo{Id: FILE_ID, Name: MTA_NAME} + communicationInstance := communicationInstanceMock{getNodesResponse: nodes, uploadFileResponse: fileInfo} + + utils := newTmsTestsUtils() + utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content")) + + mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH) + utils.AddFile(MTA_YAML_PATH_LOCAL, mtaYamlBytes) + + mtaExtDescriptorBytes, _ := os.ReadFile(MTA_EXT_DESCRIPTOR_PATH) + utils.AddFile(MTA_EXT_DESCRIPTOR_PATH_LOCAL, mtaExtDescriptorBytes) + + nodeNameExtDescriptorMapping := map[string]interface{}{NODE_NAME: MTA_EXT_DESCRIPTOR_PATH_LOCAL} + config := tmsExportOptions{MtaPath: MTA_PATH_LOCAL, CustomDescription: CUSTOM_DESCRIPTION, NamedUser: NAMED_USER, NodeName: NODE_NAME, MtaVersion: MTA_VERSION, NodeExtDescriptorMapping: nodeNameExtDescriptorMapping} + + // test + err := runTmsExport(config, &communicationInstance, utils) + + // assert + assert.NoError(t, err) + }) + + t.Run("error path: error while uploading file", func(t *testing.T) { + t.Parallel() + + // init + nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}} + communicationInstance := communicationInstanceMock{getNodesResponse: nodes, isErrorOnUploadFile: true} + + utils := newTmsTestsUtils() + utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content")) + + mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH) + utils.AddFile(MTA_YAML_PATH_LOCAL, mtaYamlBytes) + + mtaExtDescriptorBytes, _ := os.ReadFile(MTA_EXT_DESCRIPTOR_PATH) + utils.AddFile(MTA_EXT_DESCRIPTOR_PATH_LOCAL, mtaExtDescriptorBytes) + + nodeNameExtDescriptorMapping := map[string]interface{}{NODE_NAME: MTA_EXT_DESCRIPTOR_PATH_LOCAL} + config := tmsExportOptions{MtaPath: MTA_PATH_LOCAL, CustomDescription: CUSTOM_DESCRIPTION, NamedUser: NAMED_USER, NodeName: NODE_NAME, MtaVersion: MTA_VERSION, NodeExtDescriptorMapping: nodeNameExtDescriptorMapping} + + // test + err := runTmsExport(config, &communicationInstance, utils) + + // assert + assert.EqualError(t, err, "failed to upload file: Something went wrong on uploading file") + }) + + t.Run("error path: error while uploading MTA extension descriptor to node", func(t *testing.T) { + t.Parallel() + + // init + nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}} + communicationInstance := communicationInstanceMock{getNodesResponse: nodes, isErrorOnUploadMtaExtDescriptorToNode: true} + + utils := newTmsTestsUtils() + utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content")) + + mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH) + utils.AddFile(MTA_YAML_PATH_LOCAL, mtaYamlBytes) + + mtaExtDescriptorBytes, _ := os.ReadFile(MTA_EXT_DESCRIPTOR_PATH) + utils.AddFile(MTA_EXT_DESCRIPTOR_PATH_LOCAL, mtaExtDescriptorBytes) + + nodeNameExtDescriptorMapping := map[string]interface{}{NODE_NAME: MTA_EXT_DESCRIPTOR_PATH_LOCAL} + config := tmsExportOptions{MtaPath: MTA_PATH_LOCAL, CustomDescription: CUSTOM_DESCRIPTION, NamedUser: NAMED_USER, NodeName: NODE_NAME, MtaVersion: MTA_VERSION, NodeExtDescriptorMapping: nodeNameExtDescriptorMapping} + + // test + err := runTmsExport(config, &communicationInstance, utils) + + // assert + assert.EqualError(t, err, "failed to upload MTA extension descriptor to node: Something went wrong on uploading MTA extension descriptor to node") + }) + + t.Run("error path: error while exporting file to node", func(t *testing.T) { + t.Parallel() + + // init + nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}} + fileInfo := tms.FileInfo{Id: FILE_ID, Name: MTA_NAME} + communicationInstance := communicationInstanceMock{getNodesResponse: nodes, uploadFileResponse: fileInfo, isErrorOnExportFileToNode: true} + + utils := newTmsTestsUtils() + utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content")) + + mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH) + utils.AddFile(MTA_YAML_PATH_LOCAL, mtaYamlBytes) + + mtaExtDescriptorBytes, _ := os.ReadFile(MTA_EXT_DESCRIPTOR_PATH) + utils.AddFile(MTA_EXT_DESCRIPTOR_PATH_LOCAL, mtaExtDescriptorBytes) + + nodeNameExtDescriptorMapping := map[string]interface{}{NODE_NAME: MTA_EXT_DESCRIPTOR_PATH_LOCAL} + config := tmsExportOptions{MtaPath: MTA_PATH_LOCAL, CustomDescription: CUSTOM_DESCRIPTION, NamedUser: NAMED_USER, NodeName: NODE_NAME, MtaVersion: MTA_VERSION, NodeExtDescriptorMapping: nodeNameExtDescriptorMapping} + + // test + err := runTmsExport(config, &communicationInstance, utils) + + // assert + assert.EqualError(t, err, "failed to export file to node: Something went wrong on exporting file to node") + }) +} diff --git a/cmd/tmsUpload.go b/cmd/tmsUpload.go index 14e39d07e..3fc053dac 100644 --- a/cmd/tmsUpload.go +++ b/cmd/tmsUpload.go @@ -1,205 +1,37 @@ package cmd import ( - "encoding/json" "fmt" - "net/url" - "sort" - "strconv" - "github.com/SAP/jenkins-library/pkg/command" - piperHttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" - "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/telemetry" "github.com/SAP/jenkins-library/pkg/tms" - "github.com/ghodss/yaml" - "github.com/pkg/errors" ) -type uaa struct { - Url string `json:"url"` - ClientId string `json:"clientid"` - ClientSecret string `json:"clientsecret"` -} +func tmsUpload(uploadConfig tmsUploadOptions, telemetryData *telemetry.CustomData, influx *tmsUploadInflux) { + utils := tms.NewTmsUtils() + config := convertUploadOptions(uploadConfig) + communicationInstance := tms.SetupCommunication(config) -type serviceKey struct { - Uaa uaa `json:"uaa"` - Uri string `json:"uri"` -} - -type tmsUploadUtils interface { - command.ExecRunner - - FileExists(filename string) (bool, error) - FileRead(path string) ([]byte, error) - - // Add more methods here, or embed additional interfaces, or remove/replace as required. - // The tmsUploadUtils interface should be descriptive of your runtime dependencies, - // i.e. include everything you need to be able to mock in tests. - // Unit tests shall be executable in parallel (not depend on global state), and don't (re-)test dependencies. -} - -type tmsUploadUtilsBundle struct { - *command.Command - *piperutils.Files - - // Embed more structs as necessary to implement methods or interfaces you add to tmsUploadUtils. - // Structs embedded in this way must each have a unique set of methods attached. - // If there is no struct which implements the method you need, attach the method to - // tmsUploadUtilsBundle and forward to the implementation of the dependency. -} - -func newTmsUploadUtils() tmsUploadUtils { - utils := tmsUploadUtilsBundle{ - Command: &command.Command{}, - Files: &piperutils.Files{}, - } - // Reroute command output to logging framework - utils.Stdout(log.Writer()) - utils.Stderr(log.Writer()) - return &utils -} - -func tmsUpload(config tmsUploadOptions, telemetryData *telemetry.CustomData, influx *tmsUploadInflux) { - // Utils can be used wherever the command.ExecRunner interface is expected. - // It can also be used for example as a mavenExecRunner. - utils := newTmsUploadUtils() - client := &piperHttp.Client{} - proxy := config.Proxy - options := piperHttp.ClientOptions{} - if proxy != "" { - transportProxy, err := url.Parse(proxy) - if err != nil { - log.Entry().WithError(err).Fatalf("Failed to parse proxy string %v into a URL structure", proxy) - } - - options = piperHttp.ClientOptions{TransportProxy: transportProxy} - client.SetOptions(options) - if GeneralConfig.Verbose { - log.Entry().Infof("HTTP client instructed to use %v proxy", proxy) - } - } - - serviceKey, err := unmarshalServiceKey(config.TmsServiceKey) + err := runTmsUpload(uploadConfig, communicationInstance, utils) if err != nil { - log.Entry().WithError(err).Fatal("Failed to unmarshal TMS service key") - } - log.RegisterSecret(serviceKey.Uaa.ClientSecret) - - if GeneralConfig.Verbose { - log.Entry().Info("Will be used for communication:") - log.Entry().Infof("- client id: %v", serviceKey.Uaa.ClientId) - log.Entry().Infof("- TMS URL: %v", serviceKey.Uri) - log.Entry().Infof("- UAA URL: %v", serviceKey.Uaa.Url) - } - - communicationInstance, err := tms.NewCommunicationInstance(client, serviceKey.Uri, serviceKey.Uaa.Url, serviceKey.Uaa.ClientId, serviceKey.Uaa.ClientSecret, GeneralConfig.Verbose, options) - if err != nil { - log.Entry().WithError(err).Fatal("Failed to prepare client for talking with TMS") - } - - if err := runTmsUpload(config, communicationInstance, utils); err != nil { log.Entry().WithError(err).Fatal("Failed to run tmsUpload step") } } -func runTmsUpload(config tmsUploadOptions, communicationInstance tms.CommunicationInterface, utils tmsUploadUtils) error { - mtaPath := config.MtaPath - exists, _ := utils.FileExists(mtaPath) - if !exists { - log.SetErrorCategory(log.ErrorConfiguration) - return fmt.Errorf("mta file %s not found", mtaPath) - } - - description := tms.DEFAULT_TR_DESCRIPTION - if config.CustomDescription != "" { - description = config.CustomDescription - } - - namedUser := config.NamedUser - nodeName := config.NodeName - mtaVersion := config.MtaVersion - nodeNameExtDescriptorMapping := config.NodeExtDescriptorMapping - - if GeneralConfig.Verbose { - log.Entry().Info("The step will use the following values:") - log.Entry().Infof("- description: %v", description) - - if len(nodeNameExtDescriptorMapping) > 0 { - log.Entry().Infof("- mapping between node names and MTA extension descriptor file paths: %v", nodeNameExtDescriptorMapping) - } - log.Entry().Infof("- MTA path: %v", mtaPath) - log.Entry().Infof("- MTA version: %v", mtaVersion) - if namedUser != "" { - log.Entry().Infof("- named user: %v", namedUser) - } - log.Entry().Infof("- node name: %v", nodeName) - } - - if len(nodeNameExtDescriptorMapping) > 0 { - nodes, errGetNodes := communicationInstance.GetNodes() - if errGetNodes != nil { - log.SetErrorCategory(log.ErrorService) - return fmt.Errorf("failed to get nodes: %w", errGetNodes) - } - - mtaYamlMap, errGetMtaYamlAsMap := getYamlAsMap(utils, "mta.yaml") - if errGetMtaYamlAsMap != nil { - log.SetErrorCategory(log.ErrorConfiguration) - return fmt.Errorf("failed to get mta.yaml as map: %w", errGetMtaYamlAsMap) - } - _, isIdParameterInMap := mtaYamlMap["ID"] - _, isVersionParameterInMap := mtaYamlMap["version"] - if !isIdParameterInMap || !isVersionParameterInMap { - var errorMessage string - if !isIdParameterInMap { - errorMessage += "parameter 'ID' is not found in mta.yaml\n" - } - if !isVersionParameterInMap { - errorMessage += "parameter 'version' is not found in mta.yaml\n" - } - log.SetErrorCategory(log.ErrorConfiguration) - return errors.New(errorMessage) - } - - // validate the whole mapping and then throw errors together, so that user can get them after a single pipeline run - nodeIdExtDescriptorMapping, errGetNodeIdExtDescriptorMapping := formNodeIdExtDescriptorMappingWithValidation(utils, nodeNameExtDescriptorMapping, nodes, mtaYamlMap, mtaVersion) - if errGetNodeIdExtDescriptorMapping != nil { - log.SetErrorCategory(log.ErrorConfiguration) - return errGetNodeIdExtDescriptorMapping - } - - for nodeId, mtaExtDescriptorPath := range nodeIdExtDescriptorMapping { - obtainedMtaExtDescriptor, errGetMtaExtDescriptor := communicationInstance.GetMtaExtDescriptor(nodeId, fmt.Sprintf("%v", mtaYamlMap["ID"]), mtaVersion) - if errGetMtaExtDescriptor != nil { - log.SetErrorCategory(log.ErrorService) - return fmt.Errorf("failed to get MTA extension descriptor: %w", errGetMtaExtDescriptor) - } - - if obtainedMtaExtDescriptor != (tms.MtaExtDescriptor{}) { - _, errUpdateMtaExtDescriptor := communicationInstance.UpdateMtaExtDescriptor(nodeId, obtainedMtaExtDescriptor.Id, mtaExtDescriptorPath, mtaVersion, description, namedUser) - if errUpdateMtaExtDescriptor != nil { - log.SetErrorCategory(log.ErrorService) - return fmt.Errorf("failed to update MTA extension descriptor: %w", errUpdateMtaExtDescriptor) - } - } else { - _, errUploadMtaExtDescriptor := communicationInstance.UploadMtaExtDescriptorToNode(nodeId, mtaExtDescriptorPath, mtaVersion, description, namedUser) - if errUploadMtaExtDescriptor != nil { - log.SetErrorCategory(log.ErrorService) - return fmt.Errorf("failed to upload MTA extension descriptor to node: %w", errUploadMtaExtDescriptor) - } - } - } - } - - fileInfo, errUploadFile := communicationInstance.UploadFile(mtaPath, namedUser) +func runTmsUpload(uploadConfig tmsUploadOptions, communicationInstance tms.CommunicationInterface, utils tms.TmsUtils) error { + config := convertUploadOptions(uploadConfig) + fileId, errUploadFile := tms.UploadFile(config, communicationInstance, utils) if errUploadFile != nil { - log.SetErrorCategory(log.ErrorService) - return fmt.Errorf("failed to upload file: %w", errUploadFile) + return errUploadFile } - _, errUploadFileToNode := communicationInstance.UploadFileToNode(nodeName, strconv.FormatInt(fileInfo.Id, 10), description, namedUser) + errUploadDescriptors := tms.UploadDescriptors(config, communicationInstance, utils) + if errUploadDescriptors != nil { + return errUploadDescriptors + } + + _, errUploadFileToNode := communicationInstance.UploadFileToNode(config.NodeName, fileId, config.CustomDescription, config.NamedUser) if errUploadFileToNode != nil { log.SetErrorCategory(log.ErrorService) return fmt.Errorf("failed to upload file to node: %w", errUploadFileToNode) @@ -208,87 +40,20 @@ func runTmsUpload(config tmsUploadOptions, communicationInstance tms.Communicati return nil } -func formNodeIdExtDescriptorMappingWithValidation(utils tmsUploadUtils, nodeNameExtDescriptorMapping map[string]interface{}, nodes []tms.Node, mtaYamlMap map[string]interface{}, mtaVersion string) (map[int64]string, error) { - var wrongMtaIdExtDescriptors []string - var wrongExtDescriptorPaths []string - var wrongNodeNames []string - var errorMessage string - - nodeIdExtDescriptorMapping := make(map[int64]string) - for nodeName, mappedValue := range nodeNameExtDescriptorMapping { - mappedValueString := fmt.Sprintf("%v", mappedValue) - exists, _ := utils.FileExists(mappedValueString) - if exists { - extDescriptorMap, errGetYamlAsMap := getYamlAsMap(utils, mappedValueString) - if errGetYamlAsMap == nil { - if fmt.Sprintf("%v", mtaYamlMap["ID"]) != fmt.Sprintf("%v", extDescriptorMap["extends"]) { - wrongMtaIdExtDescriptors = append(wrongMtaIdExtDescriptors, mappedValueString) - } - } else { - wrappedErr := errors.Wrapf(errGetYamlAsMap, "tried to parse %v as yaml, but got an error", mappedValueString) - errorMessage += fmt.Sprintf("%v\n", wrappedErr) - } - } else { - wrongExtDescriptorPaths = append(wrongExtDescriptorPaths, mappedValueString) - } - - isNodeFound := false - for _, node := range nodes { - if node.Name == nodeName { - nodeIdExtDescriptorMapping[node.Id] = mappedValueString - isNodeFound = true - break - } - } - if !isNodeFound { - wrongNodeNames = append(wrongNodeNames, nodeName) - } - } - - if mtaVersion != "*" && mtaVersion != mtaYamlMap["version"] { - errorMessage += "parameter 'mtaVersion' does not match the MTA version in mta.yaml\n" - } - - if len(wrongMtaIdExtDescriptors) > 0 || len(wrongExtDescriptorPaths) > 0 || len(wrongNodeNames) > 0 { - if len(wrongMtaIdExtDescriptors) > 0 { - sort.Strings(wrongMtaIdExtDescriptors) - errorMessage += fmt.Sprintf("parameter 'extends' in MTA extension descriptor files %v is not the same as MTA ID or is missing at all\n", wrongMtaIdExtDescriptors) - } - if len(wrongExtDescriptorPaths) > 0 { - sort.Strings(wrongExtDescriptorPaths) - errorMessage += fmt.Sprintf("MTA extension descriptor files %v do not exist\n", wrongExtDescriptorPaths) - } - if len(wrongNodeNames) > 0 { - sort.Strings(wrongNodeNames) - errorMessage += fmt.Sprintf("nodes %v do not exist. Please check node names provided in 'nodeExtDescriptorMapping' parameter or create these nodes\n", wrongNodeNames) - } - } - - if errorMessage == "" { - return nodeIdExtDescriptorMapping, nil - } else { - return nil, errors.New(errorMessage) +func convertUploadOptions(uploadConfig tmsUploadOptions) tms.Options { + var config tms.Options + config.TmsServiceKey = uploadConfig.TmsServiceKey + config.CustomDescription = uploadConfig.CustomDescription + if config.CustomDescription == "" { + config.CustomDescription = tms.DEFAULT_TR_DESCRIPTION } -} - -func getYamlAsMap(utils tmsUploadUtils, yamlPath string) (map[string]interface{}, error) { - var result map[string]interface{} - bytes, err := utils.FileRead(yamlPath) - if err != nil { - return result, err - } - err = yaml.Unmarshal(bytes, &result) - if err != nil { - return result, err - } - - return result, nil -} - -func unmarshalServiceKey(serviceKeyJson string) (serviceKey serviceKey, err error) { - err = json.Unmarshal([]byte(serviceKeyJson), &serviceKey) - if err != nil { - return - } - return + config.NamedUser = uploadConfig.NamedUser + config.NodeName = uploadConfig.NodeName + config.MtaPath = uploadConfig.MtaPath + config.MtaVersion = uploadConfig.MtaVersion + config.NodeExtDescriptorMapping = uploadConfig.NodeExtDescriptorMapping + config.Proxy = uploadConfig.Proxy + config.StashContent = uploadConfig.StashContent + config.Verbose = GeneralConfig.Verbose + return config } diff --git a/cmd/tmsUpload_generated.go b/cmd/tmsUpload_generated.go index 2c9cb568e..f4213c862 100644 --- a/cmd/tmsUpload_generated.go +++ b/cmd/tmsUpload_generated.go @@ -77,7 +77,7 @@ func TmsUploadCommand() *cobra.Command { var createTmsUploadCmd = &cobra.Command{ Use: STEP_NAME, Short: "This step allows you to upload an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape.", - Long: `This step allows you to upload an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape. + Long: `This step allows you to upload an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape. The MTA file is attached to a new transport request which is added directly to the import queue of the specified transport node. TMS lets you manage transports between SAP Business Technology Platform accounts in Neo and Cloud Foundry, such as from DEV to TEST and PROD accounts. For more information, see [official documentation of SAP Cloud Transport Management service](https://help.sap.com/viewer/p/TRANSPORT_MANAGEMENT_SERVICE) diff --git a/cmd/tmsUpload_test.go b/cmd/tmsUpload_test.go index 0dfed7230..2c3f43722 100644 --- a/cmd/tmsUpload_test.go +++ b/cmd/tmsUpload_test.go @@ -1,6 +1,7 @@ package cmd import ( + "encoding/json" "fmt" "os" "strconv" @@ -39,13 +40,13 @@ const WRONG_MTA_VERSION = "3.2.1" const LAST_CHANGED_AT = "2021-11-16T13:06:05.711Z" const INVALID_INPUT_MSG = "Invalid input parameter(s) when getting MTA extension descriptor" -type tmsUploadMockUtils struct { +type tmsMockUtils struct { *mock.ExecMockRunner *mock.FilesMock } -func newTmsUploadTestsUtils() tmsUploadMockUtils { - utils := tmsUploadMockUtils{ +func newTmsTestsUtils() tmsMockUtils { + utils := tmsMockUtils{ ExecMockRunner: &mock.ExecMockRunner{}, FilesMock: &mock.FilesMock{}, } @@ -59,12 +60,14 @@ type communicationInstanceMock struct { uploadMtaExtDescriptorToNodeResponse tms.MtaExtDescriptor uploadFileResponse tms.FileInfo uploadFileToNodeResponse tms.NodeUploadResponseEntity + exportFileToNodeResponse tms.NodeUploadResponseEntity isErrorOnGetNodes bool isErrorOnGetMtaExtDescriptor bool isErrorOnUpdateMtaExtDescriptor bool isErrorOnUploadMtaExtDescriptorToNode bool isErrorOnUploadFile bool isErrorOnUploadFileToNode bool + isErrorOnExportFileToNode bool } func (cim *communicationInstanceMock) GetNodes() ([]tms.Node, error) { @@ -141,6 +144,14 @@ func (cim *communicationInstanceMock) UploadFileToNode(nodeName, fileId, descrip } } +func mapToJson(m map[string]interface{}) (string, error) { + b, err := json.Marshal(m) + if err != nil { + return "", err + } + return string(b), nil +} + func TestRunTmsUpload(t *testing.T) { t.Parallel() @@ -152,7 +163,7 @@ func TestRunTmsUpload(t *testing.T) { fileInfo := tms.FileInfo{Id: FILE_ID, Name: MTA_NAME} communicationInstance := communicationInstanceMock{getNodesResponse: nodes, uploadFileResponse: fileInfo} - utils := newTmsUploadTestsUtils() + utils := newTmsTestsUtils() utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content")) mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH) @@ -171,14 +182,14 @@ func TestRunTmsUpload(t *testing.T) { assert.NoError(t, err) }) - t.Run("happy path: no mapping between node nmaes and MTA extension descriptors is provided -> only upload file and upload file to node calls will be executed", func(t *testing.T) { + t.Run("happy path: no mapping between node names and MTA extension descriptors is provided -> only upload file and upload file to node calls will be executed", func(t *testing.T) { t.Parallel() // init fileInfo := tms.FileInfo{Id: FILE_ID, Name: MTA_NAME} communicationInstance := communicationInstanceMock{uploadFileResponse: fileInfo} - utils := newTmsUploadTestsUtils() + utils := newTmsTestsUtils() utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content")) config := tmsUploadOptions{MtaPath: MTA_PATH_LOCAL, CustomDescription: CUSTOM_DESCRIPTION, NamedUser: NAMED_USER, NodeName: NODE_NAME, MtaVersion: MTA_VERSION} @@ -199,7 +210,7 @@ func TestRunTmsUpload(t *testing.T) { fileInfo := tms.FileInfo{Id: FILE_ID, Name: MTA_NAME} communicationInstance := communicationInstanceMock{getNodesResponse: nodes, getMtaExtDescriptorResponse: mtaExtDescriptor, uploadFileResponse: fileInfo} - utils := newTmsUploadTestsUtils() + utils := newTmsTestsUtils() utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content")) mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH) @@ -223,7 +234,7 @@ func TestRunTmsUpload(t *testing.T) { // init communicationInstance := communicationInstanceMock{} - utils := newTmsUploadTestsUtils() + utils := newTmsTestsUtils() nodeNameExtDescriptorMapping := map[string]interface{}{NODE_NAME: MTA_EXT_DESCRIPTOR_PATH_LOCAL} config := tmsUploadOptions{MtaPath: MTA_PATH_LOCAL, CustomDescription: CUSTOM_DESCRIPTION, NamedUser: NAMED_USER, NodeName: NODE_NAME, MtaVersion: MTA_VERSION, NodeExtDescriptorMapping: nodeNameExtDescriptorMapping} @@ -240,7 +251,7 @@ func TestRunTmsUpload(t *testing.T) { // init communicationInstance := communicationInstanceMock{isErrorOnGetNodes: true} - utils := newTmsUploadTestsUtils() + utils := newTmsTestsUtils() utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content")) nodeNameExtDescriptorMapping := map[string]interface{}{NODE_NAME: MTA_EXT_DESCRIPTOR_PATH_LOCAL} @@ -259,7 +270,7 @@ func TestRunTmsUpload(t *testing.T) { // init nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}} communicationInstance := communicationInstanceMock{getNodesResponse: nodes} - utils := newTmsUploadTestsUtils() + utils := newTmsTestsUtils() utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content")) nodeNameExtDescriptorMapping := map[string]interface{}{NODE_NAME: MTA_EXT_DESCRIPTOR_PATH_LOCAL} @@ -278,7 +289,7 @@ func TestRunTmsUpload(t *testing.T) { // init nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}} communicationInstance := communicationInstanceMock{getNodesResponse: nodes} - utils := newTmsUploadTestsUtils() + utils := newTmsTestsUtils() utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content")) mtaYamlBytes, _ := os.ReadFile(INVALID_MTA_YAML_PATH) @@ -300,7 +311,7 @@ func TestRunTmsUpload(t *testing.T) { // init nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}} communicationInstance := communicationInstanceMock{getNodesResponse: nodes} - utils := newTmsUploadTestsUtils() + utils := newTmsTestsUtils() utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content")) mtaYamlBytes, _ := os.ReadFile(INVALID_MTA_YAML_PATH_2) @@ -326,7 +337,7 @@ func TestRunTmsUpload(t *testing.T) { // init nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}} communicationInstance := communicationInstanceMock{getNodesResponse: nodes} - utils := newTmsUploadTestsUtils() + utils := newTmsTestsUtils() utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content")) mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH) @@ -366,7 +377,7 @@ func TestRunTmsUpload(t *testing.T) { // init nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}} communicationInstance := communicationInstanceMock{getNodesResponse: nodes, isErrorOnGetMtaExtDescriptor: true} - utils := newTmsUploadTestsUtils() + utils := newTmsTestsUtils() utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content")) mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH) @@ -393,7 +404,7 @@ func TestRunTmsUpload(t *testing.T) { mtaExtDescriptor := tms.MtaExtDescriptor{Id: ID_OF_MTA_EXT_DESCRIPTOR, Description: "Some existing description", MtaId: MTA_ID, MtaExtId: MTA_EXT_ID, MtaVersion: MTA_VERSION, LastChangedAt: LAST_CHANGED_AT} communicationInstance := communicationInstanceMock{getNodesResponse: nodes, getMtaExtDescriptorResponse: mtaExtDescriptor, isErrorOnUpdateMtaExtDescriptor: true} - utils := newTmsUploadTestsUtils() + utils := newTmsTestsUtils() utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content")) mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH) @@ -419,7 +430,7 @@ func TestRunTmsUpload(t *testing.T) { nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}} communicationInstance := communicationInstanceMock{getNodesResponse: nodes, isErrorOnUploadMtaExtDescriptorToNode: true} - utils := newTmsUploadTestsUtils() + utils := newTmsTestsUtils() utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content")) mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH) @@ -445,7 +456,7 @@ func TestRunTmsUpload(t *testing.T) { nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}} communicationInstance := communicationInstanceMock{getNodesResponse: nodes, isErrorOnUploadFile: true} - utils := newTmsUploadTestsUtils() + utils := newTmsTestsUtils() utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content")) mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH) @@ -472,7 +483,7 @@ func TestRunTmsUpload(t *testing.T) { fileInfo := tms.FileInfo{Id: FILE_ID, Name: MTA_NAME} communicationInstance := communicationInstanceMock{getNodesResponse: nodes, uploadFileResponse: fileInfo, isErrorOnUploadFileToNode: true} - utils := newTmsUploadTestsUtils() + utils := newTmsTestsUtils() utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content")) mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH) diff --git a/documentation/docs/steps/tmsExport.md b/documentation/docs/steps/tmsExport.md new file mode 100644 index 000000000..3d8f3171a --- /dev/null +++ b/documentation/docs/steps/tmsExport.md @@ -0,0 +1,9 @@ +# ${docGenStepName} + +## ${docGenDescription} + +## ${docGenParameters} + +## ${docGenConfiguration} + +## ${docJenkinsPluginDependencies} diff --git a/documentation/mkdocs.yml b/documentation/mkdocs.yml index aed9c1563..a00df76e0 100644 --- a/documentation/mkdocs.yml +++ b/documentation/mkdocs.yml @@ -168,6 +168,7 @@ nav: - spinnakerTriggerPipeline: steps/spinnakerTriggerPipeline.md - testsPublishResults: steps/testsPublishResults.md - tmsUpload: steps/tmsUpload.md + - tmsExport: steps/tmsExport.md - transportRequestDocIDFromGit: steps/transportRequestDocIDFromGit.md - transportRequestReqIDFromGit: steps/transportRequestReqIDFromGit.md - transportRequestUploadCTS: steps/transportRequestUploadCTS.md diff --git a/integration/github_actions_integration_test_list.yml b/integration/github_actions_integration_test_list.yml index 6fc58d856..fc9acd2df 100644 --- a/integration/github_actions_integration_test_list.yml +++ b/integration/github_actions_integration_test_list.yml @@ -20,6 +20,7 @@ run: - '"TestMTAIntegration"' # - '"TestNexusIntegration"' - '"TestTmsUploadIntegration"' + - '"TestTmsExportIntegration"' # these are light-weighted tests, so we can use only one pod to reduce resource consumption - '"Test(Gauge|GCS|GitHub|GitOps|Influx|NPM|Piper|Python|Sonar|Vault|Karma)Integration"' diff --git a/integration/integration_tms_export_test.go b/integration/integration_tms_export_test.go new file mode 100644 index 000000000..8ee561708 --- /dev/null +++ b/integration/integration_tms_export_test.go @@ -0,0 +1,56 @@ +//go:build integration +// +build integration + +// can be executed with +// go test -v -tags integration -run TestTmsExportIntegration ./integration/... + +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTmsExportIntegrationYaml(t *testing.T) { + // success case: run with custom config + readEnv() + container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{ + Image: "devxci/mbtci-java11-node14", + User: "root", + TestDir: []string{"testdata", "TestTmsIntegration"}, + Environment: map[string]string{"PIPER_tmsServiceKey": tmsServiceKey}, + }) + defer container.terminate(t) + + err := container.whenRunningPiperCommand("tmsExport", "--customConfig=.pipeline/export_config.yml") + if err != nil { + t.Fatalf("Piper command failed %s", err) + } + + container.assertHasOutput(t, "tmsExport - File uploaded successfully") + container.assertHasOutput(t, "tmsExport - MTA extension descriptor updated successfully") + container.assertHasOutput(t, "tmsExport - Node export executed successfully") + container.assertHasOutput(t, "tmsExport - SUCCESS") +} + +func TestTmsExportIntegrationBinFailDescription(t *testing.T) { + // error case: run cmd with invalid description + readEnv() + container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{ + Image: "devxci/mbtci-java11-node14", + User: "root", + TestDir: []string{"testdata", "TestTmsIntegration"}, + Environment: map[string]string{"PIPER_tmsServiceKey": tmsServiceKey}, + }) + defer container.terminate(t) + + err := container.whenRunningPiperCommand("tmsExport", + "--mtaPath=scv_x.mtar", + "--nodeName=PIPER-TEST", + "--customDescription={Bad description}") + + assert.Error(t, err, "Did expect error") + container.assertHasOutput(t, "error tmsExport - HTTP request failed with error") + container.assertHasOutput(t, "Failed to run tmsExport - failed to export file to node") +} diff --git a/integration/integration_tms_upload_test.go b/integration/integration_tms_upload_test.go index 014adeb05..e7b93dc7b 100644 --- a/integration/integration_tms_upload_test.go +++ b/integration/integration_tms_upload_test.go @@ -2,7 +2,7 @@ // +build integration // can be executed with -// go test -v -tags integration -run TestTmsUploadIntegration ./integration +// go test -v -tags integration -run TestTmsIntegration ./integration package main @@ -31,7 +31,7 @@ func TestTmsUploadIntegrationBinSuccess(t *testing.T) { container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{ Image: "devxci/mbtci-java11-node14", User: "root", - TestDir: []string{"testdata", "TestTmsUploadIntegration"}, + TestDir: []string{"testdata", "TestTmsIntegration"}, Environment: map[string]string{"PIPER_tmsServiceKey": tmsServiceKey}, }) defer container.terminate(t) @@ -57,7 +57,7 @@ func TestTmsUploadIntegrationBinNoDescriptionSuccess(t *testing.T) { container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{ Image: "devxci/mbtci-java11-node14", User: "root", - TestDir: []string{"testdata", "TestTmsUploadIntegration"}, + TestDir: []string{"testdata", "TestTmsIntegration"}, Environment: map[string]string{"PIPER_tmsServiceKey": tmsServiceKey}, }) defer container.terminate(t) @@ -82,7 +82,7 @@ func TestTmsUploadIntegrationBinFailParam(t *testing.T) { container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{ Image: "devxci/mbtci-java11-node14", User: "root", - TestDir: []string{"testdata", "TestTmsUploadIntegration"}, + TestDir: []string{"testdata", "TestTmsIntegration"}, }) defer container.terminate(t) @@ -104,7 +104,7 @@ func TestTmsUploadIntegrationBinFailDescription(t *testing.T) { container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{ Image: "devxci/mbtci-java11-node14", User: "root", - TestDir: []string{"testdata", "TestTmsUploadIntegration"}, + TestDir: []string{"testdata", "TestTmsIntegration"}, Environment: map[string]string{"PIPER_tmsServiceKey": tmsServiceKey}, }) defer container.terminate(t) @@ -125,12 +125,12 @@ func TestTmsUploadIntegrationYaml(t *testing.T) { container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{ Image: "devxci/mbtci-java11-node14", User: "root", - TestDir: []string{"testdata", "TestTmsUploadIntegration"}, + TestDir: []string{"testdata", "TestTmsIntegration"}, Environment: map[string]string{"PIPER_tmsServiceKey": tmsServiceKey}, }) defer container.terminate(t) - err := container.whenRunningPiperCommand("tmsUpload", "--customConfig=.pipeline/tms_integration_test_config.yml") + err := container.whenRunningPiperCommand("tmsUpload", "--customConfig=.pipeline/upload_config.yml") if err != nil { t.Fatalf("Piper command failed %s", err) } diff --git a/integration/testdata/TestTmsIntegration/.pipeline/export_config.yml b/integration/testdata/TestTmsIntegration/.pipeline/export_config.yml new file mode 100644 index 000000000..b0447a578 --- /dev/null +++ b/integration/testdata/TestTmsIntegration/.pipeline/export_config.yml @@ -0,0 +1,12 @@ +general: + verbose: true +steps: + tmsExport: + nodeName: PIPER-TEST + namedUser: Piper-Export + mtaPath: scv_x.mtar + verbose: true + mtaVersion: 1.0.0 + nodeExtDescriptorMapping: + PIPER-TEST: 'scv_x.mtaext' + PIPER-PROD: 'scv_x.mtaext' diff --git a/integration/testdata/TestTmsUploadIntegration/.pipeline/tms_integration_test_config.yml b/integration/testdata/TestTmsIntegration/.pipeline/upload_config.yml similarity index 86% rename from integration/testdata/TestTmsUploadIntegration/.pipeline/tms_integration_test_config.yml rename to integration/testdata/TestTmsIntegration/.pipeline/upload_config.yml index 04f7cebc7..80a70e9bd 100644 --- a/integration/testdata/TestTmsUploadIntegration/.pipeline/tms_integration_test_config.yml +++ b/integration/testdata/TestTmsIntegration/.pipeline/upload_config.yml @@ -4,6 +4,7 @@ steps: tmsUpload: useGoStep: true nodeName: PIPER-TEST + namedUser: Piper-Upload mtaPath: scv_x.mtar verbose: true mtaVersion: 1.0.0 diff --git a/integration/testdata/TestTmsUploadIntegration/mta.yaml b/integration/testdata/TestTmsIntegration/mta.yaml similarity index 100% rename from integration/testdata/TestTmsUploadIntegration/mta.yaml rename to integration/testdata/TestTmsIntegration/mta.yaml diff --git a/integration/testdata/TestTmsUploadIntegration/scv_x.mtaext b/integration/testdata/TestTmsIntegration/scv_x.mtaext similarity index 100% rename from integration/testdata/TestTmsUploadIntegration/scv_x.mtaext rename to integration/testdata/TestTmsIntegration/scv_x.mtaext diff --git a/integration/testdata/TestTmsUploadIntegration/scv_x.mtar b/integration/testdata/TestTmsIntegration/scv_x.mtar similarity index 100% rename from integration/testdata/TestTmsUploadIntegration/scv_x.mtar rename to integration/testdata/TestTmsIntegration/scv_x.mtar diff --git a/pkg/tms/tms.go b/pkg/tms/tmsClient.go similarity index 83% rename from pkg/tms/tms.go rename to pkg/tms/tmsClient.go index 0639eb2b6..43842d07d 100644 --- a/pkg/tms/tms.go +++ b/pkg/tms/tmsClient.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net/http" "net/url" "os" @@ -14,89 +13,8 @@ import ( piperHttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" + "github.com/SAP/jenkins-library/pkg/xsuaa" "github.com/pkg/errors" - "github.com/sirupsen/logrus" -) - -type AuthToken struct { - TokenType string `json:"token_type"` - AccessToken string `json:"access_token"` - ExpiresIn int `json:"expires_in"` -} - -type CommunicationInstance struct { - tmsUrl string - uaaUrl string - clientId string - clientSecret string - httpClient piperHttp.Uploader - logger *logrus.Entry - isVerbose bool -} - -type Node struct { - Id int64 `json:"id"` - Name string `json:"name"` -} - -type nodes struct { - Nodes []Node `json:"nodes"` -} - -type MtaExtDescriptor struct { - Id int64 `json:"id"` - Description string `json:"description"` - MtaId string `json:"mtaId"` - MtaExtId string `json:"mtaExtId"` - MtaVersion string `json:"mtaVersion"` - LastChangedAt string `json:"lastChangedAt"` -} - -type mtaExtDescriptors struct { - MtaExtDescriptors []MtaExtDescriptor `json:"mtaExtDescriptors"` -} - -type FileInfo struct { - Id int64 `json:"fileId"` - Name string `json:"fileName"` -} - -type NodeUploadResponseEntity struct { - TransportRequestId int64 `json:"transportRequestId"` - TransportRequestDescription string `json:"transportRequestDescription"` - QueueEntries []QueueEntry `json:"queueEntries"` -} - -type QueueEntry struct { - Id int64 `json:"queueId"` - NodeId int64 `json:"nodeId"` - NodeName string `json:"nodeName"` -} - -type NodeUploadRequestEntity struct { - ContentType string `json:"contentType"` - StorageType string `json:"storageType"` - NodeName string `json:"nodeName"` - Description string `json:"description"` - NamedUser string `json:"namedUser"` - Entries []Entry `json:"entries"` -} - -type Entry struct { - Uri string `json:"uri"` -} - -type CommunicationInterface interface { - GetNodes() ([]Node, error) - GetMtaExtDescriptor(nodeId int64, mtaId, mtaVersion string) (MtaExtDescriptor, error) - UpdateMtaExtDescriptor(nodeId, idOfMtaExtDescriptor int64, file, mtaVersion, description, namedUser string) (MtaExtDescriptor, error) - UploadMtaExtDescriptorToNode(nodeId int64, file, mtaVersion, description, namedUser string) (MtaExtDescriptor, error) - UploadFile(file, namedUser string) (FileInfo, error) - UploadFileToNode(nodeName, fileId, description, namedUser string) (NodeUploadResponseEntity, error) -} - -const ( - DEFAULT_TR_DESCRIPTION = "Created by Piper" ) // NewCommunicationInstance returns CommunicationInstance structure with http client prepared for communication with TMS backend @@ -147,7 +65,7 @@ func (communicationInstance *CommunicationInstance) getOAuthToken() (string, err return "", err } - var token AuthToken + var token xsuaa.AuthToken json.Unmarshal(data, &token) if communicationInstance.isVerbose { @@ -160,8 +78,8 @@ func (communicationInstance *CommunicationInstance) getOAuthToken() (string, err func sendRequest(communicationInstance *CommunicationInstance, method, urlPathAndQuery string, body io.Reader, header http.Header, expectedStatusCode int, isTowardsUaa bool) ([]byte, error) { var requestBody io.Reader if body != nil { - closer := ioutil.NopCloser(body) - bodyBytes, _ := ioutil.ReadAll(closer) + closer := io.NopCloser(body) + bodyBytes, _ := io.ReadAll(closer) requestBody = bytes.NewBuffer(bodyBytes) defer closer.Close() } @@ -185,7 +103,7 @@ func sendRequest(communicationInstance *CommunicationInstance, method, urlPathAn return nil, fmt.Errorf("unexpected positive HTTP status code %v, while it was expected %v", response.StatusCode, expectedStatusCode) } - data, _ := ioutil.ReadAll(response.Body) + data, _ := io.ReadAll(response.Body) if !isTowardsUaa && communicationInstance.isVerbose { communicationInstance.logger.Debugf("Valid response body: %v", string(data)) } @@ -195,7 +113,7 @@ func sendRequest(communicationInstance *CommunicationInstance, method, urlPathAn func (communicationInstance *CommunicationInstance) logResponseBody(response *http.Response) { if response != nil && response.Body != nil { - data, _ := ioutil.ReadAll(response.Body) + data, _ := io.ReadAll(response.Body) communicationInstance.logger.Errorf("Response body: %s", data) response.Body.Close() } @@ -289,6 +207,36 @@ func (communicationInstance *CommunicationInstance) UploadFileToNode(nodeName, f } +func (communicationInstance *CommunicationInstance) ExportFileToNode(nodeName, fileId, description, namedUser string) (NodeUploadResponseEntity, error) { + if communicationInstance.isVerbose { + communicationInstance.logger.Info("Node export started") + communicationInstance.logger.Infof("tmsUrl: %v, nodeName: %v, fileId: %v, description: %v, namedUser: %v", communicationInstance.tmsUrl, nodeName, fileId, description, namedUser) + } + + header := http.Header{} + header.Add("Content-Type", "application/json") + + var nodeUploadResponseEntity NodeUploadResponseEntity + entry := Entry{Uri: fileId} + body := NodeUploadRequestEntity{ContentType: "MTA", StorageType: "FILE", NodeName: nodeName, Description: description, NamedUser: namedUser, Entries: []Entry{entry}} + bodyBytes, errMarshaling := json.Marshal(body) + if errMarshaling != nil { + return nodeUploadResponseEntity, errors.Wrapf(errMarshaling, "unable to marshal request body %v", body) + } + + data, errSendRequest := sendRequest(communicationInstance, http.MethodPost, "/v2/nodes/export", bytes.NewReader(bodyBytes), header, http.StatusOK, false) + if errSendRequest != nil { + return nodeUploadResponseEntity, errSendRequest + } + + json.Unmarshal(data, &nodeUploadResponseEntity) + if communicationInstance.isVerbose { + communicationInstance.logger.Info("Node export executed successfully") + } + return nodeUploadResponseEntity, nil + +} + func (communicationInstance *CommunicationInstance) UpdateMtaExtDescriptor(nodeId, idOfMtaExtDescriptor int64, file, mtaVersion, description, namedUser string) (MtaExtDescriptor, error) { if communicationInstance.isVerbose { communicationInstance.logger.Info("Update of MTA extension descriptor started") @@ -408,7 +356,7 @@ func upload(communicationInstance *CommunicationInstance, uploadRequestData pipe return nil, fmt.Errorf("unexpected positive HTTP status code %v, while it was expected %v", response.StatusCode, expectedStatusCode) } - data, _ := ioutil.ReadAll(response.Body) + data, _ := io.ReadAll(response.Body) if communicationInstance.isVerbose { communicationInstance.logger.Debugf("Valid response body: %v", string(data)) } diff --git a/pkg/tms/tms_test.go b/pkg/tms/tmsClient_test.go similarity index 98% rename from pkg/tms/tms_test.go rename to pkg/tms/tmsClient_test.go index d17c4c50e..971687aa5 100644 --- a/pkg/tms/tms_test.go +++ b/pkg/tms/tmsClient_test.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "net/url" "os" @@ -48,21 +47,21 @@ func (um *uploaderMock) SendRequest(method, url string, body io.Reader, header h if um.httpStatusCode >= 300 { httpError = fmt.Errorf("http error %v", um.httpStatusCode) } - return &http.Response{StatusCode: um.httpStatusCode, Body: ioutil.NopCloser(strings.NewReader(um.responseBody))}, httpError + return &http.Response{StatusCode: um.httpStatusCode, Body: io.NopCloser(strings.NewReader(um.responseBody))}, httpError } func (um *uploaderMock) UploadFile(url, file, fieldName string, header http.Header, cookies []*http.Cookie, uploadType string) (*http.Response, error) { um.httpMethod = http.MethodPost um.urlCalled = url um.header = header - return &http.Response{StatusCode: um.httpStatusCode, Body: ioutil.NopCloser(bytes.NewReader([]byte(um.responseBody)))}, nil + return &http.Response{StatusCode: um.httpStatusCode, Body: io.NopCloser(bytes.NewReader([]byte(um.responseBody)))}, nil } func (um *uploaderMock) UploadRequest(method, url, file, fieldName string, header http.Header, cookies []*http.Cookie, uploadType string) (*http.Response, error) { um.httpMethod = http.MethodPost um.urlCalled = url um.header = header - return &http.Response{StatusCode: um.httpStatusCode, Body: ioutil.NopCloser(bytes.NewReader([]byte(um.responseBody)))}, nil + return &http.Response{StatusCode: um.httpStatusCode, Body: io.NopCloser(bytes.NewReader([]byte(um.responseBody)))}, nil } func (um *uploaderMock) Upload(uploadRequestData piperHttp.UploadRequestData) (*http.Response, error) { @@ -84,7 +83,7 @@ func (um *uploaderMock) Upload(uploadRequestData piperHttp.UploadRequestData) (* if um.httpStatusCode >= 300 { httpError = fmt.Errorf("http error %v", um.httpStatusCode) } - return &http.Response{StatusCode: um.httpStatusCode, Body: ioutil.NopCloser(strings.NewReader(um.responseBody))}, httpError + return &http.Response{StatusCode: um.httpStatusCode, Body: io.NopCloser(strings.NewReader(um.responseBody))}, httpError } func (um *uploaderMock) SetOptions(options piperHttp.ClientOptions) { diff --git a/pkg/tms/tmsUtils.go b/pkg/tms/tmsUtils.go new file mode 100644 index 000000000..a270d78af --- /dev/null +++ b/pkg/tms/tmsUtils.go @@ -0,0 +1,358 @@ +package tms + +import ( + "encoding/json" + "fmt" + "net/url" + "sort" + "strconv" + + "github.com/SAP/jenkins-library/pkg/command" + piperHttp "github.com/SAP/jenkins-library/pkg/http" + "github.com/SAP/jenkins-library/pkg/log" + "github.com/SAP/jenkins-library/pkg/piperutils" + "github.com/ghodss/yaml" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +type TmsUtils interface { + command.ExecRunner + FileExists(filename string) (bool, error) + FileRead(path string) ([]byte, error) +} + +type uaa struct { + Url string `json:"url"` + ClientId string `json:"clientid"` + ClientSecret string `json:"clientsecret"` +} + +type serviceKey struct { + Uaa uaa `json:"uaa"` + Uri string `json:"uri"` +} + +type CommunicationInstance struct { + tmsUrl string + uaaUrl string + clientId string + clientSecret string + httpClient piperHttp.Uploader + logger *logrus.Entry + isVerbose bool +} + +type Node struct { + Id int64 `json:"id"` + Name string `json:"name"` +} + +type nodes struct { + Nodes []Node `json:"nodes"` +} + +type MtaExtDescriptor struct { + Id int64 `json:"id"` + Description string `json:"description"` + MtaId string `json:"mtaId"` + MtaExtId string `json:"mtaExtId"` + MtaVersion string `json:"mtaVersion"` + LastChangedAt string `json:"lastChangedAt"` +} + +type mtaExtDescriptors struct { + MtaExtDescriptors []MtaExtDescriptor `json:"mtaExtDescriptors"` +} + +type FileInfo struct { + Id int64 `json:"fileId"` + Name string `json:"fileName"` +} + +type NodeUploadResponseEntity struct { + TransportRequestId int64 `json:"transportRequestId"` + TransportRequestDescription string `json:"transportRequestDescription"` + QueueEntries []QueueEntry `json:"queueEntries"` +} + +type QueueEntry struct { + Id int64 `json:"queueId"` + NodeId int64 `json:"nodeId"` + NodeName string `json:"nodeName"` +} + +type NodeUploadRequestEntity struct { + ContentType string `json:"contentType"` + StorageType string `json:"storageType"` + NodeName string `json:"nodeName"` + Description string `json:"description"` + NamedUser string `json:"namedUser"` + Entries []Entry `json:"entries"` +} + +type Entry struct { + Uri string `json:"uri"` +} + +type CommunicationInterface interface { + GetNodes() ([]Node, error) + GetMtaExtDescriptor(nodeId int64, mtaId, mtaVersion string) (MtaExtDescriptor, error) + UpdateMtaExtDescriptor(nodeId, idOfMtaExtDescriptor int64, file, mtaVersion, description, namedUser string) (MtaExtDescriptor, error) + UploadMtaExtDescriptorToNode(nodeId int64, file, mtaVersion, description, namedUser string) (MtaExtDescriptor, error) + UploadFile(file, namedUser string) (FileInfo, error) + UploadFileToNode(nodeName, fileId, description, namedUser string) (NodeUploadResponseEntity, error) + ExportFileToNode(nodeName, fileId, description, namedUser string) (NodeUploadResponseEntity, error) +} + +type Options struct { + TmsServiceKey string `json:"tmsServiceKey,omitempty"` + CustomDescription string `json:"customDescription,omitempty"` + NamedUser string `json:"namedUser,omitempty"` + NodeName string `json:"nodeName,omitempty"` + MtaPath string `json:"mtaPath,omitempty"` + MtaVersion string `json:"mtaVersion,omitempty"` + NodeExtDescriptorMapping map[string]interface{} `json:"nodeExtDescriptorMapping,omitempty"` + Proxy string `json:"proxy,omitempty"` + StashContent []string `json:"stashContent,omitempty"` + Verbose bool +} + +type tmsUtilsBundle struct { + *command.Command + *piperutils.Files +} + +const DEFAULT_TR_DESCRIPTION = "Created by Piper" + +func NewTmsUtils() TmsUtils { + utils := tmsUtilsBundle{ + Command: &command.Command{}, + Files: &piperutils.Files{}, + } + // Reroute command output to logging framework + utils.Stdout(log.Writer()) + utils.Stderr(log.Writer()) + return &utils +} + +func unmarshalServiceKey(serviceKeyJson string) (serviceKey serviceKey, err error) { + err = json.Unmarshal([]byte(serviceKeyJson), &serviceKey) + if err != nil { + return + } + return +} + +func FormNodeIdExtDescriptorMappingWithValidation(utils TmsUtils, nodeNameExtDescriptorMapping map[string]interface{}, nodes []Node, mtaYamlMap map[string]interface{}, mtaVersion string) (map[int64]string, error) { + var wrongMtaIdExtDescriptors []string + var wrongExtDescriptorPaths []string + var wrongNodeNames []string + var errorMessage string + + nodeIdExtDescriptorMapping := make(map[int64]string) + for nodeName, mappedValue := range nodeNameExtDescriptorMapping { + mappedValueString := fmt.Sprintf("%v", mappedValue) + exists, _ := utils.FileExists(mappedValueString) + if exists { + extDescriptorMap, errGetYamlAsMap := GetYamlAsMap(utils, mappedValueString) + if errGetYamlAsMap == nil { + if fmt.Sprintf("%v", mtaYamlMap["ID"]) != fmt.Sprintf("%v", extDescriptorMap["extends"]) { + wrongMtaIdExtDescriptors = append(wrongMtaIdExtDescriptors, mappedValueString) + } + } else { + wrappedErr := errors.Wrapf(errGetYamlAsMap, "tried to parse %v as yaml, but got an error", mappedValueString) + errorMessage += fmt.Sprintf("%v\n", wrappedErr) + } + } else { + wrongExtDescriptorPaths = append(wrongExtDescriptorPaths, mappedValueString) + } + + isNodeFound := false + for _, node := range nodes { + if node.Name == nodeName { + nodeIdExtDescriptorMapping[node.Id] = mappedValueString + isNodeFound = true + break + } + } + if !isNodeFound { + wrongNodeNames = append(wrongNodeNames, nodeName) + } + } + + if mtaVersion != "*" && mtaVersion != mtaYamlMap["version"] { + errorMessage += "parameter 'mtaVersion' does not match the MTA version in mta.yaml\n" + } + + if len(wrongMtaIdExtDescriptors) > 0 || len(wrongExtDescriptorPaths) > 0 || len(wrongNodeNames) > 0 { + if len(wrongMtaIdExtDescriptors) > 0 { + sort.Strings(wrongMtaIdExtDescriptors) + errorMessage += fmt.Sprintf("parameter 'extends' in MTA extension descriptor files %v is not the same as MTA ID or is missing at all\n", wrongMtaIdExtDescriptors) + } + if len(wrongExtDescriptorPaths) > 0 { + sort.Strings(wrongExtDescriptorPaths) + errorMessage += fmt.Sprintf("MTA extension descriptor files %v do not exist\n", wrongExtDescriptorPaths) + } + if len(wrongNodeNames) > 0 { + sort.Strings(wrongNodeNames) + errorMessage += fmt.Sprintf("nodes %v do not exist. Please check node names provided in 'nodeExtDescriptorMapping' parameter or create these nodes\n", wrongNodeNames) + } + } + + if errorMessage == "" { + return nodeIdExtDescriptorMapping, nil + } else { + return nil, errors.New(errorMessage) + } +} + +func GetYamlAsMap(utils TmsUtils, yamlPath string) (map[string]interface{}, error) { + var result map[string]interface{} + bytes, err := utils.FileRead(yamlPath) + if err != nil { + return result, err + } + err = yaml.Unmarshal(bytes, &result) + if err != nil { + return result, err + } + + return result, nil +} + +func SetupCommunication(config Options) (communicationInstance CommunicationInterface) { + client := &piperHttp.Client{} + proxy := config.Proxy + options := piperHttp.ClientOptions{} + if proxy != "" { + transportProxy, err := url.Parse(proxy) + if err != nil { + log.Entry().WithError(err).Fatalf("Failed to parse proxy string %v into a URL structure", proxy) + } + + options = piperHttp.ClientOptions{TransportProxy: transportProxy} + client.SetOptions(options) + if config.Verbose { + log.Entry().Infof("HTTP client instructed to use %v proxy", proxy) + } + } + + serviceKey, err := unmarshalServiceKey(config.TmsServiceKey) + if err != nil { + log.Entry().WithError(err).Fatal("Failed to unmarshal TMS service key") + } + log.RegisterSecret(serviceKey.Uaa.ClientSecret) + + if config.Verbose { + log.Entry().Info("Will be used for communication:") + log.Entry().Infof("- client id: %v", serviceKey.Uaa.ClientId) + log.Entry().Infof("- TMS URL: %v", serviceKey.Uri) + log.Entry().Infof("- UAA URL: %v", serviceKey.Uaa.Url) + } + + commuInstance, err := NewCommunicationInstance(client, serviceKey.Uri, serviceKey.Uaa.Url, serviceKey.Uaa.ClientId, serviceKey.Uaa.ClientSecret, config.Verbose, options) + if err != nil { + log.Entry().WithError(err).Fatal("Failed to prepare client for talking with TMS") + } + return commuInstance +} + +func UploadDescriptors(config Options, communicationInstance CommunicationInterface, utils TmsUtils) error { + description := config.CustomDescription + namedUser := config.NamedUser + nodeName := config.NodeName + mtaVersion := config.MtaVersion + nodeNameExtDescriptorMapping := config.NodeExtDescriptorMapping + mtaPath := config.MtaPath + + if config.Verbose { + log.Entry().Info("The step will use the following values:") + log.Entry().Infof("- description: %v", config.CustomDescription) + + if len(nodeNameExtDescriptorMapping) > 0 { + log.Entry().Infof("- mapping between node names and MTA extension descriptor file paths: %v", nodeNameExtDescriptorMapping) + } + log.Entry().Infof("- MTA path: %v", mtaPath) + log.Entry().Infof("- MTA version: %v", mtaVersion) + if namedUser != "" { + log.Entry().Infof("- named user: %v", namedUser) + } + log.Entry().Infof("- node name: %v", nodeName) + } + + if len(nodeNameExtDescriptorMapping) > 0 { + nodes, errGetNodes := communicationInstance.GetNodes() + if errGetNodes != nil { + log.SetErrorCategory(log.ErrorService) + return fmt.Errorf("failed to get nodes: %w", errGetNodes) + } + + mtaYamlMap, errGetMtaYamlAsMap := GetYamlAsMap(utils, "mta.yaml") + if errGetMtaYamlAsMap != nil { + log.SetErrorCategory(log.ErrorConfiguration) + return fmt.Errorf("failed to get mta.yaml as map: %w", errGetMtaYamlAsMap) + } + _, isIdParameterInMap := mtaYamlMap["ID"] + _, isVersionParameterInMap := mtaYamlMap["version"] + if !isIdParameterInMap || !isVersionParameterInMap { + var errorMessage string + if !isIdParameterInMap { + errorMessage += "parameter 'ID' is not found in mta.yaml\n" + } + if !isVersionParameterInMap { + errorMessage += "parameter 'version' is not found in mta.yaml\n" + } + log.SetErrorCategory(log.ErrorConfiguration) + return errors.New(errorMessage) + } + + // validate the whole mapping and then throw errors together, so that user can get them after a single pipeline run + nodeIdExtDescriptorMapping, errGetNodeIdExtDescriptorMapping := FormNodeIdExtDescriptorMappingWithValidation(utils, nodeNameExtDescriptorMapping, nodes, mtaYamlMap, mtaVersion) + if errGetNodeIdExtDescriptorMapping != nil { + log.SetErrorCategory(log.ErrorConfiguration) + return errGetNodeIdExtDescriptorMapping + } + + for nodeId, mtaExtDescriptorPath := range nodeIdExtDescriptorMapping { + obtainedMtaExtDescriptor, errGetMtaExtDescriptor := communicationInstance.GetMtaExtDescriptor(nodeId, fmt.Sprintf("%v", mtaYamlMap["ID"]), mtaVersion) + if errGetMtaExtDescriptor != nil { + log.SetErrorCategory(log.ErrorService) + return fmt.Errorf("failed to get MTA extension descriptor: %w", errGetMtaExtDescriptor) + } + + if obtainedMtaExtDescriptor != (MtaExtDescriptor{}) { + _, errUpdateMtaExtDescriptor := communicationInstance.UpdateMtaExtDescriptor(nodeId, obtainedMtaExtDescriptor.Id, mtaExtDescriptorPath, mtaVersion, description, namedUser) + if errUpdateMtaExtDescriptor != nil { + log.SetErrorCategory(log.ErrorService) + return fmt.Errorf("failed to update MTA extension descriptor: %w", errUpdateMtaExtDescriptor) + } + } else { + _, errUploadMtaExtDescriptor := communicationInstance.UploadMtaExtDescriptorToNode(nodeId, mtaExtDescriptorPath, mtaVersion, description, namedUser) + if errUploadMtaExtDescriptor != nil { + log.SetErrorCategory(log.ErrorService) + return fmt.Errorf("failed to upload MTA extension descriptor to node: %w", errUploadMtaExtDescriptor) + } + } + } + } + return nil +} + +func UploadFile(config Options, communicationInstance CommunicationInterface, utils TmsUtils) (string, error) { + mtaPath := config.MtaPath + exists, _ := utils.FileExists(mtaPath) + if !exists { + log.SetErrorCategory(log.ErrorConfiguration) + return "", fmt.Errorf("mta file %s not found", mtaPath) + } + + fileInfo, errUploadFile := communicationInstance.UploadFile(mtaPath, config.NamedUser) + if errUploadFile != nil { + log.SetErrorCategory(log.ErrorService) + return "", fmt.Errorf("failed to upload file: %w", errUploadFile) + } + + fileId := strconv.FormatInt(fileInfo.Id, 10) + return fileId, nil +} diff --git a/resources/metadata/tmsExport.yaml b/resources/metadata/tmsExport.yaml new file mode 100644 index 000000000..4472f3785 --- /dev/null +++ b/resources/metadata/tmsExport.yaml @@ -0,0 +1,102 @@ +metadata: + name: tmsExport + description: This step allows you to export an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape. + longDescription: |- + This step allows you to export an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape. The MTA file is attached to a new transport request which is added to the import queues of the follow-on transport nodes of the specified export node. + + TMS lets you manage transports between SAP Business Technology Platform accounts in Neo and Cloud Foundry, such as from DEV to TEST and PROD accounts. + For more information, see [official documentation of SAP Cloud Transport Management service](https://help.sap.com/viewer/p/TRANSPORT_MANAGEMENT_SERVICE) + + !!! note "Prerequisites" + * You have subscribed to and set up TMS, as described in [Initial Setup](https://help.sap.com/viewer/7f7160ec0d8546c6b3eab72fb5ad6fd8/Cloud/en-US/66fd7283c62f48adb23c56fb48c84a60.html), which includes the configuration of your transport landscape. + * A corresponding service key has been created, as described in [Set Up the Environment to Transport Content Archives directly in an Application](https://help.sap.com/viewer/7f7160ec0d8546c6b3eab72fb5ad6fd8/Cloud/en-US/8d9490792ed14f1bbf8a6ac08a6bca64.html). This service key (JSON) must be stored as a secret text within the Jenkins secure store or provided as value of tmsServiceKey parameter. +spec: + inputs: + secrets: + - name: credentialsId + description: Jenkins 'Secret text' credentials ID containing service key for SAP Cloud Transport Management service. + type: jenkins + resources: + - name: buildResult + type: stash + params: + - name: tmsServiceKey + type: string + description: Service key JSON string to access the SAP Cloud Transport Management service instance APIs. If not specified and if pipeline is running on Jenkins, service key, stored under ID provided with credentialsId parameter, is used. + scope: + - PARAMETERS + - STEPS + - STAGES + mandatory: true + secret: true + resourceRef: + - name: credentialsId + type: secret + param: tmsServiceKey + - name: customDescription + type: string + description: Can be used as the description of a transport request. Will overwrite the default, which is corresponding Git commit ID. + scope: + - PARAMETERS + - STEPS + - STAGES + resourceRef: + - name: commonPipelineEnvironment + param: git/commitId + - name: namedUser + type: string + description: Defines the named user to execute transport request with. The default value is 'Piper-Pipeline'. If pipeline is running on Jenkins, the name of the user, who started the job, is tried to be used at first. + default: Piper-Pipeline + scope: + - PARAMETERS + - STEPS + - STAGES + - name: nodeName + type: string + description: Defines the name of the export node - starting node in TMS landscape. The transport request is added to the queues of the follow-on nodes of export node. + scope: + - PARAMETERS + - STEPS + - STAGES + mandatory: true + - name: mtaPath + type: string + description: Defines the relative path to *.mtar file for the export to the SAP Cloud Transport Management service. If not specified, it will use the *.mtar file created in mtaBuild. + scope: + - PARAMETERS + - STEPS + - STAGES + resourceRef: + - name: commonPipelineEnvironment + param: mtarFilePath + - name: mtaVersion + type: string + description: Defines the version of the MTA for which the MTA extension descriptor will be used. You can use an asterisk (*) to accept any MTA version, or use a specific version compliant with SemVer 2.0, e.g. 1.0.0 (see semver.org). If the parameter is not configured, an asterisk is used. + default: "*" + scope: + - PARAMETERS + - STEPS + - STAGES + - name: nodeExtDescriptorMapping + type: map[string]interface{} + description: 'Available only for transports in Cloud Foundry environment. Defines a mapping between a transport node name and an MTA extension descriptor file path that you want to use for the transport node, e.g. nodeExtDescriptorMapping: {"nodeName": "example.mtaext", "nodeName2": "example2.mtaext"}.' + scope: + - PARAMETERS + - STEPS + - STAGES + - name: proxy + type: string + description: Proxy URL which should be used for communication with the SAP Cloud Transport Management service backend. + scope: + - PARAMETERS + - STEPS + - STAGES + outputs: + resources: + - name: influx + type: influx + params: + - name: step_data + fields: + - name: tms + type: bool diff --git a/resources/metadata/tmsUpload.yaml b/resources/metadata/tmsUpload.yaml index ac0147fb3..afe53f971 100644 --- a/resources/metadata/tmsUpload.yaml +++ b/resources/metadata/tmsUpload.yaml @@ -2,7 +2,7 @@ metadata: name: tmsUpload description: This step allows you to upload an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape. longDescription: |- - This step allows you to upload an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape. + This step allows you to upload an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape. The MTA file is attached to a new transport request which is added directly to the import queue of the specified transport node. TMS lets you manage transports between SAP Business Technology Platform accounts in Neo and Cloud Foundry, such as from DEV to TEST and PROD accounts. For more information, see [official documentation of SAP Cloud Transport Management service](https://help.sap.com/viewer/p/TRANSPORT_MANAGEMENT_SERVICE) diff --git a/test/groovy/CommonStepsTest.groovy b/test/groovy/CommonStepsTest.groovy index 57417d810..a1800f8ae 100644 --- a/test/groovy/CommonStepsTest.groovy +++ b/test/groovy/CommonStepsTest.groovy @@ -225,6 +225,8 @@ public class CommonStepsTest extends BasePiperTest{ 'awsS3Upload', 'ansSendEvent', 'apiProviderList', //implementing new golang pattern without fields + 'tmsUpload', + 'tmsExport', ] @Test diff --git a/vars/tmsExport.groovy b/vars/tmsExport.groovy new file mode 100644 index 000000000..f486429d0 --- /dev/null +++ b/vars/tmsExport.groovy @@ -0,0 +1,13 @@ +import groovy.transform.Field +import com.sap.piper.JenkinsUtils + +@Field String STEP_NAME = getClass().getName() +@Field String METADATA_FILE = 'metadata/tmsExport.yaml' + +void call(Map parameters = [:]) { + List credentials = [ + [type: 'token', id: 'credentialsId', env: ['PIPER_tmsServiceKey']] + ] + + piperExecuteBin(parameters, STEP_NAME, METADATA_FILE, credentials, false, false, true) +}