jenkins/sap-jenkins-library
1
0
Fork 0
mirror of https://github.com/SAP/jenkins-library.git synced 2025-01-04 04:07:16 +02:00
Code Issues Releases Activity

Tms export (#4160)

Browse Source 
* Change parameter type of nodeExtDescriptorMapping

(cherry picked from commit ca7ce0485a)

* Remove usage of the depricated ioutil package

(cherry picked from commit 9821915b33)

* Fix cmd failure if neither git/commitId nor customDescription are
provided

(cherry picked from commit c362681e45)

* Fix unit test

(cherry picked from commit 53a90aabb5)

* Step metadata, step code generation

* change type of nodeExtDescriptorMapping for export

* Refactoring and export implementation

* integration test

* Add export step

* Integration test

* format

* discard piper.go

* Review related changes

* restore piper.go

* remove unused method

* Extend documentation

* Add parameter useGoStep to tmsUpload.groovy

* Regenerate steps

* Rename function

* refactor constants

* Add error path tests

* Move some code to tms package

* Move more code to tms

* Combine tmsUpload, tmsUtils

* Add groovy wrapper

* add parameters to groovy step

* add import

* jenkinsUtils instance

* comment namedUser logic in groovy

* namedUser param

* remove logic for namedUser param

* Remove TMS integration tests

* discard changes in tmsUpload.groovy

* Remove parameters

* Restore parameters

* Change type of NodeExtDescriptorMapping to map[string]interface{}

* tmsUpload: Change type of NodeExtDescriptorMapping to map

* Resolve ioutil deprecation

* Review related changes

* Formatting

* Review related improvements

* Add tmsUtils test

* Formatting tmsUtils_test

* Remove parameters from groovy wrapper

* Remove tmsUtils_test

* Add TMS steps to fieldRelatedWhitelist

* Add integration test

* Add test to github_actions_integration_test_list.yml

* Move test helper method

* Step documentation placeholder

* Remove parameter StashContent

* Restore cmd/integrationArtifactTransport.go

---------

Co-authored-by: Oliver Feldmann <oliver.feldmann@sap.com>
This commit is contained in:
gerstneralex 2023-03-27 16:55:29 +02:00 committed by GitHub
parent 92a782a6c3
commit f5c33d51bb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 1196 additions and 386 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/metadata_generated.go
View File

@ -101,6 +101,7 @@ func GetAllStepMetadata() map[string]config.StepData {
"shellExecute": shellExecuteMetadata(),
"sonarExecuteScan": sonarExecuteScanMetadata(),
"terraformExecute": terraformExecuteMetadata(),
"tmsExport": tmsExportMetadata(),
"tmsUpload": tmsUploadMetadata(),
"transportRequestDocIDFromGit": transportRequestDocIDFromGitMetadata(),
"transportRequestReqIDFromGit": transportRequestReqIDFromGitMetadata(),

1
cmd/piper.go
View File

@ -192,6 +192,7 @@ func Execute() {
rootCmd.AddCommand(AnsSendEventCommand())
rootCmd.AddCommand(ApiProviderListCommand())
rootCmd.AddCommand(TmsUploadCommand())
rootCmd.AddCommand(TmsExportCommand())
rootCmd.AddCommand(IntegrationArtifactTransportCommand())
addRootFlags(rootCmd)

65
cmd/tmsExport.go Normal file
View File

@ -0,0 +1,65 @@
package cmd
import (
"fmt"
"github.com/SAP/jenkins-library/pkg/command"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/SAP/jenkins-library/pkg/telemetry"
"github.com/SAP/jenkins-library/pkg/tms"
)
type tmsExportUtilsBundle struct {
*command.Command
*piperutils.Files
}
func tmsExport(exportConfig tmsExportOptions, telemetryData *telemetry.CustomData, influx *tmsExportInflux) {
utils := tms.NewTmsUtils()
config := convertExportOptions(exportConfig)
communicationInstance := tms.SetupCommunication(config)
err := runTmsExport(exportConfig, communicationInstance, utils)
if err != nil {
log.Entry().WithError(err).Fatal("Failed to run tmsExport")
}
}
func runTmsExport(exportConfig tmsExportOptions, communicationInstance tms.CommunicationInterface, utils tms.TmsUtils) error {
config := convertExportOptions(exportConfig)
fileId, errUploadFile := tms.UploadFile(config, communicationInstance, utils)
if errUploadFile != nil {
return errUploadFile
}
errUploadDescriptors := tms.UploadDescriptors(config, communicationInstance, utils)
if errUploadDescriptors != nil {
return errUploadDescriptors
}
_, errExportFileToNode := communicationInstance.ExportFileToNode(config.NodeName, fileId, config.CustomDescription, config.NamedUser)
if errExportFileToNode != nil {
log.SetErrorCategory(log.ErrorService)
return fmt.Errorf("failed to export file to node: %w", errExportFileToNode)
}
return nil
}
func convertExportOptions(exportConfig tmsExportOptions) tms.Options {
var config tms.Options
config.TmsServiceKey = exportConfig.TmsServiceKey
config.CustomDescription = exportConfig.CustomDescription
if config.CustomDescription == "" {
config.CustomDescription = tms.DEFAULT_TR_DESCRIPTION
}
config.NamedUser = exportConfig.NamedUser
config.NodeName = exportConfig.NodeName
config.MtaPath = exportConfig.MtaPath
config.MtaVersion = exportConfig.MtaVersion
config.NodeExtDescriptorMapping = exportConfig.NodeExtDescriptorMapping
config.Proxy = exportConfig.Proxy
config.Verbose = GeneralConfig.Verbose
return config
}

300
cmd/tmsExport_generated.go Normal file
View File

@ -0,0 +1,300 @@
// Code generated by piper's step-generator. DO NOT EDIT.
package cmd
import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/SAP/jenkins-library/pkg/config"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/piperenv"
"github.com/SAP/jenkins-library/pkg/splunk"
"github.com/SAP/jenkins-library/pkg/telemetry"
"github.com/SAP/jenkins-library/pkg/validation"
"github.com/spf13/cobra"
)
type tmsExportOptions struct {
TmsServiceKey string `json:"tmsServiceKey,omitempty"`
CustomDescription string `json:"customDescription,omitempty"`
NamedUser string `json:"namedUser,omitempty"`
NodeName string `json:"nodeName,omitempty"`
MtaPath string `json:"mtaPath,omitempty"`
MtaVersion string `json:"mtaVersion,omitempty"`
NodeExtDescriptorMapping map[string]interface{} `json:"nodeExtDescriptorMapping,omitempty"`
Proxy string `json:"proxy,omitempty"`
}
type tmsExportInflux struct {
step_data struct {
fields struct {
tms bool
}
tags struct {
}
}
}
func (i *tmsExportInflux) persist(path, resourceName string) {
measurementContent := []struct {
measurement string
valType string
name string
value interface{}
}{
{valType: config.InfluxField, measurement: "step_data", name: "tms", value: i.step_data.fields.tms},
}
errCount := 0
for _, metric := range measurementContent {
err := piperenv.SetResourceParameter(path, resourceName, filepath.Join(metric.measurement, fmt.Sprintf("%vs", metric.valType), metric.name), metric.value)
if err != nil {
log.Entry().WithError(err).Error("Error persisting influx environment.")
errCount++
}
}
if errCount > 0 {
log.Entry().Error("failed to persist Influx environment")
}
}
// TmsExportCommand This step allows you to export an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape.
func TmsExportCommand() *cobra.Command {
const STEP_NAME = "tmsExport"
metadata := tmsExportMetadata()
var stepConfig tmsExportOptions
var startTime time.Time
var influx tmsExportInflux
var logCollector *log.CollectorHook
var splunkClient *splunk.Splunk
telemetryClient := &telemetry.Telemetry{}
var createTmsExportCmd = &cobra.Command{
Use: STEP_NAME,
Short: "This step allows you to export an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape.",
Long: `This step allows you to export an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape. The MTA file is attached to a new transport request which is added to the import queues of the follow-on transport nodes of the specified export node.
TMS lets you manage transports between SAP Business Technology Platform accounts in Neo and Cloud Foundry, such as from DEV to TEST and PROD accounts.
For more information, see [official documentation of SAP Cloud Transport Management service](https://help.sap.com/viewer/p/TRANSPORT_MANAGEMENT_SERVICE)
!!! note "Prerequisites"
* You have subscribed to and set up TMS, as described in [Initial Setup](https://help.sap.com/viewer/7f7160ec0d8546c6b3eab72fb5ad6fd8/Cloud/en-US/66fd7283c62f48adb23c56fb48c84a60.html), which includes the configuration of your transport landscape.
* A corresponding service key has been created, as described in [Set Up the Environment to Transport Content Archives directly in an Application](https://help.sap.com/viewer/7f7160ec0d8546c6b3eab72fb5ad6fd8/Cloud/en-US/8d9490792ed14f1bbf8a6ac08a6bca64.html). This service key (JSON) must be stored as a secret text within the Jenkins secure store or provided as value of tmsServiceKey parameter.`,
PreRunE: func(cmd *cobra.Command, _ []string) error {
startTime = time.Now()
log.SetStepName(STEP_NAME)
log.SetVerbose(GeneralConfig.Verbose)
GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens)
path, _ := os.Getwd()
fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path}
log.RegisterHook(fatalHook)
err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile)
if err != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return err
}
log.RegisterSecret(stepConfig.TmsServiceKey)
if len(GeneralConfig.HookConfig.SentryConfig.Dsn) > 0 {
sentryHook := log.NewSentryHook(GeneralConfig.HookConfig.SentryConfig.Dsn, GeneralConfig.CorrelationID)
log.RegisterHook(&sentryHook)
}
if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 {
splunkClient = &splunk.Splunk{}
logCollector = &log.CollectorHook{CorrelationID: GeneralConfig.CorrelationID}
log.RegisterHook(logCollector)
}
if err = log.RegisterANSHookIfConfigured(GeneralConfig.CorrelationID); err != nil {
log.Entry().WithError(err).Warn("failed to set up SAP Alert Notification Service log hook")
}
validation, err := validation.New(validation.WithJSONNamesForStructFields(), validation.WithPredefinedErrorMessages())
if err != nil {
return err
}
if err = validation.ValidateStruct(stepConfig); err != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return err
}
return nil
},
Run: func(_ *cobra.Command, _ []string) {
stepTelemetryData := telemetry.CustomData{}
stepTelemetryData.ErrorCode = "1"
handler := func() {
influx.persist(GeneralConfig.EnvRootPath, "influx")
config.RemoveVaultSecretFiles()
stepTelemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds())
stepTelemetryData.ErrorCategory = log.GetErrorCategory().String()
stepTelemetryData.PiperCommitHash = GitCommit
telemetryClient.SetData(&stepTelemetryData)
telemetryClient.Send()
if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 {
splunkClient.Send(telemetryClient.GetData(), logCollector)
}
}
log.DeferExitHandler(handler)
defer handler()
telemetryClient.Initialize(GeneralConfig.NoTelemetry, STEP_NAME)
if len(GeneralConfig.HookConfig.SplunkConfig.Dsn) > 0 {
splunkClient.Initialize(GeneralConfig.CorrelationID,
GeneralConfig.HookConfig.SplunkConfig.Dsn,
GeneralConfig.HookConfig.SplunkConfig.Token,
GeneralConfig.HookConfig.SplunkConfig.Index,
GeneralConfig.HookConfig.SplunkConfig.SendLogs)
}
tmsExport(stepConfig, &stepTelemetryData, &influx)
stepTelemetryData.ErrorCode = "0"
log.Entry().Info("SUCCESS")
},
}
addTmsExportFlags(createTmsExportCmd, &stepConfig)
return createTmsExportCmd
}
func addTmsExportFlags(cmd *cobra.Command, stepConfig *tmsExportOptions) {
cmd.Flags().StringVar(&stepConfig.TmsServiceKey, "tmsServiceKey", os.Getenv("PIPER_tmsServiceKey"), "Service key JSON string to access the SAP Cloud Transport Management service instance APIs. If not specified and if pipeline is running on Jenkins, service key, stored under ID provided with credentialsId parameter, is used.")
cmd.Flags().StringVar(&stepConfig.CustomDescription, "customDescription", os.Getenv("PIPER_customDescription"), "Can be used as the description of a transport request. Will overwrite the default, which is corresponding Git commit ID.")
cmd.Flags().StringVar(&stepConfig.NamedUser, "namedUser", `Piper-Pipeline`, "Defines the named user to execute transport request with. The default value is 'Piper-Pipeline'. If pipeline is running on Jenkins, the name of the user, who started the job, is tried to be used at first.")
cmd.Flags().StringVar(&stepConfig.NodeName, "nodeName", os.Getenv("PIPER_nodeName"), "Defines the name of the export node - starting node in TMS landscape. The transport request is added to the queues of the follow-on nodes of export node.")
cmd.Flags().StringVar(&stepConfig.MtaPath, "mtaPath", os.Getenv("PIPER_mtaPath"), "Defines the relative path to *.mtar file for the export to the SAP Cloud Transport Management service. If not specified, it will use the *.mtar file created in mtaBuild.")
cmd.Flags().StringVar(&stepConfig.MtaVersion, "mtaVersion", `*`, "Defines the version of the MTA for which the MTA extension descriptor will be used. You can use an asterisk (*) to accept any MTA version, or use a specific version compliant with SemVer 2.0, e.g. 1.0.0 (see semver.org). If the parameter is not configured, an asterisk is used.")
cmd.Flags().StringVar(&stepConfig.Proxy, "proxy", os.Getenv("PIPER_proxy"), "Proxy URL which should be used for communication with the SAP Cloud Transport Management service backend.")
cmd.MarkFlagRequired("tmsServiceKey")
cmd.MarkFlagRequired("nodeName")
}
// retrieve step metadata
func tmsExportMetadata() config.StepData {
var theMetaData = config.StepData{
Metadata: config.StepMetadata{
Name: "tmsExport",
Aliases: []config.Alias{},
Description: "This step allows you to export an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape.",
},
Spec: config.StepSpec{
Inputs: config.StepInputs{
Secrets: []config.StepSecrets{
{Name: "credentialsId", Description: "Jenkins 'Secret text' credentials ID containing service key for SAP Cloud Transport Management service.", Type: "jenkins"},
},
Resources: []config.StepResources{
{Name: "buildResult", Type: "stash"},
},
Parameters: []config.StepParameters{
{
Name: "tmsServiceKey",
ResourceRef: []config.ResourceReference{
{
Name: "credentialsId",
Param: "tmsServiceKey",
Type: "secret",
},
},
Scope: []string{"PARAMETERS", "STEPS", "STAGES"},
Type: "string",
Mandatory: true,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_tmsServiceKey"),
},
{
Name: "customDescription",
ResourceRef: []config.ResourceReference{
{
Name: "commonPipelineEnvironment",
Param: "git/commitId",
},
},
Scope: []string{"PARAMETERS", "STEPS", "STAGES"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_customDescription"),
},
{
Name: "namedUser",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STEPS", "STAGES"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Default: `Piper-Pipeline`,
},
{
Name: "nodeName",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STEPS", "STAGES"},
Type: "string",
Mandatory: true,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_nodeName"),
},
{
Name: "mtaPath",
ResourceRef: []config.ResourceReference{
{
Name: "commonPipelineEnvironment",
Param: "mtarFilePath",
},
},
Scope: []string{"PARAMETERS", "STEPS", "STAGES"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_mtaPath"),
},
{
Name: "mtaVersion",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STEPS", "STAGES"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Default: `*`,
},
{
Name: "nodeExtDescriptorMapping",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STEPS", "STAGES"},
Type: "map[string]interface{}",
Mandatory: false,
Aliases: []config.Alias{},
},
{
Name: "proxy",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STEPS", "STAGES"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
Default: os.Getenv("PIPER_proxy"),
},
},
},
Outputs: config.StepOutputs{
Resources: []config.StepResources{
{
Name: "influx",
Type: "influx",
Parameters: []map[string]interface{}{
{"name": "step_data", "fields": []map[string]string{{"name": "tms"}}},
},
},
},
},
},
}
return theMetaData
}

17
cmd/tmsExport_generated_test.go Normal file
View File

@ -0,0 +1,17 @@
package cmd
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestTmsExportCommand(t *testing.T) {
t.Parallel()
testCmd := TmsExportCommand()
// only high level testing performed - details are tested in step generation procedure
assert.Equal(t, "tmsExport", testCmd.Use, "command name incorrect")
}

148
cmd/tmsExport_test.go Normal file
View File

@ -0,0 +1,148 @@
package cmd
import (
"os"
"strconv"
"testing"
"github.com/SAP/jenkins-library/pkg/mock"
"github.com/SAP/jenkins-library/pkg/tms"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
)
type tmsExportMockUtils struct {
*mock.ExecMockRunner
*mock.FilesMock
}
func newTmsExportTestsUtils() tmsExportMockUtils {
utils := tmsExportMockUtils{
ExecMockRunner: &mock.ExecMockRunner{},
FilesMock: &mock.FilesMock{},
}
return utils
}
func (cim *communicationInstanceMock) ExportFileToNode(nodeName, fileId, description, namedUser string) (tms.NodeUploadResponseEntity, error) {
var nodeUploadResponseEntity tms.NodeUploadResponseEntity
if description != CUSTOM_DESCRIPTION || nodeName != NODE_NAME || fileId != strconv.FormatInt(FILE_ID, 10) || namedUser != NAMED_USER {
return nodeUploadResponseEntity, errors.New(INVALID_INPUT_MSG)
}
if cim.isErrorOnExportFileToNode {
return nodeUploadResponseEntity, errors.New("Something went wrong on exporting file to node")
} else {
return cim.exportFileToNodeResponse, nil
}
}
func TestRunTmsExport(t *testing.T) {
t.Parallel()
t.Run("happy path: 1. get nodes 2. get MTA ext descriptor -> nothing obtained 3. upload MTA ext descriptor to node 4. upload file 5. export file to node", func(t *testing.T) {
t.Parallel()
// init
nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}}
fileInfo := tms.FileInfo{Id: FILE_ID, Name: MTA_NAME}
communicationInstance := communicationInstanceMock{getNodesResponse: nodes, uploadFileResponse: fileInfo}
utils := newTmsTestsUtils()
utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content"))
mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH)
utils.AddFile(MTA_YAML_PATH_LOCAL, mtaYamlBytes)
mtaExtDescriptorBytes, _ := os.ReadFile(MTA_EXT_DESCRIPTOR_PATH)
utils.AddFile(MTA_EXT_DESCRIPTOR_PATH_LOCAL, mtaExtDescriptorBytes)
nodeNameExtDescriptorMapping := map[string]interface{}{NODE_NAME: MTA_EXT_DESCRIPTOR_PATH_LOCAL}
config := tmsExportOptions{MtaPath: MTA_PATH_LOCAL, CustomDescription: CUSTOM_DESCRIPTION, NamedUser: NAMED_USER, NodeName: NODE_NAME, MtaVersion: MTA_VERSION, NodeExtDescriptorMapping: nodeNameExtDescriptorMapping}
// test
err := runTmsExport(config, &communicationInstance, utils)
// assert
assert.NoError(t, err)
})
t.Run("error path: error while uploading file", func(t *testing.T) {
t.Parallel()
// init
nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}}
communicationInstance := communicationInstanceMock{getNodesResponse: nodes, isErrorOnUploadFile: true}
utils := newTmsTestsUtils()
utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content"))
mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH)
utils.AddFile(MTA_YAML_PATH_LOCAL, mtaYamlBytes)
mtaExtDescriptorBytes, _ := os.ReadFile(MTA_EXT_DESCRIPTOR_PATH)
utils.AddFile(MTA_EXT_DESCRIPTOR_PATH_LOCAL, mtaExtDescriptorBytes)
nodeNameExtDescriptorMapping := map[string]interface{}{NODE_NAME: MTA_EXT_DESCRIPTOR_PATH_LOCAL}
config := tmsExportOptions{MtaPath: MTA_PATH_LOCAL, CustomDescription: CUSTOM_DESCRIPTION, NamedUser: NAMED_USER, NodeName: NODE_NAME, MtaVersion: MTA_VERSION, NodeExtDescriptorMapping: nodeNameExtDescriptorMapping}
// test
err := runTmsExport(config, &communicationInstance, utils)
// assert
assert.EqualError(t, err, "failed to upload file: Something went wrong on uploading file")
})
t.Run("error path: error while uploading MTA extension descriptor to node", func(t *testing.T) {
t.Parallel()
// init
nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}}
communicationInstance := communicationInstanceMock{getNodesResponse: nodes, isErrorOnUploadMtaExtDescriptorToNode: true}
utils := newTmsTestsUtils()
utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content"))
mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH)
utils.AddFile(MTA_YAML_PATH_LOCAL, mtaYamlBytes)
mtaExtDescriptorBytes, _ := os.ReadFile(MTA_EXT_DESCRIPTOR_PATH)
utils.AddFile(MTA_EXT_DESCRIPTOR_PATH_LOCAL, mtaExtDescriptorBytes)
nodeNameExtDescriptorMapping := map[string]interface{}{NODE_NAME: MTA_EXT_DESCRIPTOR_PATH_LOCAL}
config := tmsExportOptions{MtaPath: MTA_PATH_LOCAL, CustomDescription: CUSTOM_DESCRIPTION, NamedUser: NAMED_USER, NodeName: NODE_NAME, MtaVersion: MTA_VERSION, NodeExtDescriptorMapping: nodeNameExtDescriptorMapping}
// test
err := runTmsExport(config, &communicationInstance, utils)
// assert
assert.EqualError(t, err, "failed to upload MTA extension descriptor to node: Something went wrong on uploading MTA extension descriptor to node")
})
t.Run("error path: error while exporting file to node", func(t *testing.T) {
t.Parallel()
// init
nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}}
fileInfo := tms.FileInfo{Id: FILE_ID, Name: MTA_NAME}
communicationInstance := communicationInstanceMock{getNodesResponse: nodes, uploadFileResponse: fileInfo, isErrorOnExportFileToNode: true}
utils := newTmsTestsUtils()
utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content"))
mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH)
utils.AddFile(MTA_YAML_PATH_LOCAL, mtaYamlBytes)
mtaExtDescriptorBytes, _ := os.ReadFile(MTA_EXT_DESCRIPTOR_PATH)
utils.AddFile(MTA_EXT_DESCRIPTOR_PATH_LOCAL, mtaExtDescriptorBytes)
nodeNameExtDescriptorMapping := map[string]interface{}{NODE_NAME: MTA_EXT_DESCRIPTOR_PATH_LOCAL}
config := tmsExportOptions{MtaPath: MTA_PATH_LOCAL, CustomDescription: CUSTOM_DESCRIPTION, NamedUser: NAMED_USER, NodeName: NODE_NAME, MtaVersion: MTA_VERSION, NodeExtDescriptorMapping: nodeNameExtDescriptorMapping}
// test
err := runTmsExport(config, &communicationInstance, utils)
// assert
assert.EqualError(t, err, "failed to export file to node: Something went wrong on exporting file to node")
})
}

295
cmd/tmsUpload.go
View File

@ -1,205 +1,37 @@
package cmd
import (
"encoding/json"
"fmt"
"net/url"
"sort"
"strconv"
"github.com/SAP/jenkins-library/pkg/command"
piperHttp "github.com/SAP/jenkins-library/pkg/http"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/SAP/jenkins-library/pkg/telemetry"
"github.com/SAP/jenkins-library/pkg/tms"
"github.com/ghodss/yaml"
"github.com/pkg/errors"
)
type uaa struct {
Url string `json:"url"`
ClientId string `json:"clientid"`
ClientSecret string `json:"clientsecret"`
}
func tmsUpload(uploadConfig tmsUploadOptions, telemetryData *telemetry.CustomData, influx *tmsUploadInflux) {
utils := tms.NewTmsUtils()
config := convertUploadOptions(uploadConfig)
communicationInstance := tms.SetupCommunication(config)
type serviceKey struct {
Uaa uaa `json:"uaa"`
Uri string `json:"uri"`
}
type tmsUploadUtils interface {
command.ExecRunner
FileExists(filename string) (bool, error)
FileRead(path string) ([]byte, error)
// Add more methods here, or embed additional interfaces, or remove/replace as required.
// The tmsUploadUtils interface should be descriptive of your runtime dependencies,
// i.e. include everything you need to be able to mock in tests.
// Unit tests shall be executable in parallel (not depend on global state), and don't (re-)test dependencies.
}
type tmsUploadUtilsBundle struct {
*command.Command
*piperutils.Files
// Embed more structs as necessary to implement methods or interfaces you add to tmsUploadUtils.
// Structs embedded in this way must each have a unique set of methods attached.
// If there is no struct which implements the method you need, attach the method to
// tmsUploadUtilsBundle and forward to the implementation of the dependency.
}
func newTmsUploadUtils() tmsUploadUtils {
utils := tmsUploadUtilsBundle{
Command: &command.Command{},
Files: &piperutils.Files{},
}
// Reroute command output to logging framework
utils.Stdout(log.Writer())
utils.Stderr(log.Writer())
return &utils
}
func tmsUpload(config tmsUploadOptions, telemetryData *telemetry.CustomData, influx *tmsUploadInflux) {
// Utils can be used wherever the command.ExecRunner interface is expected.
// It can also be used for example as a mavenExecRunner.
utils := newTmsUploadUtils()
client := &piperHttp.Client{}
proxy := config.Proxy
options := piperHttp.ClientOptions{}
if proxy != "" {
transportProxy, err := url.Parse(proxy)
if err != nil {
log.Entry().WithError(err).Fatalf("Failed to parse proxy string %v into a URL structure", proxy)
}
options = piperHttp.ClientOptions{TransportProxy: transportProxy}
client.SetOptions(options)
if GeneralConfig.Verbose {
log.Entry().Infof("HTTP client instructed to use %v proxy", proxy)
}
}
serviceKey, err := unmarshalServiceKey(config.TmsServiceKey)
err := runTmsUpload(uploadConfig, communicationInstance, utils)
if err != nil {
log.Entry().WithError(err).Fatal("Failed to unmarshal TMS service key")
}
log.RegisterSecret(serviceKey.Uaa.ClientSecret)
if GeneralConfig.Verbose {
log.Entry().Info("Will be used for communication:")
log.Entry().Infof("- client id: %v", serviceKey.Uaa.ClientId)
log.Entry().Infof("- TMS URL: %v", serviceKey.Uri)
log.Entry().Infof("- UAA URL: %v", serviceKey.Uaa.Url)
}
communicationInstance, err := tms.NewCommunicationInstance(client, serviceKey.Uri, serviceKey.Uaa.Url, serviceKey.Uaa.ClientId, serviceKey.Uaa.ClientSecret, GeneralConfig.Verbose, options)
if err != nil {
log.Entry().WithError(err).Fatal("Failed to prepare client for talking with TMS")
}
if err := runTmsUpload(config, communicationInstance, utils); err != nil {
log.Entry().WithError(err).Fatal("Failed to run tmsUpload step")
}
}
func runTmsUpload(config tmsUploadOptions, communicationInstance tms.CommunicationInterface, utils tmsUploadUtils) error {
mtaPath := config.MtaPath
exists, _ := utils.FileExists(mtaPath)
if !exists {
log.SetErrorCategory(log.ErrorConfiguration)
return fmt.Errorf("mta file %s not found", mtaPath)
}
description := tms.DEFAULT_TR_DESCRIPTION
if config.CustomDescription != "" {
description = config.CustomDescription
}
namedUser := config.NamedUser
nodeName := config.NodeName
mtaVersion := config.MtaVersion
nodeNameExtDescriptorMapping := config.NodeExtDescriptorMapping
if GeneralConfig.Verbose {
log.Entry().Info("The step will use the following values:")
log.Entry().Infof("- description: %v", description)
if len(nodeNameExtDescriptorMapping) > 0 {
log.Entry().Infof("- mapping between node names and MTA extension descriptor file paths: %v", nodeNameExtDescriptorMapping)
}
log.Entry().Infof("- MTA path: %v", mtaPath)
log.Entry().Infof("- MTA version: %v", mtaVersion)
if namedUser != "" {
log.Entry().Infof("- named user: %v", namedUser)
}
log.Entry().Infof("- node name: %v", nodeName)
}
if len(nodeNameExtDescriptorMapping) > 0 {
nodes, errGetNodes := communicationInstance.GetNodes()
if errGetNodes != nil {
log.SetErrorCategory(log.ErrorService)
return fmt.Errorf("failed to get nodes: %w", errGetNodes)
}
mtaYamlMap, errGetMtaYamlAsMap := getYamlAsMap(utils, "mta.yaml")
if errGetMtaYamlAsMap != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return fmt.Errorf("failed to get mta.yaml as map: %w", errGetMtaYamlAsMap)
}
_, isIdParameterInMap := mtaYamlMap["ID"]
_, isVersionParameterInMap := mtaYamlMap["version"]
if !isIdParameterInMap || !isVersionParameterInMap {
var errorMessage string
if !isIdParameterInMap {
errorMessage += "parameter 'ID' is not found in mta.yaml\n"
}
if !isVersionParameterInMap {
errorMessage += "parameter 'version' is not found in mta.yaml\n"
}
log.SetErrorCategory(log.ErrorConfiguration)
return errors.New(errorMessage)
}
// validate the whole mapping and then throw errors together, so that user can get them after a single pipeline run
nodeIdExtDescriptorMapping, errGetNodeIdExtDescriptorMapping := formNodeIdExtDescriptorMappingWithValidation(utils, nodeNameExtDescriptorMapping, nodes, mtaYamlMap, mtaVersion)
if errGetNodeIdExtDescriptorMapping != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return errGetNodeIdExtDescriptorMapping
}
for nodeId, mtaExtDescriptorPath := range nodeIdExtDescriptorMapping {
obtainedMtaExtDescriptor, errGetMtaExtDescriptor := communicationInstance.GetMtaExtDescriptor(nodeId, fmt.Sprintf("%v", mtaYamlMap["ID"]), mtaVersion)
if errGetMtaExtDescriptor != nil {
log.SetErrorCategory(log.ErrorService)
return fmt.Errorf("failed to get MTA extension descriptor: %w", errGetMtaExtDescriptor)
}
if obtainedMtaExtDescriptor != (tms.MtaExtDescriptor{}) {
_, errUpdateMtaExtDescriptor := communicationInstance.UpdateMtaExtDescriptor(nodeId, obtainedMtaExtDescriptor.Id, mtaExtDescriptorPath, mtaVersion, description, namedUser)
if errUpdateMtaExtDescriptor != nil {
log.SetErrorCategory(log.ErrorService)
return fmt.Errorf("failed to update MTA extension descriptor: %w", errUpdateMtaExtDescriptor)
}
} else {
_, errUploadMtaExtDescriptor := communicationInstance.UploadMtaExtDescriptorToNode(nodeId, mtaExtDescriptorPath, mtaVersion, description, namedUser)
if errUploadMtaExtDescriptor != nil {
log.SetErrorCategory(log.ErrorService)
return fmt.Errorf("failed to upload MTA extension descriptor to node: %w", errUploadMtaExtDescriptor)
}
}
}
}
fileInfo, errUploadFile := communicationInstance.UploadFile(mtaPath, namedUser)
func runTmsUpload(uploadConfig tmsUploadOptions, communicationInstance tms.CommunicationInterface, utils tms.TmsUtils) error {
config := convertUploadOptions(uploadConfig)
fileId, errUploadFile := tms.UploadFile(config, communicationInstance, utils)
if errUploadFile != nil {
log.SetErrorCategory(log.ErrorService)
return fmt.Errorf("failed to upload file: %w", errUploadFile)
return errUploadFile
}
_, errUploadFileToNode := communicationInstance.UploadFileToNode(nodeName, strconv.FormatInt(fileInfo.Id, 10), description, namedUser)
errUploadDescriptors := tms.UploadDescriptors(config, communicationInstance, utils)
if errUploadDescriptors != nil {
return errUploadDescriptors
}
_, errUploadFileToNode := communicationInstance.UploadFileToNode(config.NodeName, fileId, config.CustomDescription, config.NamedUser)
if errUploadFileToNode != nil {
log.SetErrorCategory(log.ErrorService)
return fmt.Errorf("failed to upload file to node: %w", errUploadFileToNode)
@ -208,87 +40,20 @@ func runTmsUpload(config tmsUploadOptions, communicationInstance tms.Communicati
return nil
}
func formNodeIdExtDescriptorMappingWithValidation(utils tmsUploadUtils, nodeNameExtDescriptorMapping map[string]interface{}, nodes []tms.Node, mtaYamlMap map[string]interface{}, mtaVersion string) (map[int64]string, error) {
var wrongMtaIdExtDescriptors []string
var wrongExtDescriptorPaths []string
var wrongNodeNames []string
var errorMessage string
nodeIdExtDescriptorMapping := make(map[int64]string)
for nodeName, mappedValue := range nodeNameExtDescriptorMapping {
mappedValueString := fmt.Sprintf("%v", mappedValue)
exists, _ := utils.FileExists(mappedValueString)
if exists {
extDescriptorMap, errGetYamlAsMap := getYamlAsMap(utils, mappedValueString)
if errGetYamlAsMap == nil {
if fmt.Sprintf("%v", mtaYamlMap["ID"]) != fmt.Sprintf("%v", extDescriptorMap["extends"]) {
wrongMtaIdExtDescriptors = append(wrongMtaIdExtDescriptors, mappedValueString)
}
} else {
wrappedErr := errors.Wrapf(errGetYamlAsMap, "tried to parse %v as yaml, but got an error", mappedValueString)
errorMessage += fmt.Sprintf("%v\n", wrappedErr)
}
} else {
wrongExtDescriptorPaths = append(wrongExtDescriptorPaths, mappedValueString)
}
isNodeFound := false
for _, node := range nodes {
if node.Name == nodeName {
nodeIdExtDescriptorMapping[node.Id] = mappedValueString
isNodeFound = true
break
}
}
if !isNodeFound {
wrongNodeNames = append(wrongNodeNames, nodeName)
}
}
if mtaVersion != "*" && mtaVersion != mtaYamlMap["version"] {
errorMessage += "parameter 'mtaVersion' does not match the MTA version in mta.yaml\n"
}
if len(wrongMtaIdExtDescriptors) > 0 || len(wrongExtDescriptorPaths) > 0 || len(wrongNodeNames) > 0 {
if len(wrongMtaIdExtDescriptors) > 0 {
sort.Strings(wrongMtaIdExtDescriptors)
errorMessage += fmt.Sprintf("parameter 'extends' in MTA extension descriptor files %v is not the same as MTA ID or is missing at all\n", wrongMtaIdExtDescriptors)
}
if len(wrongExtDescriptorPaths) > 0 {
sort.Strings(wrongExtDescriptorPaths)
errorMessage += fmt.Sprintf("MTA extension descriptor files %v do not exist\n", wrongExtDescriptorPaths)
}
if len(wrongNodeNames) > 0 {
sort.Strings(wrongNodeNames)
errorMessage += fmt.Sprintf("nodes %v do not exist. Please check node names provided in 'nodeExtDescriptorMapping' parameter or create these nodes\n", wrongNodeNames)
}
}
if errorMessage == "" {
return nodeIdExtDescriptorMapping, nil
} else {
return nil, errors.New(errorMessage)
func convertUploadOptions(uploadConfig tmsUploadOptions) tms.Options {
var config tms.Options
config.TmsServiceKey = uploadConfig.TmsServiceKey
config.CustomDescription = uploadConfig.CustomDescription
if config.CustomDescription == "" {
config.CustomDescription = tms.DEFAULT_TR_DESCRIPTION
}
}
func getYamlAsMap(utils tmsUploadUtils, yamlPath string) (map[string]interface{}, error) {
var result map[string]interface{}
bytes, err := utils.FileRead(yamlPath)
if err != nil {
return result, err
}
err = yaml.Unmarshal(bytes, &result)
if err != nil {
return result, err
}
return result, nil
}
func unmarshalServiceKey(serviceKeyJson string) (serviceKey serviceKey, err error) {
err = json.Unmarshal([]byte(serviceKeyJson), &serviceKey)
if err != nil {
return
}
return
config.NamedUser = uploadConfig.NamedUser
config.NodeName = uploadConfig.NodeName
config.MtaPath = uploadConfig.MtaPath
config.MtaVersion = uploadConfig.MtaVersion
config.NodeExtDescriptorMapping = uploadConfig.NodeExtDescriptorMapping
config.Proxy = uploadConfig.Proxy
config.StashContent = uploadConfig.StashContent
config.Verbose = GeneralConfig.Verbose
return config
}

2
cmd/tmsUpload_generated.go
View File

@ -77,7 +77,7 @@ func TmsUploadCommand() *cobra.Command {
var createTmsUploadCmd = &cobra.Command{
Use: STEP_NAME,
Short: "This step allows you to upload an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape.",
Long: `This step allows you to upload an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape.
Long: `This step allows you to upload an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape. The MTA file is attached to a new transport request which is added directly to the import queue of the specified transport node.
TMS lets you manage transports between SAP Business Technology Platform accounts in Neo and Cloud Foundry, such as from DEV to TEST and PROD accounts.
For more information, see [official documentation of SAP Cloud Transport Management service](https://help.sap.com/viewer/p/TRANSPORT_MANAGEMENT_SERVICE)

47
cmd/tmsUpload_test.go
View File

@ -1,6 +1,7 @@
package cmd
import (
"encoding/json"
"fmt"
"os"
"strconv"
@ -39,13 +40,13 @@ const WRONG_MTA_VERSION = "3.2.1"
const LAST_CHANGED_AT = "2021-11-16T13:06:05.711Z"
const INVALID_INPUT_MSG = "Invalid input parameter(s) when getting MTA extension descriptor"
type tmsUploadMockUtils struct {
type tmsMockUtils struct {
*mock.ExecMockRunner
*mock.FilesMock
}
func newTmsUploadTestsUtils() tmsUploadMockUtils {
utils := tmsUploadMockUtils{
func newTmsTestsUtils() tmsMockUtils {
utils := tmsMockUtils{
ExecMockRunner: &mock.ExecMockRunner{},
FilesMock: &mock.FilesMock{},
}
@ -59,12 +60,14 @@ type communicationInstanceMock struct {
uploadMtaExtDescriptorToNodeResponse tms.MtaExtDescriptor
uploadFileResponse tms.FileInfo
uploadFileToNodeResponse tms.NodeUploadResponseEntity
exportFileToNodeResponse tms.NodeUploadResponseEntity
isErrorOnGetNodes bool
isErrorOnGetMtaExtDescriptor bool
isErrorOnUpdateMtaExtDescriptor bool
isErrorOnUploadMtaExtDescriptorToNode bool
isErrorOnUploadFile bool
isErrorOnUploadFileToNode bool
isErrorOnExportFileToNode bool
}
func (cim *communicationInstanceMock) GetNodes() ([]tms.Node, error) {
@ -141,6 +144,14 @@ func (cim *communicationInstanceMock) UploadFileToNode(nodeName, fileId, descrip
}
}
func mapToJson(m map[string]interface{}) (string, error) {
b, err := json.Marshal(m)
if err != nil {
return "", err
}
return string(b), nil
}
func TestRunTmsUpload(t *testing.T) {
t.Parallel()
@ -152,7 +163,7 @@ func TestRunTmsUpload(t *testing.T) {
fileInfo := tms.FileInfo{Id: FILE_ID, Name: MTA_NAME}
communicationInstance := communicationInstanceMock{getNodesResponse: nodes, uploadFileResponse: fileInfo}
utils := newTmsUploadTestsUtils()
utils := newTmsTestsUtils()
utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content"))
mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH)
@ -171,14 +182,14 @@ func TestRunTmsUpload(t *testing.T) {
assert.NoError(t, err)
})
t.Run("happy path: no mapping between node nmaes and MTA extension descriptors is provided -> only upload file and upload file to node calls will be executed", func(t *testing.T) {
t.Run("happy path: no mapping between node names and MTA extension descriptors is provided -> only upload file and upload file to node calls will be executed", func(t *testing.T) {
t.Parallel()
// init
fileInfo := tms.FileInfo{Id: FILE_ID, Name: MTA_NAME}
communicationInstance := communicationInstanceMock{uploadFileResponse: fileInfo}
utils := newTmsUploadTestsUtils()
utils := newTmsTestsUtils()
utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content"))
config := tmsUploadOptions{MtaPath: MTA_PATH_LOCAL, CustomDescription: CUSTOM_DESCRIPTION, NamedUser: NAMED_USER, NodeName: NODE_NAME, MtaVersion: MTA_VERSION}
@ -199,7 +210,7 @@ func TestRunTmsUpload(t *testing.T) {
fileInfo := tms.FileInfo{Id: FILE_ID, Name: MTA_NAME}
communicationInstance := communicationInstanceMock{getNodesResponse: nodes, getMtaExtDescriptorResponse: mtaExtDescriptor, uploadFileResponse: fileInfo}
utils := newTmsUploadTestsUtils()
utils := newTmsTestsUtils()
utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content"))
mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH)
@ -223,7 +234,7 @@ func TestRunTmsUpload(t *testing.T) {
// init
communicationInstance := communicationInstanceMock{}
utils := newTmsUploadTestsUtils()
utils := newTmsTestsUtils()
nodeNameExtDescriptorMapping := map[string]interface{}{NODE_NAME: MTA_EXT_DESCRIPTOR_PATH_LOCAL}
config := tmsUploadOptions{MtaPath: MTA_PATH_LOCAL, CustomDescription: CUSTOM_DESCRIPTION, NamedUser: NAMED_USER, NodeName: NODE_NAME, MtaVersion: MTA_VERSION, NodeExtDescriptorMapping: nodeNameExtDescriptorMapping}
@ -240,7 +251,7 @@ func TestRunTmsUpload(t *testing.T) {
// init
communicationInstance := communicationInstanceMock{isErrorOnGetNodes: true}
utils := newTmsUploadTestsUtils()
utils := newTmsTestsUtils()
utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content"))
nodeNameExtDescriptorMapping := map[string]interface{}{NODE_NAME: MTA_EXT_DESCRIPTOR_PATH_LOCAL}
@ -259,7 +270,7 @@ func TestRunTmsUpload(t *testing.T) {
// init
nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}}
communicationInstance := communicationInstanceMock{getNodesResponse: nodes}
utils := newTmsUploadTestsUtils()
utils := newTmsTestsUtils()
utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content"))
nodeNameExtDescriptorMapping := map[string]interface{}{NODE_NAME: MTA_EXT_DESCRIPTOR_PATH_LOCAL}
@ -278,7 +289,7 @@ func TestRunTmsUpload(t *testing.T) {
// init
nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}}
communicationInstance := communicationInstanceMock{getNodesResponse: nodes}
utils := newTmsUploadTestsUtils()
utils := newTmsTestsUtils()
utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content"))
mtaYamlBytes, _ := os.ReadFile(INVALID_MTA_YAML_PATH)
@ -300,7 +311,7 @@ func TestRunTmsUpload(t *testing.T) {
// init
nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}}
communicationInstance := communicationInstanceMock{getNodesResponse: nodes}
utils := newTmsUploadTestsUtils()
utils := newTmsTestsUtils()
utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content"))
mtaYamlBytes, _ := os.ReadFile(INVALID_MTA_YAML_PATH_2)
@ -326,7 +337,7 @@ func TestRunTmsUpload(t *testing.T) {
// init
nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}}
communicationInstance := communicationInstanceMock{getNodesResponse: nodes}
utils := newTmsUploadTestsUtils()
utils := newTmsTestsUtils()
utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content"))
mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH)
@ -366,7 +377,7 @@ func TestRunTmsUpload(t *testing.T) {
// init
nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}}
communicationInstance := communicationInstanceMock{getNodesResponse: nodes, isErrorOnGetMtaExtDescriptor: true}
utils := newTmsUploadTestsUtils()
utils := newTmsTestsUtils()
utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content"))
mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH)
@ -393,7 +404,7 @@ func TestRunTmsUpload(t *testing.T) {
mtaExtDescriptor := tms.MtaExtDescriptor{Id: ID_OF_MTA_EXT_DESCRIPTOR, Description: "Some existing description", MtaId: MTA_ID, MtaExtId: MTA_EXT_ID, MtaVersion: MTA_VERSION, LastChangedAt: LAST_CHANGED_AT}
communicationInstance := communicationInstanceMock{getNodesResponse: nodes, getMtaExtDescriptorResponse: mtaExtDescriptor, isErrorOnUpdateMtaExtDescriptor: true}
utils := newTmsUploadTestsUtils()
utils := newTmsTestsUtils()
utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content"))
mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH)
@ -419,7 +430,7 @@ func TestRunTmsUpload(t *testing.T) {
nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}}
communicationInstance := communicationInstanceMock{getNodesResponse: nodes, isErrorOnUploadMtaExtDescriptorToNode: true}
utils := newTmsUploadTestsUtils()
utils := newTmsTestsUtils()
utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content"))
mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH)
@ -445,7 +456,7 @@ func TestRunTmsUpload(t *testing.T) {
nodes := []tms.Node{{Id: NODE_ID, Name: NODE_NAME}}
communicationInstance := communicationInstanceMock{getNodesResponse: nodes, isErrorOnUploadFile: true}
utils := newTmsUploadTestsUtils()
utils := newTmsTestsUtils()
utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content"))
mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH)
@ -472,7 +483,7 @@ func TestRunTmsUpload(t *testing.T) {
fileInfo := tms.FileInfo{Id: FILE_ID, Name: MTA_NAME}
communicationInstance := communicationInstanceMock{getNodesResponse: nodes, uploadFileResponse: fileInfo, isErrorOnUploadFileToNode: true}
utils := newTmsUploadTestsUtils()
utils := newTmsTestsUtils()
utils.AddFile(MTA_PATH_LOCAL, []byte("dummy content"))
mtaYamlBytes, _ := os.ReadFile(MTA_YAML_PATH)

9
documentation/docs/steps/tmsExport.md Normal file
View File

@ -0,0 +1,9 @@
# ${docGenStepName}
## ${docGenDescription}
## ${docGenParameters}
## ${docGenConfiguration}
## ${docJenkinsPluginDependencies}

1
documentation/mkdocs.yml
View File

@ -168,6 +168,7 @@ nav:
- spinnakerTriggerPipeline: steps/spinnakerTriggerPipeline.md
- testsPublishResults: steps/testsPublishResults.md
- tmsUpload: steps/tmsUpload.md
- tmsExport: steps/tmsExport.md
- transportRequestDocIDFromGit: steps/transportRequestDocIDFromGit.md
- transportRequestReqIDFromGit: steps/transportRequestReqIDFromGit.md
- transportRequestUploadCTS: steps/transportRequestUploadCTS.md

1
integration/github_actions_integration_test_list.yml
View File

@ -20,6 +20,7 @@ run:
- '"TestMTAIntegration"'
# - '"TestNexusIntegration"'
- '"TestTmsUploadIntegration"'
- '"TestTmsExportIntegration"'
# these are light-weighted tests, so we can use only one pod to reduce resource consumption
- '"Test(Gauge|GCS|GitHub|GitOps|Influx|NPM|Piper|Python|Sonar|Vault|Karma)Integration"'

56
integration/integration_tms_export_test.go Normal file
View File

@ -0,0 +1,56 @@
//go:build integration
// +build integration
// can be executed with
// go test -v -tags integration -run TestTmsExportIntegration ./integration/...
package main
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestTmsExportIntegrationYaml(t *testing.T) {
// success case: run with custom config
readEnv()
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
Image: "devxci/mbtci-java11-node14",
User: "root",
TestDir: []string{"testdata", "TestTmsIntegration"},
Environment: map[string]string{"PIPER_tmsServiceKey": tmsServiceKey},
})
defer container.terminate(t)
err := container.whenRunningPiperCommand("tmsExport", "--customConfig=.pipeline/export_config.yml")
if err != nil {
t.Fatalf("Piper command failed %s", err)
}
container.assertHasOutput(t, "tmsExport - File uploaded successfully")
container.assertHasOutput(t, "tmsExport - MTA extension descriptor updated successfully")
container.assertHasOutput(t, "tmsExport - Node export executed successfully")
container.assertHasOutput(t, "tmsExport - SUCCESS")
}
func TestTmsExportIntegrationBinFailDescription(t *testing.T) {
// error case: run cmd with invalid description
readEnv()
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
Image: "devxci/mbtci-java11-node14",
User: "root",
TestDir: []string{"testdata", "TestTmsIntegration"},
Environment: map[string]string{"PIPER_tmsServiceKey": tmsServiceKey},
})
defer container.terminate(t)
err := container.whenRunningPiperCommand("tmsExport",
"--mtaPath=scv_x.mtar",
"--nodeName=PIPER-TEST",
"--customDescription={Bad description}")
assert.Error(t, err, "Did expect error")
container.assertHasOutput(t, "error tmsExport - HTTP request failed with error")
container.assertHasOutput(t, "Failed to run tmsExport - failed to export file to node")
}

14
integration/integration_tms_upload_test.go
View File

@ -2,7 +2,7 @@
// +build integration
// can be executed with
// go test -v -tags integration -run TestTmsUploadIntegration ./integration
// go test -v -tags integration -run TestTmsIntegration ./integration
package main
@ -31,7 +31,7 @@ func TestTmsUploadIntegrationBinSuccess(t *testing.T) {
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
Image: "devxci/mbtci-java11-node14",
User: "root",
TestDir: []string{"testdata", "TestTmsUploadIntegration"},
TestDir: []string{"testdata", "TestTmsIntegration"},
Environment: map[string]string{"PIPER_tmsServiceKey": tmsServiceKey},
})
defer container.terminate(t)
@ -57,7 +57,7 @@ func TestTmsUploadIntegrationBinNoDescriptionSuccess(t *testing.T) {
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
Image: "devxci/mbtci-java11-node14",
User: "root",
TestDir: []string{"testdata", "TestTmsUploadIntegration"},
TestDir: []string{"testdata", "TestTmsIntegration"},
Environment: map[string]string{"PIPER_tmsServiceKey": tmsServiceKey},
})
defer container.terminate(t)
@ -82,7 +82,7 @@ func TestTmsUploadIntegrationBinFailParam(t *testing.T) {
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
Image: "devxci/mbtci-java11-node14",
User: "root",
TestDir: []string{"testdata", "TestTmsUploadIntegration"},
TestDir: []string{"testdata", "TestTmsIntegration"},
})
defer container.terminate(t)
@ -104,7 +104,7 @@ func TestTmsUploadIntegrationBinFailDescription(t *testing.T) {
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
Image: "devxci/mbtci-java11-node14",
User: "root",
TestDir: []string{"testdata", "TestTmsUploadIntegration"},
TestDir: []string{"testdata", "TestTmsIntegration"},
Environment: map[string]string{"PIPER_tmsServiceKey": tmsServiceKey},
})
defer container.terminate(t)
@ -125,12 +125,12 @@ func TestTmsUploadIntegrationYaml(t *testing.T) {
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
Image: "devxci/mbtci-java11-node14",
User: "root",
TestDir: []string{"testdata", "TestTmsUploadIntegration"},
TestDir: []string{"testdata", "TestTmsIntegration"},
Environment: map[string]string{"PIPER_tmsServiceKey": tmsServiceKey},
})
defer container.terminate(t)
err := container.whenRunningPiperCommand("tmsUpload", "--customConfig=.pipeline/tms_integration_test_config.yml")
err := container.whenRunningPiperCommand("tmsUpload", "--customConfig=.pipeline/upload_config.yml")
if err != nil {
t.Fatalf("Piper command failed %s", err)
}

12
integration/testdata/TestTmsIntegration/.pipeline/export_config.yml vendored Normal file
View File

@ -0,0 +1,12 @@
general:
verbose: true
steps:
tmsExport:
nodeName: PIPER-TEST
namedUser: Piper-Export
mtaPath: scv_x.mtar
verbose: true
mtaVersion: 1.0.0
nodeExtDescriptorMapping:
PIPER-TEST: 'scv_x.mtaext'
PIPER-PROD: 'scv_x.mtaext'

1
integration/testdata/TestTmsUploadIntegration/.pipeline/tms_integration_test_config.yml → integration/testdata/TestTmsIntegration/.pipeline/upload_config.yml vendored
View File

@ -4,6 +4,7 @@ steps:
tmsUpload:
useGoStep: true
nodeName: PIPER-TEST
namedUser: Piper-Upload
mtaPath: scv_x.mtar
verbose: true
mtaVersion: 1.0.0

0
integration/testdata/TestTmsUploadIntegration/mta.yaml → integration/testdata/TestTmsIntegration/mta.yaml vendored
View File

0
integration/testdata/TestTmsUploadIntegration/scv_x.mtaext → integration/testdata/TestTmsIntegration/scv_x.mtaext vendored
View File

0
integration/testdata/TestTmsUploadIntegration/scv_x.mtar → integration/testdata/TestTmsIntegration/scv_x.mtar vendored
View File

126
pkg/tms/tms.go → pkg/tms/tmsClient.go
View File

@ -6,7 +6,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
@ -14,89 +13,8 @@ import (
piperHttp "github.com/SAP/jenkins-library/pkg/http"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/xsuaa"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
type AuthToken struct {
TokenType string `json:"token_type"`
AccessToken string `json:"access_token"`
ExpiresIn int `json:"expires_in"`
}
type CommunicationInstance struct {
tmsUrl string
uaaUrl string
clientId string
clientSecret string
httpClient piperHttp.Uploader
logger *logrus.Entry
isVerbose bool
}
type Node struct {
Id int64 `json:"id"`
Name string `json:"name"`
}
type nodes struct {
Nodes []Node `json:"nodes"`
}
type MtaExtDescriptor struct {
Id int64 `json:"id"`
Description string `json:"description"`
MtaId string `json:"mtaId"`
MtaExtId string `json:"mtaExtId"`
MtaVersion string `json:"mtaVersion"`
LastChangedAt string `json:"lastChangedAt"`
}
type mtaExtDescriptors struct {
MtaExtDescriptors []MtaExtDescriptor `json:"mtaExtDescriptors"`
}
type FileInfo struct {
Id int64 `json:"fileId"`
Name string `json:"fileName"`
}
type NodeUploadResponseEntity struct {
TransportRequestId int64 `json:"transportRequestId"`
TransportRequestDescription string `json:"transportRequestDescription"`
QueueEntries []QueueEntry `json:"queueEntries"`
}
type QueueEntry struct {
Id int64 `json:"queueId"`
NodeId int64 `json:"nodeId"`
NodeName string `json:"nodeName"`
}
type NodeUploadRequestEntity struct {
ContentType string `json:"contentType"`
StorageType string `json:"storageType"`
NodeName string `json:"nodeName"`
Description string `json:"description"`
NamedUser string `json:"namedUser"`
Entries []Entry `json:"entries"`
}
type Entry struct {
Uri string `json:"uri"`
}
type CommunicationInterface interface {
GetNodes() ([]Node, error)
GetMtaExtDescriptor(nodeId int64, mtaId, mtaVersion string) (MtaExtDescriptor, error)
UpdateMtaExtDescriptor(nodeId, idOfMtaExtDescriptor int64, file, mtaVersion, description, namedUser string) (MtaExtDescriptor, error)
UploadMtaExtDescriptorToNode(nodeId int64, file, mtaVersion, description, namedUser string) (MtaExtDescriptor, error)
UploadFile(file, namedUser string) (FileInfo, error)
UploadFileToNode(nodeName, fileId, description, namedUser string) (NodeUploadResponseEntity, error)
}
const (
DEFAULT_TR_DESCRIPTION = "Created by Piper"
)
// NewCommunicationInstance returns CommunicationInstance structure with http client prepared for communication with TMS backend
@ -147,7 +65,7 @@ func (communicationInstance *CommunicationInstance) getOAuthToken() (string, err
return "", err
}
var token AuthToken
var token xsuaa.AuthToken
json.Unmarshal(data, &token)
if communicationInstance.isVerbose {
@ -160,8 +78,8 @@ func (communicationInstance *CommunicationInstance) getOAuthToken() (string, err
func sendRequest(communicationInstance *CommunicationInstance, method, urlPathAndQuery string, body io.Reader, header http.Header, expectedStatusCode int, isTowardsUaa bool) ([]byte, error) {
var requestBody io.Reader
if body != nil {
closer := ioutil.NopCloser(body)
bodyBytes, _ := ioutil.ReadAll(closer)
closer := io.NopCloser(body)
bodyBytes, _ := io.ReadAll(closer)
requestBody = bytes.NewBuffer(bodyBytes)
defer closer.Close()
}
@ -185,7 +103,7 @@ func sendRequest(communicationInstance *CommunicationInstance, method, urlPathAn
return nil, fmt.Errorf("unexpected positive HTTP status code %v, while it was expected %v", response.StatusCode, expectedStatusCode)
}
data, _ := ioutil.ReadAll(response.Body)
data, _ := io.ReadAll(response.Body)
if !isTowardsUaa && communicationInstance.isVerbose {
communicationInstance.logger.Debugf("Valid response body: %v", string(data))
}
@ -195,7 +113,7 @@ func sendRequest(communicationInstance *CommunicationInstance, method, urlPathAn
func (communicationInstance *CommunicationInstance) logResponseBody(response *http.Response) {
if response != nil && response.Body != nil {
data, _ := ioutil.ReadAll(response.Body)
data, _ := io.ReadAll(response.Body)
communicationInstance.logger.Errorf("Response body: %s", data)
response.Body.Close()
}
@ -289,6 +207,36 @@ func (communicationInstance *CommunicationInstance) UploadFileToNode(nodeName, f
}
func (communicationInstance *CommunicationInstance) ExportFileToNode(nodeName, fileId, description, namedUser string) (NodeUploadResponseEntity, error) {
if communicationInstance.isVerbose {
communicationInstance.logger.Info("Node export started")
communicationInstance.logger.Infof("tmsUrl: %v, nodeName: %v, fileId: %v, description: %v, namedUser: %v", communicationInstance.tmsUrl, nodeName, fileId, description, namedUser)
}
header := http.Header{}
header.Add("Content-Type", "application/json")
var nodeUploadResponseEntity NodeUploadResponseEntity
entry := Entry{Uri: fileId}
body := NodeUploadRequestEntity{ContentType: "MTA", StorageType: "FILE", NodeName: nodeName, Description: description, NamedUser: namedUser, Entries: []Entry{entry}}
bodyBytes, errMarshaling := json.Marshal(body)
if errMarshaling != nil {
return nodeUploadResponseEntity, errors.Wrapf(errMarshaling, "unable to marshal request body %v", body)
}
data, errSendRequest := sendRequest(communicationInstance, http.MethodPost, "/v2/nodes/export", bytes.NewReader(bodyBytes), header, http.StatusOK, false)
if errSendRequest != nil {
return nodeUploadResponseEntity, errSendRequest
}
json.Unmarshal(data, &nodeUploadResponseEntity)
if communicationInstance.isVerbose {
communicationInstance.logger.Info("Node export executed successfully")
}
return nodeUploadResponseEntity, nil
}
func (communicationInstance *CommunicationInstance) UpdateMtaExtDescriptor(nodeId, idOfMtaExtDescriptor int64, file, mtaVersion, description, namedUser string) (MtaExtDescriptor, error) {
if communicationInstance.isVerbose {
communicationInstance.logger.Info("Update of MTA extension descriptor started")
@ -408,7 +356,7 @@ func upload(communicationInstance *CommunicationInstance, uploadRequestData pipe
return nil, fmt.Errorf("unexpected positive HTTP status code %v, while it was expected %v", response.StatusCode, expectedStatusCode)
}
data, _ := ioutil.ReadAll(response.Body)
data, _ := io.ReadAll(response.Body)
if communicationInstance.isVerbose {
communicationInstance.logger.Debugf("Valid response body: %v", string(data))
}

9
pkg/tms/tms_test.go → pkg/tms/tmsClient_test.go
View File

@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
@ -48,21 +47,21 @@ func (um *uploaderMock) SendRequest(method, url string, body io.Reader, header h
if um.httpStatusCode >= 300 {
httpError = fmt.Errorf("http error %v", um.httpStatusCode)
}
return &http.Response{StatusCode: um.httpStatusCode, Body: ioutil.NopCloser(strings.NewReader(um.responseBody))}, httpError
return &http.Response{StatusCode: um.httpStatusCode, Body: io.NopCloser(strings.NewReader(um.responseBody))}, httpError
}
func (um *uploaderMock) UploadFile(url, file, fieldName string, header http.Header, cookies []*http.Cookie, uploadType string) (*http.Response, error) {
um.httpMethod = http.MethodPost
um.urlCalled = url
um.header = header
return &http.Response{StatusCode: um.httpStatusCode, Body: ioutil.NopCloser(bytes.NewReader([]byte(um.responseBody)))}, nil
return &http.Response{StatusCode: um.httpStatusCode, Body: io.NopCloser(bytes.NewReader([]byte(um.responseBody)))}, nil
}
func (um *uploaderMock) UploadRequest(method, url, file, fieldName string, header http.Header, cookies []*http.Cookie, uploadType string) (*http.Response, error) {
um.httpMethod = http.MethodPost
um.urlCalled = url
um.header = header
return &http.Response{StatusCode: um.httpStatusCode, Body: ioutil.NopCloser(bytes.NewReader([]byte(um.responseBody)))}, nil
return &http.Response{StatusCode: um.httpStatusCode, Body: io.NopCloser(bytes.NewReader([]byte(um.responseBody)))}, nil
}
func (um *uploaderMock) Upload(uploadRequestData piperHttp.UploadRequestData) (*http.Response, error) {
@ -84,7 +83,7 @@ func (um *uploaderMock) Upload(uploadRequestData piperHttp.UploadRequestData) (*
if um.httpStatusCode >= 300 {
httpError = fmt.Errorf("http error %v", um.httpStatusCode)
}
return &http.Response{StatusCode: um.httpStatusCode, Body: ioutil.NopCloser(strings.NewReader(um.responseBody))}, httpError
return &http.Response{StatusCode: um.httpStatusCode, Body: io.NopCloser(strings.NewReader(um.responseBody))}, httpError
}
func (um *uploaderMock) SetOptions(options piperHttp.ClientOptions) {

358
pkg/tms/tmsUtils.go Normal file
View File

@ -0,0 +1,358 @@
package tms
import (
"encoding/json"
"fmt"
"net/url"
"sort"
"strconv"
"github.com/SAP/jenkins-library/pkg/command"
piperHttp "github.com/SAP/jenkins-library/pkg/http"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/ghodss/yaml"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
type TmsUtils interface {
command.ExecRunner
FileExists(filename string) (bool, error)
FileRead(path string) ([]byte, error)
}
type uaa struct {
Url string `json:"url"`
ClientId string `json:"clientid"`
ClientSecret string `json:"clientsecret"`
}
type serviceKey struct {
Uaa uaa `json:"uaa"`
Uri string `json:"uri"`
}
type CommunicationInstance struct {
tmsUrl string
uaaUrl string
clientId string
clientSecret string
httpClient piperHttp.Uploader
logger *logrus.Entry
isVerbose bool
}
type Node struct {
Id int64 `json:"id"`
Name string `json:"name"`
}
type nodes struct {
Nodes []Node `json:"nodes"`
}
type MtaExtDescriptor struct {
Id int64 `json:"id"`
Description string `json:"description"`
MtaId string `json:"mtaId"`
MtaExtId string `json:"mtaExtId"`
MtaVersion string `json:"mtaVersion"`
LastChangedAt string `json:"lastChangedAt"`
}
type mtaExtDescriptors struct {
MtaExtDescriptors []MtaExtDescriptor `json:"mtaExtDescriptors"`
}
type FileInfo struct {
Id int64 `json:"fileId"`
Name string `json:"fileName"`
}
type NodeUploadResponseEntity struct {
TransportRequestId int64 `json:"transportRequestId"`
TransportRequestDescription string `json:"transportRequestDescription"`
QueueEntries []QueueEntry `json:"queueEntries"`
}
type QueueEntry struct {
Id int64 `json:"queueId"`
NodeId int64 `json:"nodeId"`
NodeName string `json:"nodeName"`
}
type NodeUploadRequestEntity struct {
ContentType string `json:"contentType"`
StorageType string `json:"storageType"`
NodeName string `json:"nodeName"`
Description string `json:"description"`
NamedUser string `json:"namedUser"`
Entries []Entry `json:"entries"`
}
type Entry struct {
Uri string `json:"uri"`
}
type CommunicationInterface interface {
GetNodes() ([]Node, error)
GetMtaExtDescriptor(nodeId int64, mtaId, mtaVersion string) (MtaExtDescriptor, error)
UpdateMtaExtDescriptor(nodeId, idOfMtaExtDescriptor int64, file, mtaVersion, description, namedUser string) (MtaExtDescriptor, error)
UploadMtaExtDescriptorToNode(nodeId int64, file, mtaVersion, description, namedUser string) (MtaExtDescriptor, error)
UploadFile(file, namedUser string) (FileInfo, error)
UploadFileToNode(nodeName, fileId, description, namedUser string) (NodeUploadResponseEntity, error)
ExportFileToNode(nodeName, fileId, description, namedUser string) (NodeUploadResponseEntity, error)
}
type Options struct {
TmsServiceKey string `json:"tmsServiceKey,omitempty"`
CustomDescription string `json:"customDescription,omitempty"`
NamedUser string `json:"namedUser,omitempty"`
NodeName string `json:"nodeName,omitempty"`
MtaPath string `json:"mtaPath,omitempty"`
MtaVersion string `json:"mtaVersion,omitempty"`
NodeExtDescriptorMapping map[string]interface{} `json:"nodeExtDescriptorMapping,omitempty"`
Proxy string `json:"proxy,omitempty"`
StashContent []string `json:"stashContent,omitempty"`
Verbose bool
}
type tmsUtilsBundle struct {
*command.Command
*piperutils.Files
}
const DEFAULT_TR_DESCRIPTION = "Created by Piper"
func NewTmsUtils() TmsUtils {
utils := tmsUtilsBundle{
Command: &command.Command{},
Files: &piperutils.Files{},
}
// Reroute command output to logging framework
utils.Stdout(log.Writer())
utils.Stderr(log.Writer())
return &utils
}
func unmarshalServiceKey(serviceKeyJson string) (serviceKey serviceKey, err error) {
err = json.Unmarshal([]byte(serviceKeyJson), &serviceKey)
if err != nil {
return
}
return
}
func FormNodeIdExtDescriptorMappingWithValidation(utils TmsUtils, nodeNameExtDescriptorMapping map[string]interface{}, nodes []Node, mtaYamlMap map[string]interface{}, mtaVersion string) (map[int64]string, error) {
var wrongMtaIdExtDescriptors []string
var wrongExtDescriptorPaths []string
var wrongNodeNames []string
var errorMessage string
nodeIdExtDescriptorMapping := make(map[int64]string)
for nodeName, mappedValue := range nodeNameExtDescriptorMapping {
mappedValueString := fmt.Sprintf("%v", mappedValue)
exists, _ := utils.FileExists(mappedValueString)
if exists {
extDescriptorMap, errGetYamlAsMap := GetYamlAsMap(utils, mappedValueString)
if errGetYamlAsMap == nil {
if fmt.Sprintf("%v", mtaYamlMap["ID"]) != fmt.Sprintf("%v", extDescriptorMap["extends"]) {
wrongMtaIdExtDescriptors = append(wrongMtaIdExtDescriptors, mappedValueString)
}
} else {
wrappedErr := errors.Wrapf(errGetYamlAsMap, "tried to parse %v as yaml, but got an error", mappedValueString)
errorMessage += fmt.Sprintf("%v\n", wrappedErr)
}
} else {
wrongExtDescriptorPaths = append(wrongExtDescriptorPaths, mappedValueString)
}
isNodeFound := false
for _, node := range nodes {
if node.Name == nodeName {
nodeIdExtDescriptorMapping[node.Id] = mappedValueString
isNodeFound = true
break
}
}
if !isNodeFound {
wrongNodeNames = append(wrongNodeNames, nodeName)
}
}
if mtaVersion != "*" && mtaVersion != mtaYamlMap["version"] {
errorMessage += "parameter 'mtaVersion' does not match the MTA version in mta.yaml\n"
}
if len(wrongMtaIdExtDescriptors) > 0 || len(wrongExtDescriptorPaths) > 0 || len(wrongNodeNames) > 0 {
if len(wrongMtaIdExtDescriptors) > 0 {
sort.Strings(wrongMtaIdExtDescriptors)
errorMessage += fmt.Sprintf("parameter 'extends' in MTA extension descriptor files %v is not the same as MTA ID or is missing at all\n", wrongMtaIdExtDescriptors)
}
if len(wrongExtDescriptorPaths) > 0 {
sort.Strings(wrongExtDescriptorPaths)
errorMessage += fmt.Sprintf("MTA extension descriptor files %v do not exist\n", wrongExtDescriptorPaths)
}
if len(wrongNodeNames) > 0 {
sort.Strings(wrongNodeNames)
errorMessage += fmt.Sprintf("nodes %v do not exist. Please check node names provided in 'nodeExtDescriptorMapping' parameter or create these nodes\n", wrongNodeNames)
}
}
if errorMessage == "" {
return nodeIdExtDescriptorMapping, nil
} else {
return nil, errors.New(errorMessage)
}
}
func GetYamlAsMap(utils TmsUtils, yamlPath string) (map[string]interface{}, error) {
var result map[string]interface{}
bytes, err := utils.FileRead(yamlPath)
if err != nil {
return result, err
}
err = yaml.Unmarshal(bytes, &result)
if err != nil {
return result, err
}
return result, nil
}
func SetupCommunication(config Options) (communicationInstance CommunicationInterface) {
client := &piperHttp.Client{}
proxy := config.Proxy
options := piperHttp.ClientOptions{}
if proxy != "" {
transportProxy, err := url.Parse(proxy)
if err != nil {
log.Entry().WithError(err).Fatalf("Failed to parse proxy string %v into a URL structure", proxy)
}
options = piperHttp.ClientOptions{TransportProxy: transportProxy}
client.SetOptions(options)
if config.Verbose {
log.Entry().Infof("HTTP client instructed to use %v proxy", proxy)
}
}
serviceKey, err := unmarshalServiceKey(config.TmsServiceKey)
if err != nil {
log.Entry().WithError(err).Fatal("Failed to unmarshal TMS service key")
}
log.RegisterSecret(serviceKey.Uaa.ClientSecret)
if config.Verbose {
log.Entry().Info("Will be used for communication:")
log.Entry().Infof("- client id: %v", serviceKey.Uaa.ClientId)
log.Entry().Infof("- TMS URL: %v", serviceKey.Uri)
log.Entry().Infof("- UAA URL: %v", serviceKey.Uaa.Url)
}
commuInstance, err := NewCommunicationInstance(client, serviceKey.Uri, serviceKey.Uaa.Url, serviceKey.Uaa.ClientId, serviceKey.Uaa.ClientSecret, config.Verbose, options)
if err != nil {
log.Entry().WithError(err).Fatal("Failed to prepare client for talking with TMS")
}
return commuInstance
}
func UploadDescriptors(config Options, communicationInstance CommunicationInterface, utils TmsUtils) error {
description := config.CustomDescription
namedUser := config.NamedUser
nodeName := config.NodeName
mtaVersion := config.MtaVersion
nodeNameExtDescriptorMapping := config.NodeExtDescriptorMapping
mtaPath := config.MtaPath
if config.Verbose {
log.Entry().Info("The step will use the following values:")
log.Entry().Infof("- description: %v", config.CustomDescription)
if len(nodeNameExtDescriptorMapping) > 0 {
log.Entry().Infof("- mapping between node names and MTA extension descriptor file paths: %v", nodeNameExtDescriptorMapping)
}
log.Entry().Infof("- MTA path: %v", mtaPath)
log.Entry().Infof("- MTA version: %v", mtaVersion)
if namedUser != "" {
log.Entry().Infof("- named user: %v", namedUser)
}
log.Entry().Infof("- node name: %v", nodeName)
}
if len(nodeNameExtDescriptorMapping) > 0 {
nodes, errGetNodes := communicationInstance.GetNodes()
if errGetNodes != nil {
log.SetErrorCategory(log.ErrorService)
return fmt.Errorf("failed to get nodes: %w", errGetNodes)
}
mtaYamlMap, errGetMtaYamlAsMap := GetYamlAsMap(utils, "mta.yaml")
if errGetMtaYamlAsMap != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return fmt.Errorf("failed to get mta.yaml as map: %w", errGetMtaYamlAsMap)
}
_, isIdParameterInMap := mtaYamlMap["ID"]
_, isVersionParameterInMap := mtaYamlMap["version"]
if !isIdParameterInMap || !isVersionParameterInMap {
var errorMessage string
if !isIdParameterInMap {
errorMessage += "parameter 'ID' is not found in mta.yaml\n"
}
if !isVersionParameterInMap {
errorMessage += "parameter 'version' is not found in mta.yaml\n"
}
log.SetErrorCategory(log.ErrorConfiguration)
return errors.New(errorMessage)
}
// validate the whole mapping and then throw errors together, so that user can get them after a single pipeline run
nodeIdExtDescriptorMapping, errGetNodeIdExtDescriptorMapping := FormNodeIdExtDescriptorMappingWithValidation(utils, nodeNameExtDescriptorMapping, nodes, mtaYamlMap, mtaVersion)
if errGetNodeIdExtDescriptorMapping != nil {
log.SetErrorCategory(log.ErrorConfiguration)
return errGetNodeIdExtDescriptorMapping
}
for nodeId, mtaExtDescriptorPath := range nodeIdExtDescriptorMapping {
obtainedMtaExtDescriptor, errGetMtaExtDescriptor := communicationInstance.GetMtaExtDescriptor(nodeId, fmt.Sprintf("%v", mtaYamlMap["ID"]), mtaVersion)
if errGetMtaExtDescriptor != nil {
log.SetErrorCategory(log.ErrorService)
return fmt.Errorf("failed to get MTA extension descriptor: %w", errGetMtaExtDescriptor)
}
if obtainedMtaExtDescriptor != (MtaExtDescriptor{}) {
_, errUpdateMtaExtDescriptor := communicationInstance.UpdateMtaExtDescriptor(nodeId, obtainedMtaExtDescriptor.Id, mtaExtDescriptorPath, mtaVersion, description, namedUser)
if errUpdateMtaExtDescriptor != nil {
log.SetErrorCategory(log.ErrorService)
return fmt.Errorf("failed to update MTA extension descriptor: %w", errUpdateMtaExtDescriptor)
}
} else {
_, errUploadMtaExtDescriptor := communicationInstance.UploadMtaExtDescriptorToNode(nodeId, mtaExtDescriptorPath, mtaVersion, description, namedUser)
if errUploadMtaExtDescriptor != nil {
log.SetErrorCategory(log.ErrorService)
return fmt.Errorf("failed to upload MTA extension descriptor to node: %w", errUploadMtaExtDescriptor)
}
}
}
}
return nil
}
func UploadFile(config Options, communicationInstance CommunicationInterface, utils TmsUtils) (string, error) {
mtaPath := config.MtaPath
exists, _ := utils.FileExists(mtaPath)
if !exists {
log.SetErrorCategory(log.ErrorConfiguration)
return "", fmt.Errorf("mta file %s not found", mtaPath)
}
fileInfo, errUploadFile := communicationInstance.UploadFile(mtaPath, config.NamedUser)
if errUploadFile != nil {
log.SetErrorCategory(log.ErrorService)
return "", fmt.Errorf("failed to upload file: %w", errUploadFile)
}
fileId := strconv.FormatInt(fileInfo.Id, 10)
return fileId, nil
}

102
resources/metadata/tmsExport.yaml Normal file
View File

@ -0,0 +1,102 @@
metadata:
name: tmsExport
description: This step allows you to export an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape.
longDescription: |-
This step allows you to export an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape. The MTA file is attached to a new transport request which is added to the import queues of the follow-on transport nodes of the specified export node.
TMS lets you manage transports between SAP Business Technology Platform accounts in Neo and Cloud Foundry, such as from DEV to TEST and PROD accounts.
For more information, see [official documentation of SAP Cloud Transport Management service](https://help.sap.com/viewer/p/TRANSPORT_MANAGEMENT_SERVICE)
!!! note "Prerequisites"
* You have subscribed to and set up TMS, as described in [Initial Setup](https://help.sap.com/viewer/7f7160ec0d8546c6b3eab72fb5ad6fd8/Cloud/en-US/66fd7283c62f48adb23c56fb48c84a60.html), which includes the configuration of your transport landscape.
* A corresponding service key has been created, as described in [Set Up the Environment to Transport Content Archives directly in an Application](https://help.sap.com/viewer/7f7160ec0d8546c6b3eab72fb5ad6fd8/Cloud/en-US/8d9490792ed14f1bbf8a6ac08a6bca64.html). This service key (JSON) must be stored as a secret text within the Jenkins secure store or provided as value of tmsServiceKey parameter.
spec:
inputs:
secrets:
- name: credentialsId
description: Jenkins 'Secret text' credentials ID containing service key for SAP Cloud Transport Management service.
type: jenkins
resources:
- name: buildResult
type: stash
params:
- name: tmsServiceKey
type: string
description: Service key JSON string to access the SAP Cloud Transport Management service instance APIs. If not specified and if pipeline is running on Jenkins, service key, stored under ID provided with credentialsId parameter, is used.
scope:
- PARAMETERS
- STEPS
- STAGES
mandatory: true
secret: true
resourceRef:
- name: credentialsId
type: secret
param: tmsServiceKey
- name: customDescription
type: string
description: Can be used as the description of a transport request. Will overwrite the default, which is corresponding Git commit ID.
scope:
- PARAMETERS
- STEPS
- STAGES
resourceRef:
- name: commonPipelineEnvironment
param: git/commitId
- name: namedUser
type: string
description: Defines the named user to execute transport request with. The default value is 'Piper-Pipeline'. If pipeline is running on Jenkins, the name of the user, who started the job, is tried to be used at first.
default: Piper-Pipeline
scope:
- PARAMETERS
- STEPS
- STAGES
- name: nodeName
type: string
description: Defines the name of the export node - starting node in TMS landscape. The transport request is added to the queues of the follow-on nodes of export node.
scope:
- PARAMETERS
- STEPS
- STAGES
mandatory: true
- name: mtaPath
type: string
description: Defines the relative path to *.mtar file for the export to the SAP Cloud Transport Management service. If not specified, it will use the *.mtar file created in mtaBuild.
scope:
- PARAMETERS
- STEPS
- STAGES
resourceRef:
- name: commonPipelineEnvironment
param: mtarFilePath
- name: mtaVersion
type: string
description: Defines the version of the MTA for which the MTA extension descriptor will be used. You can use an asterisk (*) to accept any MTA version, or use a specific version compliant with SemVer 2.0, e.g. 1.0.0 (see semver.org). If the parameter is not configured, an asterisk is used.
default: "*"
scope:
- PARAMETERS
- STEPS
- STAGES
- name: nodeExtDescriptorMapping
type: map[string]interface{}
description: 'Available only for transports in Cloud Foundry environment. Defines a mapping between a transport node name and an MTA extension descriptor file path that you want to use for the transport node, e.g. nodeExtDescriptorMapping: {"nodeName": "example.mtaext", "nodeName2": "example2.mtaext"}.'
scope:
- PARAMETERS
- STEPS
- STAGES
- name: proxy
type: string
description: Proxy URL which should be used for communication with the SAP Cloud Transport Management service backend.
scope:
- PARAMETERS
- STEPS
- STAGES
outputs:
resources:
- name: influx
type: influx
params:
- name: step_data
fields:
- name: tms
type: bool

2
resources/metadata/tmsUpload.yaml
View File

@ -2,7 +2,7 @@ metadata:
name: tmsUpload
description: This step allows you to upload an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape.
longDescription: |-
This step allows you to upload an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape.
This step allows you to upload an MTA file (multi-target application archive) and multiple MTA extension descriptors into a TMS (SAP Cloud Transport Management service) landscape for further TMS-controlled distribution through a TMS-configured landscape. The MTA file is attached to a new transport request which is added directly to the import queue of the specified transport node.
TMS lets you manage transports between SAP Business Technology Platform accounts in Neo and Cloud Foundry, such as from DEV to TEST and PROD accounts.
For more information, see [official documentation of SAP Cloud Transport Management service](https://help.sap.com/viewer/p/TRANSPORT_MANAGEMENT_SERVICE)

2
test/groovy/CommonStepsTest.groovy
View File

@ -225,6 +225,8 @@ public class CommonStepsTest extends BasePiperTest{
'awsS3Upload',
'ansSendEvent',
'apiProviderList', //implementing new golang pattern without fields
'tmsUpload',
'tmsExport',
]
@Test

13
vars/tmsExport.groovy Normal file
View File

@ -0,0 +1,13 @@
import groovy.transform.Field
import com.sap.piper.JenkinsUtils
@Field String STEP_NAME = getClass().getName()
@Field String METADATA_FILE = 'metadata/tmsExport.yaml'
void call(Map parameters = [:]) {
List credentials = [
[type: 'token', id: 'credentialsId', env: ['PIPER_tmsServiceKey']]
]
piperExecuteBin(parameters, STEP_NAME, METADATA_FILE, credentials, false, false, true)
}