sap-jenkins-library/pkg/tms/tmsUtils.go

package tms

import (
	"encoding/json"
	"fmt"
	"net/url"
	"sort"
	"strconv"

	"github.com/SAP/jenkins-library/pkg/command"
	piperHttp "github.com/SAP/jenkins-library/pkg/http"
	"github.com/SAP/jenkins-library/pkg/log"
	"github.com/SAP/jenkins-library/pkg/piperutils"
	"github.com/ghodss/yaml"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type TmsUtils interface {
	command.ExecRunner
	FileExists(filename string) (bool, error)
	FileRead(path string) ([]byte, error)
}

type uaa struct {
	Url          string `json:"url"`
	ClientId     string `json:"clientid"`
	ClientSecret string `json:"clientsecret"`
}

type serviceKey struct {
	Uaa uaa    `json:"uaa"`
	Uri string `json:"uri"`
}

type CommunicationInstance struct {
	tmsUrl       string
	uaaUrl       string
	clientId     string
	clientSecret string
	httpClient   piperHttp.Uploader
	logger       *logrus.Entry
	isVerbose    bool
}

type Node struct {
	Id   int64  `json:"id"`
	Name string `json:"name"`
}

type nodes struct {
	Nodes []Node `json:"nodes"`
}

type MtaExtDescriptor struct {
	Id            int64  `json:"id"`
	Description   string `json:"description"`
	MtaId         string `json:"mtaId"`
	MtaExtId      string `json:"mtaExtId"`
	MtaVersion    string `json:"mtaVersion"`
	LastChangedAt string `json:"lastChangedAt"`
}

type mtaExtDescriptors struct {
	MtaExtDescriptors []MtaExtDescriptor `json:"mtaExtDescriptors"`
}

type FileInfo struct {
	Id   int64  `json:"fileId"`
	Name string `json:"fileName"`
}

type NodeUploadResponseEntity struct {
	TransportRequestId          int64        `json:"transportRequestId"`
	TransportRequestDescription string       `json:"transportRequestDescription"`
	QueueEntries                []QueueEntry `json:"queueEntries"`
}

type QueueEntry struct {
	Id       int64  `json:"queueId"`
	NodeId   int64  `json:"nodeId"`
	NodeName string `json:"nodeName"`
}

type NodeUploadRequestEntity struct {
	ContentType string  `json:"contentType"`
	StorageType string  `json:"storageType"`
	NodeName    string  `json:"nodeName"`
	Description string  `json:"description"`
	NamedUser   string  `json:"namedUser"`
	Entries     []Entry `json:"entries"`
}

type Entry struct {
	Uri string `json:"uri"`
}

type CommunicationInterface interface {
	GetNodes() ([]Node, error)
	GetMtaExtDescriptor(nodeId int64, mtaId, mtaVersion string) (MtaExtDescriptor, error)
	UpdateMtaExtDescriptor(nodeId, idOfMtaExtDescriptor int64, file, mtaVersion, description, namedUser string) (MtaExtDescriptor, error)
	UploadMtaExtDescriptorToNode(nodeId int64, file, mtaVersion, description, namedUser string) (MtaExtDescriptor, error)
	UploadFile(file, namedUser string) (FileInfo, error)
	UploadFileToNode(nodeName, fileId, description, namedUser string) (NodeUploadResponseEntity, error)
	ExportFileToNode(nodeName, fileId, description, namedUser string) (NodeUploadResponseEntity, error)
}

type Options struct {
	TmsServiceKey            string                 `json:"tmsServiceKey,omitempty"`
	CustomDescription        string                 `json:"customDescription,omitempty"`
	NamedUser                string                 `json:"namedUser,omitempty"`
	NodeName                 string                 `json:"nodeName,omitempty"`
	MtaPath                  string                 `json:"mtaPath,omitempty"`
	MtaVersion               string                 `json:"mtaVersion,omitempty"`
	NodeExtDescriptorMapping map[string]interface{} `json:"nodeExtDescriptorMapping,omitempty"`
	Proxy                    string                 `json:"proxy,omitempty"`
	StashContent             []string               `json:"stashContent,omitempty"`
	Verbose                  bool
}

type tmsUtilsBundle struct {
	*command.Command
	*piperutils.Files
}

const DEFAULT_TR_DESCRIPTION = "Created by Piper"

func NewTmsUtils() TmsUtils {
	utils := tmsUtilsBundle{
		Command: &command.Command{},
		Files:   &piperutils.Files{},
	}
	// Reroute command output to logging framework
	utils.Stdout(log.Writer())
	utils.Stderr(log.Writer())
	return &utils
}

func unmarshalServiceKey(serviceKeyJson string) (serviceKey serviceKey, err error) {
	err = json.Unmarshal([]byte(serviceKeyJson), &serviceKey)
	if err != nil {
		return
	}
	return
}

func FormNodeIdExtDescriptorMappingWithValidation(utils TmsUtils, nodeNameExtDescriptorMapping map[string]interface{}, nodes []Node, mtaYamlMap map[string]interface{}, mtaVersion string) (map[int64]string, error) {
	var wrongMtaIdExtDescriptors []string
	var wrongExtDescriptorPaths []string
	var wrongNodeNames []string
	var errorMessage string

	nodeIdExtDescriptorMapping := make(map[int64]string)
	for nodeName, mappedValue := range nodeNameExtDescriptorMapping {
		mappedValueString := fmt.Sprintf("%v", mappedValue)
		exists, _ := utils.FileExists(mappedValueString)
		if exists {
			extDescriptorMap, errGetYamlAsMap := GetYamlAsMap(utils, mappedValueString)
			if errGetYamlAsMap == nil {
				if fmt.Sprintf("%v", mtaYamlMap["ID"]) != fmt.Sprintf("%v", extDescriptorMap["extends"]) {
					wrongMtaIdExtDescriptors = append(wrongMtaIdExtDescriptors, mappedValueString)
				}
			} else {
				wrappedErr := errors.Wrapf(errGetYamlAsMap, "tried to parse %v as yaml, but got an error", mappedValueString)
				errorMessage += fmt.Sprintf("%v\n", wrappedErr)
			}
		} else {
			wrongExtDescriptorPaths = append(wrongExtDescriptorPaths, mappedValueString)
		}

		isNodeFound := false
		for _, node := range nodes {
			if node.Name == nodeName {
				nodeIdExtDescriptorMapping[node.Id] = mappedValueString
				isNodeFound = true
				break
			}
		}
		if !isNodeFound {
			wrongNodeNames = append(wrongNodeNames, nodeName)
		}
	}

	if mtaVersion != "*" && mtaVersion != mtaYamlMap["version"] {
		errorMessage += "parameter 'mtaVersion' does not match the MTA version in mta.yaml\n"
	}

	if len(wrongMtaIdExtDescriptors) > 0 || len(wrongExtDescriptorPaths) > 0 || len(wrongNodeNames) > 0 {
		if len(wrongMtaIdExtDescriptors) > 0 {
			sort.Strings(wrongMtaIdExtDescriptors)
			errorMessage += fmt.Sprintf("parameter 'extends' in MTA extension descriptor files %v is not the same as MTA ID or is missing at all\n", wrongMtaIdExtDescriptors)
		}
		if len(wrongExtDescriptorPaths) > 0 {
			sort.Strings(wrongExtDescriptorPaths)
			errorMessage += fmt.Sprintf("MTA extension descriptor files %v do not exist\n", wrongExtDescriptorPaths)
		}
		if len(wrongNodeNames) > 0 {
			sort.Strings(wrongNodeNames)
			errorMessage += fmt.Sprintf("nodes %v do not exist. Please check node names provided in 'nodeExtDescriptorMapping' parameter or create these nodes\n", wrongNodeNames)
		}
	}

	if errorMessage == "" {
		return nodeIdExtDescriptorMapping, nil
	} else {
		return nil, errors.New(errorMessage)
	}
}

func GetYamlAsMap(utils TmsUtils, yamlPath string) (map[string]interface{}, error) {
	var result map[string]interface{}
	bytes, err := utils.FileRead(yamlPath)
	if err != nil {
		return result, err
	}
	err = yaml.Unmarshal(bytes, &result)
	if err != nil {
		return result, err
	}

	return result, nil
}

func SetupCommunication(config Options) (communicationInstance CommunicationInterface) {
	client := &piperHttp.Client{}
	proxy := config.Proxy
	options := piperHttp.ClientOptions{}
	if proxy != "" {
		transportProxy, err := url.Parse(proxy)
		if err != nil {
			log.Entry().WithError(err).Fatalf("Failed to parse proxy string %v into a URL structure", proxy)
		}

		options = piperHttp.ClientOptions{TransportProxy: transportProxy}
		client.SetOptions(options)
		if config.Verbose {
			log.Entry().Infof("HTTP client instructed to use %v proxy", proxy)
		}
	}

	serviceKey, err := unmarshalServiceKey(config.TmsServiceKey)
	if err != nil {
		log.Entry().WithError(err).Fatal("Failed to unmarshal TMS service key")
	}
	log.RegisterSecret(serviceKey.Uaa.ClientSecret)

	if config.Verbose {
		log.Entry().Info("Will be used for communication:")
		log.Entry().Infof("- client id: %v", serviceKey.Uaa.ClientId)
		log.Entry().Infof("- TMS URL: %v", serviceKey.Uri)
		log.Entry().Infof("- UAA URL: %v", serviceKey.Uaa.Url)
	}

	commuInstance, err := NewCommunicationInstance(client, serviceKey.Uri, serviceKey.Uaa.Url, serviceKey.Uaa.ClientId, serviceKey.Uaa.ClientSecret, config.Verbose, options)
	if err != nil {
		log.Entry().WithError(err).Fatal("Failed to prepare client for talking with TMS")
	}
	return commuInstance
}

func UploadDescriptors(config Options, communicationInstance CommunicationInterface, utils TmsUtils) error {
	description := config.CustomDescription
	namedUser := config.NamedUser
	nodeName := config.NodeName
	mtaVersion := config.MtaVersion
	nodeNameExtDescriptorMapping := config.NodeExtDescriptorMapping
	mtaPath := config.MtaPath

	if config.Verbose {
		log.Entry().Info("The step will use the following values:")
		log.Entry().Infof("- description: %v", config.CustomDescription)

		if len(nodeNameExtDescriptorMapping) > 0 {
			log.Entry().Infof("- mapping between node names and MTA extension descriptor file paths: %v", nodeNameExtDescriptorMapping)
		}
		log.Entry().Infof("- MTA path: %v", mtaPath)
		log.Entry().Infof("- MTA version: %v", mtaVersion)
		if namedUser != "" {
			log.Entry().Infof("- named user: %v", namedUser)
		}
		log.Entry().Infof("- node name: %v", nodeName)
	}

	if len(nodeNameExtDescriptorMapping) > 0 {
		nodes, errGetNodes := communicationInstance.GetNodes()
		if errGetNodes != nil {
			log.SetErrorCategory(log.ErrorService)
			return fmt.Errorf("failed to get nodes: %w", errGetNodes)
		}

		mtaYamlMap, errGetMtaYamlAsMap := GetYamlAsMap(utils, "mta.yaml")
		if errGetMtaYamlAsMap != nil {
			log.SetErrorCategory(log.ErrorConfiguration)
			return fmt.Errorf("failed to get mta.yaml as map: %w", errGetMtaYamlAsMap)
		}
		_, isIdParameterInMap := mtaYamlMap["ID"]
		_, isVersionParameterInMap := mtaYamlMap["version"]
		if !isIdParameterInMap || !isVersionParameterInMap {
			var errorMessage string
			if !isIdParameterInMap {
				errorMessage += "parameter 'ID' is not found in mta.yaml\n"
			}
			if !isVersionParameterInMap {
				errorMessage += "parameter 'version' is not found in mta.yaml\n"
			}
			log.SetErrorCategory(log.ErrorConfiguration)
			return errors.New(errorMessage)
		}

		// validate the whole mapping and then throw errors together, so that user can get them after a single pipeline run
		nodeIdExtDescriptorMapping, errGetNodeIdExtDescriptorMapping := FormNodeIdExtDescriptorMappingWithValidation(utils, nodeNameExtDescriptorMapping, nodes, mtaYamlMap, mtaVersion)
		if errGetNodeIdExtDescriptorMapping != nil {
			log.SetErrorCategory(log.ErrorConfiguration)
			return errGetNodeIdExtDescriptorMapping
		}

		for nodeId, mtaExtDescriptorPath := range nodeIdExtDescriptorMapping {
			obtainedMtaExtDescriptor, errGetMtaExtDescriptor := communicationInstance.GetMtaExtDescriptor(nodeId, fmt.Sprintf("%v", mtaYamlMap["ID"]), mtaVersion)
			if errGetMtaExtDescriptor != nil {
				log.SetErrorCategory(log.ErrorService)
				return fmt.Errorf("failed to get MTA extension descriptor: %w", errGetMtaExtDescriptor)
			}

			if obtainedMtaExtDescriptor != (MtaExtDescriptor{}) {
				_, errUpdateMtaExtDescriptor := communicationInstance.UpdateMtaExtDescriptor(nodeId, obtainedMtaExtDescriptor.Id, mtaExtDescriptorPath, mtaVersion, description, namedUser)
				if errUpdateMtaExtDescriptor != nil {
					log.SetErrorCategory(log.ErrorService)
					return fmt.Errorf("failed to update MTA extension descriptor: %w", errUpdateMtaExtDescriptor)
				}
			} else {
				_, errUploadMtaExtDescriptor := communicationInstance.UploadMtaExtDescriptorToNode(nodeId, mtaExtDescriptorPath, mtaVersion, description, namedUser)
				if errUploadMtaExtDescriptor != nil {
					log.SetErrorCategory(log.ErrorService)
					return fmt.Errorf("failed to upload MTA extension descriptor to node: %w", errUploadMtaExtDescriptor)
				}
			}
		}
	}
	return nil
}

func UploadFile(config Options, communicationInstance CommunicationInterface, utils TmsUtils) (string, error) {
	mtaPath := config.MtaPath
	exists, _ := utils.FileExists(mtaPath)
	if !exists {
		log.SetErrorCategory(log.ErrorConfiguration)
		return "", fmt.Errorf("mta file %s not found", mtaPath)
	}

	fileInfo, errUploadFile := communicationInstance.UploadFile(mtaPath, config.NamedUser)
	if errUploadFile != nil {
		log.SetErrorCategory(log.ErrorService)
		return "", fmt.Errorf("failed to upload file: %w", errUploadFile)
	}

	fileId := strconv.FormatInt(fileInfo.Id, 10)
	return fileId, nil
}