package cmd import ( "encoding/json" "net/url" "reflect" "strings" "time" abapbuild "github.com/SAP/jenkins-library/pkg/abap/build" "github.com/SAP/jenkins-library/pkg/abaputils" "github.com/SAP/jenkins-library/pkg/command" piperhttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/telemetry" "github.com/pkg/errors" ) type abapEnvironmentBuildUtils interface { command.ExecRunner abaputils.Communication abapbuild.HTTPSendLoader piperutils.FileUtils getMaxRuntime() time.Duration getPollingInterval() time.Duration publish() } type abapEnvironmentBuildUtilsBundle struct { *command.Command *piperhttp.Client *abaputils.AbapUtils *piperutils.Files maxRuntime time.Duration pollingInterval time.Duration storePublish publish } type publish struct { stepName string workspace string reports []piperutils.Path links []piperutils.Path } func (p *publish) publish(utils piperutils.FileUtils) { if p.stepName != "" { piperutils.PersistReportsAndLinks(p.stepName, p.workspace, utils, p.reports, p.links) } } func (aEBUB *abapEnvironmentBuildUtilsBundle) publish() { aEBUB.storePublish.publish(aEBUB) } func (aEBUB *abapEnvironmentBuildUtilsBundle) getMaxRuntime() time.Duration { return aEBUB.maxRuntime } func (aEBUB *abapEnvironmentBuildUtilsBundle) getPollingInterval() time.Duration { return aEBUB.pollingInterval } func (aEBUB *abapEnvironmentBuildUtilsBundle) PersistReportsAndLinks(stepName, workspace string, reports, links []piperutils.Path) { // abapbuild.PersistReportsAndLinks(stepName, workspace, reports, links) if aEBUB.storePublish.stepName == "" { aEBUB.storePublish.stepName = stepName aEBUB.storePublish.workspace = workspace aEBUB.storePublish.reports = reports aEBUB.storePublish.links = links } else { aEBUB.storePublish.reports = append(aEBUB.storePublish.reports, reports...) aEBUB.storePublish.links = append(aEBUB.storePublish.reports, links...) } } func newAbapEnvironmentBuildUtils(maxRuntime time.Duration, pollingInterval time.Duration) abapEnvironmentBuildUtils { utils := abapEnvironmentBuildUtilsBundle{ Command: &command.Command{}, Client: &piperhttp.Client{}, AbapUtils: &abaputils.AbapUtils{ Exec: &command.Command{}, }, maxRuntime: maxRuntime * time.Minute, pollingInterval: pollingInterval * time.Second, storePublish: publish{}, } // Reroute command output to logging framework utils.Stdout(log.Writer()) utils.Stderr(log.Writer()) return &utils } func abapEnvironmentBuild(config abapEnvironmentBuildOptions, telemetryData *telemetry.CustomData, cpe *abapEnvironmentBuildCommonPipelineEnvironment) { utils := newAbapEnvironmentBuildUtils(time.Duration(config.MaxRuntimeInMinutes), time.Duration(config.PollingIntervalInSeconds)) if err := runAbapEnvironmentBuild(&config, telemetryData, utils, cpe); err != nil { log.Entry().WithError(err).Fatal("step execution failed") } } func runAbapEnvironmentBuild(config *abapEnvironmentBuildOptions, telemetryData *telemetry.CustomData, utils abapEnvironmentBuildUtils, cpe *abapEnvironmentBuildCommonPipelineEnvironment) error { log.Entry().Info("╔════════════════════════════════╗") log.Entry().Info("║ abapEnvironmentBuild ║") log.Entry().Info("╠════════════════════════════════╣") log.Entry().Infof("║ %-30v ║", config.Phase) log.Entry().Info("╙────────────────────────────────╜") conn := new(abapbuild.Connector) if err := initConnection(conn, config, utils); err != nil { return errors.Wrap(err, "Connector initialization for communication with the ABAP system failed") } valuesList, err := evaluateAddonDescriptor(config) if err != nil { return errors.Wrap(err, "Error during the evaluation of the AddonDescriptor") } finalValues, err := runBuilds(conn, config, utils, valuesList) // files should be published, even if an error occured utils.publish() if err != nil { return err } cpe.abap.buildValues, err = convertValuesForCPE(finalValues) if err != nil { return errors.Wrap(err, "Error during the conversion of the values for the commonPipelineenvironment") } return nil } func runBuilds(conn *abapbuild.Connector, config *abapEnvironmentBuildOptions, utils abapEnvironmentBuildUtils, valuesList [][]abapbuild.Value) ([]abapbuild.Value, error) { var finalValues []abapbuild.Value // No addonDescriptor involved if len(valuesList) == 0 { values, err := generateValuesOnlyFromConfig(config) if err != nil { return finalValues, errors.Wrap(err, "Generating the values from config failed") } finalValues, err = runBuild(conn, config, utils, values) if err != nil { return finalValues, errors.Wrap(err, "Error during execution of build framework") } } else { // Run several times for each repository in the addonDescriptor var errstrings []string vE := valuesEvaluator{} vE.m = make(map[string]string) for _, values := range valuesList { cummulatedValues, err := generateValuesWithAddonDescriptor(config, values) if err != nil { return finalValues, errors.Wrap(err, "Error generating input values") } finalValuesForOneBuild, err := runBuild(conn, config, utils, cummulatedValues) if err != nil { err = errors.Wrapf(err, "Build with input values %s failed", values2string(values)) if config.StopOnFirstError { return finalValues, err } errstrings = append(errstrings, err.Error()) } finalValuesForOneBuild = removeAddonDescriptorValues(finalValuesForOneBuild, values) // This means: probably values are duplicated, but the first one wins -> perhaps change this in the future if needed if err := vE.appendValuesIfNotPresent(finalValuesForOneBuild, false); err != nil { errstrings = append(errstrings, err.Error()) } } finalValues = vE.generateValueSlice() if len(errstrings) > 0 { finalError := errors.Errorf("%d out %d build runs failed:\n%s", len(errstrings), len(valuesList), (strings.Join(errstrings, "\n"))) return finalValues, finalError } } return finalValues, nil } func initConnection(conn *abapbuild.Connector, config *abapEnvironmentBuildOptions, utils abapEnvironmentBuildUtils) error { var connConfig abapbuild.ConnectorConfiguration connConfig.CfAPIEndpoint = config.CfAPIEndpoint connConfig.CfOrg = config.CfOrg connConfig.CfSpace = config.CfSpace connConfig.CfServiceInstance = config.CfServiceInstance connConfig.CfServiceKeyName = config.CfServiceKeyName connConfig.Host = config.Host connConfig.Username = config.Username connConfig.Password = config.Password connConfig.MaxRuntimeInMinutes = config.MaxRuntimeInMinutes connConfig.CertificateNames = config.CertificateNames connConfig.Parameters = url.Values{} if len(config.AbapSourceClient) != 0 { connConfig.Parameters.Add("sap-client", config.AbapSourceClient) } if err := conn.InitBuildFramework(connConfig, utils, utils); err != nil { return err } conn.MaxRuntime = utils.getMaxRuntime() conn.PollingInterval = utils.getPollingInterval() return nil } // ***********************************Run Build*************************************************************** func runBuild(conn *abapbuild.Connector, config *abapEnvironmentBuildOptions, utils abapEnvironmentBuildUtils, values []abapbuild.Value) ([]abapbuild.Value, error) { var finalValues []abapbuild.Value var inputValues abapbuild.Values inputValues.Values = values build := myBuild{ Build: abapbuild.Build{ Connector: *conn, }, abapEnvironmentBuildOptions: config, } if err := build.Start(inputValues); err != nil { return finalValues, err } if err := build.Poll(); err != nil { return finalValues, errors.Wrap(err, "Error during the polling for the final state of the build run") } if err := build.PrintLogs(); err != nil { return finalValues, errors.Wrap(err, "Error printing the logs") } errBuildRun := build.EvaluteIfBuildSuccessful() if err := build.Download(); err != nil { if errBuildRun != nil { errWraped := errors.Errorf("Download failed after execution of build failed: %v. Build error: %v", err, errBuildRun) return finalValues, errWraped } return finalValues, err } if err := build.Publish(utils); err != nil { return finalValues, err } finalValues, err := build.GetFinalValues() if err != nil { return finalValues, err } return finalValues, errBuildRun } type myBuild struct { abapbuild.Build *abapEnvironmentBuildOptions } func (b *myBuild) Start(values abapbuild.Values) error { if err := b.Build.Start(b.abapEnvironmentBuildOptions.Phase, values); err != nil { return errors.Wrap(err, "Error starting the build framework") } return nil } func (b *myBuild) EvaluteIfBuildSuccessful() error { if err := b.Build.EvaluteIfBuildSuccessful(b.TreatWarningsAsError); err != nil { return errors.Wrap(err, "Build ended without success") } return nil } func (b *myBuild) Download() error { if b.DownloadAllResultFiles { if err := b.DownloadAllResults(b.SubDirectoryForDownload, b.FilenamePrefixForDownload); err != nil { return errors.Wrap(err, "Error during the download of the result files") } } else { if err := b.DownloadResults(b.DownloadResultFilenames, b.SubDirectoryForDownload, b.FilenamePrefixForDownload); err != nil { return errors.Wrapf(err, "Error during the download of the result files %s", b.DownloadResultFilenames) } } return nil } func (b *myBuild) Publish(utils abapEnvironmentBuildUtils) error { if b.PublishAllDownloadedResultFiles { b.PublishAllDownloadedResults("abapEnvironmentBuild", utils) } else { if err := b.PublishDownloadedResults("abapEnvironmentBuild", b.PublishResultFilenames, utils); err != nil { return errors.Wrapf(err, "Error during the publish of the result files %s", b.PublishResultFilenames) } } return nil } func (b *myBuild) GetFinalValues() ([]abapbuild.Value, error) { var values []abapbuild.Value if err := b.GetValues(); err != nil { return values, errors.Wrapf(err, "Error getting the values from build framework") } return b.Build.Values, nil } // **********************************Values Handling************************************************************** func convertValuesForCPE(values []abapbuild.Value) (string, error) { type cpeValue struct { ValueID string `json:"value_id"` Value string `json:"value"` } var cpeValues []cpeValue byt, err := json.Marshal(&values) if err != nil { return "", errors.Wrapf(err, "Error converting the values from the build framework") } if err := json.Unmarshal(byt, &cpeValues); err != nil { return "", errors.Wrapf(err, "Error converting the values from the build framework into the structure for the commonPipelineEnvironment") } jsonBytes, err := json.Marshal(cpeValues) if err != nil { return "", errors.Wrapf(err, "Error converting the converted values") } return string(jsonBytes), nil } func removeAddonDescriptorValues(finalValuesFromBuild []abapbuild.Value, valuesFromAddonDescriptor []abapbuild.Value) []abapbuild.Value { var finalValues []abapbuild.Value mapForAddonDescriptorValues := make(map[string]string) for _, value := range valuesFromAddonDescriptor { mapForAddonDescriptorValues[value.ValueID] = value.Value } for _, value := range finalValuesFromBuild { _, present := mapForAddonDescriptorValues[value.ValueID] if !present { finalValues = append(finalValues, value) } } return finalValues } func generateValuesWithAddonDescriptor(config *abapEnvironmentBuildOptions, repoValues []abapbuild.Value) ([]abapbuild.Value, error) { var values []abapbuild.Value vE := valuesEvaluator{} // values from config if err := vE.initialize(config.Values); err != nil { return values, err } // values from addondescriptor if err := vE.appendValuesIfNotPresent(repoValues, true); err != nil { return values, err } // values from commonepipelineEnvironment if err := vE.appendStringValuesIfNotPresent(config.CpeValues, false); err != nil { return values, err } values = vE.generateValueSlice() return values, nil } func generateValuesOnlyFromConfig(config *abapEnvironmentBuildOptions) ([]abapbuild.Value, error) { return generateValuesWithAddonDescriptor(config, []abapbuild.Value{}) } func generateValuesFromString(stringValues string) ([]abapbuild.Value, error) { var values []abapbuild.Value if len(stringValues) > 0 { if err := json.Unmarshal([]byte(stringValues), &values); err != nil { log.SetErrorCategory(log.ErrorConfiguration) return values, errors.Wrapf(err, "Could not convert the values %s", stringValues) } } return values, nil } type valuesEvaluator struct { m map[string]string } func (vE *valuesEvaluator) initialize(stringValues string) error { values, err := generateValuesFromString(stringValues) if err != nil { return errors.Wrapf(err, "Error converting the vales from the config") } vE.m = make(map[string]string) for _, value := range values { if (len(value.ValueID) == 0) || (len(value.Value) == 0) { log.SetErrorCategory(log.ErrorConfiguration) return errors.Errorf("Values %s from config have not the right format", stringValues) } _, present := vE.m[value.ValueID] if present { log.SetErrorCategory(log.ErrorConfiguration) return errors.Errorf("Value_id %s is not unique in the config", value.ValueID) } vE.m[value.ValueID] = value.Value } return nil } func (vE *valuesEvaluator) appendStringValuesIfNotPresent(stringValues string, throwErrorIfPresent bool) error { var values []abapbuild.Value values, err := generateValuesFromString(stringValues) if err != nil { return errors.Wrapf(err, "Error converting the vales from the commonPipelineEnvironment") } if err := vE.appendValuesIfNotPresent(values, throwErrorIfPresent); err != nil { return err } return nil } func (vE *valuesEvaluator) appendValuesIfNotPresent(values []abapbuild.Value, throwErrorIfPresent bool) error { for _, value := range values { if value.ValueID == "PHASE" || value.ValueID == "BUILD_FRAMEWORK_MODE" { continue } _, present := vE.m[value.ValueID] if present { if throwErrorIfPresent { return errors.Errorf("Value_id %s already existed in the config", value.ValueID) } log.Entry().Infof("Value '%s':'%s' already existed -> discard this value", value.ValueID, value.Value) } else { vE.m[value.ValueID] = value.Value } } return nil } func (vE *valuesEvaluator) generateValueSlice() []abapbuild.Value { var values []abapbuild.Value var value abapbuild.Value for k, v := range vE.m { value.ValueID = k value.Value = v values = append(values, value) } return values } // **********************************Evaluate AddonDescriptor************************************************************** type myRepo struct { abaputils.Repository } type condition struct { Field string `json:"field"` Operator string `json:"operator"` Value string `json:"value"` } type useField struct { Use string `json:"use"` Rename string `json:"renameTo"` } func evaluateAddonDescriptor(config *abapEnvironmentBuildOptions) ([][]abapbuild.Value, error) { var listOfValuesList [][]abapbuild.Value if len(config.AddonDescriptor) == 0 && len(config.UseFieldsOfAddonDescriptor) > 0 { return listOfValuesList, errors.New("Config contains UseFieldsOfAddonDescriptor but no addonDescriptor is provided in the commonPipelineEnvironment") } if len(config.AddonDescriptor) > 0 { addonDescriptor := new(abaputils.AddonDescriptor) if err := addonDescriptor.InitFromJSONstring(config.AddonDescriptor); err != nil { log.SetErrorCategory(log.ErrorConfiguration) return listOfValuesList, errors.Wrap(err, "Error during the conversion of the AddonDescriptor") } for _, repo := range addonDescriptor.Repositories { myRepo := myRepo{ Repository: repo, } use, err := myRepo.checkCondition(config) if err != nil { return listOfValuesList, errors.Wrapf(err, "Checking of ConditionOnAddonDescriptor failed") } if use { values, err := myRepo.generateValues(config) if err != nil { return listOfValuesList, errors.Wrap(err, "Error generating values from AddonDescriptor") } if len(values) > 0 { listOfValuesList = append(listOfValuesList, values) } } } } return listOfValuesList, nil } func (mR *myRepo) checkCondition(config *abapEnvironmentBuildOptions) (bool, error) { var conditions []condition if len(config.ConditionOnAddonDescriptor) > 0 { if err := json.Unmarshal([]byte(config.ConditionOnAddonDescriptor), &conditions); err != nil { log.SetErrorCategory(log.ErrorConfiguration) return false, errors.Wrapf(err, "Conversion of ConditionOnAddonDescriptor in the config failed") } for _, cond := range conditions { if cond.Field == "" || cond.Operator == "" || cond.Value == "" { log.SetErrorCategory(log.ErrorConfiguration) return false, errors.Errorf("Invalid condition for field %s with operator %s and value %s", cond.Field, cond.Operator, cond.Value) } use, err := mR.amI(cond.Field, cond.Operator, cond.Value) if err != nil { return false, errors.Wrapf(err, "Checking the field %s failed", cond.Field) } if !use { log.Entry().Infof("addonDescriptor with the name %s does not fulfil the requierement %s%s%s from the ConditionOnAddonDescriptor, therefore it is not used", mR.Name, cond.Field, cond.Operator, cond.Value) return false, nil } log.Entry().Infof("addonDescriptor with the name %s does fulfil the requierement %s%s%s in the ConditionOnAddonDescriptor", mR.Name, cond.Field, cond.Operator, cond.Value) } } return true, nil } func (mR *myRepo) generateValues(config *abapEnvironmentBuildOptions) ([]abapbuild.Value, error) { var values []abapbuild.Value var useFields []useField if len(config.UseFieldsOfAddonDescriptor) == 0 { log.Entry().Infof("UseFieldsOfAddonDescriptor is empty, nothing is used from the addonDescriptor") } else { if err := json.Unmarshal([]byte(config.UseFieldsOfAddonDescriptor), &useFields); err != nil { log.SetErrorCategory(log.ErrorConfiguration) return values, errors.Wrapf(err, "Conversion of UseFieldsOfAddonDescriptor in the config failed") } m := make(map[string]string) for _, uF := range useFields { if uF.Use == "" || uF.Rename == "" { log.SetErrorCategory(log.ErrorConfiguration) return values, errors.Errorf("Invalid UseFieldsOfAddonDescriptor for use %s and renameTo %s", uF.Use, uF.Rename) } m[uF.Use] = uF.Rename } fields := reflect.ValueOf(mR.Repository) typeOfS := fields.Type() for i := 0; i < fields.NumField(); i++ { var value abapbuild.Value ValueID := typeOfS.Field(i).Name rename, present := m[ValueID] if present { log.Entry().Infof("Use field %s from addonDescriptor and rename it to %s, the value is %s", ValueID, rename, fields.Field(i).String()) value.ValueID = rename value.Value = fields.Field(i).String() values = append(values, value) } } if len(values) != len(useFields) { log.SetErrorCategory(log.ErrorConfiguration) return values, errors.Errorf("Not all fields in UseFieldsOfAddonDescriptor have been found. Probably a 'use' was used which does not exist") } } return values, nil } func (mR *myRepo) getField(field string) string { r := reflect.ValueOf(mR) f := reflect.Indirect(r).FieldByName(field) return string(f.String()) } func (mR *myRepo) amI(field string, operator string, comp string) (bool, error) { operators := OperatorCallback{ "==": Equal, "!=": Unequal, } name := mR.getField(field) if fn, ok := operators[operator]; ok { return fn(name, comp), nil } log.SetErrorCategory(log.ErrorConfiguration) return false, errors.Errorf("Invalid operator %s", operator) } type OperatorCallback map[string]func(string, string) bool func Equal(a, b string) bool { return a == b } func Unequal(a, b string) bool { return a != b } func values2string(values []abapbuild.Value) string { var result string for index, value := range values { if index > 0 { result = result + "; " } result = result + value.ValueID + " = " + value.Value } return result }