abapEnvironmentAssemblePackages new Features (#2661)

* Unit Test Assemble Package

* Remove obsolete lines

dust wiping

* climate change

* climate change #2

* climate change #3

* climate change #4

* climate change #5

* NSPC serial builds

* Actual Delivery Commit

* Download Delivery_logs.zip

* Publish Result

* Testing

* !Polling

* Provide Commit to BF only if set

* dust wiping

* More Dust to Wipe

* Publish more than on file

* Write Log for Publish

* fix unit test (now Dummy Entries)

* save one line of code for climate change

* Update cmd/abapEnvironmentAssemblePackages.go

Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com>

* More Detailed Log Messages

Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com>

cmd/abapEnvironmentAssembleConfirm.go

@@ -100,6 +100,29 @@ func startingConfirm(repos []abaputils.Repository, conn abapbuild.Connector, del
 	return builds, nil
 }
 
+func polling(builds []buildWithRepository, maxRuntimeInMinutes time.Duration, pollIntervalsInSeconds time.Duration) error {
+	timeout := time.After(maxRuntimeInMinutes)
+	ticker := time.Tick(pollIntervalsInSeconds)
+	for {
+		select {
+		case <-timeout:
+			return errors.New("Timed out")
+		case <-ticker:
+			var allFinished bool = true
+			for i := range builds {
+				builds[i].build.Get()
+				if !builds[i].build.IsFinished() {
+					log.Entry().Infof("Assembly of %s is not yet finished, check again in %s", builds[i].repo.PackageName, pollIntervalsInSeconds)
+					allFinished = false
+				}
+			}
+			if allFinished {
+				return nil
+			}
+		}
+	}
+}
+
 func (b *buildWithRepository) startConfirm() error {
 	if b.repo.Name == "" || b.repo.Namespace == "" || b.repo.PackageName == "" {
 		return errors.New("Parameters missing. Please provide software component name, namespace and packagename")

cmd/abapEnvironmentAssembleConfirm_test.go

@@ -9,6 +9,23 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func TestPolling(t *testing.T) {
+	t.Run("Run polling", func(t *testing.T) {
+		var repo abaputils.Repository
+		b := testSetup(&abapbuild.ClMock{}, "ABIFNLDCSQPOVMXK4DNPBDRW2M")
+		var buildsWithRepo []buildWithRepository
+		bWR := buildWithRepository{
+			build: b,
+			repo:  repo,
+		}
+		buildsWithRepo = append(buildsWithRepo, bWR)
+		timeout := time.Duration(600 * time.Second)
+		pollInterval := time.Duration(1 * time.Second)
+		err := polling(buildsWithRepo, timeout, pollInterval)
+		assert.NoError(t, err)
+		assert.Equal(t, abapbuild.Finished, buildsWithRepo[0].build.RunState)
+	})
+}
 func TestStartingConfirm(t *testing.T) {
 	t.Run("Run starting", func(t *testing.T) {
 		client := &abapbuild.ClMock{

cmd/abapEnvironmentAssemblePackages.go

@@ -11,6 +11,7 @@ import (
 	"github.com/SAP/jenkins-library/pkg/command"
 	piperhttp "github.com/SAP/jenkins-library/pkg/http"
 	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/SAP/jenkins-library/pkg/piperutils"
 	"github.com/SAP/jenkins-library/pkg/telemetry"
 	"github.com/pkg/errors"
 )
@@ -54,7 +55,7 @@ func runAbapEnvironmentAssemblePackages(config *abapEnvironmentAssemblePackagesO
 
 	err := conn.InitBuildFramework(connConfig, com, client)
 	if err != nil {
-		return errors.Wrap(err, "Connector initialization failed")
+		return errors.Wrap(err, "Connector initialization for communication with the ABAP system failed")
 	}
 
 	var addonDescriptor abaputils.AddonDescriptor
@@ -63,28 +64,35 @@ func runAbapEnvironmentAssemblePackages(config *abapEnvironmentAssemblePackagesO
 		return errors.Wrap(err, "Reading AddonDescriptor failed [Make sure abapAddonAssemblyKit...CheckCVs|CheckPV|ReserveNextPackages steps have been run before]")
 	}
 
-	delayBetweenPostsInSeconds := time.Duration(3 * time.Second)
-	builds, buildsAlreadyReleased, err := starting(addonDescriptor.Repositories, *conn, delayBetweenPostsInSeconds)
-	if err != nil {
-		return errors.Wrap(err, "Starting Builds failed")
-	}
 	maxRuntimeInMinutes := time.Duration(config.MaxRuntimeInMinutes) * time.Minute
 	pollIntervalsInMilliseconds := time.Duration(config.PollIntervalsInMilliseconds) * time.Millisecond
-	err = polling(builds, maxRuntimeInMinutes, pollIntervalsInMilliseconds)
+	builds, err := executeBuilds(addonDescriptor.Repositories, *conn, maxRuntimeInMinutes, pollIntervalsInMilliseconds)
 	if err != nil {
-		return errors.Wrap(err, "Polling failed")
+		return errors.Wrap(err, "Starting Builds for Repositories with reserved AAKaaS packages failed")
 	}
 
 	err = checkIfFailedAndPrintLogs(builds)
 	if err != nil {
-		return errors.Wrap(err, "Check if failed and Printing Logs failed")
+		return errors.Wrap(err, "Checking for failed Builds and Printing Build Logs failed")
 	}
-	reposBackToCPE, err := downloadSARXML(builds)
+
+	var filesToPublish []piperutils.Path
+	filesToPublish, err = downloadResultToFile(builds, "SAR_XML", filesToPublish)
 	if err != nil {
-		return errors.Wrap(err, "Download SAR XML failed")
+		return errors.Wrap(err, "Download of Build Artifact SAR_XML failed")
 	}
-	// also write the already released packages back to cpe
-	for _, b := range buildsAlreadyReleased {
+
+	filesToPublish, err = downloadResultToFile(builds, "DELIVERY_LOGS.ZIP", filesToPublish)
+	if err != nil {
+		//changed result storage with 2105, thus ignore errors for now
+		log.Entry().Error(errors.Wrap(err, "Download of Build Artifact DELIVERY_LOGS.ZIP failed"))
+	}
+
+	log.Entry().Infof("Publsihing %v files", len(filesToPublish))
+	piperutils.PersistReportsAndLinks("abapEnvironmentAssemblePackages", "", filesToPublish, nil)
+
+	var reposBackToCPE []abaputils.Repository
+	for _, b := range builds {
 		reposBackToCPE = append(reposBackToCPE, b.repo)
 	}
 	addonDescriptor.Repositories = reposBackToCPE
@@ -94,89 +102,129 @@ func runAbapEnvironmentAssemblePackages(config *abapEnvironmentAssemblePackagesO
 	return nil
 }
 
-func starting(repos []abaputils.Repository, conn abapbuild.Connector, delayBetweenPostsInSeconds time.Duration) ([]buildWithRepository, []buildWithRepository, error) {
+func executeBuilds(repos []abaputils.Repository, conn abapbuild.Connector, maxRuntimeInMinutes time.Duration, pollIntervalsInMilliseconds time.Duration) ([]buildWithRepository, error) {
 	var builds []buildWithRepository
-	var buildsAlreadyReleased []buildWithRepository
+
 	for _, repo := range repos {
-		assemblyBuild := abapbuild.Build{
-			Connector: conn,
-		}
+
 		buildRepo := buildWithRepository{
-			build: assemblyBuild,
-			repo:  repo,
+			build: abapbuild.Build{
+				Connector: conn,
+			},
+			repo: repo,
 		}
+
 		if repo.Status == "P" {
 			err := buildRepo.start()
 			if err != nil {
-				return builds, buildsAlreadyReleased, err
+				buildRepo.build.RunState = abapbuild.Failed
+				log.Entry().Error(err)
+				log.Entry().Info("Continueing with other builds (if any)")
+			} else {
+				err = buildRepo.waitToBeFinished(maxRuntimeInMinutes, pollIntervalsInMilliseconds)
+				if err != nil {
+					buildRepo.build.RunState = abapbuild.Failed
+					log.Entry().Error(err)
+					log.Entry().Error("Continuing with other builds (if any) but keep in Mind that even if this build finishes beyond timeout the result is not trustworthy due to possible side effects!")
+				}
 			}
-			builds = append(builds, buildRepo)
 		} else {
 			log.Entry().Infof("Packages %s is in status '%s'. No need to run the assembly", repo.PackageName, repo.Status)
-			buildsAlreadyReleased = append(buildsAlreadyReleased, buildRepo)
 		}
 
-		//as batch events in the ABAP Backend need a little time
-		time.Sleep(delayBetweenPostsInSeconds)
+		builds = append(builds, buildRepo)
 	}
-	return builds, buildsAlreadyReleased, nil
+	return builds, nil
 }
 
-func polling(builds []buildWithRepository, maxRuntimeInMinutes time.Duration, pollIntervalsInSeconds time.Duration) error {
+func (br *buildWithRepository) waitToBeFinished(maxRuntimeInMinutes time.Duration, pollIntervalsInMilliseconds time.Duration) error {
 	timeout := time.After(maxRuntimeInMinutes)
-	ticker := time.Tick(pollIntervalsInSeconds)
+	ticker := time.Tick(pollIntervalsInMilliseconds)
 	for {
 		select {
 		case <-timeout:
 			return errors.Errorf("Timed out: (max Runtime %v reached)", maxRuntimeInMinutes)
 		case <-ticker:
-			var allFinished bool = true
-			for i := range builds {
-				builds[i].build.Get()
-				if !builds[i].build.IsFinished() {
-					log.Entry().Infof("Assembly of %s is not yet finished, check again in %s", builds[i].repo.PackageName, pollIntervalsInSeconds)
-					allFinished = false
-				}
-			}
-			if allFinished {
+			br.build.Get()
+			if !br.build.IsFinished() {
+				log.Entry().Infof("Assembly of %s is not yet finished, check again in %s", br.repo.PackageName, pollIntervalsInMilliseconds)
+			} else {
 				return nil
 			}
 		}
 	}
 }
 
-func (b *buildWithRepository) start() error {
-	if b.repo.Name == "" || b.repo.Version == "" || b.repo.SpLevel == "" || b.repo.Namespace == "" || b.repo.PackageType == "" || b.repo.PackageName == "" {
+func (br *buildWithRepository) start() error {
+	if br.repo.Name == "" || br.repo.Version == "" || br.repo.SpLevel == "" || br.repo.Namespace == "" || br.repo.PackageType == "" || br.repo.PackageName == "" {
 		return errors.New("Parameters missing. Please provide software component name, version, sp-level, namespace, packagetype and packagename")
 	}
 	valuesInput := abapbuild.Values{
 		Values: []abapbuild.Value{
 			{
 				ValueID: "SWC",
-				Value:   b.repo.Name,
+				Value:   br.repo.Name,
 			},
 			{
 				ValueID: "CVERS",
-				Value:   b.repo.Name + "." + b.repo.Version + "." + b.repo.SpLevel,
+				Value:   br.repo.Name + "." + br.repo.Version + "." + br.repo.SpLevel,
 			},
 			{
 				ValueID: "NAMESPACE",
-				Value:   b.repo.Namespace,
+				Value:   br.repo.Namespace,
 			},
 			{
-				ValueID: "PACKAGE_NAME_" + b.repo.PackageType,
-				Value:   b.repo.PackageName,
+				ValueID: "PACKAGE_NAME_" + br.repo.PackageType,
+				Value:   br.repo.PackageName,
 			},
 		},
 	}
-	if b.repo.PredecessorCommitID != "" {
+	if br.repo.PredecessorCommitID != "" {
 		valuesInput.Values = append(valuesInput.Values,
 			abapbuild.Value{ValueID: "PREVIOUS_DELIVERY_COMMIT",
-				Value: b.repo.PredecessorCommitID})
+				Value: br.repo.PredecessorCommitID})
 	}
-	phase := "BUILD_" + b.repo.PackageType
-	log.Entry().Infof("Starting assembly of package %s", b.repo.PackageName)
-	return b.build.Start(phase, valuesInput)
+	if br.repo.CommitID != "" {
+		valuesInput.Values = append(valuesInput.Values,
+			abapbuild.Value{ValueID: "ACTUAL_DELIVERY_COMMIT",
+				Value: br.repo.CommitID})
+	}
+	phase := "BUILD_" + br.repo.PackageType
+	log.Entry().Infof("Starting assembly of package %s", br.repo.PackageName)
+	return br.build.Start(phase, valuesInput)
+}
+
+func downloadResultToFile(builds []buildWithRepository, resultName string, filesToPublish []piperutils.Path) ([]piperutils.Path, error) {
+	envPath := filepath.Join(GeneralConfig.EnvRootPath, "commonPipelineEnvironment", "abap")
+
+	for i, b := range builds {
+		if b.repo.Status != "P" {
+			continue
+		}
+		buildResult, err := b.build.GetResult(resultName)
+		if err != nil {
+			return filesToPublish, err
+		}
+		var fileName string
+		if len(buildResult.AdditionalInfo) <= 255 {
+			fileName = buildResult.AdditionalInfo
+		} else {
+			fileName = buildResult.Name
+		}
+		downloadPath := filepath.Join(envPath, path.Base(fileName))
+		log.Entry().Infof("Downloading %s file %s to %s", resultName, path.Base(fileName), downloadPath)
+		err = buildResult.Download(downloadPath)
+		if err != nil {
+			return filesToPublish, err
+		}
+		if resultName == "SAR_XML" {
+			builds[i].repo.SarXMLFilePath = downloadPath
+		}
+
+		log.Entry().Infof("Add %s to be published", resultName)
+		filesToPublish = append(filesToPublish, piperutils.Path{Target: downloadPath, Name: resultName, Mandatory: true})
+	}
+	return filesToPublish, nil
 }
 
 func checkIfFailedAndPrintLogs(builds []buildWithRepository) error {

cmd/abapEnvironmentAssemblePackages_test.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -37,9 +36,7 @@ func TestCheckIfFailedAndPrintLogsWithError(t *testing.T) {
 		err := checkIfFailedAndPrintLogs(buildsWithRepo)
 		assert.Error(t, err)
 	})
-}
 
-func TestCheckIfFailedAndPrintLogs(t *testing.T) {
 	t.Run("checkIfFailedAndPrintLogs", func(t *testing.T) {
 		var repo abaputils.Repository
 		b := testSetup(&abapbuild.ClMock{}, "ABIFNLDCSQPOVMXK4DNPBDRW2M")
@@ -55,40 +52,8 @@ func TestCheckIfFailedAndPrintLogs(t *testing.T) {
 	})
 }
 
-func TestStarting(t *testing.T) {
-	t.Run("Run starting", func(t *testing.T) {
-		client := &abapbuild.ClMock{
-			Token: "MyToken",
-		}
-		conn := new(abapbuild.Connector)
-		conn.Client = client
-		conn.Header = make(map[string][]string)
-		var repos []abaputils.Repository
-		repo := abaputils.Repository{
-			Name:        "RepoA",
-			Version:     "0001",
-			PackageName: "Package",
-			PackageType: "AOI",
-			SpLevel:     "0000",
-			PatchLevel:  "0000",
-			Status:      "P",
-			Namespace:   "/DEMO/",
-		}
-		repos = append(repos, repo)
-		repo.Status = "R"
-		repos = append(repos, repo)
-
-		builds, buildsAlreadyReleased, err := starting(repos, *conn, time.Duration(0*time.Second))
-		assert.NoError(t, err)
-		assert.Equal(t, 1, len(builds))
-		assert.Equal(t, 1, len(buildsAlreadyReleased))
-		assert.Equal(t, abapbuild.Accepted, builds[0].build.RunState)
-		assert.Equal(t, abapbuild.RunState(""), buildsAlreadyReleased[0].build.RunState)
-	})
-}
-
 func TestStartingInvalidInput(t *testing.T) {
-	t.Run("Run starting", func(t *testing.T) {
+	t.Run("Run starting with Invalid Input", func(t *testing.T) {
 		client := &abapbuild.ClMock{
 			Token: "MyToken",
 		}
@@ -101,47 +66,22 @@ func TestStartingInvalidInput(t *testing.T) {
 			Status: "P",
 		}
 		repos = append(repos, repo)
-		_, _, err := starting(repos, *conn, time.Duration(0*time.Second))
-		assert.Error(t, err)
-	})
-}
-
-func TestPolling(t *testing.T) {
-	t.Run("Run polling", func(t *testing.T) {
-		var repo abaputils.Repository
-		b := testSetup(&abapbuild.ClMock{}, "ABIFNLDCSQPOVMXK4DNPBDRW2M")
-		var buildsWithRepo []buildWithRepository
-		bWR := buildWithRepository{
-			build: b,
-			repo:  repo,
-		}
-		buildsWithRepo = append(buildsWithRepo, bWR)
-		timeout := time.Duration(600 * time.Second)
-		pollInterval := time.Duration(1 * time.Second)
-		err := polling(buildsWithRepo, timeout, pollInterval)
+		builds, err := executeBuilds(repos, *conn, time.Duration(0*time.Second), time.Duration(1*time.Millisecond))
 		assert.NoError(t, err)
-		assert.Equal(t, abapbuild.Finished, buildsWithRepo[0].build.RunState)
-	})
-}
-
-func TestDownloadSARXML(t *testing.T) {
-	t.Run("Run downloadSARXML", func(t *testing.T) {
-		var repo abaputils.Repository
-		b := testSetup(&abapbuild.ClMock{}, "ABIFNLDCSQPOVMXK4DNPBDRW2M")
-		var buildsWithRepo []buildWithRepository
-		bWR := buildWithRepository{
-			build: b,
-			repo:  repo,
-		}
-		buildsWithRepo = append(buildsWithRepo, bWR)
-		repos, err := downloadSARXML(buildsWithRepo)
-		assert.NoError(t, err)
-		downloadPath := filepath.Join(GeneralConfig.EnvRootPath, "commonPipelineEnvironment", "abap", "SAPK-001AAINITAPC1.SAR")
-		assert.Equal(t, downloadPath, repos[0].SarXMLFilePath)
+		assert.Equal(t, 1, len(builds))
+		assert.Equal(t, abapbuild.Failed, builds[0].build.RunState)
 	})
 }
 
 func TestStep(t *testing.T) {
+	autils := &abaputils.AUtilsMock{
+		ReturnedConnectionDetailsHTTP: abaputils.ConnectionDetailsHTTP{
+			URL: `/sap/opu/odata/BUILD/CORE_SRV`,
+		},
+	}
+	client := abapbuild.GetBuildMockClient()
+	cpe := &abapEnvironmentAssemblePackagesCommonPipelineEnvironment{}
+
 	t.Run("abapEnvironmentAssemblePackages: nothing to do", func(t *testing.T) {
 
 		config := &abapEnvironmentAssemblePackagesOptions{
@@ -150,10 +90,6 @@ func TestStep(t *testing.T) {
 			PollIntervalsInMilliseconds: 1,
 		}
 
-		autils := &abaputils.AUtilsMock{}
-		client := abapbuild.GetBuildMockClient()
-		cpe := &abapEnvironmentAssemblePackagesCommonPipelineEnvironment{}
-
 		err := runAbapEnvironmentAssemblePackages(config, nil, autils, &client, cpe)
 		assert.NoError(t, err)
 	})
@@ -165,16 +101,6 @@ func TestStep(t *testing.T) {
 			PollIntervalsInMilliseconds: 1,
 		}
 
-		autils := &abaputils.AUtilsMock{
-			ReturnedConnectionDetailsHTTP: abaputils.ConnectionDetailsHTTP{
-				URL: `/sap/opu/odata/BUILD/CORE_SRV`,
-			},
-		}
-
-		client := abapbuild.GetBuildMockClient()
-
-		cpe := &abapEnvironmentAssemblePackagesCommonPipelineEnvironment{}
-
 		err := runAbapEnvironmentAssemblePackages(config, nil, autils, &client, cpe)
 		assert.NoError(t, err)
 		assert.Contains(t, cpe.abap.addonDescriptor, `SAPK-001AAINITAPC1.SAR`)

pkg/abap/build/bfw.go

@@ -327,6 +327,10 @@ func (t *task) getResults() error {
 		for i := range t.Results {
 			t.Results[i].connector = t.connector
 		}
+		if len(t.Results) == 0 {
+			//prevent 2nd GET request - no new results will occure...
+			t.Results = append(t.Results, Result{Name: "Dummy"})
+		}
 	}
 	return nil
 }

pkg/abap/build/bfw_test.go

@@ -101,7 +101,7 @@ func TestGetResults(t *testing.T) {
 		b := testSetup(&ClMock{}, "ABIFNLDCSQPOVMXK4DNPBDRW2M")
 		err := b.getResults()
 		assert.NoError(t, err)
-		assert.Equal(t, 0, len(b.Tasks[0].Results))
+		assert.Equal(t, 1, len(b.Tasks[0].Results))
 		assert.Equal(t, 2, len(b.Tasks[1].Results))
 		assert.Equal(t, "image/jpeg", b.Tasks[1].Results[0].Mimetype)
 		assert.Equal(t, "application/octet-stream", b.Tasks[1].Results[1].Mimetype)

pkg/abap/build/mockClient.go

@@ -35,7 +35,7 @@ func NewMockClient() MockClient {
 func (mc *MockClient) AddResponse(method, url string, response http.Response) {
 	responseList, ok := mc.Data[method+url]
 	if !ok {
-		responseList = make([]http.Response, 1)
+		responseList = make([]http.Response, 0)
 	}
 	responseList = append(responseList, response)
 
@@ -73,7 +73,7 @@ func (mc *MockClient) getResponse(method, url string) (http.Response, bool) {
 	if !ok {
 		return http.Response{}, false
 	}
-	response := responseList[1]
+	response := responseList[0]
 	if len(responseList) > 1 {
 		mc.Data[method+url] = responseList[1:]
 	} else {
@@ -1453,6 +1453,32 @@ var buildGetTask11Result = mockData{
 	statusCode: 200,
 }
 var buildGetTask12Result = mockData{
+	method: `GET`,
+	url:    `/sap/opu/odata/BUILD/CORE_SRV/tasks(build_id='AKO22FYOFYPOXHOBVKXUTX3A3Q',task_id=12)/results`,
+	body: `{
+	"d" : {
+		"results" : [
+			{
+				"__metadata" : {
+					"id" : "https://7aa9d1a3-a876-464e-b59a-f26104452461.abap.stagingaws.hanavlab.ondemand.com/sap/opu/odata/BUILD/CORE_SRV/results(build_id='AKO22FYOFYPOXHOBVKXUTX3A3Q',task_id=12,name='DELIVERY_LOGS.ZIP')",
+					"uri" : "https://7aa9d1a3-a876-464e-b59a-f26104452461.abap.stagingaws.hanavlab.ondemand.com/sap/opu/odata/BUILD/CORE_SRV/results(build_id='AKO22FYOFYPOXHOBVKXUTX3A3Q',task_id=12,name='DELIVERY_LOGS.ZIP')",
+					"type" : "BUILD.CORE_SRV.xBUILDxVIEW_RESULTSType",
+					"content_type" : "application/x-zip-compressed",
+					"media_src" : "https://7aa9d1a3-a876-464e-b59a-f26104452461.abap.stagingaws.hanavlab.ondemand.com/sap/opu/odata/BUILD/CORE_SRV/results(build_id='AKO22FYOFYPOXHOBVKXUTX3A3Q',task_id=12,name='DELIVERY_LOGS.ZIP')/$value"
+				},
+				"build_id" : "AKO22FYOFYPOXHOBVKXUTX3A3Q",
+				"task_id" : 12,
+				"name" : "DELIVERY_LOGS.ZIP",
+				"additional_info" : "SAPK-001AAINITAPC1.zip",
+				"mimetype" : "application/x-zip-compressed"
+			}
+		]
+	}
+}`,
+	statusCode: 200,
+}
+
+var buildGetTask12ResultOrig = mockData{
 	method: `GET`,
 	url:    `/sap/opu/odata/BUILD/CORE_SRV/tasks(build_id='AKO22FYOFYPOXHOBVKXUTX3A3Q',task_id=12)/results`,
 	body: `{