RobustConfirm (#3179)

confirm no longer done based on package status but now based on boolean indicator which is set during assembly step.
Thus confirm can now be placed after release packages.

cmd/abapEnvironmentAssembleConfirm.go

@@ -54,14 +54,14 @@ func runAbapEnvironmentAssembleConfirm(config *abapEnvironmentAssembleConfirmOpt
 	if err != nil {
 		return err
 	}
-	delayBetweenPostsInSeconds := time.Duration(3 * time.Second)
+	delayBetweenPosts := time.Duration(3 * time.Second)
-	builds, err := startingConfirm(addonDescriptor.Repositories, *conn, delayBetweenPostsInSeconds)
+	builds, err := startingConfirm(addonDescriptor.Repositories, *conn, delayBetweenPosts)
 	if err != nil {
 		return err
 	}
 	maxRuntimeInMinutes := time.Duration(config.MaxRuntimeInMinutes) * time.Minute
-	pollIntervalsInSeconds := time.Duration(60 * time.Second)
+	pollInterval := time.Duration(60 * time.Second)
-	err = polling(builds, maxRuntimeInMinutes, pollIntervalsInSeconds)
+	err = polling(builds, maxRuntimeInMinutes, pollInterval)
 	if err != nil {
 		return err
 	}
@@ -72,9 +72,8 @@ func runAbapEnvironmentAssembleConfirm(config *abapEnvironmentAssembleConfirmOpt
 	return nil
 }
 
-func startingConfirm(repos []abaputils.Repository, conn abapbuild.Connector, delayBetweenPostsInSeconds time.Duration) ([]buildWithRepository, error) {
+func startingConfirm(repos []abaputils.Repository, conn abapbuild.Connector, delayBetweenPosts time.Duration) ([]buildWithRepository, error) {
-	var builds []buildWithRepository
+	var confirmedBuilds []buildWithRepository
-	var buildsAlreadyReleased []buildWithRepository
 	for _, repo := range repos {
 		assemblyBuild := abapbuild.Build{
 			Connector: conn,
@@ -83,26 +82,25 @@ func startingConfirm(repos []abaputils.Repository, conn abapbuild.Connector, del
 			build: assemblyBuild,
 			repo:  repo,
 		}
-		if repo.Status != "R" {
+		if repo.InBuildScope {
 			err := buildRepo.startConfirm()
 			if err != nil {
-				return builds, err
+				return confirmedBuilds, err
 			}
-			builds = append(builds, buildRepo)
+			confirmedBuilds = append(confirmedBuilds, buildRepo)
 		} else {
-			log.Entry().Infof("Packages %s is in status '%s'. No need assembly done, no need to confirm", repo.PackageName, repo.Status)
+			log.Entry().Infof("Packages %s was not assembled in this pipeline run, thus no need to confirm", repo.PackageName)
-			buildsAlreadyReleased = append(buildsAlreadyReleased, buildRepo)
 		}
 
 		//as batch events in the ABAP Backend need a little time
-		time.Sleep(delayBetweenPostsInSeconds)
+		time.Sleep(delayBetweenPosts)
 	}
-	return builds, nil
+	return confirmedBuilds, nil
 }
 
-func polling(builds []buildWithRepository, maxRuntimeInMinutes time.Duration, pollIntervalsInSeconds time.Duration) error {
+func polling(builds []buildWithRepository, maxRuntimeInMinutes time.Duration, pollInterval time.Duration) error {
 	timeout := time.After(maxRuntimeInMinutes)
-	ticker := time.Tick(pollIntervalsInSeconds)
+	ticker := time.Tick(pollInterval)
 	for {
 		select {
 		case <-timeout:
@@ -112,7 +110,7 @@ func polling(builds []buildWithRepository, maxRuntimeInMinutes time.Duration, po
 			for i := range builds {
 				builds[i].build.Get()
 				if !builds[i].build.IsFinished() {
-					log.Entry().Infof("Assembly of %s is not yet finished, check again in %s", builds[i].repo.PackageName, pollIntervalsInSeconds)
+					log.Entry().Infof("Assembly of %s is not yet finished, check again in %s", builds[i].repo.PackageName, pollInterval)
 					allFinished = false
 				}
 			}

cmd/abapEnvironmentAssembleConfirm_test.go

@@ -47,6 +47,7 @@ func TestStartingConfirm(t *testing.T) {
 		}
 		repos = append(repos, repo)
 		repo.Status = "R"
+		repo.InBuildScope = true
 		repos = append(repos, repo)
 
 		builds, err := startingConfirm(repos, *conn, time.Duration(0*time.Second))
@@ -66,8 +67,8 @@ func TestStartingConfirmInvalidInput(t *testing.T) {
 		conn.Header = make(map[string][]string)
 		var repos []abaputils.Repository
 		repo := abaputils.Repository{
-			Name:   "RepoA",
+			Name:         "RepoA",
-			Status: "P",
+			InBuildScope: true,
 		}
 		repos = append(repos, repo)
 		_, err := startingConfirm(repos, *conn, time.Duration(0*time.Second))

cmd/abapEnvironmentAssemblePackages.go

@@ -65,8 +65,8 @@ func runAbapEnvironmentAssemblePackages(config *abapEnvironmentAssemblePackagesO
 	}
 
 	maxRuntimeInMinutes := time.Duration(config.MaxRuntimeInMinutes) * time.Minute
-	pollIntervalsInMilliseconds := time.Duration(config.PollIntervalsInMilliseconds) * time.Millisecond
+	pollInterval := time.Duration(config.PollIntervalsInMilliseconds) * time.Millisecond
-	builds, err := executeBuilds(addonDescriptor.Repositories, *conn, maxRuntimeInMinutes, pollIntervalsInMilliseconds)
+	builds, err := executeBuilds(addonDescriptor.Repositories, *conn, maxRuntimeInMinutes, pollInterval)
 	if err != nil {
 		return errors.Wrap(err, "Starting Builds for Repositories with reserved AAKaaS packages failed")
 	}
@@ -102,7 +102,7 @@ func runAbapEnvironmentAssemblePackages(config *abapEnvironmentAssemblePackagesO
 	return nil
 }
 
-func executeBuilds(repos []abaputils.Repository, conn abapbuild.Connector, maxRuntimeInMinutes time.Duration, pollIntervalsInMilliseconds time.Duration) ([]buildWithRepository, error) {
+func executeBuilds(repos []abaputils.Repository, conn abapbuild.Connector, maxRuntimeInMinutes time.Duration, pollInterval time.Duration) ([]buildWithRepository, error) {
 	var builds []buildWithRepository
 
 	for _, repo := range repos {
@@ -115,13 +115,14 @@ func executeBuilds(repos []abaputils.Repository, conn abapbuild.Connector, maxRu
 		}
 
 		if repo.Status == "P" {
+			buildRepo.repo.InBuildScope = true
 			err := buildRepo.start()
 			if err != nil {
 				buildRepo.build.RunState = abapbuild.Failed
 				log.Entry().Error(err)
 				log.Entry().Info("Continueing with other builds (if any)")
 			} else {
-				err = buildRepo.waitToBeFinished(maxRuntimeInMinutes, pollIntervalsInMilliseconds)
+				err = buildRepo.waitToBeFinished(maxRuntimeInMinutes, pollInterval)
 				if err != nil {
 					buildRepo.build.RunState = abapbuild.Failed
 					log.Entry().Error(err)
@@ -137,9 +138,9 @@ func executeBuilds(repos []abaputils.Repository, conn abapbuild.Connector, maxRu
 	return builds, nil
 }
 
-func (br *buildWithRepository) waitToBeFinished(maxRuntimeInMinutes time.Duration, pollIntervalsInMilliseconds time.Duration) error {
+func (br *buildWithRepository) waitToBeFinished(maxRuntimeInMinutes time.Duration, pollInterval time.Duration) error {
 	timeout := time.After(maxRuntimeInMinutes)
-	ticker := time.Tick(pollIntervalsInMilliseconds)
+	ticker := time.Tick(pollInterval)
 	for {
 		select {
 		case <-timeout:
@@ -147,7 +148,7 @@ func (br *buildWithRepository) waitToBeFinished(maxRuntimeInMinutes time.Duratio
 		case <-ticker:
 			br.build.Get()
 			if !br.build.IsFinished() {
-				log.Entry().Infof("Assembly of %s is not yet finished, check again in %s", br.repo.PackageName, pollIntervalsInMilliseconds)
+				log.Entry().Infof("Assembly of %s is not yet finished, check again in %s", br.repo.PackageName, pollInterval)
 			} else {
 				return nil
 			}
@@ -251,25 +252,3 @@ func checkIfFailedAndPrintLogs(builds []buildWithRepository) error {
 	}
 	return nil
 }
-
-func downloadSARXML(builds []buildWithRepository) ([]abaputils.Repository, error) {
-	var reposBackToCPE []abaputils.Repository
-	resultName := "SAR_XML"
-	envPath := filepath.Join(GeneralConfig.EnvRootPath, "commonPipelineEnvironment", "abap")
-	for i, b := range builds {
-		resultSARXML, err := b.build.GetResult(resultName)
-		if err != nil {
-			return reposBackToCPE, err
-		}
-		sarPackage := resultSARXML.AdditionalInfo
-		downloadPath := filepath.Join(envPath, path.Base(sarPackage))
-		log.Entry().Infof("Downloading SAR file %s to %s", path.Base(sarPackage), downloadPath)
-		err = resultSARXML.Download(downloadPath)
-		if err != nil {
-			return reposBackToCPE, err
-		}
-		builds[i].repo.SarXMLFilePath = downloadPath
-		reposBackToCPE = append(reposBackToCPE, builds[i].repo)
-	}
-	return reposBackToCPE, nil
-}

cmd/abapEnvironmentAssemblePackages_test.go

@@ -92,6 +92,7 @@ func TestStep(t *testing.T) {
 
 		err := runAbapEnvironmentAssemblePackages(config, nil, autils, &client, cpe)
 		assert.NoError(t, err)
+		assert.Contains(t, cpe.abap.addonDescriptor, `"InBuildScope":false`)
 	})
 	t.Run("abapEnvironmentAssemblePackages: build", func(t *testing.T) {
 
@@ -104,6 +105,7 @@ func TestStep(t *testing.T) {
 		err := runAbapEnvironmentAssemblePackages(config, nil, autils, &client, cpe)
 		assert.NoError(t, err)
 		assert.Contains(t, cpe.abap.addonDescriptor, `SAPK-001AAINITAPC1.SAR`)
+		assert.Contains(t, cpe.abap.addonDescriptor, `"InBuildScope":true`)
 	})
 }
 

documentation/docs/pipelines/abapEnvironment/configuration.md

@@ -12,7 +12,7 @@ You can have a look at different pipeline configurations in our [SAP-samples rep
 | [Prepare System](stages/prepareSystem.md)           | [abapEnvironmentCreateSystem](https://sap.github.io/jenkins-library/steps/abapEnvironmentCreateSystem/), [cloudFoundryCreateServiceKey](https://sap.github.io/jenkins-library/steps/cloudFoundryCreateServiceKey/)|
 | [Clone Repositories](stages/cloneRepositories.md)       | [abapEnvironmentPullGitRepo](https://sap.github.io/jenkins-library/steps/abapEnvironmentPullGitRepo/)|
 | [ATC](stages/ATC.md)                      | [abapEnvironmentRunATCCheck](https://sap.github.io/jenkins-library/steps/abapEnvironmentRunATCCheck/)|
-| [Build](stages/build.md)                    | [cloudFoundryCreateServiceKey](https://sap.github.io/jenkins-library/steps/cloudFoundryCreateServiceKey/), [abapAddonAssemblyKitReserveNextPackages](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitReserveNextPackages/), [abapEnvironmentAssemblePackages](https://sap.github.io/jenkins-library/steps/abapEnvironmentAssemblePackages/), [abapAddonAssemblyKitRegisterPackages](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitRegisterPackages/), [abapEnvironmentAssembleConfirm](https://sap.github.io/jenkins-library/steps/abapEnvironmentAssembleConfirm/), [abapAddonAssemblyKitReleasePackages](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitReleasePackages/), [abapAddonAssemblyKitCreateTargetVector](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitCreateTargetVector/), [abapAddonAssemblyKitPublishTargetVector](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitPublishTargetVector/)|
+| [Build](stages/build.md)                    | [cloudFoundryCreateServiceKey](https://sap.github.io/jenkins-library/steps/cloudFoundryCreateServiceKey/), [abapAddonAssemblyKitReserveNextPackages](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitReserveNextPackages/), [abapEnvironmentAssemblePackages](https://sap.github.io/jenkins-library/steps/abapEnvironmentAssemblePackages/), [abapAddonAssemblyKitRegisterPackages](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitRegisterPackages/), [abapAddonAssemblyKitReleasePackages](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitReleasePackages/), [abapEnvironmentAssembleConfirm](https://sap.github.io/jenkins-library/steps/abapEnvironmentAssembleConfirm/), [abapAddonAssemblyKitCreateTargetVector](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitCreateTargetVector/), [abapAddonAssemblyKitPublishTargetVector](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitPublishTargetVector/)|
 | [Integration Tests](stages/integrationTest.md)        | [cloudFoundryCreateService](https://sap.github.io/jenkins-library/steps/cloudFoundryCreateService/)|
 | [Confirm](stages/confirm.md)                  | -     |
 | [Publish](stages/publish.md)                  | [abapAddonAssemblyKitPublishTargetVector](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitPublishTargetVector/)|

documentation/docs/pipelines/abapEnvironment/introduction.md

@@ -28,7 +28,7 @@ The following stages and steps are part of the pipeline:
 | [Prepare System](stages/prepareSystem.md)           | [abapEnvironmentCreateSystem](https://sap.github.io/jenkins-library/steps/abapEnvironmentCreateSystem/), [cloudFoundryCreateServiceKey](https://sap.github.io/jenkins-library/steps/cloudFoundryCreateServiceKey/)|
 | [Clone Repositories](stages/cloneRepositories.md)       | [abapEnvironmentPullGitRepo](https://sap.github.io/jenkins-library/steps/abapEnvironmentPullGitRepo/)|
 | [ATC](stages/ATC.md)                      | [abapEnvironmentRunATCCheck](https://sap.github.io/jenkins-library/steps/abapEnvironmentRunATCCheck/)|
-| [Build](stages/build.md)                    | [cloudFoundryCreateServiceKey](https://sap.github.io/jenkins-library/steps/cloudFoundryCreateServiceKey/), [abapAddonAssemblyKitReserveNextPackages](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitReserveNextPackages/), [abapEnvironmentAssemblePackages](https://sap.github.io/jenkins-library/steps/abapEnvironmentAssemblePackages/), [abapAddonAssemblyKitRegisterPackages](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitRegisterPackages/), [abapEnvironmentAssembleConfirm](https://sap.github.io/jenkins-library/steps/abapEnvironmentAssembleConfirm/), [abapAddonAssemblyKitReleasePackages](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitReleasePackages/), [abapAddonAssemblyKitCreateTargetVector](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitCreateTargetVector/), [abapAddonAssemblyKitPublishTargetVector](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitPublishTargetVector/)|
+| [Build](stages/build.md)                    | [cloudFoundryCreateServiceKey](https://sap.github.io/jenkins-library/steps/cloudFoundryCreateServiceKey/), [abapAddonAssemblyKitReserveNextPackages](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitReserveNextPackages/), [abapEnvironmentAssemblePackages](https://sap.github.io/jenkins-library/steps/abapEnvironmentAssemblePackages/), [abapAddonAssemblyKitRegisterPackages](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitRegisterPackages/), [abapAddonAssemblyKitReleasePackages](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitReleasePackages/), [abapEnvironmentAssembleConfirm](https://sap.github.io/jenkins-library/steps/abapEnvironmentAssembleConfirm/), [abapAddonAssemblyKitCreateTargetVector](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitCreateTargetVector/), [abapAddonAssemblyKitPublishTargetVector](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitPublishTargetVector/)|
 | [Integration Tests](stages/integrationTest.md)        | [cloudFoundryCreateService](https://sap.github.io/jenkins-library/steps/cloudFoundryCreateService/)|
 | [Confirm](stages/confirm.md)                  | -     |
 | [Publish](stages/publish.md)                  | [abapAddonAssemblyKitPublishTargetVector](https://sap.github.io/jenkins-library/steps/abapAddonAssemblyKitPublishTargetVector/)|

documentation/docs/pipelines/abapEnvironment/stages/build.md

@@ -10,8 +10,8 @@ The following steps are executed in this stage:
 - [abapAddonAssemblyKitReserveNextPackages](../../../steps/abapAddonAssemblyKitReserveNextPackages.md)
 - [abapEnvironmentAssemblePackages](../../../steps/abapEnvironmentAssemblePackages.md)
 - [abapAddonAssemblyKitRegisterPackages](../../../steps/abapAddonAssemblyKitRegisterPackages.md)
-- [abapEnvironmentAssembleConfirm](../../../steps/abapEnvironmentAssembleConfirm.md)
 - [abapAddonAssemblyKitReleasePackages](../../../steps/abapAddonAssemblyKitReleasePackages.md)
+- [abapEnvironmentAssembleConfirm](../../../steps/abapEnvironmentAssembleConfirm.md)
 - [abapAddonAssemblyKitCreateTargetVector](../../../steps/abapAddonAssemblyKitCreateTargetVector.md)
 - [abapAddonAssemblyKitPublishTargetVector](../../../steps/abapAddonAssemblyKitPublishTargetVector.md)
 

pkg/abaputils/addonDescriptor.go

@@ -51,6 +51,7 @@ type Repository struct {
 	Namespace           string
 	SarXMLFilePath      string
 	Languages           []string `json:"languages"`
+	InBuildScope        bool
 }
 
 // ReadAddonDescriptorType is the type for ReadAddonDescriptor for mocking
@@ -116,7 +117,7 @@ func (me *AddonDescriptor) initFromYmlFile(FileName string, readFile readFileFun
 func CheckAddonDescriptorForRepositories(addonDescriptor AddonDescriptor) error {
 	//checking if parsing went wrong
 	if len(addonDescriptor.Repositories) == 0 {
-		return errors.New(fmt.Sprintf("AddonDescriptor doesn't contain any repositories"))
+		return errors.New("AddonDescriptor doesn't contain any repositories")
 	}
 	//
 	emptyRepositoryCounter := 0
@@ -125,7 +126,7 @@ func CheckAddonDescriptorForRepositories(addonDescriptor AddonDescriptor) error
 			emptyRepositoryCounter++
 		}
 		if counter+1 == len(addonDescriptor.Repositories) && emptyRepositoryCounter == len(addonDescriptor.Repositories) {
-			return errors.New(fmt.Sprintf("AddonDescriptor doesn't contain any repositories"))
+			return errors.New("AddonDescriptor doesn't contain any repositories")
 		}
 	}
 	return nil
@@ -159,3 +160,13 @@ func (me *Repository) GetAakAasLanguageVector() string {
 	}
 	return languageVector
 }
+
+func (me *AddonDescriptor) GetRepositoriesInBuildScope() []Repository {
+	var RepositoriesInBuildScope []Repository
+	for _, repo := range me.Repositories {
+		if repo.InBuildScope {
+			RepositoriesInBuildScope = append(RepositoriesInBuildScope, repo)
+		}
+	}
+	return RepositoriesInBuildScope
+}

pkg/abaputils/addonDescriptor_test.go

@@ -14,18 +14,29 @@ func readFileMock(FileName string) ([]byte, error) {
 }
 
 func TestAddonDescriptorNew(t *testing.T) {
-	t.Run("Import addon.yml", func(t *testing.T) {
+	var addonDescriptor AddonDescriptor
-		var addonDescriptor AddonDescriptor
+	err := addonDescriptor.initFromYmlFile(TestAddonDescriptorYAML, readFileMock)
-		err := addonDescriptor.initFromYmlFile(TestAddonDescriptorYAML, readFileMock)
+	assert.NoError(t, err)
-		CheckAddonDescriptorForRepositories(addonDescriptor)
+	err = CheckAddonDescriptorForRepositories(addonDescriptor)
+	assert.NoError(t, err)
 
-		assert.NoError(t, err)
+	t.Run("Import addon.yml", func(t *testing.T) {
 		assert.Equal(t, "/DMO/myAddonProduct", addonDescriptor.AddonProduct)
 		assert.Equal(t, "/DMO/REPO_A", addonDescriptor.Repositories[0].Name)
 		assert.Equal(t, "JEK8S273S", addonDescriptor.Repositories[1].CommitID)
 		assert.Equal(t, "FR", addonDescriptor.Repositories[1].Languages[2])
 		assert.Equal(t, `ISO-DEENFR`, addonDescriptor.Repositories[1].GetAakAasLanguageVector())
 	})
+
+	t.Run("getRepositoriesInBuildScope", func(t *testing.T) {
+		assert.Equal(t, 2, len(addonDescriptor.Repositories))
+		addonDescriptor.Repositories[1].InBuildScope = true
+
+		repos := addonDescriptor.GetRepositoriesInBuildScope()
+		assert.Equal(t, 1, len(repos))
+		assert.Equal(t, "/DMO/REPO_B", repos[0].Name)
+	})
+
 }
 
 var TestAddonDescriptorYAML = `---
@@ -130,6 +141,7 @@ func TestReadAddonDescriptor(t *testing.T) {
       - repo: 'testRepo2'`
 
 		err = ioutil.WriteFile("repositories.yml", []byte(manifestFileString), 0644)
+		assert.NoError(t, err)
 
 		addonDescriptor, err := ReadAddonDescriptor("repositories.yml")
 

vars/abapEnvironmentPipelineStageBuild.groovy

@@ -10,8 +10,8 @@ import static com.sap.piper.Prerequisites.checkScript
     'abapAddonAssemblyKitReserveNextPackages',
     'abapEnvironmentAssemblePackages',
     'abapAddonAssemblyKitRegisterPackages',
-    'abapEnvironmentAssembleConfirm',
     'abapAddonAssemblyKitReleasePackages',
+    'abapEnvironmentAssembleConfirm',
     'abapAddonAssemblyKitCreateTargetVector',
     'abapAddonAssemblyKitPublishTargetVector'
 ]
@@ -29,8 +29,8 @@ void call(Map parameters = [:]) {
         abapAddonAssemblyKitReserveNextPackages script: parameters.script
         abapEnvironmentAssemblePackages script: parameters.script
         abapAddonAssemblyKitRegisterPackages script: parameters.script
-        abapEnvironmentAssembleConfirm script: parameters.script
         abapAddonAssemblyKitReleasePackages script: parameters.script
+        abapEnvironmentAssembleConfirm script: parameters.script
         abapAddonAssemblyKitCreateTargetVector script: parameters.script
         abapAddonAssemblyKitPublishTargetVector(script: parameters.script, targetVectorScope: 'T')
     }