AAKaaS upload in chunks (#2422)

* Upload in Chunks

* fix unit tests for file upload in chunks

* upload in chunks review round 1

* Upload in Chunks - review round 2 - comments

Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com>

cmd/abapAddonAssemblyKitRegisterPackages.go

@@ -64,16 +64,21 @@ func uploadSarFiles(repos []abaputils.Repository, conn abapbuild.Connector, read
 				return errors.New("Parameter missing. Please provide the path to the SAR file")
 			}
 			filename := filepath.Base(repos[i].SarXMLFilePath)
-			conn.Header["Content-Filename"] = []string{filename}
+			log.Entry().Infof("Trying to read file %s", repos[i].SarXMLFilePath)
 			sarFile, err := readFileFunc(repos[i].SarXMLFilePath)
 			if err != nil {
 				return err
 			}
-			log.Entry().Infof("Upload SAR file %s", filename)
-			err = conn.UploadSarFile("/odata/aas_file_upload", sarFile)
+			log.Entry().Infof("... %d bytes read", len(sarFile))
+			if len(sarFile) == 0 {
+				return errors.New("File has no content - 0 bytes")
+			}
+			log.Entry().Infof("Upload SAR file %s in chunks", filename)
+			err = conn.UploadSarFileInChunks("/odata/aas_file_upload", filename, sarFile)
 			if err != nil {
 				return err
 			}
+			log.Entry().Info("...done")
 		} else {
 			log.Entry().Infof("Package %s has status %s, cannot upload the SAR file of this package", repos[i].PackageName, repos[i].Status)
 		}
@@ -93,11 +98,14 @@ func registerPackages(repos []abaputils.Repository, conn abapbuild.Connector) ([
 		var pack aakaas.Package
 		pack.InitPackage(repos[i], conn)
 		if repos[i].Status == string(aakaas.PackageStatusPlanned) {
+			log.Entry().Infof("Trying to Register Package %s", pack.PackageName)
 			err := pack.Register()
 			if err != nil {
 				return repos, err
 			}
+			log.Entry().Info("...done, take over new status")
 			pack.ChangeStatus(&repos[i])
+			log.Entry().Info("...done")
 		} else {
 			log.Entry().Infof("Package %s has status %s, cannot register this package", pack.PackageName, pack.Status)
 		}

cmd/abapAddonAssemblyKitRegisterPackages_test.go

@@ -12,11 +12,12 @@ import (
 )
 
 func mockReader(path string) ([]byte, error) {
-	var file []byte
 	if path == "exists" {
-		return file, nil
+		return []byte("test"), nil
+	} else if path == "null" {
+		return []byte(""), nil
 	}
-	return file, errors.New("error reading the file")
+	return nil, errors.New("error reading the file")
 }
 
 func TestRegisterPackagesStep(t *testing.T) {
@@ -48,7 +49,25 @@ func TestRegisterPackagesStep(t *testing.T) {
 		json.Unmarshal([]byte(cpe.abap.addonDescriptor), &addonDescriptorFinal)
 		assert.Equal(t, "L", addonDescriptorFinal.Repositories[0].Status)
 	})
+	t.Run("step error - null file", func(t *testing.T) {
+		client := &abaputils.ClientMock{
+			BodyList: []string{responseRegisterPackagesPost, "myToken", "dummyResponseUpload"},
+		}
+		addonDescriptor := abaputils.AddonDescriptor{
+			Repositories: []abaputils.Repository{
+				{
+					PackageName:    "SAPK-002AAINDRNMSPC",
+					Status:         "P",
+					SarXMLFilePath: "null",
+				},
+			},
+		}
+		adoDesc, _ := json.Marshal(addonDescriptor)
+		config.AddonDescriptor = string(adoDesc)
+		err := runAbapAddonAssemblyKitRegisterPackages(&config, nil, client, &cpe, mockReader)
 
+		assert.Error(t, err, "Did expect error")
+	})
 	t.Run("step error - uploadSarFiles", func(t *testing.T) {
 		client := &abaputils.ClientMock{
 			Body:  "ErrorBody",

pkg/abap/aakaas/package.go

@@ -3,6 +3,7 @@ package aakaas
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 
 	abapbuild "github.com/SAP/jenkins-library/pkg/abap/build"
 	"github.com/SAP/jenkins-library/pkg/abaputils"
@@ -126,7 +127,10 @@ func (p *Package) Register() error {
 	}
 
 	var jPck jsonPackage
-	json.Unmarshal(body, &jPck)
+	err = json.Unmarshal(body, &jPck)
+	if err != nil {
+		return fmt.Errorf("failed to parse package status from response: %w", err)
+	}
 	p.Status = jPck.Package.Status
 	log.Entry().Infof("Package status %s", p.Status)
 	return nil

pkg/abap/build/connector.go

@@ -5,10 +5,12 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/cookiejar"
+	"strconv"
 	"time"
 
 	"github.com/SAP/jenkins-library/pkg/abaputils"
 	piperhttp "github.com/SAP/jenkins-library/pkg/http"
+	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/pkg/errors"
 )
 
@@ -174,3 +176,40 @@ func (conn Connector) UploadSarFile(appendum string, sarFile []byte) error {
 	defer response.Body.Close()
 	return nil
 }
+
+// UploadSarFileInChunks : upload *.sar file in chunks
+func (conn Connector) UploadSarFileInChunks(appendum string, fileName string, sarFile []byte) error {
+	//Maybe Next Refactoring step to read the file in chunks, too?
+	//In case it turns out to be not reliable add a retry mechanism
+
+	url := conn.Baseurl + appendum
+
+	header := make(map[string][]string)
+	header["Content-Disposition"] = []string{"form-data; name=\"file\"; filename=\"" + fileName + "\""}
+
+	//chunkSize := 10000 // 10KB for testing
+	//chunkSize := 1000000 //1MB for Testing,
+	chunkSize := 10000000 //10MB
+	log.Entry().Infof("Upload in chunks of %d bytes", chunkSize)
+
+	sarFileBuffer := bytes.NewBuffer(sarFile)
+	fileSize := sarFileBuffer.Len()
+
+	for sarFileBuffer.Len() > 0 {
+		startOffset := fileSize - sarFileBuffer.Len()
+		nextChunk := bytes.NewBuffer(sarFileBuffer.Next(chunkSize))
+		endOffset := fileSize - sarFileBuffer.Len()
+		header["Content-Range"] = []string{"bytes " + strconv.Itoa(startOffset) + " - " + strconv.Itoa(endOffset) + " / " + strconv.Itoa(fileSize)}
+		log.Entry().Info(header["Content-Range"])
+
+		response, err := conn.Client.SendRequest("POST", url, nextChunk, header, nil)
+		if err != nil {
+			errorbody, _ := ioutil.ReadAll(response.Body)
+			response.Body.Close()
+			return errors.Wrapf(err, "Upload of SAR file failed: %v", string(errorbody))
+		}
+
+		response.Body.Close()
+	}
+	return nil
+}