1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2024-12-12 10:55:20 +02:00

New log entities for Pull & Clone (#3517)

* WIP

* New Logs

* Improving

* Determine log output based on available entities

* Increase width

* Add line

* Adapt TestPollEntity

* Format

* Fix query

* Adapt tests

* Fix test

* Improve formatting

* Retern early in case of no logs

* Remove duplicate log
This commit is contained in:
Daniel Mieg 2022-02-11 10:16:40 +01:00 committed by GitHub
parent be10e14c88
commit f08ff92171
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 217 additions and 39 deletions

View File

@ -86,7 +86,6 @@ func checkoutBranches(repositories []abaputils.Repository, checkoutConnectionDet
if err != nil {
break
}
finishCheckoutLogs(repo.Branch, repo.Name)
}
return err
}
@ -141,8 +140,7 @@ func triggerCheckout(repositoryName string, branchName string, checkoutConnectio
return uriConnectionDetails, err
}
expandLog := "?$expand=to_Execution_log,to_Transport_log"
uriConnectionDetails.URL = body.Metadata.URI + expandLog
uriConnectionDetails.URL = body.Metadata.URI
return uriConnectionDetails, nil
}

View File

@ -1,6 +1,7 @@
package cmd
import (
"fmt"
"io/ioutil"
"os"
"testing"
@ -32,8 +33,11 @@ func TestCheckoutBranchStep(t *testing.T) {
BranchName: "testBranch",
}
logResultSuccess := fmt.Sprintf(`{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Success", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }`)
client := &abaputils.ClientMock{
BodyList: []string{
logResultSuccess,
`{"d" : { "EntitySets" : [ "LogOverviews" ] } }`,
`{"d" : { "status" : "S" } }`,
`{"d" : { "status" : "S" } }`,
`{"d" : { "status" : "S" } }`,
@ -57,8 +61,11 @@ func TestCheckoutBranchStep(t *testing.T) {
config := abapEnvironmentCheckoutBranchOptions{}
logResultError := fmt.Sprintf(`{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Error", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }`)
client := &abaputils.ClientMock{
BodyList: []string{
logResultError,
`{"d" : { "EntitySets" : [ "LogOverviews" ] } }`,
`{"d" : { "status" : "E" } }`,
`{"d" : { "status" : "E" } }`,
`{"d" : { "status" : "E" } }`,
@ -92,8 +99,11 @@ func TestCheckoutBranchStep(t *testing.T) {
BranchName: "testBranch",
}
logResultError := fmt.Sprintf(`{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Error", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }`)
client := &abaputils.ClientMock{
BodyList: []string{
logResultError,
`{"d" : { "EntitySets" : [ "LogOverviews" ] } }`,
`{"d" : { "status" : "E" } }`,
`{"d" : { "status" : "E" } }`,
`{"d" : { "status" : "E" } }`,
@ -263,7 +273,7 @@ func TestTriggerCheckout(t *testing.T) {
// given
receivedURI := "example.com/Branches"
uriExpected := receivedURI + "?$expand=to_Execution_log,to_Transport_log"
uriExpected := receivedURI
tokenExpected := "myToken"
client := &abaputils.ClientMock{

View File

@ -146,7 +146,7 @@ func triggerClone(repo abaputils.Repository, cloneConnectionDetails abaputils.Co
// The entity "Clones" does not allow for polling. To poll the progress, the related entity "Pull" has to be called
// While "Clones" has the key fields UUID, SC_NAME and BRANCH_NAME, "Pull" only has the key field UUID
uriConnectionDetails.URL = uriConnectionDetails.URL + "/sap/opu/odata/sap/MANAGE_GIT_REPOSITORY/Pull(uuid=guid'" + body.UUID + "')" + "?$expand=to_Execution_log,to_Transport_log"
uriConnectionDetails.URL = uriConnectionDetails.URL + "/sap/opu/odata/sap/MANAGE_GIT_REPOSITORY/Pull(uuid=guid'" + body.UUID + "')"
return uriConnectionDetails, nil
}

View File

@ -1,6 +1,7 @@
package cmd
import (
"fmt"
"io/ioutil"
"os"
"testing"
@ -57,16 +58,23 @@ repositories:
Repositories: "filename.yaml",
}
logResultSuccess := fmt.Sprintf(`{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Success", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }`)
client := &abaputils.ClientMock{
BodyList: []string{
logResultSuccess,
`{"d" : { "EntitySets" : [ "LogOverviews" ] } }`,
`{"d" : { "status" : "S" } }`,
`{"d" : { "status" : "R" } }`,
`{"d" : { "status" : "R" } }`,
`{"d" : { "status" : "R" } }`,
logResultSuccess,
`{"d" : { "EntitySets" : [ "LogOverviews" ] } }`,
`{"d" : { "status" : "S" } }`,
`{"d" : { "status" : "R" } }`,
`{"d" : { "status" : "R" } }`,
`{"d" : { "status" : "R" } }`,
logResultSuccess,
`{"d" : { "EntitySets" : [ "LogOverviews" ] } }`,
`{"d" : { "status" : "S" } }`,
`{"d" : { "status" : "R" } }`,
`{"d" : { "status" : "R" } }`,
@ -161,8 +169,11 @@ repositories:
Repositories: "filename.yaml",
}
logResultError := fmt.Sprintf(`{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Error", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }`)
client := &abaputils.ClientMock{
BodyList: []string{
logResultError,
`{"d" : { "EntitySets" : [ "LogOverviews" ] } }`,
`{"d" : { "status" : "E" } }`,
`{"d" : { "status" : "R" } }`,
`{"d" : { "status" : "R" } }`,

View File

@ -163,8 +163,7 @@ func triggerPull(repo abaputils.Repository, pullConnectionDetails abaputils.Conn
return uriConnectionDetails, err
}
expandLog := "?$expand=to_Execution_log,to_Transport_log"
uriConnectionDetails.URL = body.Metadata.URI + expandLog
uriConnectionDetails.URL = body.Metadata.URI
return uriConnectionDetails, nil
}

View File

@ -1,6 +1,7 @@
package cmd
import (
"fmt"
"io/ioutil"
"os"
"testing"
@ -32,8 +33,11 @@ func TestPullStep(t *testing.T) {
RepositoryNames: []string{"testRepo1"},
}
logResultSuccess := fmt.Sprintf(`{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Success", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }`)
client := &abaputils.ClientMock{
BodyList: []string{
logResultSuccess,
`{"d" : { "EntitySets" : [ "LogOverviews" ] } }`,
`{"d" : { "status" : "S" } }`,
`{"d" : { "status" : "R" } }`,
`{"d" : { "status" : "R" } }`,
@ -167,8 +171,11 @@ repositories:
Repositories: "filename.yaml",
}
logResultError := fmt.Sprintf(`{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Error", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }`)
client := &abaputils.ClientMock{
BodyList: []string{
logResultError,
`{"d" : { "EntitySets" : [ "LogOverviews" ] } }`,
`{"d" : { "status" : "E" } }`,
`{"d" : { "status" : "R" } }`,
`{"d" : { "status" : "R" } }`,
@ -225,9 +232,11 @@ repositories:
Repositories: "filename.yaml",
IgnoreCommit: true,
}
logResultError := fmt.Sprintf(`{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Error", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }`)
client := &abaputils.ClientMock{
BodyList: []string{
logResultError,
`{"d" : { "EntitySets" : [ "LogOverviews" ] } }`,
`{"d" : { "status" : "E" } }`,
`{"d" : { "status" : "R" } }`,
`{"d" : { "status" : "R" } }`,
@ -349,7 +358,7 @@ func TestTriggerPull(t *testing.T) {
t.Run("Test trigger pull: success case", func(t *testing.T) {
receivedURI := "example.com/Entity"
uriExpected := receivedURI + "?$expand=to_Execution_log,to_Transport_log"
uriExpected := receivedURI
tokenExpected := "myToken"
client := &abaputils.ClientMock{

View File

@ -51,8 +51,10 @@ func (abaputils *AbapUtils) GetAbapCommunicationArrangementInfo(options AbapEnvi
var hostOdataURL = options.Host + oDataURL
if match {
connectionDetails.URL = hostOdataURL
connectionDetails.Host = options.Host
} else {
connectionDetails.URL = "https://" + hostOdataURL
connectionDetails.Host = "https://" + options.Host
}
connectionDetails.User = options.Username
connectionDetails.Password = options.Password
@ -67,6 +69,7 @@ func (abaputils *AbapUtils) GetAbapCommunicationArrangementInfo(options AbapEnvi
if error != nil {
return connectionDetails, errors.Wrap(error, "Read service key failed")
}
connectionDetails.Host = abapServiceKey.URL
connectionDetails.URL = abapServiceKey.URL + oDataURL
connectionDetails.User = abapServiceKey.Abap.Username
connectionDetails.Password = abapServiceKey.Abap.Password
@ -266,6 +269,7 @@ type AbapMetadata struct {
// ConnectionDetailsHTTP contains fields for HTTP connections including the XCSRF token
type ConnectionDetailsHTTP struct {
Host string
User string `json:"user"`
Password string `json:"password"`
URL string `json:"url"`

View File

@ -20,37 +20,24 @@ func PollEntity(repositoryName string, connectionDetails ConnectionDetailsHTTP,
var status string = "R"
for {
var resp, err = GetHTTPResponse("GET", connectionDetails, nil, client)
pullEntity, responseStatus, err := GetPullStatus(repositoryName, connectionDetails, client)
if err != nil {
log.SetErrorCategory(log.ErrorInfrastructure)
err = HandleHTTPError(resp, err, "Could not pull the Repository / Software Component "+repositoryName, connectionDetails)
return "", err
return status, err
}
defer resp.Body.Close()
status = pullEntity.Status
log.Entry().WithField("StatusCode", responseStatus).Info("Pull Status: " + pullEntity.StatusDescription)
if pullEntity.Status != "R" {
// Parse response
var abapResp map[string]*json.RawMessage
var body PullEntity
bodyText, _ := ioutil.ReadAll(resp.Body)
json.Unmarshal(bodyText, &abapResp)
json.Unmarshal(*abapResp["d"], &body)
if reflect.DeepEqual(PullEntity{}, body) {
log.Entry().WithField("StatusCode", resp.Status).WithField("repositoryName", repositoryName).Error("Could not pull the Repository / Software Component")
log.SetErrorCategory(log.ErrorInfrastructure)
var err = errors.New("Request to ABAP System not successful")
return "", err
}
status = body.Status
log.Entry().WithField("StatusCode", resp.Status).Info("Pull Status: " + body.StatusDescription)
if body.Status != "R" {
if body.Status == "E" {
log.SetErrorCategory(log.ErrorUndefined)
PrintLogs(body, true)
if serviceContainsNewLogEntities(connectionDetails, client) {
PrintLogs(repositoryName, connectionDetails, client)
} else {
PrintLogs(body, false)
// Fallback
if pullEntity.Status == "E" {
log.SetErrorCategory(log.ErrorUndefined)
PrintLegacyLogs(repositoryName, connectionDetails, client, true)
} else {
PrintLegacyLogs(repositoryName, connectionDetails, client, false)
}
}
break
}
@ -59,9 +46,108 @@ func PollEntity(repositoryName string, connectionDetails ConnectionDetailsHTTP,
return status, nil
}
// PrintLogs sorts and formats the received transport and execution log of an import
func PrintLogs(entity PullEntity, errorOnSystem bool) {
func serviceContainsNewLogEntities(connectionDetails ConnectionDetailsHTTP, client piperhttp.Sender) (newLogEntitiesAvailable bool) {
newLogEntitiesAvailable = false
details := connectionDetails
details.URL = details.Host + "/sap/opu/odata/sap/MANAGE_GIT_REPOSITORY/"
resp, err := GetHTTPResponse("GET", details, nil, client)
if err != nil {
return
}
defer resp.Body.Close()
var entitySet EntitySetsForManageGitRepository
// Parse response
var abapResp map[string]*json.RawMessage
bodyText, _ := ioutil.ReadAll(resp.Body)
json.Unmarshal(bodyText, &abapResp)
json.Unmarshal(*abapResp["d"], &entitySet)
for _, entitySet := range entitySet.EntitySets {
if entitySet == "LogOverviews" || entitySet == "LogProtocols" {
return true
}
}
return
}
func PrintLogs(repositoryName string, connectionDetails ConnectionDetailsHTTP, client piperhttp.Sender) {
connectionDetails.URL = connectionDetails.URL + "?$expand=to_Log_Overview,to_Log_Overview/to_Log_Protocol"
entity, _, err := GetPullStatus(repositoryName, connectionDetails, client)
if err != nil {
return
}
if len(entity.ToLogOverview.Results) == 0 {
// return if no logs are available
return
}
// Sort logs
sort.SliceStable(entity.ToLogOverview.Results, func(i, j int) bool {
return entity.ToLogOverview.Results[i].Index < entity.ToLogOverview.Results[j].Index
})
// Print Overview
log.Entry().Infof("\n")
log.Entry().Infof("-----------------------------------------------------------------------")
log.Entry().Infof("| %-22s | %10s | %-29s |", "Phase", "Status", "Timestamp")
log.Entry().Infof("-----------------------------------------------------------------------")
for _, logEntry := range entity.ToLogOverview.Results {
log.Entry().Infof("| %-22s | %10s | %-29s |", logEntry.Name, logEntry.Status, ConvertTime(logEntry.Timestamp))
}
log.Entry().Infof("-----------------------------------------------------------------------")
// Print Details
for _, logEntryForDetails := range entity.ToLogOverview.Results {
printLog(logEntryForDetails)
}
log.Entry().Infof("-------------------------")
return
}
func printLog(logEntry LogResultsV2) {
sort.SliceStable(logEntry.ToLogProtocol.Results, func(i, j int) bool {
return logEntry.ToLogProtocol.Results[i].ProtocolLine < logEntry.ToLogProtocol.Results[j].ProtocolLine
})
if logEntry.Status != `Success` {
log.Entry().Infof("\n")
log.Entry().Infof("-------------------------")
log.Entry().Infof("%s (%v)", logEntry.Name, ConvertTime(logEntry.Timestamp))
log.Entry().Infof("-------------------------")
for _, entry := range logEntry.ToLogProtocol.Results {
log.Entry().Info(entry.Description)
}
} else {
log.Entry().Debugf("\n")
log.Entry().Debugf("-------------------------")
log.Entry().Debugf("%s (%v)", logEntry.Name, ConvertTime(logEntry.Timestamp))
log.Entry().Debugf("-------------------------")
for _, entry := range logEntry.ToLogProtocol.Results {
log.Entry().Debug(entry.Description)
}
}
}
// PrintLegacyLogs sorts and formats the received transport and execution log of an import; Deprecated with SAP BTP, ABAP Environment release 2205
func PrintLegacyLogs(repositoryName string, connectionDetails ConnectionDetailsHTTP, client piperhttp.Sender, errorOnSystem bool) {
connectionDetails.URL = connectionDetails.URL + "?$expand=to_Transport_log,to_Execution_log"
entity, _, err := GetPullStatus(repositoryName, connectionDetails, client)
if err != nil {
return
}
// Sort logs
sort.SliceStable(entity.ToExecutionLog.Results, func(i, j int) bool {
return entity.ToExecutionLog.Results[i].Index < entity.ToExecutionLog.Results[j].Index
@ -108,6 +194,31 @@ func PrintLogs(entity PullEntity, errorOnSystem bool) {
}
func GetPullStatus(repositoryName string, connectionDetails ConnectionDetailsHTTP, client piperhttp.Sender) (body PullEntity, status string, err error) {
resp, err := GetHTTPResponse("GET", connectionDetails, nil, client)
if err != nil {
log.SetErrorCategory(log.ErrorInfrastructure)
err = HandleHTTPError(resp, err, "Could not pull the Repository / Software Component "+repositoryName, connectionDetails)
return body, resp.Status, err
}
defer resp.Body.Close()
// Parse response
var abapResp map[string]*json.RawMessage
bodyText, _ := ioutil.ReadAll(resp.Body)
json.Unmarshal(bodyText, &abapResp)
json.Unmarshal(*abapResp["d"], &body)
if reflect.DeepEqual(PullEntity{}, body) {
log.Entry().WithField("StatusCode", resp.Status).WithField("repositoryName", repositoryName).Error("Could not pull the Repository / Software Component")
log.SetErrorCategory(log.ErrorInfrastructure)
var err = errors.New("Request to ABAP System not successful")
return body, resp.Status, err
}
return body, resp.Status, nil
}
//GetRepositories for parsing one or multiple branches and repositories from repositories file or branchName and repositoryName configuration
func GetRepositories(config *RepositoriesConfig) ([]Repository, error) {
var repositories = make([]Repository, 0)
@ -211,6 +322,7 @@ type PullEntity struct {
ChangeTime string `json:"change_time"`
ToExecutionLog AbapLogs `json:"to_Execution_log"`
ToTransportLog AbapLogs `json:"to_Transport_log"`
ToLogOverview AbapLogsV2 `json:"to_Log_Overview"`
}
// BranchEntity struct for the Branch entity A4C_A2G_GHA_SC_BRANCH
@ -249,6 +361,31 @@ type AbapLogs struct {
Results []LogResults `json:"results"`
}
type AbapLogsV2 struct {
Results []LogResultsV2 `json:"results"`
}
type LogResultsV2 struct {
Metadata AbapMetadata `json:"__metadata"`
Index int `json:"log_index"`
Name string `json:"log_name"`
Status string `json:"type_of_found_issues"`
Timestamp string `json:"timestamp"`
ToLogProtocol LogProtocolResults `json:"to_Log_Protocol"`
}
type LogProtocolResults struct {
Results []LogProtocol `json:"results"`
}
type LogProtocol struct {
Metadata AbapMetadata `json:"__metadata"`
OverviewIndex int `json:"log_index"`
ProtocolLine int `json:"index_no"`
Type string `json:"type"`
Description string `json:"descr"`
}
// LogResults struct for Execution and Transport Log entities A4C_A2G_GHA_SC_LOG_EXE and A4C_A2G_GHA_SC_LOG_TP
type LogResults struct {
Index string `json:"index_no"`
@ -264,3 +401,7 @@ type RepositoriesConfig struct {
RepositoryNames []string
Repositories string
}
type EntitySetsForManageGitRepository struct {
EntitySets []string `json:"EntitySets"`
}

View File

@ -1,6 +1,7 @@
package abaputils
import (
"fmt"
"io/ioutil"
"os"
"testing"
@ -12,8 +13,11 @@ func TestPollEntity(t *testing.T) {
t.Run("Test poll entity - success case", func(t *testing.T) {
logResultSuccess := fmt.Sprintf(`{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Success", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }`)
client := &ClientMock{
BodyList: []string{
logResultSuccess,
`{"d" : { "EntitySets" : [ "LogOverviews" ] } }`,
`{"d" : { "status" : "S" } }`,
`{"d" : { "status" : "R" } }`,
},
@ -47,9 +51,11 @@ func TestPollEntity(t *testing.T) {
})
t.Run("Test poll entity - error case", func(t *testing.T) {
logResultError := fmt.Sprintf(`{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Error", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }`)
client := &ClientMock{
BodyList: []string{
logResultError,
`{"d" : { "EntitySets" : [ "LogOverviews" ] } }`,
`{"d" : { "status" : "E" } }`,
`{"d" : { "status" : "R" } }`,
},