feat: http report creation for build steps (#3888)

* URL logging feature for execution step provided

cmd/cnbBuild.go

@@ -129,9 +129,11 @@ func setCustomBuildpacks(bpacks []string, dockerCreds string, utils cnbutils.Bui
 
 func newCnbBuildUtils() cnbutils.BuildUtils {
 	utils := cnbBuildUtilsBundle{
-		Command: &command.Command{},
-		Files:   &piperutils.Files{},
-		Client:  &docker.Client{},
+		Command: &command.Command{
+			StepName: "cnbBuild",
+		},
+		Files:  &piperutils.Files{},
+		Client: &docker.Client{},
 	}
 	utils.Stdout(log.Writer())
 	utils.Stderr(log.Writer())

cmd/golangBuild.go

@@ -90,7 +90,9 @@ func newGolangBuildUtils(config golangBuildOptions) golangBuildUtils {
 	httpClient.SetOptions(httpClientOptions)
 
 	utils := golangBuildUtilsBundle{
-		Command:  &command.Command{},
+		Command: &command.Command{
+			StepName: "golangBuild",
+		},
 		Files:    &piperutils.Files{},
 		Uploader: &httpClient,
 		Client: &goget.ClientImpl{

cmd/gradleExecuteBuild.go

@@ -100,8 +100,10 @@ type gradleExecuteBuildUtilsBundle struct {
 
 func newGradleExecuteBuildUtils() gradleExecuteBuildUtils {
 	utils := gradleExecuteBuildUtilsBundle{
-		Command: &command.Command{},
-		Files:   &piperutils.Files{},
+		Command: &command.Command{
+			StepName: "gradleExecuteBuild",
+		},
+		Files: &piperutils.Files{},
 	}
 	utils.Stdout(log.Writer())
 	utils.Stderr(log.Writer())

cmd/mavenBuild.go

@@ -134,7 +134,9 @@ func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomDat
 			}
 
 			downloadClient := &piperhttp.Client{}
-			runner := &command.Command{}
+			runner := &command.Command{
+				StepName: "mavenBuild",
+			}
 			fileUtils := &piperutils.Files{}
 			if len(config.CustomTLSCertificateLinks) > 0 {
 				if err := loadRemoteRepoCertificates(config.CustomTLSCertificateLinks, downloadClient, &deployFlags, runner, fileUtils, config.JavaCaCertFilePath); err != nil {

cmd/mtaBuild.go

@@ -120,9 +120,11 @@ func (bundle *mtaBuildUtilsBundle) DownloadAndCopySettingsFiles(globalSettingsFi
 
 func newMtaBuildUtilsBundle() mtaBuildUtils {
 	utils := mtaBuildUtilsBundle{
-		Command: &command.Command{},
-		Files:   &piperutils.Files{},
-		Client:  &piperhttp.Client{},
+		Command: &command.Command{
+			StepName: "mtaBuild",
+		},
+		Files:  &piperutils.Files{},
+		Client: &piperhttp.Client{},
 	}
 	utils.Stdout(log.Writer())
 	utils.Stderr(log.Writer())

cmd/pythonBuild.go

@@ -29,8 +29,10 @@ type pythonBuildUtilsBundle struct {
 
 func newPythonBuildUtils() pythonBuildUtils {
 	utils := pythonBuildUtilsBundle{
-		Command: &command.Command{},
-		Files:   &piperutils.Files{},
+		Command: &command.Command{
+			StepName: "pythonBuild",
+		},
+		Files: &piperutils.Files{},
 	}
 	// Reroute command output to logging framework
 	utils.Stdout(log.Writer())

go.mod

@@ -57,6 +57,7 @@ require (
 	gopkg.in/ini.v1 v1.66.2
 	gopkg.in/yaml.v2 v2.4.0
 	helm.sh/helm/v3 v3.8.0
+	mvdan.cc/xurls/v2 v2.4.0
 )
 
 require (

go.sum

@@ -1657,8 +1657,9 @@ github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
-github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
+github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg=
+github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
 github.com/rs/zerolog v1.4.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc/go.mod h1:HFLT6i9iR4QBOF5rdCyjddC9t59ArqWJV2xx+jwcCMo=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
@@ -2661,6 +2662,8 @@ k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
 k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs=
 k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 layeh.com/radius v0.0.0-20190322222518-890bc1058917 h1:BDXFaFzUt5EIqe/4wrTc4AcYZWP6iC6Ult+jQWLh5eU=
+mvdan.cc/xurls/v2 v2.4.0 h1:tzxjVAj+wSBmDcF6zBB7/myTy3gX9xvi8Tyr28AuQgc=
+mvdan.cc/xurls/v2 v2.4.0/go.mod h1:+GEjq9uNjqs8LQfM9nVnM8rff0OQ5Iash5rzX+N1CSg=
 oras.land/oras-go v1.1.0 h1:tfWM1RT7PzUwWphqHU6ptPU3ZhwVnSw/9nEGf519rYg=
 oras.land/oras-go v1.1.0/go.mod h1:1A7vR/0KknT2UkJVWh+xMi95I/AhK8ZrxrnUSmXN0bQ=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

pkg/command/command.go

@@ -7,7 +7,6 @@ import (
 	"io"
 	"os"
 	"os/exec"
-	"regexp"
 	"strings"
 	"syscall"
 
@@ -16,14 +15,10 @@ import (
 	"github.com/pkg/errors"
 )
 
-const (
-	RelaxedURLRegEx = `(?:\b)((http(s?):\/\/)?(((www\.)?[a-zA-Z0-9\.\-\_]+(\.[a-zA-Z]{2,3})+)|((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))(\/[a-zA-Z0-9\_\-\.\/\?\%\#\&\=]*)?)(?:\s|\b)`
-)
-
 // Command defines the information required for executing a call to any executable
 type Command struct {
 	ErrorCategoryMapping map[string][]string
-	URLReportFileName    string
+	StepName             string
 	dir                  string
 	stdin                io.Reader
 	stdout               io.Writer
@@ -230,7 +225,7 @@ func (c *Command) startCmd(cmd *exec.Cmd) (*execution, error) {
 		return nil, errors.Wrap(err, "starting command failed")
 	}
 
-	execution := execution{cmd: cmd}
+	execution := execution{cmd: cmd, ul: log.NewURLLogger(c.StepName)}
 	execution.wg.Add(2)
 
 	srcOut := stdout
@@ -261,12 +256,12 @@ func (c *Command) startCmd(cmd *exec.Cmd) (*execution, error) {
 	}
 
 	go func() {
-		if c.URLReportFileName != "" {
+		if c.StepName != "" {
 			var buf bytes.Buffer
 			br := bufio.NewWriter(&buf)
 			_, execution.errCopyStdout = piperutils.CopyData(io.MultiWriter(c.stdout, br), srcOut)
 			br.Flush()
-			handleURLs(buf.String(), c.URLReportFileName)
+			execution.ul.Parse(buf)
 		} else {
 			_, execution.errCopyStdout = piperutils.CopyData(c.stdout, srcOut)
 		}
@@ -274,12 +269,12 @@ func (c *Command) startCmd(cmd *exec.Cmd) (*execution, error) {
 	}()
 
 	go func() {
-		if c.URLReportFileName != "" {
+		if c.StepName != "" {
 			var buf bytes.Buffer
 			bw := bufio.NewWriter(&buf)
 			_, execution.errCopyStderr = piperutils.CopyData(io.MultiWriter(c.stderr, bw), srcErr)
 			bw.Flush()
-			handleURLs(buf.String(), c.URLReportFileName)
+			execution.ul.Parse(buf)
 		} else {
 			_, execution.errCopyStderr = piperutils.CopyData(c.stderr, srcErr)
 		}
@@ -289,22 +284,6 @@ func (c *Command) startCmd(cmd *exec.Cmd) (*execution, error) {
 	return &execution, nil
 }
 
-func handleURLs(s, file string) {
-	reg := regexp.MustCompile(RelaxedURLRegEx)
-	matches := reg.FindAllStringSubmatch(s, -1)
-	f, err := os.Create(file)
-	if err != nil {
-		log.Entry().WithError(err).Info("failed to create url report file")
-	}
-	defer f.Close()
-	for _, match := range matches {
-		_, err = f.WriteString(match[1] + "\n")
-		if err != nil {
-			log.Entry().WithError(err).Info("failed to write record to url report")
-		}
-	}
-}
-
 func (c *Command) scanLog(in io.Reader) {
 	scanner := bufio.NewScanner(in)
 	scanner.Split(scanShortLines)

pkg/command/execution.go

@@ -1,6 +1,7 @@
 package command
 
 import (
+	"github.com/SAP/jenkins-library/pkg/log"
 	"os/exec"
 	"sync"
 )
@@ -11,6 +12,7 @@ type execution struct {
 	wg            sync.WaitGroup
 	errCopyStdout error
 	errCopyStderr error
+	ul            *log.URLLogger
 }
 
 func (execution *execution) Kill() error {
@@ -19,6 +21,7 @@ func (execution *execution) Kill() error {
 
 func (execution *execution) Wait() error {
 	execution.wg.Wait()
+	execution.ul.WriteURLsLogToJSON()
 	return execution.cmd.Wait()
 }
 

pkg/kubernetes/utils.go

@@ -62,6 +62,7 @@ func NewDeployUtilsBundle(customTLSCertificateLinks []string) DeployUtils {
 					"Error: release * failed, * timed out waiting for the condition",
 				},
 			},
+			StepName: "helmExecute",
 		},
 		Files:  &piperutils.Files{},
 		Client: &piperhttp.Client{},

pkg/log/url.go

@@ -0,0 +1,97 @@
+package log
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"mvdan.cc/xurls/v2"
+	"os"
+	"sync"
+)
+
+type (
+	step struct {
+		Step map[string]u `json:"step"`
+	}
+	u struct {
+		URLs []string `json:"url"`
+	}
+)
+
+const (
+	urlLogFileName = "url-log.json"
+)
+
+type URLLogger struct {
+	buf struct {
+		data [][]byte
+		sync.RWMutex
+	}
+	stepName string
+}
+
+func NewURLLogger(stepName string) *URLLogger {
+	return &URLLogger{stepName: stepName}
+}
+
+func (cl *URLLogger) WriteURLsLogToJSON() error {
+	cl.buf.Lock()
+	defer cl.buf.Unlock()
+	if len(cl.buf.data) == 0 {
+		return nil
+	}
+	file, err := os.OpenFile(urlLogFileName, os.O_CREATE|os.O_RDWR, 0600)
+	if err != nil {
+		return fmt.Errorf("failed to open file: %w", err)
+	}
+	defer func() {
+		dErr := file.Close()
+		if dErr != nil {
+			err = fmt.Errorf("can't close file: %w", dErr)
+		}
+	}()
+	fileBuf, err := ioutil.ReadAll(file)
+	if err != nil {
+		return fmt.Errorf("can't read from gile: %w", err)
+	}
+	urlsLog := step{make(map[string]u)}
+	if len(fileBuf) != 0 {
+		err = json.Unmarshal(fileBuf, &urlsLog)
+		if err != nil {
+			return fmt.Errorf("can't unmarshal log: %w", err)
+		}
+		fileBuf = fileBuf[:0]
+	}
+	var urls []string
+	if stepLogs, ok := urlsLog.Step[cl.stepName]; ok {
+		urls = stepLogs.URLs
+	}
+	for _, url := range cl.buf.data {
+		urls = append(urls, string(url))
+	}
+	urlsLog.Step[cl.stepName] = u{urls}
+	encoderBuf := bytes.NewBuffer(fileBuf)
+	jsonEncoder := json.NewEncoder(encoderBuf)
+	jsonEncoder.SetEscapeHTML(false)
+	jsonEncoder.SetIndent("", " ")
+	err = jsonEncoder.Encode(urlsLog)
+	if err != nil {
+		return fmt.Errorf("json encode error: %w", err)
+	}
+	_, err = file.WriteAt(encoderBuf.Bytes(), 0)
+	if err != nil {
+		return fmt.Errorf("failed to write log: %w", err)
+	}
+	return err
+}
+
+func (cl *URLLogger) Parse(buf bytes.Buffer) {
+	cl.buf.Lock()
+	defer cl.buf.Unlock()
+	cl.buf.data = append(cl.buf.data, parseURLs(buf.Bytes())...)
+}
+
+func parseURLs(src []byte) [][]byte {
+	return xurls.Strict().FindAll(src, -1)
+}

pkg/npm/npm.go

@@ -74,7 +74,9 @@ type utilsBundle struct {
 // GetExecRunner returns an execRunner if it's not yet initialized
 func (u *utilsBundle) GetExecRunner() ExecRunner {
 	if u.execRunner == nil {
-		u.execRunner = &command.Command{}
+		u.execRunner = &command.Command{
+			StepName: "npmExecuteScripts",
+		}
 		u.execRunner.Stdout(log.Writer())
 		u.execRunner.Stderr(log.Writer())
 	}