Run npm scripts in virtual frame buffer and extend command.go to run executable asynchronously (#1669)

Co-authored-by: Stephan Aßmus <stephan.assmus@sap.com>
Co-authored-by: Florian Wilhelm <florian.wilhelm02@sap.com>

cmd/fortifyExecuteScan.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/bmatcuk/doublestar"
+	"io"
 	"io/ioutil"
 	"math"
 	"os"
@@ -35,6 +36,13 @@ type pullRequestService interface {
 	ListPullRequestsWithCommit(ctx context.Context, owner, repo, sha string, opts *github.PullRequestListOptions) ([]*github.PullRequest, *github.Response, error)
 }
 
+type fortifyExecRunner interface {
+	Stdout(out io.Writer)
+	Stderr(err io.Writer)
+	SetDir(d string)
+	RunExecutable(e string, p ...string) error
+}
+
 const checkString = "<---CHECK FORTIFY---"
 const classpathFileName = "fortify-execute-scan-cp.txt"
 
@@ -51,7 +59,7 @@ func fortifyExecuteScan(config fortifyExecuteScanOptions, telemetryData *telemet
 	}
 }
 
-func runFortifyScan(config fortifyExecuteScanOptions, sys fortify.System, command execRunner, telemetryData *telemetry.CustomData, influx *fortifyExecuteScanInflux, auditStatus map[string]string) error {
+func runFortifyScan(config fortifyExecuteScanOptions, sys fortify.System, command fortifyExecRunner, telemetryData *telemetry.CustomData, influx *fortifyExecuteScanInflux, auditStatus map[string]string) error {
 	log.Entry().Debugf("Running Fortify scan against SSC at %v", config.ServerURL)
 	artifact, err := versioning.GetArtifact(config.BuildTool, config.BuildDescriptorFile, &versioning.Options{}, command)
 	if err != nil {
@@ -447,7 +455,7 @@ func calculateTimeDifferenceToLastUpload(uploadDate models.Iso8601MilliDateTime,
 	return absoluteSeconds
 }
 
-func executeTemplatedCommand(command execRunner, cmdTemplate []string, context map[string]string) {
+func executeTemplatedCommand(command fortifyExecRunner, cmdTemplate []string, context map[string]string) {
 	for index, cmdTemplatePart := range cmdTemplate {
 		result, err := piperutils.ExecuteTemplate(cmdTemplatePart, context)
 		if err != nil {
@@ -461,7 +469,7 @@ func executeTemplatedCommand(command execRunner, cmdTemplate []string, context m
 	}
 }
 
-func autoresolvePipClasspath(executable string, parameters []string, file string, command execRunner) string {
+func autoresolvePipClasspath(executable string, parameters []string, file string, command fortifyExecRunner) string {
 	// redirect stdout and create cp file from command output
 	outfile, err := os.Create(file)
 	if err != nil {
@@ -477,7 +485,7 @@ func autoresolvePipClasspath(executable string, parameters []string, file string
 	return readClasspathFile(file)
 }
 
-func autoresolveMavenClasspath(config fortifyExecuteScanOptions, file string, command execRunner) string {
+func autoresolveMavenClasspath(config fortifyExecuteScanOptions, file string, command fortifyExecRunner) string {
 	if filepath.IsAbs(file) {
 		log.Entry().Warnf("Passing an absolute path for -Dmdep.outputFile results in the classpath only for the last module in multi-module maven projects.")
 	}
@@ -552,7 +560,7 @@ func removeDuplicates(contents, separator string) string {
 	return contents
 }
 
-func triggerFortifyScan(config fortifyExecuteScanOptions, command execRunner, buildID, buildLabel, buildProject string) {
+func triggerFortifyScan(config fortifyExecuteScanOptions, command fortifyExecRunner, buildID, buildLabel, buildProject string) {
 	var err error = nil
 	// Do special Python related prep
 	pipVersion := "pip3"
@@ -639,7 +647,7 @@ func populateMavenTranslate(config *fortifyExecuteScanOptions, classpath string)
 	return string(translateJSON), err
 }
 
-func translateProject(config *fortifyExecuteScanOptions, command execRunner, buildID, classpath string) {
+func translateProject(config *fortifyExecuteScanOptions, command fortifyExecRunner, buildID, classpath string) {
 	var translateList []map[string]string
 	json.Unmarshal([]byte(config.Translate), &translateList)
 	log.Entry().Debugf("Translating with options: %v", translateList)
@@ -651,7 +659,7 @@ func translateProject(config *fortifyExecuteScanOptions, command execRunner, bui
 	}
 }
 
-func handleSingleTranslate(config *fortifyExecuteScanOptions, command execRunner, buildID string, t map[string]string) {
+func handleSingleTranslate(config *fortifyExecuteScanOptions, command fortifyExecRunner, buildID string, t map[string]string) {
 	if t != nil {
 		log.Entry().Debugf("Handling translate config %v", t)
 		translateOptions := []string{
@@ -672,7 +680,7 @@ func handleSingleTranslate(config *fortifyExecuteScanOptions, command execRunner
 	}
 }
 
-func scanProject(config *fortifyExecuteScanOptions, command execRunner, buildID, buildLabel, buildProject string) {
+func scanProject(config *fortifyExecuteScanOptions, command fortifyExecRunner, buildID, buildLabel, buildProject string) {
 	var scanOptions = []string{
 		"-verbose",
 		"-64",

cmd/interfaces.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"github.com/SAP/jenkins-library/pkg/command"
 	"io"
 )
 
@@ -14,6 +15,7 @@ type runner interface {
 type execRunner interface {
 	runner
 	RunExecutable(e string, p ...string) error
+	RunExecutableInBackground(executable string, params ...string) (command.Execution, error)
 }
 
 type shellRunner interface {

cmd/npmExecuteScripts.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"fmt"
 	"github.com/SAP/jenkins-library/pkg/command"
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/SAP/jenkins-library/pkg/npm"
@@ -71,6 +72,15 @@ func runNpmExecuteScripts(utils npmExecuteScriptsUtilsInterface, options *npmExe
 		return err
 	}
 
+	if options.VirtualFrameBuffer {
+		cmd, err := execRunner.RunExecutableInBackground("Xvfb", "-ac", ":99", "-screen", "0", "1280x1024x16")
+		if err != nil {
+			return fmt.Errorf("failed to start virtual frame buffer%w", err)
+		}
+		defer cmd.Kill()
+		execRunner.SetEnv([]string{"DISPLAY=:99"})
+	}
+
 	for _, file := range packageJSONFiles {
 		dir := path.Dir(file)
 		err = utils.chdir(dir)

cmd/npmExecuteScripts_generated.go

@@ -18,6 +18,7 @@ type npmExecuteScriptsOptions struct {
 	RunScripts         []string `json:"runScripts,omitempty"`
 	DefaultNpmRegistry string   `json:"defaultNpmRegistry,omitempty"`
 	SapNpmRegistry     string   `json:"sapNpmRegistry,omitempty"`
+	VirtualFrameBuffer bool     `json:"virtualFrameBuffer,omitempty"`
 }
 
 // NpmExecuteScriptsCommand Execute npm run scripts on all npm packages in a project
@@ -78,6 +79,7 @@ func addNpmExecuteScriptsFlags(cmd *cobra.Command, stepConfig *npmExecuteScripts
 	cmd.Flags().StringSliceVar(&stepConfig.RunScripts, "runScripts", []string{}, "List of additional run scripts to execute from package.json.")
 	cmd.Flags().StringVar(&stepConfig.DefaultNpmRegistry, "defaultNpmRegistry", os.Getenv("PIPER_defaultNpmRegistry"), "URL of the npm registry to use. Defaults to https://registry.npmjs.org/")
 	cmd.Flags().StringVar(&stepConfig.SapNpmRegistry, "sapNpmRegistry", `https://npm.sap.com`, "The default npm registry URL to be used as the remote mirror for the SAP npm packages.")
+	cmd.Flags().BoolVar(&stepConfig.VirtualFrameBuffer, "virtualFrameBuffer", false, "(Linux only) Start a virtual frame buffer in the background. This allows you to run a web browser without the need for an X server. Note that xvfb needs to be installed in the execution environment.")
 
 }
 
@@ -123,6 +125,14 @@ func npmExecuteScriptsMetadata() config.StepData {
 						Mandatory:   false,
 						Aliases:     []config.Alias{},
 					},
+					{
+						Name:        "virtualFrameBuffer",
+						ResourceRef: []config.ResourceReference{},
+						Scope:       []string{"PARAMETERS", "STAGES", "STEPS"},
+						Type:        "bool",
+						Mandatory:   false,
+						Aliases:     []config.Alias{},
+					},
 				},
 			},
 		},

cmd/npmExecuteScripts_test.go

@@ -129,6 +129,29 @@ func TestNpmExecuteScripts(t *testing.T) {
 		assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"run-script", "foo", "--if-present"}}, utils.execRunner.Calls[3])
 		assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"run-script", "bar", "--if-present"}}, utils.execRunner.Calls[4])
 	})
+
+	t.Run("Call run-scripts with virtual frame buffer", func(t *testing.T) {
+		utils := newNpmExecuteScriptsMockUtilsBundle()
+		utils.files["package.json"] = []byte("{\"scripts\": { \"foo\": \"\" } }")
+		options := npmExecuteScriptsOptions{}
+		options.Install = false
+		options.RunScripts = []string{"foo"}
+		options.VirtualFrameBuffer = true
+
+		err := runNpmExecuteScripts(&utils, &options)
+
+		assert.Contains(t, utils.execRunner.Env, "DISPLAY=:99")
+		assert.NoError(t, err)
+		if assert.Len(t, utils.execRunner.Calls, 4) {
+			xvfbCall := utils.execRunner.Calls[0]
+			assert.Equal(t, "Xvfb", xvfbCall.Exec)
+			assert.Equal(t, []string{"-ac", ":99", "-screen", "0", "1280x1024x16"}, xvfbCall.Params)
+			assert.True(t, xvfbCall.Async)
+			assert.True(t, xvfbCall.Execution.Killed)
+
+			assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"run-script", "foo", "--if-present"}}, utils.execRunner.Calls[3])
+		}
+	})
 }
 
 func newNpmExecuteScriptsMockUtilsBundle() npmExecuteScriptsMockUtilsBundle {

pkg/command/command.go

@@ -4,13 +4,11 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/pkg/errors"
 	"io"
 	"os"
 	"os/exec"
 	"strings"
-	"sync"
-
-	"github.com/pkg/errors"
 )
 
 // Command defines the information required for executing a call to any executable
@@ -92,6 +90,32 @@ func (c *Command) RunExecutable(executable string, params ...string) error {
 	return nil
 }
 
+// RunExecutableInBackground runs the specified executable with parameters in the background non blocking
+// !! While the cmd.Env is applied during command execution, it is NOT involved when the actual executable is resolved.
+//    Thus the executable needs to be on the PATH of the current process and it is not sufficient to alter the PATH on cmd.Env.
+func (c *Command) RunExecutableInBackground(executable string, params ...string) (Execution, error) {
+
+	_out, _err := prepareOut(c.stdout, c.stderr)
+
+	cmd := ExecCommand(executable, params...)
+
+	if len(c.dir) > 0 {
+		cmd.Dir = c.dir
+	}
+
+	log.Entry().Infof("running command: %v %v", executable, strings.Join(params, (" ")))
+
+	appendEnvironment(cmd, c.env)
+
+	execution, err := startCmd(cmd, _out, _err)
+
+	if err != nil {
+		return nil, errors.Wrapf(err, "starting command '%v' failed", executable)
+	}
+
+	return execution, nil
+}
+
 func appendEnvironment(cmd *exec.Cmd, env []string) {
 
 	if len(env) > 0 {
@@ -119,46 +143,52 @@ func appendEnvironment(cmd *exec.Cmd, env []string) {
 	}
 }
 
-func runCmd(cmd *exec.Cmd, _out, _err io.Writer) error {
+func startCmd(cmd *exec.Cmd, _out, _err io.Writer) (*execution, error) {
 
 	stdout, stderr, err := cmdPipes(cmd)
 
 	if err != nil {
-		return errors.Wrap(err, "getting commmand pipes failed")
+		return nil, errors.Wrap(err, "getting command pipes failed")
 	}
 
 	err = cmd.Start()
 	if err != nil {
-		return errors.Wrap(err, "starting command failed")
+		return nil, errors.Wrap(err, "starting command failed")
 	}
 
-	var wg sync.WaitGroup
-	wg.Add(2)
-
-	var errStdout, errStderr error
+	execution := execution{cmd: cmd}
+	execution.wg.Add(2)
 
 	go func() {
-		_, errStdout = io.Copy(_out, stdout)
-		wg.Done()
+		_, execution.errCopyStdout = io.Copy(_out, stdout)
+		execution.wg.Done()
 	}()
 
 	go func() {
-		_, errStderr = io.Copy(_err, stderr)
-		wg.Done()
+		_, execution.errCopyStderr = io.Copy(_err, stderr)
+		execution.wg.Done()
 	}()
 
-	wg.Wait()
+	return &execution, nil
+}
 
-	err = cmd.Wait()
+func runCmd(cmd *exec.Cmd, _out, _err io.Writer) error {
+
+	execution, err := startCmd(cmd, _out, _err)
+	if err != nil {
+		return err
+	}
+
+	err = execution.Wait()
+
+	if execution.errCopyStdout != nil || execution.errCopyStderr != nil {
+		return fmt.Errorf("failed to capture stdout/stderr: '%v'/'%v'", execution.errCopyStdout, execution.errCopyStderr)
+	}
 
 	if err != nil {
 		return errors.Wrap(err, "cmd.Run() failed")
 	}
 
-	if errStdout != nil || errStderr != nil {
-		return fmt.Errorf("failed to capture stdout/stderr: '%v'/'%v'", errStdout, errStderr)
-	}
-
 	return nil
 }
 

pkg/command/execution.go

@@ -0,0 +1,29 @@
+package command
+
+import (
+	"os/exec"
+	"sync"
+)
+
+//errCopyStdout and errCopyStderr are filled after the command execution after Wait() terminates
+type execution struct {
+	cmd           *exec.Cmd
+	wg            sync.WaitGroup
+	errCopyStdout error
+	errCopyStderr error
+}
+
+func (execution *execution) Kill() error {
+	return execution.cmd.Process.Kill()
+}
+
+func (execution *execution) Wait() error {
+	execution.wg.Wait()
+	return execution.cmd.Wait()
+}
+
+// Execution references a background process which is started by RunExecutableInBackground
+type Execution interface {
+	Kill() error
+	Wait() error
+}

pkg/mock/runner.go

@@ -3,6 +3,7 @@
 package mock
 
 import (
+	"github.com/SAP/jenkins-library/pkg/command"
 	"io"
 	"io/ioutil"
 	"regexp"
@@ -20,8 +21,14 @@ type ExecMockRunner struct {
 }
 
 type ExecCall struct {
-	Exec   string
-	Params []string
+	Execution *Execution
+	Async     bool
+	Exec      string
+	Params    []string
+}
+
+type Execution struct {
+	Killed bool
 }
 
 type ShellMockRunner struct {
@@ -53,6 +60,21 @@ func (m *ExecMockRunner) RunExecutable(e string, p ...string) error {
 	return handleCall(c, m.StdoutReturn, m.ShouldFailOnCommand, m.stdout)
 }
 
+func (m *ExecMockRunner) RunExecutableInBackground(e string, p ...string) (command.Execution, error) {
+
+	execution := Execution{}
+	exec := ExecCall{Exec: e, Params: p, Async: true, Execution: &execution}
+	m.Calls = append(m.Calls, exec)
+
+	c := strings.Join(append([]string{e}, p...), " ")
+
+	err := handleCall(c, m.StdoutReturn, m.ShouldFailOnCommand, m.stdout)
+	if err != nil {
+		return nil, err
+	}
+	return &execution, nil
+}
+
 func (m *ExecMockRunner) Stdout(out io.Writer) {
 	m.stdout = out
 }
@@ -81,6 +103,15 @@ func (m *ShellMockRunner) RunShell(s string, c string) error {
 	return handleCall(c, m.StdoutReturn, m.ShouldFailOnCommand, m.stdout)
 }
 
+func (e *Execution) Kill() error {
+	e.Killed = true
+	return nil
+}
+
+func (e *Execution) Wait() error {
+	return nil
+}
+
 func handleCall(call string, stdoutReturn map[string]string, shouldFailOnCommand map[string]error, stdout io.Writer) error {
 
 	if stdoutReturn != nil {

resources/metadata/npmExecuteScripts.yaml

@@ -38,6 +38,13 @@ spec:
           - STAGES
           - STEPS
         default: https://npm.sap.com
+      - name: virtualFrameBuffer
+        type: bool
+        description: (Linux only) Start a virtual frame buffer in the background. This allows you to run a web browser without the need for an X server. Note that xvfb needs to be installed in the execution environment.
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
   containers:
     - name: node
       image: node:12-buster-slim