2019-10-25 14:58:59 +02:00
|
|
|
package command
|
|
|
|
|
|
|
|
import (
|
2020-06-24 10:04:05 +02:00
|
|
|
"bufio"
|
2019-10-25 14:58:59 +02:00
|
|
|
"bytes"
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"os"
|
|
|
|
"os/exec"
|
2020-04-22 11:06:00 +02:00
|
|
|
"strings"
|
2020-06-30 12:57:27 +02:00
|
|
|
"syscall"
|
2020-06-24 10:04:05 +02:00
|
|
|
|
|
|
|
"github.com/SAP/jenkins-library/pkg/log"
|
2021-06-23 14:41:52 +02:00
|
|
|
"github.com/SAP/jenkins-library/pkg/piperutils"
|
2020-06-24 10:04:05 +02:00
|
|
|
"github.com/pkg/errors"
|
2019-10-25 14:58:59 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
// Command defines the information required for executing a call to any executable
|
|
|
|
type Command struct {
|
2020-06-24 10:04:05 +02:00
|
|
|
ErrorCategoryMapping map[string][]string
|
2022-08-05 13:08:19 +02:00
|
|
|
StepName string
|
2020-06-24 10:04:05 +02:00
|
|
|
dir string
|
2021-04-29 16:50:23 +02:00
|
|
|
stdin io.Reader
|
2020-06-24 10:04:05 +02:00
|
|
|
stdout io.Writer
|
|
|
|
stderr io.Writer
|
|
|
|
env []string
|
2020-06-30 12:57:27 +02:00
|
|
|
exitCode int
|
2019-10-25 14:58:59 +02:00
|
|
|
}
|
|
|
|
|
2020-07-01 11:28:16 +02:00
|
|
|
type runner interface {
|
2020-07-28 17:06:17 +02:00
|
|
|
SetDir(dir string)
|
|
|
|
SetEnv(env []string)
|
|
|
|
AppendEnv(env []string)
|
2021-04-29 16:50:23 +02:00
|
|
|
Stdin(in io.Reader)
|
2020-07-01 11:28:16 +02:00
|
|
|
Stdout(out io.Writer)
|
|
|
|
Stderr(err io.Writer)
|
2021-02-05 11:54:38 +02:00
|
|
|
GetStdout() io.Writer
|
|
|
|
GetStderr() io.Writer
|
2020-07-01 11:28:16 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// ExecRunner mock for intercepting calls to executables
|
|
|
|
type ExecRunner interface {
|
|
|
|
runner
|
2020-07-28 17:06:17 +02:00
|
|
|
RunExecutable(executable string, params ...string) error
|
2020-07-01 11:28:16 +02:00
|
|
|
RunExecutableInBackground(executable string, params ...string) (Execution, error)
|
|
|
|
}
|
|
|
|
|
|
|
|
// ShellRunner mock for intercepting shell calls
|
|
|
|
type ShellRunner interface {
|
|
|
|
runner
|
2020-07-28 17:06:17 +02:00
|
|
|
RunShell(shell string, command string) error
|
2020-07-01 11:28:16 +02:00
|
|
|
}
|
|
|
|
|
2020-02-27 13:11:22 +02:00
|
|
|
// SetDir sets the working directory for the execution
|
2020-07-28 17:06:17 +02:00
|
|
|
func (c *Command) SetDir(dir string) {
|
|
|
|
c.dir = dir
|
2019-10-25 14:58:59 +02:00
|
|
|
}
|
|
|
|
|
2020-02-27 13:11:22 +02:00
|
|
|
// SetEnv sets explicit environment variables to be used for execution
|
2020-07-28 17:06:17 +02:00
|
|
|
func (c *Command) SetEnv(env []string) {
|
|
|
|
c.env = env
|
|
|
|
}
|
|
|
|
|
|
|
|
// AppendEnv appends environment variables to be used for execution
|
|
|
|
func (c *Command) AppendEnv(env []string) {
|
|
|
|
c.env = append(c.env, env...)
|
2020-01-24 15:30:27 +02:00
|
|
|
}
|
|
|
|
|
2022-02-08 12:55:01 +02:00
|
|
|
func (c *Command) GetOsEnv() []string {
|
|
|
|
return os.Environ()
|
|
|
|
}
|
|
|
|
|
2021-04-29 16:50:23 +02:00
|
|
|
// Stdin ..
|
|
|
|
func (c *Command) Stdin(stdin io.Reader) {
|
|
|
|
c.stdin = stdin
|
|
|
|
}
|
|
|
|
|
2019-11-19 15:29:53 +02:00
|
|
|
// Stdout ..
|
|
|
|
func (c *Command) Stdout(stdout io.Writer) {
|
|
|
|
c.stdout = stdout
|
|
|
|
}
|
|
|
|
|
|
|
|
// Stderr ..
|
|
|
|
func (c *Command) Stderr(stderr io.Writer) {
|
|
|
|
c.stderr = stderr
|
|
|
|
}
|
|
|
|
|
2021-02-05 11:54:38 +02:00
|
|
|
// GetStdout Returns the writer for stdout
|
|
|
|
func (c *Command) GetStdout() io.Writer {
|
|
|
|
return c.stdout
|
|
|
|
}
|
|
|
|
|
|
|
|
//GetStderr Retursn the writer for stderr
|
|
|
|
func (c *Command) GetStderr() io.Writer {
|
|
|
|
return c.stderr
|
|
|
|
}
|
|
|
|
|
2019-10-25 14:58:59 +02:00
|
|
|
// ExecCommand defines how to execute os commands
|
|
|
|
var ExecCommand = exec.Command
|
|
|
|
|
|
|
|
// RunShell runs the specified command on the shell
|
|
|
|
func (c *Command) RunShell(shell, script string) error {
|
|
|
|
|
2020-06-24 10:04:05 +02:00
|
|
|
c.prepareOut()
|
2019-10-25 14:58:59 +02:00
|
|
|
|
|
|
|
cmd := ExecCommand(shell)
|
|
|
|
|
2020-01-24 15:30:27 +02:00
|
|
|
if len(c.dir) > 0 {
|
|
|
|
cmd.Dir = c.dir
|
|
|
|
}
|
|
|
|
|
2020-03-03 12:15:24 +02:00
|
|
|
appendEnvironment(cmd, c.env)
|
2020-01-24 15:30:27 +02:00
|
|
|
|
2019-10-25 14:58:59 +02:00
|
|
|
in := bytes.Buffer{}
|
|
|
|
in.Write([]byte(script))
|
|
|
|
cmd.Stdin = &in
|
|
|
|
|
2020-04-22 11:06:00 +02:00
|
|
|
log.Entry().Infof("running shell script: %v %v", shell, script)
|
|
|
|
|
2020-06-24 10:04:05 +02:00
|
|
|
if err := c.runCmd(cmd); err != nil {
|
2019-10-25 14:58:59 +02:00
|
|
|
return errors.Wrapf(err, "running shell script failed with %v", shell)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// RunExecutable runs the specified executable with parameters
|
2020-03-12 10:00:51 +02:00
|
|
|
// !! While the cmd.Env is applied during command execution, it is NOT involved when the actual executable is resolved.
|
|
|
|
// Thus the executable needs to be on the PATH of the current process and it is not sufficient to alter the PATH on cmd.Env.
|
2019-10-25 14:58:59 +02:00
|
|
|
func (c *Command) RunExecutable(executable string, params ...string) error {
|
|
|
|
|
2020-06-24 10:04:05 +02:00
|
|
|
c.prepareOut()
|
2019-10-25 14:58:59 +02:00
|
|
|
|
|
|
|
cmd := ExecCommand(executable, params...)
|
|
|
|
|
|
|
|
if len(c.dir) > 0 {
|
|
|
|
cmd.Dir = c.dir
|
|
|
|
}
|
|
|
|
|
2020-04-22 11:06:00 +02:00
|
|
|
log.Entry().Infof("running command: %v %v", executable, strings.Join(params, (" ")))
|
|
|
|
|
2020-03-03 12:15:24 +02:00
|
|
|
appendEnvironment(cmd, c.env)
|
2020-01-24 15:30:27 +02:00
|
|
|
|
2021-04-29 16:50:23 +02:00
|
|
|
if c.stdin != nil {
|
|
|
|
cmd.Stdin = c.stdin
|
|
|
|
}
|
|
|
|
|
2020-06-24 10:04:05 +02:00
|
|
|
if err := c.runCmd(cmd); err != nil {
|
2019-10-25 14:58:59 +02:00
|
|
|
return errors.Wrapf(err, "running command '%v' failed", executable)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-06-16 11:42:51 +02:00
|
|
|
// RunExecutableInBackground runs the specified executable with parameters in the background non blocking
|
|
|
|
// !! While the cmd.Env is applied during command execution, it is NOT involved when the actual executable is resolved.
|
|
|
|
// Thus the executable needs to be on the PATH of the current process and it is not sufficient to alter the PATH on cmd.Env.
|
|
|
|
func (c *Command) RunExecutableInBackground(executable string, params ...string) (Execution, error) {
|
|
|
|
|
2020-06-24 10:04:05 +02:00
|
|
|
c.prepareOut()
|
2020-06-16 11:42:51 +02:00
|
|
|
|
|
|
|
cmd := ExecCommand(executable, params...)
|
|
|
|
|
|
|
|
if len(c.dir) > 0 {
|
|
|
|
cmd.Dir = c.dir
|
|
|
|
}
|
|
|
|
|
|
|
|
log.Entry().Infof("running command: %v %v", executable, strings.Join(params, (" ")))
|
|
|
|
|
|
|
|
appendEnvironment(cmd, c.env)
|
|
|
|
|
2021-04-29 16:50:23 +02:00
|
|
|
if c.stdin != nil {
|
|
|
|
cmd.Stdin = c.stdin
|
|
|
|
}
|
|
|
|
|
2020-06-24 10:04:05 +02:00
|
|
|
execution, err := c.startCmd(cmd)
|
2020-06-16 11:42:51 +02:00
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrapf(err, "starting command '%v' failed", executable)
|
|
|
|
}
|
|
|
|
|
|
|
|
return execution, nil
|
|
|
|
}
|
|
|
|
|
2020-06-30 12:57:27 +02:00
|
|
|
// GetExitCode allows to retrieve the exit code of a command execution
|
|
|
|
func (c *Command) GetExitCode() int {
|
|
|
|
return c.exitCode
|
|
|
|
}
|
|
|
|
|
2020-03-03 12:15:24 +02:00
|
|
|
func appendEnvironment(cmd *exec.Cmd, env []string) {
|
|
|
|
|
|
|
|
if len(env) > 0 {
|
|
|
|
|
|
|
|
// When cmd.Env is nil the environment variables from the current
|
|
|
|
// process are also used by the forked process. Our environment variables
|
|
|
|
// should not replace the existing environment, but they should be appended.
|
|
|
|
// Hence we populate cmd.Env first with the current environment in case we
|
|
|
|
// find it empty. In case there is already something, we append to that environment.
|
|
|
|
// In that case we assume the current values of `cmd.Env` has either been setup based
|
|
|
|
// on `os.Environ()` or that was initialized in another way for a good reason.
|
|
|
|
//
|
|
|
|
// In case we have the same environment variable as in the current environment (`os.Environ()`)
|
|
|
|
// and in `env`, the environment variable from `env` is effectively used since this is the
|
|
|
|
// later one. There is no merging between both environment variables.
|
|
|
|
//
|
|
|
|
// cf. https://golang.org/pkg/os/exec/#Command
|
|
|
|
// If Env contains duplicate environment keys, only the last
|
|
|
|
// value in the slice for each duplicate key is used.
|
|
|
|
|
|
|
|
if len(cmd.Env) == 0 {
|
|
|
|
cmd.Env = os.Environ()
|
|
|
|
}
|
|
|
|
cmd.Env = append(cmd.Env, env...)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-06-24 10:04:05 +02:00
|
|
|
func (c *Command) startCmd(cmd *exec.Cmd) (*execution, error) {
|
2019-10-25 14:58:59 +02:00
|
|
|
|
|
|
|
stdout, stderr, err := cmdPipes(cmd)
|
|
|
|
|
|
|
|
if err != nil {
|
2020-06-16 11:42:51 +02:00
|
|
|
return nil, errors.Wrap(err, "getting command pipes failed")
|
2019-10-25 14:58:59 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
err = cmd.Start()
|
|
|
|
if err != nil {
|
2020-06-16 11:42:51 +02:00
|
|
|
return nil, errors.Wrap(err, "starting command failed")
|
2019-10-25 14:58:59 +02:00
|
|
|
}
|
|
|
|
|
2022-08-05 13:08:19 +02:00
|
|
|
execution := execution{cmd: cmd, ul: log.NewURLLogger(c.StepName)}
|
2020-06-16 11:42:51 +02:00
|
|
|
execution.wg.Add(2)
|
2019-10-25 14:58:59 +02:00
|
|
|
|
2020-06-24 10:04:05 +02:00
|
|
|
srcOut := stdout
|
|
|
|
srcErr := stderr
|
|
|
|
|
|
|
|
if c.ErrorCategoryMapping != nil {
|
|
|
|
prOut, pwOut := io.Pipe()
|
|
|
|
trOut := io.TeeReader(stdout, pwOut)
|
|
|
|
srcOut = prOut
|
|
|
|
|
|
|
|
prErr, pwErr := io.Pipe()
|
|
|
|
trErr := io.TeeReader(stderr, pwErr)
|
|
|
|
srcErr = prErr
|
|
|
|
|
|
|
|
execution.wg.Add(2)
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
defer execution.wg.Done()
|
|
|
|
defer pwOut.Close()
|
|
|
|
c.scanLog(trOut)
|
|
|
|
}()
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
defer execution.wg.Done()
|
|
|
|
defer pwErr.Close()
|
|
|
|
c.scanLog(trErr)
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
2019-10-25 14:58:59 +02:00
|
|
|
go func() {
|
2022-08-05 13:08:19 +02:00
|
|
|
if c.StepName != "" {
|
2022-02-25 16:35:44 +02:00
|
|
|
var buf bytes.Buffer
|
|
|
|
br := bufio.NewWriter(&buf)
|
|
|
|
_, execution.errCopyStdout = piperutils.CopyData(io.MultiWriter(c.stdout, br), srcOut)
|
|
|
|
br.Flush()
|
2022-08-05 13:08:19 +02:00
|
|
|
execution.ul.Parse(buf)
|
2022-02-25 16:35:44 +02:00
|
|
|
} else {
|
|
|
|
_, execution.errCopyStdout = piperutils.CopyData(c.stdout, srcOut)
|
|
|
|
}
|
2020-06-16 11:42:51 +02:00
|
|
|
execution.wg.Done()
|
2019-10-25 14:58:59 +02:00
|
|
|
}()
|
|
|
|
|
|
|
|
go func() {
|
2022-08-05 13:08:19 +02:00
|
|
|
if c.StepName != "" {
|
2022-02-25 16:35:44 +02:00
|
|
|
var buf bytes.Buffer
|
|
|
|
bw := bufio.NewWriter(&buf)
|
|
|
|
_, execution.errCopyStderr = piperutils.CopyData(io.MultiWriter(c.stderr, bw), srcErr)
|
|
|
|
bw.Flush()
|
2022-08-05 13:08:19 +02:00
|
|
|
execution.ul.Parse(buf)
|
2022-02-25 16:35:44 +02:00
|
|
|
} else {
|
|
|
|
_, execution.errCopyStderr = piperutils.CopyData(c.stderr, srcErr)
|
|
|
|
}
|
2020-06-16 11:42:51 +02:00
|
|
|
execution.wg.Done()
|
2019-10-25 14:58:59 +02:00
|
|
|
}()
|
|
|
|
|
2020-06-16 11:42:51 +02:00
|
|
|
return &execution, nil
|
|
|
|
}
|
2019-10-25 14:58:59 +02:00
|
|
|
|
2020-06-24 10:04:05 +02:00
|
|
|
func (c *Command) scanLog(in io.Reader) {
|
|
|
|
scanner := bufio.NewScanner(in)
|
|
|
|
scanner.Split(scanShortLines)
|
|
|
|
for scanner.Scan() {
|
|
|
|
line := scanner.Text()
|
|
|
|
c.parseConsoleErrors(line)
|
|
|
|
}
|
|
|
|
if err := scanner.Err(); err != nil {
|
|
|
|
log.Entry().WithError(err).Info("failed to scan log file")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func scanShortLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
|
|
|
|
lenData := len(data)
|
|
|
|
if atEOF && lenData == 0 {
|
|
|
|
return 0, nil, nil
|
|
|
|
}
|
|
|
|
if lenData > 32767 && !bytes.Contains(data[0:lenData], []byte("\n")) {
|
|
|
|
// we will neglect long output
|
|
|
|
// no use cases known where this would be relevant
|
|
|
|
// current accepted implication: error pattern would not be found
|
|
|
|
// -> resulting in wrong error categorization
|
|
|
|
return lenData, nil, nil
|
|
|
|
}
|
|
|
|
if i := bytes.IndexByte(data, '\n'); i >= 0 && i < 32767 {
|
|
|
|
// We have a full newline-terminated line with a size limit
|
|
|
|
// Size limit is required since otherwise scanner would stall
|
|
|
|
return i + 1, data[0:i], nil
|
|
|
|
}
|
|
|
|
// If we're at EOF, we have a final, non-terminated line. Return it.
|
|
|
|
if atEOF {
|
|
|
|
return len(data), data, nil
|
|
|
|
}
|
|
|
|
// Request more data.
|
|
|
|
return 0, nil, nil
|
|
|
|
}
|
2019-10-25 14:58:59 +02:00
|
|
|
|
2020-06-24 10:04:05 +02:00
|
|
|
func (c *Command) parseConsoleErrors(logLine string) {
|
|
|
|
for category, categoryErrors := range c.ErrorCategoryMapping {
|
|
|
|
for _, errorPart := range categoryErrors {
|
2020-06-30 10:48:30 +02:00
|
|
|
if matchPattern(logLine, errorPart) {
|
2020-06-24 10:04:05 +02:00
|
|
|
log.SetErrorCategory(log.ErrorCategoryByString(category))
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-06-30 10:48:30 +02:00
|
|
|
func matchPattern(text, pattern string) bool {
|
|
|
|
if len(pattern) == 0 && len(text) != 0 {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
parts := strings.Split(pattern, "*")
|
|
|
|
for _, part := range parts {
|
|
|
|
if !strings.Contains(text, part) {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2020-06-24 10:04:05 +02:00
|
|
|
func (c *Command) runCmd(cmd *exec.Cmd) error {
|
|
|
|
|
|
|
|
execution, err := c.startCmd(cmd)
|
2019-10-25 14:58:59 +02:00
|
|
|
if err != nil {
|
2020-06-16 11:42:51 +02:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
err = execution.Wait()
|
|
|
|
|
|
|
|
if execution.errCopyStdout != nil || execution.errCopyStderr != nil {
|
|
|
|
return fmt.Errorf("failed to capture stdout/stderr: '%v'/'%v'", execution.errCopyStdout, execution.errCopyStderr)
|
2019-10-25 14:58:59 +02:00
|
|
|
}
|
|
|
|
|
2020-06-16 11:42:51 +02:00
|
|
|
if err != nil {
|
2020-06-30 12:57:27 +02:00
|
|
|
// provide fallback to ensure a non 0 exit code in case of an error
|
|
|
|
c.exitCode = 1
|
|
|
|
// try to identify the detailed error code
|
|
|
|
if exitErr, ok := err.(*exec.ExitError); ok {
|
|
|
|
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
|
|
|
|
c.exitCode = status.ExitStatus()
|
|
|
|
}
|
|
|
|
}
|
2020-06-16 11:42:51 +02:00
|
|
|
return errors.Wrap(err, "cmd.Run() failed")
|
2019-10-25 14:58:59 +02:00
|
|
|
}
|
2020-06-30 12:57:27 +02:00
|
|
|
c.exitCode = 0
|
2019-10-25 14:58:59 +02:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-06-24 10:04:05 +02:00
|
|
|
func (c *Command) prepareOut() {
|
2019-10-25 14:58:59 +02:00
|
|
|
|
|
|
|
//ToDo: check use of multiwriter instead to always write into os.Stdout and os.Stdin?
|
|
|
|
//stdout := io.MultiWriter(os.Stdout, &stdoutBuf)
|
|
|
|
//stderr := io.MultiWriter(os.Stderr, &stderrBuf)
|
|
|
|
|
2020-06-24 10:04:05 +02:00
|
|
|
if c.stdout == nil {
|
|
|
|
c.stdout = os.Stdout
|
2019-10-25 14:58:59 +02:00
|
|
|
}
|
2020-06-24 10:04:05 +02:00
|
|
|
if c.stderr == nil {
|
|
|
|
c.stderr = os.Stderr
|
2019-10-25 14:58:59 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func cmdPipes(cmd *exec.Cmd) (io.ReadCloser, io.ReadCloser, error) {
|
|
|
|
stdout, err := cmd.StdoutPipe()
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, errors.Wrap(err, "getting Stdout pipe failed")
|
|
|
|
}
|
|
|
|
|
|
|
|
stderr, err := cmd.StderrPipe()
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, errors.Wrap(err, "getting Stderr pipe failed")
|
|
|
|
}
|
2021-04-29 16:50:23 +02:00
|
|
|
|
2019-10-25 14:58:59 +02:00
|
|
|
return stdout, stderr, nil
|
|
|
|
}
|