Improve step logging (#3722)

cli/exec/exec.go

@@ -39,6 +39,7 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/compiler"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/linter"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/matrix"
+	pipelineLog "go.woodpecker-ci.org/woodpecker/v2/pipeline/log"
 	"go.woodpecker-ci.org/woodpecker/v2/shared/utils"
 )
 
@@ -273,8 +274,8 @@ func convertPathForWindows(path string) string {
 	return filepath.ToSlash(path)
 }
 
+const maxLogLineLength = 1024 * 1024 // 1mb
 var defaultLogger = pipeline.Logger(func(step *backendTypes.Step, rc io.Reader) error {
-	logStream := NewLineWriter(step.Name, step.UUID)
-	_, err := io.Copy(logStream, rc)
-	return err
+	logWriter := NewLineWriter(step.Name, step.UUID)
+	return pipelineLog.CopyLineByLine(logWriter, rc, maxLogLineLength)
 })

cli/exec/line.go

@@ -16,50 +16,34 @@ package exec
 
 import (
 	"fmt"
+	"io"
 	"os"
-	"strings"
 	"time"
-
-	"go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc"
 )
 
 // LineWriter sends logs to the client.
 type LineWriter struct {
-	stepName string
-	stepUUID string
-	num      int
-	now      time.Time
-	rep      *strings.Replacer
-	lines    []*rpc.LogEntry
+	stepName  string
+	stepUUID  string
+	num       int
+	startTime time.Time
 }
 
 // NewLineWriter returns a new line reader.
-func NewLineWriter(stepName, stepUUID string) *LineWriter {
+func NewLineWriter(stepName, stepUUID string) io.WriteCloser {
 	return &LineWriter{
-		stepName: stepName,
-		stepUUID: stepUUID,
-		now:      time.Now().UTC(),
+		stepName:  stepName,
+		stepUUID:  stepUUID,
+		startTime: time.Now().UTC(),
 	}
 }
 
 func (w *LineWriter) Write(p []byte) (n int, err error) {
-	data := string(p)
-	if w.rep != nil {
-		data = w.rep.Replace(data)
-	}
-
-	line := &rpc.LogEntry{
-		Data:     data,
-		StepUUID: w.stepUUID,
-		Line:     w.num,
-		Time:     int64(time.Since(w.now).Seconds()),
-		Type:     rpc.LogEntryStdout,
-	}
-
-	fmt.Fprintf(os.Stderr, "[%s:L%d:%ds] %s", w.stepName, w.num, int64(time.Since(w.now).Seconds()), data)
-
+	fmt.Fprintf(os.Stderr, "[%s:L%d:%ds] %s", w.stepName, w.num, int64(time.Since(w.startTime).Seconds()), p)
 	w.num++
-
-	w.lines = append(w.lines, line)
 	return len(p), nil
 }
+
+func (w *LineWriter) Close() error {
+	return nil
+}