Rework log streaming and related functions (#1802)

closes #1801
closes #1815 
closes #1144
closes  #983
closes  #557
closes #1827
regression of #1791

# TODO
- [x] adjust log model
- [x] add migration for logs
- [x] send log line via grpc using step-id
- [x] save log-line to db
- [x] stream log-lines to UI
- [x] use less structs for log-data
- [x] make web UI work
  - [x] display logs loaded from db
  - [x] display streaming logs
- [ ] ~~make migration work~~ -> dedicated pull (#1828)

# TESTED
- [x] new logs are stored in database
- [x] log retrieval via cli (of new logs) works
- [x] log streaming works (tested via curl & webui)
- [x] log retrieval via web (of new logs) works

---------

Co-authored-by: 6543 <6543@obermui.de>

cli/exec/exec.go

@@ -248,7 +248,7 @@ var defaultLogger = pipeline.LogFunc(func(step *backendTypes.Step, rc multipart.
 		return err
 	}
 
-	logStream := NewLineWriter(step.Alias)
+	logStream := NewLineWriter(step.Alias, step.UUID)
 	_, err = io.Copy(logStream, part)
 	return err
 })

cli/exec/line.go

@@ -19,60 +19,44 @@ import (
 	"os"
 	"strings"
 	"time"
-)
 
-// Identifies the type of line in the logs.
-const (
-	LineStdout int = iota
-	LineStderr
-	LineExitCode
-	LineMetadata
-	LineProgress
+	"github.com/woodpecker-ci/woodpecker/pipeline/rpc"
 )
 
-// Line is a line of console output.
-type Line struct {
-	Step string `json:"step,omitempty"`
-	Time int64  `json:"time,omitempty"`
-	Type int    `json:"type,omitempty"`
-	Pos  int    `json:"pos,omitempty"`
-	Out  string `json:"out,omitempty"`
-}
-
 // LineWriter sends logs to the client.
 type LineWriter struct {
-	name  string
-	num   int
-	now   time.Time
-	rep   *strings.Replacer
-	lines []*Line
+	stepName string
+	stepUUID string
+	num      int
+	now      time.Time
+	rep      *strings.Replacer
+	lines    []*rpc.LogEntry
 }
 
 // NewLineWriter returns a new line reader.
-func NewLineWriter(name string) *LineWriter {
-	w := new(LineWriter)
-	w.name = name
-	w.num = 0
-	w.now = time.Now().UTC()
-
-	return w
+func NewLineWriter(stepName, stepUUID string) *LineWriter {
+	return &LineWriter{
+		stepName: stepName,
+		stepUUID: stepUUID,
+		now:      time.Now().UTC(),
+	}
 }
 
 func (w *LineWriter) Write(p []byte) (n int, err error) {
-	out := string(p)
+	data := string(p)
 	if w.rep != nil {
-		out = w.rep.Replace(out)
+		data = w.rep.Replace(data)
 	}
 
-	line := &Line{
-		Out:  out,
-		Step: w.name,
-		Pos:  w.num,
-		Time: int64(time.Since(w.now).Seconds()),
-		Type: LineStdout,
+	line := &rpc.LogEntry{
+		Data:     data,
+		StepUUID: w.stepUUID,
+		Line:     w.num,
+		Time:     int64(time.Since(w.now).Seconds()),
+		Type:     rpc.LogEntryStdout,
 	}
 
-	fmt.Fprintf(os.Stderr, "[%s:L%d:%ds] %s", w.name, w.num, int64(time.Since(w.now).Seconds()), out)
+	fmt.Fprintf(os.Stderr, "[%s:L%d:%ds] %s", w.stepName, w.num, int64(time.Since(w.now).Seconds()), data)
 
 	w.num++
 

cli/pipeline/logs.go

@@ -27,7 +27,7 @@ import (
 var pipelineLogsCmd = &cli.Command{
 	Name:      "logs",
 	Usage:     "show pipeline logs",
-	ArgsUsage: "<repo/name> [pipeline] [step]",
+	ArgsUsage: "<repo/name> [pipeline] [stepID]",
 	Action:    pipelineLogs,
 	Flags:     common.GlobalFlags,
 }
@@ -54,13 +54,13 @@ func pipelineLogs(c *cli.Context) error {
 		return err
 	}
 
-	logs, err := client.PipelineLogs(owner, name, number, step)
+	logs, err := client.StepLogEntries(owner, name, number, step)
 	if err != nil {
 		return err
 	}
 
 	for _, log := range logs {
-		fmt.Print(log.Output)
+		fmt.Print(string(log.Data))
 	}
 
 	return nil