woodpecker/server/api/build.go

// Copyright 2018 Drone.IO Inc.
// Copyright 2021 Informatyka Boguslawski sp. z o.o. sp.k., http://www.ib.pl/
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// This file has been modified by Informatyka Boguslawski sp. z o.o. sp.k.

package api

import (
	"bytes"
	"context"
	"database/sql"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/rs/zerolog/log"

	"github.com/woodpecker-ci/woodpecker/server"
	"github.com/woodpecker-ci/woodpecker/server/model"
	"github.com/woodpecker-ci/woodpecker/server/queue"
	"github.com/woodpecker-ci/woodpecker/server/remote"
	"github.com/woodpecker-ci/woodpecker/server/router/middleware/session"
	"github.com/woodpecker-ci/woodpecker/server/shared"
	"github.com/woodpecker-ci/woodpecker/server/store"
)

func GetBuilds(c *gin.Context) {
	repo := session.Repo(c)
	page, err := strconv.Atoi(c.DefaultQuery("page", "1"))
	if err != nil {
		_ = c.AbortWithError(http.StatusBadRequest, err)
		return
	}

	builds, err := store.FromContext(c).GetBuildList(repo, page)
	if err != nil {
		c.AbortWithStatus(http.StatusInternalServerError)
		return
	}
	c.JSON(http.StatusOK, builds)
}

func GetBuild(c *gin.Context) {
	_store := store.FromContext(c)
	if c.Param("number") == "latest" {
		GetBuildLast(c)
		return
	}

	repo := session.Repo(c)
	num, err := strconv.ParseInt(c.Param("number"), 10, 64)
	if err != nil {
		_ = c.AbortWithError(http.StatusBadRequest, err)
		return
	}

	build, err := _store.GetBuildNumber(repo, num)
	if err != nil {
		_ = c.AbortWithError(http.StatusInternalServerError, err)
		return
	}
	files, _ := _store.FileList(build)
	procs, _ := _store.ProcList(build)
	if build.Procs, err = model.Tree(procs); err != nil {
		_ = c.AbortWithError(http.StatusInternalServerError, err)
		return
	}
	build.Files = files

	c.JSON(http.StatusOK, build)
}

func GetBuildLast(c *gin.Context) {
	_store := store.FromContext(c)
	repo := session.Repo(c)
	branch := c.DefaultQuery("branch", repo.Branch)

	build, err := _store.GetBuildLast(repo, branch)
	if err != nil {
		c.String(http.StatusInternalServerError, err.Error())
		return
	}

	procs, err := _store.ProcList(build)
	if err != nil {
		_ = c.AbortWithError(http.StatusInternalServerError, err)
		return
	}
	if build.Procs, err = model.Tree(procs); err != nil {
		_ = c.AbortWithError(http.StatusInternalServerError, err)
		return
	}
	c.JSON(http.StatusOK, build)
}

func GetBuildLogs(c *gin.Context) {
	_store := store.FromContext(c)
	repo := session.Repo(c)

	// parse the build number and job sequence number from
	// the request parameter.
	num, _ := strconv.ParseInt(c.Params.ByName("number"), 10, 64)
	ppid, _ := strconv.Atoi(c.Params.ByName("pid"))
	name := c.Params.ByName("proc")

	build, err := _store.GetBuildNumber(repo, num)
	if err != nil {
		_ = c.AbortWithError(404, err)
		return
	}

	proc, err := _store.ProcChild(build, ppid, name)
	if err != nil {
		_ = c.AbortWithError(404, err)
		return
	}

	rc, err := _store.LogFind(proc)
	if err != nil {
		_ = c.AbortWithError(404, err)
		return
	}

	defer rc.Close()

	c.Header("Content-Type", "application/json")
	if _, err := io.Copy(c.Writer, rc); err != nil {
		log.Error().Err(err).Msg("could not copy log to http response")
	}
}

func GetProcLogs(c *gin.Context) {
	_store := store.FromContext(c)
	repo := session.Repo(c)

	// parse the build number and job sequence number from
	// the request parameter.
	num, _ := strconv.ParseInt(c.Params.ByName("number"), 10, 64)
	pid, _ := strconv.Atoi(c.Params.ByName("pid"))

	build, err := _store.GetBuildNumber(repo, num)
	if err != nil {
		_ = c.AbortWithError(http.StatusNotFound, err)
		return
	}

	proc, err := _store.ProcFind(build, pid)
	if err != nil {
		_ = c.AbortWithError(http.StatusNotFound, err)
		return
	}

	rc, err := _store.LogFind(proc)
	if err != nil {
		_ = c.AbortWithError(http.StatusNotFound, err)
		return
	}

	defer rc.Close()

	c.Header("Content-Type", "application/json")
	if _, err := io.Copy(c.Writer, rc); err != nil {
		log.Error().Err(err).Msg("could not copy log to http response")
	}
}

func GetBuildConfig(c *gin.Context) {
	_store := store.FromContext(c)
	repo := session.Repo(c)
	num, err := strconv.ParseInt(c.Param("number"), 10, 64)
	if err != nil {
		_ = c.AbortWithError(http.StatusBadRequest, err)
		return
	}

	build, err := _store.GetBuildNumber(repo, num)
	if err != nil {
		_ = c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	configs, err := _store.ConfigsForBuild(build.ID)
	if err != nil {
		c.String(http.StatusInternalServerError, err.Error())
		return
	}

	c.JSON(http.StatusOK, configs)
}

// DeleteBuild cancels a build
func DeleteBuild(c *gin.Context) {
	_store := store.FromContext(c)
	repo := session.Repo(c)
	num, _ := strconv.ParseInt(c.Params.ByName("number"), 10, 64)

	build, err := _store.GetBuildNumber(repo, num)
	if err != nil {
		_ = c.AbortWithError(http.StatusNotFound, err)
		return
	}

	if build.Status != model.StatusRunning && build.Status != model.StatusPending {
		c.String(http.StatusBadRequest, "Cannot cancel a non-running or non-pending build")
		return
	}

	code, err := cancelBuild(c, _store, repo, build)
	if err != nil {
		_ = c.AbortWithError(code, err)
		return
	}

	c.String(code, "")
}

// Cancel the build and returns the status.
func cancelBuild(
	ctx context.Context,
	_store store.Store,
	repo *model.Repo,
	build *model.Build,
) (int, error) {
	procs, err := _store.ProcList(build)
	if err != nil {
		return http.StatusNotFound, err
	}

	// First cancel/evict procs in the queue in one go
	var (
		procsToCancel []string
		procsToEvict  []string
	)
	for _, proc := range procs {
		if proc.PPID != 0 {
			continue
		}
		if proc.State == model.StatusRunning {
			procsToCancel = append(procsToCancel, fmt.Sprint(proc.ID))
		}
		if proc.State == model.StatusPending {
			procsToEvict = append(procsToEvict, fmt.Sprint(proc.ID))
		}
	}

	if len(procsToEvict) != 0 {
		if err := server.Config.Services.Queue.EvictAtOnce(ctx, procsToEvict); err != nil {
			log.Error().Err(err).Msgf("queue: evict_at_once: %v", procsToEvict)
		}
		if err := server.Config.Services.Queue.ErrorAtOnce(ctx, procsToEvict, queue.ErrCancel); err != nil {
			log.Error().Err(err).Msgf("queue: evict_at_once: %v", procsToEvict)
		}
	}
	if len(procsToCancel) != 0 {
		if err := server.Config.Services.Queue.ErrorAtOnce(ctx, procsToCancel, queue.ErrCancel); err != nil {
			log.Error().Err(err).Msgf("queue: evict_at_once: %v", procsToCancel)
		}
	}

	// Then update the DB status for pending builds
	// Running ones will be set when the agents stop on the cancel signal
	for _, proc := range procs {
		if proc.State == model.StatusPending {
			if proc.PPID != 0 {
				if _, err = shared.UpdateProcToStatusSkipped(_store, *proc, 0); err != nil {
					log.Error().Msgf("error: done: cannot update proc_id %d state: %s", proc.ID, err)
				}
			} else {
				if _, err = shared.UpdateProcToStatusKilled(_store, *proc); err != nil {
					log.Error().Msgf("error: done: cannot update proc_id %d state: %s", proc.ID, err)
				}
			}
		}
	}

	killedBuild, err := shared.UpdateToStatusKilled(_store, *build)
	if err != nil {
		log.Error().Err(err).Msgf("UpdateToStatusKilled: %v", build)
		return http.StatusInternalServerError, err
	}

	procs, err = _store.ProcList(killedBuild)
	if err != nil {
		return http.StatusNotFound, err
	}
	if killedBuild.Procs, err = model.Tree(procs); err != nil {
		return http.StatusInternalServerError, err
	}
	if err := publishToTopic(ctx, killedBuild, repo); err != nil {
		log.Error().Err(err).Msg("publishToTopic")
	}

	return http.StatusNoContent, nil
}

func PostApproval(c *gin.Context) {
	var (
		_store = store.FromContext(c)
		repo   = session.Repo(c)
		user   = session.User(c)
		num, _ = strconv.ParseInt(c.Params.ByName("number"), 10, 64)
	)

	build, err := _store.GetBuildNumber(repo, num)
	if err != nil {
		_ = c.AbortWithError(404, err)
		return
	}
	if build.Status != model.StatusBlocked {
		c.String(http.StatusBadRequest, "cannot decline a build with status %s", build.Status)
		return
	}

	// fetch the build file from the database
	configs, err := _store.ConfigsForBuild(build.ID)
	if err != nil {
		log.Error().Msgf("failure to get build config for %s. %s", repo.FullName, err)
		_ = c.AbortWithError(404, err)
		return
	}

	if build, err = shared.UpdateToStatusPending(_store, *build, user.Login); err != nil {
		c.String(http.StatusInternalServerError, "error updating build. %s", err)
		return
	}

	var yamls []*remote.FileMeta
	for _, y := range configs {
		yamls = append(yamls, &remote.FileMeta{Data: y.Data, Name: y.Name})
	}

	build, buildItems, err := createBuildItems(c, _store, build, user, repo, yamls, nil)
	if err != nil {
		msg := fmt.Sprintf("failure to createBuildItems for %s", repo.FullName)
		log.Error().Err(err).Msg(msg)
		c.String(http.StatusInternalServerError, msg)
		return
	}

	build, err = startBuild(c, _store, build, user, repo, buildItems)
	if err != nil {
		msg := fmt.Sprintf("failure to start build for %s", repo.FullName)
		log.Error().Err(err).Msg(msg)
		c.String(http.StatusInternalServerError, msg)
		return
	}

	c.JSON(200, build)
}

func PostDecline(c *gin.Context) {
	var (
		_store = store.FromContext(c)
		repo   = session.Repo(c)
		user   = session.User(c)
		num, _ = strconv.ParseInt(c.Params.ByName("number"), 10, 64)
	)

	build, err := _store.GetBuildNumber(repo, num)
	if err != nil {
		_ = c.AbortWithError(404, err)
		return
	}
	if build.Status != model.StatusBlocked {
		c.String(500, "cannot decline a build with status %s", build.Status)
		return
	}

	if _, err = shared.UpdateToStatusDeclined(_store, *build, user.Login); err != nil {
		c.String(500, "error updating build. %s", err)
		return
	}

	if build.Procs, err = _store.ProcList(build); err != nil {
		log.Error().Err(err).Msg("can not get proc list from store")
	}
	if build.Procs, err = model.Tree(build.Procs); err != nil {
		log.Error().Err(err).Msg("can not build tree from proc list")
	}

	if err := updateBuildStatus(c, build, repo, user); err != nil {
		log.Error().Err(err).Msg("updateBuildStatus")
	}

	if err := publishToTopic(c, build, repo); err != nil {
		log.Error().Err(err).Msg("publishToTopic")
	}

	c.JSON(200, build)
}

func GetBuildQueue(c *gin.Context) {
	out, err := store.FromContext(c).GetBuildQueue()
	if err != nil {
		c.String(500, "Error getting build queue. %s", err)
		return
	}
	c.JSON(200, out)
}

// PostBuild restarts a build
func PostBuild(c *gin.Context) {
	_remote := server.Config.Services.Remote
	_store := store.FromContext(c)
	repo := session.Repo(c)

	num, err := strconv.ParseInt(c.Param("number"), 10, 64)
	if err != nil {
		_ = c.AbortWithError(http.StatusBadRequest, err)
		return
	}

	user, err := _store.GetUser(repo.UserID)
	if err != nil {
		log.Error().Msgf("failure to find repo owner %s. %s", repo.FullName, err)
		_ = c.AbortWithError(500, err)
		return
	}

	build, err := _store.GetBuildNumber(repo, num)
	if err != nil {
		log.Error().Msgf("failure to get build %d. %s", num, err)
		_ = c.AbortWithError(404, err)
		return
	}

	switch build.Status {
	case model.StatusDeclined,
		model.StatusBlocked:
		c.String(500, "cannot restart a build with status %s", build.Status)
		return
	}

	// if the remote has a refresh token, the current access token
	// may be stale. Therefore, we should refresh prior to dispatching
	// the job.
	if refresher, ok := _remote.(remote.Refresher); ok {
		ok, err := refresher.Refresh(c, user)
		if err != nil {
			log.Error().Err(err).Msgf("refresh oauth token of user '%s' failed", user.Login)
		} else if ok {
			if err := _store.UpdateUser(user); err != nil {
				log.Error().Err(err).Msg("fail to save user to store after refresh oauth token")
			}
		}
	}

	var pipelineFiles []*remote.FileMeta

	// fetch the old pipeline config from database
	configs, err := _store.ConfigsForBuild(build.ID)
	if err != nil {
		log.Error().Msgf("failure to get build config for %s. %s", repo.FullName, err)
		_ = c.AbortWithError(404, err)
		return
	}

	for _, y := range configs {
		pipelineFiles = append(pipelineFiles, &remote.FileMeta{Data: y.Data, Name: y.Name})
	}

	// If config extension is active we should refetch the config in case something changed
	if server.Config.Services.ConfigService.IsConfigured() {
		currentFileMeta := make([]*remote.FileMeta, len(configs))
		for i, cfg := range configs {
			currentFileMeta[i] = &remote.FileMeta{Name: cfg.Name, Data: cfg.Data}
		}

		newConfig, useOld, err := server.Config.Services.ConfigService.FetchExternalConfig(c, repo, build, currentFileMeta)
		if err != nil {
			msg := fmt.Sprintf("On fetching external build config: %s", err)
			c.String(http.StatusBadRequest, msg)
			return
		}
		if !useOld {
			pipelineFiles = newConfig
		}
	}

	build.ID = 0
	build.Number = 0
	build.Parent = num
	build.Status = model.StatusPending
	build.Started = 0
	build.Finished = 0
	build.Enqueued = time.Now().UTC().Unix()
	build.Error = ""
	build.Deploy = c.DefaultQuery("deploy_to", build.Deploy)

	if event, ok := c.GetQuery("event"); ok {
		build.Event = model.WebhookEvent(event)

		if !model.ValidateWebhookEvent(build.Event) {
			msg := fmt.Sprintf("build event '%s' is invalid", event)
			c.String(http.StatusBadRequest, msg)
			return
		}
	}

	err = _store.CreateBuild(build)
	if err != nil {
		msg := fmt.Sprintf("failure to save build for %s", repo.FullName)
		log.Error().Err(err).Msg(msg)
		c.String(http.StatusInternalServerError, msg)
		return
	}

	if err := persistBuildConfigs(_store, configs, build.ID); err != nil {
		msg := fmt.Sprintf("failure to persist build config for %s.", repo.FullName)
		log.Error().Err(err).Msg(msg)
		c.String(http.StatusInternalServerError, msg)
		return
	}

	// Read query string parameters into buildParams, exclude reserved params
	envs := map[string]string{}
	for key, val := range c.Request.URL.Query() {
		switch key {
		// Skip some options of the endpoint
		case "fork", "event", "deploy_to":
			continue
		default:
			// We only accept string literals, because build parameters will be
			// injected as environment variables
			// TODO: sanitize the value
			envs[key] = val[0]
		}
	}

	build, buildItems, err := createBuildItems(c, _store, build, user, repo, pipelineFiles, envs)
	if err != nil {
		msg := fmt.Sprintf("failure to createBuildItems for %s", repo.FullName)
		log.Error().Err(err).Msg(msg)
		c.String(http.StatusInternalServerError, msg)
		return
	}

	build, err = startBuild(c, _store, build, user, repo, buildItems)
	if err != nil {
		msg := fmt.Sprintf("failure to start build for %s", repo.FullName)
		log.Error().Err(err).Msg(msg)
		c.String(http.StatusInternalServerError, msg)
		return
	}

	c.JSON(200, build)
}

func DeleteBuildLogs(c *gin.Context) {
	_store := store.FromContext(c)

	repo := session.Repo(c)
	user := session.User(c)
	num, _ := strconv.ParseInt(c.Params.ByName("number"), 10, 64)

	build, err := _store.GetBuildNumber(repo, num)
	if err != nil {
		_ = c.AbortWithError(404, err)
		return
	}

	procs, err := _store.ProcList(build)
	if err != nil {
		_ = c.AbortWithError(404, err)
		return
	}

	switch build.Status {
	case model.StatusRunning, model.StatusPending:
		c.String(400, "Cannot delete logs for a pending or running build")
		return
	}

	for _, proc := range procs {
		t := time.Now().UTC()
		buf := bytes.NewBufferString(fmt.Sprintf(deleteStr, proc.Name, user.Login, t.Format(time.UnixDate)))
		lerr := _store.LogSave(proc, buf)
		if lerr != nil {
			err = lerr
		}
	}
	if err != nil {
		c.String(400, "There was a problem deleting your logs. %s", err)
		return
	}

	c.String(204, "")
}

func createBuildItems(ctx context.Context, store store.Store, build *model.Build, user *model.User, repo *model.Repo, yamls []*remote.FileMeta, envs map[string]string) (*model.Build, []*shared.BuildItem, error) {
	netrc, err := server.Config.Services.Remote.Netrc(user, repo)
	if err != nil {
		log.Error().Err(err).Msg("Failed to generate netrc file")
	}

	// get the previous build so that we can send status change notifications
	last, err := store.GetBuildLastBefore(repo, build.Branch, build.ID)
	if err != nil && !errors.Is(err, sql.ErrNoRows) {
		log.Error().Err(err).Str("repo", repo.FullName).Msgf("Error getting last build before build number '%d'", build.Number)
	}

	secs, err := server.Config.Services.Secrets.SecretListBuild(repo, build)
	if err != nil {
		log.Error().Err(err).Msgf("Error getting secrets for %s#%d", repo.FullName, build.Number)
	}

	regs, err := server.Config.Services.Registries.RegistryList(repo)
	if err != nil {
		log.Error().Err(err).Msgf("Error getting registry credentials for %s#%d", repo.FullName, build.Number)
	}

	if envs == nil {
		envs = map[string]string{}
	}
	if server.Config.Services.Environ != nil {
		globals, _ := server.Config.Services.Environ.EnvironList(repo)
		for _, global := range globals {
			envs[global.Name] = global.Value
		}
	}

	b := shared.ProcBuilder{
		Repo:  repo,
		Curr:  build,
		Last:  last,
		Netrc: netrc,
		Secs:  secs,
		Regs:  regs,
		Envs:  envs,
		Link:  server.Config.Server.Host,
		Yamls: yamls,
	}
	buildItems, err := b.Build()
	if err != nil {
		if _, err := shared.UpdateToStatusError(store, *build, err); err != nil {
			log.Error().Err(err).Msgf("Error setting error status of build for %s#%d", repo.FullName, build.Number)
		}
		return nil, nil, err
	}

	build = shared.SetBuildStepsOnBuild(b.Curr, buildItems)

	return build, buildItems, nil
}

func cancelPreviousPipelines(
	ctx context.Context,
	_store store.Store,
	build *model.Build,
	user *model.User,
	repo *model.Repo,
) error {
	// check this event should cancel previous pipelines
	eventIncluded := false
	for _, ev := range repo.CancelPreviousPipelineEvents {
		if ev == build.Event {
			eventIncluded = true
			break
		}
	}
	if !eventIncluded {
		return nil
	}

	// get all active activeBuilds
	activeBuilds, err := _store.GetActiveBuildList(repo, -1)
	if err != nil {
		return err
	}

	buildNeedsCancel := func(active *model.Build) (bool, error) {
		// always filter on same event
		if active.Event != build.Event {
			return false, nil
		}

		// find events for the same context
		switch build.Event {
		case model.EventPush:
			return build.Branch == active.Branch, nil
		default:
			return build.Refspec == active.Refspec, nil
		}
	}

	for _, active := range activeBuilds {
		if active.ID == build.ID {
			// same build. e.g. self
			continue
		}

		cancel, err := buildNeedsCancel(active)
		if err != nil {
			log.Error().
				Err(err).
				Str("Ref", active.Ref).
				Msg("Error while trying to cancel build, skipping")
			continue
		}
		if !cancel {
			continue
		}
		_, err = cancelBuild(ctx, _store, repo, active)
		if err != nil {
			log.Error().
				Err(err).
				Str("Ref", active.Ref).
				Int64("ID", active.ID).
				Msg("Failed to cancel build")
		}
	}

	return nil
}

func startBuild(
	ctx context.Context,
	store store.Store,
	activeBuild *model.Build,
	user *model.User,
	repo *model.Repo,
	buildItems []*shared.BuildItem,
) (*model.Build, error) {
	// call to cancel previous builds if needed
	if err := cancelPreviousPipelines(ctx, store, activeBuild, user, repo); err != nil {
		// should be not breaking
		log.Error().Err(err).Msg("Failed to cancel previous builds")
	}

	if err := store.ProcCreate(activeBuild.Procs); err != nil {
		log.Error().Err(err).Str("repo", repo.FullName).Msgf("error persisting procs for %s#%d", repo.FullName, activeBuild.Number)
		return nil, err
	}

	if err := publishToTopic(ctx, activeBuild, repo); err != nil {
		log.Error().Err(err).Msg("publishToTopic")
	}

	if err := queueBuild(activeBuild, repo, buildItems); err != nil {
		log.Error().Err(err).Msg("queueBuild")
		return nil, err
	}

	if err := updateBuildStatus(ctx, activeBuild, repo, user); err != nil {
		log.Error().Err(err).Msg("updateBuildStatus")
	}

	return activeBuild, nil
}

func updateBuildStatus(ctx context.Context, build *model.Build, repo *model.Repo, user *model.User) error {
	for _, proc := range build.Procs {
		// skip child procs
		if !proc.IsParent() {
			continue
		}

		err := server.Config.Services.Remote.Status(ctx, user, repo, build, proc)
		if err != nil {
			log.Error().Err(err).Msgf("error setting commit status for %s/%d", repo.FullName, build.Number)
			return err
		}
	}

	return nil
}

func persistBuildConfigs(store store.Store, configs []*model.Config, buildID int64) error {
	for _, conf := range configs {
		buildConfig := &model.BuildConfig{
			ConfigID: conf.ID,
			BuildID:  buildID,
		}
		err := store.BuildConfigCreate(buildConfig)
		if err != nil {
			return err
		}
	}
	return nil
}

var deleteStr = `[
	{
	  "proc": %q,
	  "pos": 0,
	  "out": "logs purged by %s on %s\n"
	}
]`