mirror of
https://github.com/woodpecker-ci/woodpecker.git
synced 2024-12-12 08:23:48 +02:00
2477d2e57f
This implements #1073, adds .yaml to the accepted endings for woodpecker configs. This currently adds some more lines to the duplication (tried to compensate by fixing the other duplication in the configFetcher) as the CLI and Server are still separate.
407 lines
10 KiB
Go
407 lines
10 KiB
Go
// Copyright 2022 Woodpecker Authors
|
|
// Copyright 2018 Drone.IO Inc.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package shared
|
|
|
|
import (
|
|
"fmt"
|
|
"math/rand"
|
|
"net/url"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/drone/envsubst"
|
|
"github.com/rs/zerolog/log"
|
|
|
|
backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
|
|
"github.com/woodpecker-ci/woodpecker/pipeline/frontend"
|
|
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
|
|
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler"
|
|
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/linter"
|
|
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/matrix"
|
|
"github.com/woodpecker-ci/woodpecker/server"
|
|
"github.com/woodpecker-ci/woodpecker/server/model"
|
|
"github.com/woodpecker-ci/woodpecker/server/remote"
|
|
)
|
|
|
|
// TODO(974) move to pipeline/*
|
|
|
|
// StepBuilder Takes the hook data and the yaml and returns in internal data model
|
|
type StepBuilder struct {
|
|
Repo *model.Repo
|
|
Curr *model.Pipeline
|
|
Last *model.Pipeline
|
|
Netrc *model.Netrc
|
|
Secs []*model.Secret
|
|
Regs []*model.Registry
|
|
Link string
|
|
Yamls []*remote.FileMeta
|
|
Envs map[string]string
|
|
}
|
|
|
|
type PipelineItem struct {
|
|
Step *model.Step
|
|
Platform string
|
|
Labels map[string]string
|
|
DependsOn []string
|
|
RunsOn []string
|
|
Config *backend.Config
|
|
}
|
|
|
|
func (b *StepBuilder) Build() ([]*PipelineItem, error) {
|
|
var items []*PipelineItem
|
|
|
|
sort.Sort(remote.ByName(b.Yamls))
|
|
|
|
pidSequence := 1
|
|
|
|
for _, y := range b.Yamls {
|
|
// matrix axes
|
|
axes, err := matrix.ParseString(string(y.Data))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(axes) == 0 {
|
|
axes = append(axes, matrix.Axis{})
|
|
}
|
|
|
|
for _, axis := range axes {
|
|
step := &model.Step{
|
|
PipelineID: b.Curr.ID,
|
|
PID: pidSequence,
|
|
PGID: pidSequence,
|
|
State: model.StatusPending,
|
|
Environ: axis,
|
|
Name: SanitizePath(y.Name),
|
|
}
|
|
|
|
metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, step, b.Link)
|
|
environ := b.environmentVariables(metadata, axis)
|
|
|
|
// add global environment variables for substituting
|
|
for k, v := range b.Envs {
|
|
if _, exists := environ[k]; exists {
|
|
// don't override existing values
|
|
continue
|
|
}
|
|
environ[k] = v
|
|
}
|
|
|
|
// substitute vars
|
|
substituted, err := b.envsubst(string(y.Data), environ)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// parse yaml pipeline
|
|
parsed, err := yaml.ParseString(substituted)
|
|
if err != nil {
|
|
return nil, &yaml.PipelineParseError{Err: err}
|
|
}
|
|
|
|
// lint pipeline
|
|
if err := linter.New(
|
|
linter.WithTrusted(b.Repo.IsTrusted),
|
|
).Lint(parsed); err != nil {
|
|
return nil, &yaml.PipelineParseError{Err: err}
|
|
}
|
|
|
|
// checking if filtered.
|
|
if match, err := parsed.When.Match(metadata, true); !match && err == nil {
|
|
log.Debug().Str("pipeline", step.Name).Msg(
|
|
"Marked as skipped, dose not match metadata",
|
|
)
|
|
step.State = model.StatusSkipped
|
|
} else if err != nil {
|
|
log.Debug().Str("pipeline", step.Name).Msg(
|
|
"Pipeline config could not be parsed",
|
|
)
|
|
return nil, err
|
|
}
|
|
|
|
// TODO: deprecated branches filter => remove after some time
|
|
if !parsed.Branches.Match(b.Curr.Branch) && (b.Curr.Event != model.EventDeploy && b.Curr.Event != model.EventTag) {
|
|
log.Debug().Str("pipeline", step.Name).Msg(
|
|
"Marked as skipped, dose not match branch",
|
|
)
|
|
step.State = model.StatusSkipped
|
|
}
|
|
|
|
ir, err := b.toInternalRepresentation(parsed, environ, metadata, step.ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if len(ir.Stages) == 0 {
|
|
continue
|
|
}
|
|
|
|
item := &PipelineItem{
|
|
Step: step,
|
|
Config: ir,
|
|
Labels: parsed.Labels,
|
|
DependsOn: parsed.DependsOn,
|
|
RunsOn: parsed.RunsOn,
|
|
Platform: parsed.Platform,
|
|
}
|
|
if item.Labels == nil {
|
|
item.Labels = map[string]string{}
|
|
}
|
|
|
|
items = append(items, item)
|
|
pidSequence++
|
|
}
|
|
}
|
|
|
|
items = filterItemsWithMissingDependencies(items)
|
|
|
|
// check if at least one step can start, if list is not empty
|
|
if len(items) > 0 && !stepListContainsItemsToRun(items) {
|
|
return nil, fmt.Errorf("pipeline has no startpoint")
|
|
}
|
|
|
|
return items, nil
|
|
}
|
|
|
|
func stepListContainsItemsToRun(items []*PipelineItem) bool {
|
|
for i := range items {
|
|
if items[i].Step.State == model.StatusPending {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func filterItemsWithMissingDependencies(items []*PipelineItem) []*PipelineItem {
|
|
itemsToRemove := make([]*PipelineItem, 0)
|
|
|
|
for _, item := range items {
|
|
for _, dep := range item.DependsOn {
|
|
if !containsItemWithName(dep, items) {
|
|
itemsToRemove = append(itemsToRemove, item)
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(itemsToRemove) > 0 {
|
|
filtered := make([]*PipelineItem, 0)
|
|
for _, item := range items {
|
|
if !containsItemWithName(item.Step.Name, itemsToRemove) {
|
|
filtered = append(filtered, item)
|
|
}
|
|
}
|
|
// Recursive to handle transitive deps
|
|
return filterItemsWithMissingDependencies(filtered)
|
|
}
|
|
|
|
return items
|
|
}
|
|
|
|
func containsItemWithName(name string, items []*PipelineItem) bool {
|
|
for _, item := range items {
|
|
if name == item.Step.Name {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (b *StepBuilder) envsubst(y string, environ map[string]string) (string, error) {
|
|
return envsubst.Eval(y, func(name string) string {
|
|
env := environ[name]
|
|
if strings.Contains(env, "\n") {
|
|
env = fmt.Sprintf("%q", env)
|
|
}
|
|
return env
|
|
})
|
|
}
|
|
|
|
func (b *StepBuilder) environmentVariables(metadata frontend.Metadata, axis matrix.Axis) map[string]string {
|
|
environ := metadata.Environ()
|
|
for k, v := range axis {
|
|
environ[k] = v
|
|
}
|
|
return environ
|
|
}
|
|
|
|
func (b *StepBuilder) toInternalRepresentation(parsed *yaml.Config, environ map[string]string, metadata frontend.Metadata, stepID int64) (*backend.Config, error) {
|
|
var secrets []compiler.Secret
|
|
for _, sec := range b.Secs {
|
|
if !sec.Match(b.Curr.Event) {
|
|
continue
|
|
}
|
|
secrets = append(secrets, compiler.Secret{
|
|
Name: sec.Name,
|
|
Value: sec.Value,
|
|
Match: sec.Images,
|
|
PluginOnly: sec.PluginsOnly,
|
|
})
|
|
}
|
|
|
|
var registries []compiler.Registry
|
|
for _, reg := range b.Regs {
|
|
registries = append(registries, compiler.Registry{
|
|
Hostname: reg.Address,
|
|
Username: reg.Username,
|
|
Password: reg.Password,
|
|
Email: reg.Email,
|
|
})
|
|
}
|
|
|
|
return compiler.New(
|
|
compiler.WithEnviron(environ),
|
|
compiler.WithEnviron(b.Envs),
|
|
compiler.WithEscalated(server.Config.Pipeline.Privileged...),
|
|
compiler.WithResourceLimit(server.Config.Pipeline.Limits.MemSwapLimit, server.Config.Pipeline.Limits.MemLimit, server.Config.Pipeline.Limits.ShmSize, server.Config.Pipeline.Limits.CPUQuota, server.Config.Pipeline.Limits.CPUShares, server.Config.Pipeline.Limits.CPUSet),
|
|
compiler.WithVolumes(server.Config.Pipeline.Volumes...),
|
|
compiler.WithNetworks(server.Config.Pipeline.Networks...),
|
|
compiler.WithLocal(false),
|
|
compiler.WithOption(
|
|
compiler.WithNetrc(
|
|
b.Netrc.Login,
|
|
b.Netrc.Password,
|
|
b.Netrc.Machine,
|
|
),
|
|
b.Repo.IsSCMPrivate || server.Config.Pipeline.AuthenticatePublicRepos,
|
|
),
|
|
compiler.WithDefaultCloneImage(server.Config.Pipeline.DefaultCloneImage),
|
|
compiler.WithRegistry(registries...),
|
|
compiler.WithSecret(secrets...),
|
|
compiler.WithPrefix(
|
|
fmt.Sprintf(
|
|
"wp_%d_%d",
|
|
stepID,
|
|
rand.Int(),
|
|
),
|
|
),
|
|
compiler.WithProxy(),
|
|
compiler.WithWorkspaceFromURL("/woodpecker", b.Repo.Link),
|
|
compiler.WithMetadata(metadata),
|
|
).Compile(parsed)
|
|
}
|
|
|
|
func SetPipelineStepsOnPipeline(pipeline *model.Pipeline, pipelineItems []*PipelineItem) *model.Pipeline {
|
|
var pidSequence int
|
|
for _, item := range pipelineItems {
|
|
pipeline.Steps = append(pipeline.Steps, item.Step)
|
|
if pidSequence < item.Step.PID {
|
|
pidSequence = item.Step.PID
|
|
}
|
|
}
|
|
|
|
for _, item := range pipelineItems {
|
|
for _, stage := range item.Config.Stages {
|
|
var gid int
|
|
for _, step := range stage.Steps {
|
|
pidSequence++
|
|
if gid == 0 {
|
|
gid = pidSequence
|
|
}
|
|
step := &model.Step{
|
|
PipelineID: pipeline.ID,
|
|
Name: step.Alias,
|
|
PID: pidSequence,
|
|
PPID: item.Step.PID,
|
|
PGID: gid,
|
|
State: model.StatusPending,
|
|
}
|
|
if item.Step.State == model.StatusSkipped {
|
|
step.State = model.StatusSkipped
|
|
}
|
|
pipeline.Steps = append(pipeline.Steps, step)
|
|
}
|
|
}
|
|
}
|
|
|
|
return pipeline
|
|
}
|
|
|
|
// return the metadata from the cli context.
|
|
func metadataFromStruct(repo *model.Repo, pipeline, last *model.Pipeline, step *model.Step, link string) frontend.Metadata {
|
|
host := link
|
|
uri, err := url.Parse(link)
|
|
if err == nil {
|
|
host = uri.Host
|
|
}
|
|
return frontend.Metadata{
|
|
Repo: frontend.Repo{
|
|
Name: repo.FullName,
|
|
Link: repo.Link,
|
|
Remote: repo.Clone,
|
|
Private: repo.IsSCMPrivate,
|
|
Branch: repo.Branch,
|
|
},
|
|
Curr: metadataPipelineFromModelPipeline(pipeline, true),
|
|
Prev: metadataPipelineFromModelPipeline(last, false),
|
|
Step: frontend.Step{
|
|
Number: step.PID,
|
|
Matrix: step.Environ,
|
|
},
|
|
Sys: frontend.System{
|
|
Name: "woodpecker",
|
|
Link: link,
|
|
Host: host,
|
|
Platform: "", // will be set by pipeline platform option or by agent
|
|
},
|
|
}
|
|
}
|
|
|
|
func metadataPipelineFromModelPipeline(pipeline *model.Pipeline, includeParent bool) frontend.Pipeline {
|
|
cron := ""
|
|
if pipeline.Event == model.EventCron {
|
|
cron = pipeline.Sender
|
|
}
|
|
|
|
parent := int64(0)
|
|
if includeParent {
|
|
parent = pipeline.Parent
|
|
}
|
|
|
|
return frontend.Pipeline{
|
|
Number: pipeline.Number,
|
|
Parent: parent,
|
|
Created: pipeline.Created,
|
|
Started: pipeline.Started,
|
|
Finished: pipeline.Finished,
|
|
Status: string(pipeline.Status),
|
|
Event: string(pipeline.Event),
|
|
Link: pipeline.Link,
|
|
Target: pipeline.Deploy,
|
|
Commit: frontend.Commit{
|
|
Sha: pipeline.Commit,
|
|
Ref: pipeline.Ref,
|
|
Refspec: pipeline.Refspec,
|
|
Branch: pipeline.Branch,
|
|
Message: pipeline.Message,
|
|
Author: frontend.Author{
|
|
Name: pipeline.Author,
|
|
Email: pipeline.Email,
|
|
Avatar: pipeline.Avatar,
|
|
},
|
|
ChangedFiles: pipeline.ChangedFiles,
|
|
},
|
|
Cron: cron,
|
|
}
|
|
}
|
|
|
|
func SanitizePath(path string) string {
|
|
path = filepath.Base(path)
|
|
path = strings.TrimSuffix(path, ".yml")
|
|
path = strings.TrimSuffix(path, ".yaml")
|
|
path = strings.TrimPrefix(path, ".")
|
|
return path
|
|
}
|