2022-10-18 03:24:12 +02:00
|
|
|
// Copyright 2022 Woodpecker Authors
|
2019-06-01 10:17:02 +02:00
|
|
|
// Copyright 2018 Drone.IO Inc.
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2022-11-06 13:44:04 +02:00
|
|
|
package pipeline
|
2019-06-01 10:17:02 +02:00
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
2021-09-21 04:21:13 +02:00
|
|
|
"path/filepath"
|
2019-06-01 10:17:02 +02:00
|
|
|
"strings"
|
|
|
|
|
2023-03-21 01:48:15 +02:00
|
|
|
"github.com/oklog/ulid/v2"
|
2022-09-26 09:27:20 +02:00
|
|
|
"github.com/rs/zerolog/log"
|
2021-10-12 09:25:13 +02:00
|
|
|
|
2023-06-06 09:14:21 +02:00
|
|
|
backend_types "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
|
|
|
|
yaml_types "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
|
|
|
|
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
|
|
|
|
|
2021-09-24 13:18:34 +02:00
|
|
|
"github.com/woodpecker-ci/woodpecker/pipeline/frontend"
|
2023-06-05 00:15:07 +02:00
|
|
|
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata"
|
2021-09-24 13:18:34 +02:00
|
|
|
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
|
|
|
|
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler"
|
|
|
|
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/linter"
|
|
|
|
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/matrix"
|
2021-09-22 20:48:01 +02:00
|
|
|
"github.com/woodpecker-ci/woodpecker/server"
|
2021-09-27 19:51:55 +02:00
|
|
|
"github.com/woodpecker-ci/woodpecker/server/model"
|
2019-06-01 10:17:02 +02:00
|
|
|
)
|
|
|
|
|
2022-10-28 17:38:53 +02:00
|
|
|
// StepBuilder Takes the hook data and the yaml and returns in internal data model
|
|
|
|
type StepBuilder struct {
|
2022-05-20 05:20:17 +02:00
|
|
|
Repo *model.Repo
|
2022-10-18 03:24:12 +02:00
|
|
|
Curr *model.Pipeline
|
|
|
|
Last *model.Pipeline
|
2022-05-20 05:20:17 +02:00
|
|
|
Netrc *model.Netrc
|
|
|
|
Secs []*model.Secret
|
|
|
|
Regs []*model.Registry
|
|
|
|
Link string
|
2022-11-06 13:44:04 +02:00
|
|
|
Yamls []*forge_types.FileMeta
|
2022-05-20 05:20:17 +02:00
|
|
|
Envs map[string]string
|
2023-06-05 00:15:07 +02:00
|
|
|
Forge metadata.ServerForge
|
2019-06-01 10:17:02 +02:00
|
|
|
}
|
|
|
|
|
2022-11-06 13:44:04 +02:00
|
|
|
type Item struct {
|
2023-06-27 18:01:18 +02:00
|
|
|
Workflow *model.Workflow
|
2019-06-13 17:38:19 +02:00
|
|
|
Platform string
|
|
|
|
Labels map[string]string
|
|
|
|
DependsOn []string
|
2019-06-17 09:06:36 +02:00
|
|
|
RunsOn []string
|
2023-06-06 09:14:21 +02:00
|
|
|
Config *backend_types.Config
|
2019-06-01 10:17:02 +02:00
|
|
|
}
|
|
|
|
|
2022-11-06 13:44:04 +02:00
|
|
|
func (b *StepBuilder) Build() ([]*Item, error) {
|
|
|
|
var items []*Item
|
2019-06-01 10:17:02 +02:00
|
|
|
|
2022-11-06 13:44:04 +02:00
|
|
|
b.Yamls = forge_types.SortByName(b.Yamls)
|
2019-06-16 15:27:40 +02:00
|
|
|
|
2019-07-19 09:17:47 +02:00
|
|
|
pidSequence := 1
|
|
|
|
|
|
|
|
for _, y := range b.Yamls {
|
2019-06-05 15:58:27 +02:00
|
|
|
// matrix axes
|
2019-06-13 17:38:19 +02:00
|
|
|
axes, err := matrix.ParseString(string(y.Data))
|
2019-06-01 10:41:51 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2019-06-01 10:17:02 +02:00
|
|
|
}
|
2019-06-01 10:41:51 +02:00
|
|
|
if len(axes) == 0 {
|
|
|
|
axes = append(axes, matrix.Axis{})
|
2019-06-01 10:17:02 +02:00
|
|
|
}
|
|
|
|
|
2019-07-19 09:17:47 +02:00
|
|
|
for _, axis := range axes {
|
2023-06-27 18:01:18 +02:00
|
|
|
workflow := &model.Workflow{
|
2022-10-18 03:24:12 +02:00
|
|
|
PipelineID: b.Curr.ID,
|
|
|
|
PID: pidSequence,
|
|
|
|
State: model.StatusPending,
|
|
|
|
Environ: axis,
|
|
|
|
Name: SanitizePath(y.Name),
|
2019-06-01 10:17:02 +02:00
|
|
|
}
|
|
|
|
|
2023-06-05 00:15:07 +02:00
|
|
|
workflowMetadata := frontend.MetadataFromStruct(b.Forge, b.Repo, b.Curr, b.Last, workflow, b.Link)
|
|
|
|
environ := b.environmentVariables(workflowMetadata, axis)
|
2019-06-01 10:17:02 +02:00
|
|
|
|
2022-07-30 08:06:03 +02:00
|
|
|
// add global environment variables for substituting
|
|
|
|
for k, v := range b.Envs {
|
|
|
|
if _, exists := environ[k]; exists {
|
|
|
|
// don't override existing values
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
environ[k] = v
|
|
|
|
}
|
|
|
|
|
2019-06-05 15:58:27 +02:00
|
|
|
// substitute vars
|
2023-06-05 00:15:07 +02:00
|
|
|
substituted, err := frontend.EnvVarSubst(string(y.Data), environ)
|
2019-06-01 10:41:51 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2019-06-05 15:58:27 +02:00
|
|
|
// parse yaml pipeline
|
2019-06-13 17:38:19 +02:00
|
|
|
parsed, err := yaml.ParseString(substituted)
|
2019-06-01 10:41:51 +02:00
|
|
|
if err != nil {
|
2022-08-15 14:37:46 +02:00
|
|
|
return nil, &yaml.PipelineParseError{Err: err}
|
2019-06-01 10:41:51 +02:00
|
|
|
}
|
|
|
|
|
2019-06-05 15:58:27 +02:00
|
|
|
// lint pipeline
|
2022-01-05 18:54:44 +02:00
|
|
|
if err := linter.New(
|
2019-06-01 10:41:51 +02:00
|
|
|
linter.WithTrusted(b.Repo.IsTrusted),
|
2022-01-05 18:54:44 +02:00
|
|
|
).Lint(parsed); err != nil {
|
2022-08-15 14:37:46 +02:00
|
|
|
return nil, &yaml.PipelineParseError{Err: err}
|
2019-06-01 10:41:51 +02:00
|
|
|
}
|
|
|
|
|
2022-09-26 09:27:20 +02:00
|
|
|
// checking if filtered.
|
2023-07-02 23:45:22 +02:00
|
|
|
if match, err := parsed.When.Match(workflowMetadata, true, environ); !match && err == nil {
|
2023-04-08 13:15:28 +02:00
|
|
|
log.Debug().Str("pipeline", workflow.Name).Msg(
|
2022-09-26 09:27:20 +02:00
|
|
|
"Marked as skipped, dose not match metadata",
|
|
|
|
)
|
2023-04-08 13:15:28 +02:00
|
|
|
workflow.State = model.StatusSkipped
|
2022-10-06 01:49:23 +02:00
|
|
|
} else if err != nil {
|
2023-04-08 13:15:28 +02:00
|
|
|
log.Debug().Str("pipeline", workflow.Name).Msg(
|
2022-10-06 01:49:23 +02:00
|
|
|
"Pipeline config could not be parsed",
|
|
|
|
)
|
|
|
|
return nil, err
|
2022-09-26 09:27:20 +02:00
|
|
|
}
|
|
|
|
|
2023-06-05 00:15:07 +02:00
|
|
|
ir, err := b.toInternalRepresentation(parsed, environ, workflowMetadata, workflow.ID)
|
2022-10-06 01:49:23 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2019-06-01 10:41:51 +02:00
|
|
|
|
2022-05-20 05:20:17 +02:00
|
|
|
if len(ir.Stages) == 0 {
|
2019-07-19 09:18:40 +02:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2022-11-06 13:44:04 +02:00
|
|
|
item := &Item{
|
2023-04-08 13:15:28 +02:00
|
|
|
Workflow: workflow,
|
2019-06-13 17:38:19 +02:00
|
|
|
Config: ir,
|
|
|
|
Labels: parsed.Labels,
|
|
|
|
DependsOn: parsed.DependsOn,
|
2019-06-17 09:06:36 +02:00
|
|
|
RunsOn: parsed.RunsOn,
|
2022-05-31 01:12:18 +02:00
|
|
|
Platform: parsed.Platform,
|
2019-06-01 10:41:51 +02:00
|
|
|
}
|
|
|
|
if item.Labels == nil {
|
|
|
|
item.Labels = map[string]string{}
|
|
|
|
}
|
2019-07-19 09:18:40 +02:00
|
|
|
|
2019-06-01 10:41:51 +02:00
|
|
|
items = append(items, item)
|
2019-07-19 09:17:47 +02:00
|
|
|
pidSequence++
|
2019-06-01 10:17:02 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-07-22 14:13:46 +02:00
|
|
|
items = filterItemsWithMissingDependencies(items)
|
|
|
|
|
2022-10-28 17:38:53 +02:00
|
|
|
// check if at least one step can start, if list is not empty
|
|
|
|
if len(items) > 0 && !stepListContainsItemsToRun(items) {
|
2022-10-18 03:24:12 +02:00
|
|
|
return nil, fmt.Errorf("pipeline has no startpoint")
|
2022-01-05 18:54:44 +02:00
|
|
|
}
|
|
|
|
|
2019-06-01 10:17:02 +02:00
|
|
|
return items, nil
|
|
|
|
}
|
|
|
|
|
2022-11-06 13:44:04 +02:00
|
|
|
func stepListContainsItemsToRun(items []*Item) bool {
|
2022-01-05 18:54:44 +02:00
|
|
|
for i := range items {
|
2023-04-08 13:15:28 +02:00
|
|
|
if items[i].Workflow.State == model.StatusPending {
|
2022-01-05 18:54:44 +02:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2022-11-06 13:44:04 +02:00
|
|
|
func filterItemsWithMissingDependencies(items []*Item) []*Item {
|
|
|
|
itemsToRemove := make([]*Item, 0)
|
2019-07-22 14:13:46 +02:00
|
|
|
|
|
|
|
for _, item := range items {
|
|
|
|
for _, dep := range item.DependsOn {
|
|
|
|
if !containsItemWithName(dep, items) {
|
|
|
|
itemsToRemove = append(itemsToRemove, item)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(itemsToRemove) > 0 {
|
2022-11-06 13:44:04 +02:00
|
|
|
filtered := make([]*Item, 0)
|
2019-07-22 14:13:46 +02:00
|
|
|
for _, item := range items {
|
2023-04-08 13:15:28 +02:00
|
|
|
if !containsItemWithName(item.Workflow.Name, itemsToRemove) {
|
2019-07-22 14:13:46 +02:00
|
|
|
filtered = append(filtered, item)
|
|
|
|
}
|
|
|
|
}
|
2019-07-22 14:29:15 +02:00
|
|
|
// Recursive to handle transitive deps
|
|
|
|
return filterItemsWithMissingDependencies(filtered)
|
2019-07-22 14:13:46 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
return items
|
|
|
|
}
|
|
|
|
|
2022-11-06 13:44:04 +02:00
|
|
|
func containsItemWithName(name string, items []*Item) bool {
|
2019-07-22 14:13:46 +02:00
|
|
|
for _, item := range items {
|
2023-04-08 13:15:28 +02:00
|
|
|
if name == item.Workflow.Name {
|
2019-07-22 14:13:46 +02:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2023-06-05 00:15:07 +02:00
|
|
|
func (b *StepBuilder) environmentVariables(metadata metadata.Metadata, axis matrix.Axis) map[string]string {
|
2019-06-05 15:58:27 +02:00
|
|
|
environ := metadata.Environ()
|
|
|
|
for k, v := range axis {
|
|
|
|
environ[k] = v
|
|
|
|
}
|
|
|
|
return environ
|
|
|
|
}
|
|
|
|
|
2023-06-06 09:14:21 +02:00
|
|
|
func (b *StepBuilder) toInternalRepresentation(parsed *yaml_types.Workflow, environ map[string]string, metadata metadata.Metadata, stepID int64) (*backend_types.Config, error) {
|
2019-06-05 15:58:27 +02:00
|
|
|
var secrets []compiler.Secret
|
|
|
|
for _, sec := range b.Secs {
|
|
|
|
if !sec.Match(b.Curr.Event) {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
secrets = append(secrets, compiler.Secret{
|
2022-10-27 04:21:07 +02:00
|
|
|
Name: sec.Name,
|
|
|
|
Value: sec.Value,
|
|
|
|
Match: sec.Images,
|
|
|
|
PluginOnly: sec.PluginsOnly,
|
2019-06-05 15:58:27 +02:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
var registries []compiler.Registry
|
|
|
|
for _, reg := range b.Regs {
|
|
|
|
registries = append(registries, compiler.Registry{
|
|
|
|
Hostname: reg.Address,
|
|
|
|
Username: reg.Username,
|
|
|
|
Password: reg.Password,
|
|
|
|
Email: reg.Email,
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
return compiler.New(
|
|
|
|
compiler.WithEnviron(environ),
|
|
|
|
compiler.WithEnviron(b.Envs),
|
2023-06-05 00:15:07 +02:00
|
|
|
// TODO: server deps should be moved into StepBuilder fields and set on StepBuilder creation
|
2021-09-22 20:48:01 +02:00
|
|
|
compiler.WithEscalated(server.Config.Pipeline.Privileged...),
|
|
|
|
compiler.WithResourceLimit(server.Config.Pipeline.Limits.MemSwapLimit, server.Config.Pipeline.Limits.MemLimit, server.Config.Pipeline.Limits.ShmSize, server.Config.Pipeline.Limits.CPUQuota, server.Config.Pipeline.Limits.CPUShares, server.Config.Pipeline.Limits.CPUSet),
|
|
|
|
compiler.WithVolumes(server.Config.Pipeline.Volumes...),
|
|
|
|
compiler.WithNetworks(server.Config.Pipeline.Networks...),
|
2019-06-05 15:58:27 +02:00
|
|
|
compiler.WithLocal(false),
|
|
|
|
compiler.WithOption(
|
|
|
|
compiler.WithNetrc(
|
|
|
|
b.Netrc.Login,
|
|
|
|
b.Netrc.Password,
|
|
|
|
b.Netrc.Machine,
|
|
|
|
),
|
2022-02-08 18:55:08 +02:00
|
|
|
b.Repo.IsSCMPrivate || server.Config.Pipeline.AuthenticatePublicRepos,
|
2019-06-05 15:58:27 +02:00
|
|
|
),
|
2022-02-10 18:05:19 +02:00
|
|
|
compiler.WithDefaultCloneImage(server.Config.Pipeline.DefaultCloneImage),
|
2019-06-05 15:58:27 +02:00
|
|
|
compiler.WithRegistry(registries...),
|
|
|
|
compiler.WithSecret(secrets...),
|
|
|
|
compiler.WithPrefix(
|
|
|
|
fmt.Sprintf(
|
2023-03-21 01:48:15 +02:00
|
|
|
"wp_%s_%d",
|
2023-03-21 21:00:45 +02:00
|
|
|
strings.ToLower(ulid.Make().String()),
|
2022-10-28 17:38:53 +02:00
|
|
|
stepID,
|
2019-06-05 15:58:27 +02:00
|
|
|
),
|
|
|
|
),
|
|
|
|
compiler.WithProxy(),
|
2021-10-28 21:02:43 +02:00
|
|
|
compiler.WithWorkspaceFromURL("/woodpecker", b.Repo.Link),
|
2019-06-05 15:58:27 +02:00
|
|
|
compiler.WithMetadata(metadata),
|
2023-03-20 22:17:49 +02:00
|
|
|
compiler.WithTrusted(b.Repo.IsTrusted),
|
|
|
|
compiler.WithNetrcOnlyTrusted(b.Repo.NetrcOnlyTrusted),
|
2019-06-05 15:58:27 +02:00
|
|
|
).Compile(parsed)
|
|
|
|
}
|
|
|
|
|
2023-06-06 09:52:08 +02:00
|
|
|
// SetPipelineStepsOnPipeline is the link between pipeline representation in "pipeline package" and server
|
|
|
|
// to be specific this func currently is used to convert the pipeline.Item list (crafted by StepBuilder.Build()) into
|
|
|
|
// a pipeline that can be stored in the database by the server
|
2022-11-06 13:44:04 +02:00
|
|
|
func SetPipelineStepsOnPipeline(pipeline *model.Pipeline, pipelineItems []*Item) *model.Pipeline {
|
2019-07-23 14:10:44 +02:00
|
|
|
var pidSequence int
|
2022-10-18 03:24:12 +02:00
|
|
|
for _, item := range pipelineItems {
|
2023-04-08 13:15:28 +02:00
|
|
|
if pidSequence < item.Workflow.PID {
|
|
|
|
pidSequence = item.Workflow.PID
|
2019-07-23 14:10:44 +02:00
|
|
|
}
|
2019-07-22 13:43:18 +02:00
|
|
|
}
|
|
|
|
|
2022-10-18 03:24:12 +02:00
|
|
|
for _, item := range pipelineItems {
|
2019-06-05 10:36:16 +02:00
|
|
|
for _, stage := range item.Config.Stages {
|
|
|
|
var gid int
|
|
|
|
for _, step := range stage.Steps {
|
2019-07-23 14:10:44 +02:00
|
|
|
pidSequence++
|
2019-06-05 10:36:16 +02:00
|
|
|
if gid == 0 {
|
2019-07-23 14:10:44 +02:00
|
|
|
gid = pidSequence
|
2019-06-05 10:36:16 +02:00
|
|
|
}
|
2022-10-28 17:38:53 +02:00
|
|
|
step := &model.Step{
|
2022-10-18 03:24:12 +02:00
|
|
|
Name: step.Alias,
|
2023-06-06 09:52:08 +02:00
|
|
|
UUID: step.UUID,
|
|
|
|
PipelineID: pipeline.ID,
|
2022-10-18 03:24:12 +02:00
|
|
|
PID: pidSequence,
|
2023-04-08 13:15:28 +02:00
|
|
|
PPID: item.Workflow.PID,
|
2022-10-18 03:24:12 +02:00
|
|
|
State: model.StatusPending,
|
2023-06-29 14:56:01 +02:00
|
|
|
Failure: step.Failure,
|
2019-06-05 10:36:16 +02:00
|
|
|
}
|
2023-04-08 13:15:28 +02:00
|
|
|
if item.Workflow.State == model.StatusSkipped {
|
2022-10-28 17:38:53 +02:00
|
|
|
step.State = model.StatusSkipped
|
2019-06-19 09:36:54 +02:00
|
|
|
}
|
2023-06-27 18:01:18 +02:00
|
|
|
item.Workflow.Children = append(item.Workflow.Children, step)
|
2019-06-05 10:36:16 +02:00
|
|
|
}
|
|
|
|
}
|
2023-06-27 18:01:18 +02:00
|
|
|
pipeline.Workflows = append(pipeline.Workflows, item.Workflow)
|
2019-06-05 10:36:16 +02:00
|
|
|
}
|
2019-07-22 13:43:18 +02:00
|
|
|
|
2022-10-18 03:24:12 +02:00
|
|
|
return pipeline
|
2019-06-05 10:36:16 +02:00
|
|
|
}
|
|
|
|
|
2021-09-22 20:48:01 +02:00
|
|
|
func SanitizePath(path string) string {
|
2021-09-21 04:21:13 +02:00
|
|
|
path = filepath.Base(path)
|
2019-06-13 17:38:19 +02:00
|
|
|
path = strings.TrimSuffix(path, ".yml")
|
2022-11-03 20:12:40 +02:00
|
|
|
path = strings.TrimSuffix(path, ".yaml")
|
2019-06-13 17:38:19 +02:00
|
|
|
path = strings.TrimPrefix(path, ".")
|
|
|
|
return path
|
|
|
|
}
|