You've already forked woodpecker
mirror of
https://github.com/woodpecker-ci/woodpecker.git
synced 2025-11-29 21:48:14 +02:00
Add imports checks to linter (#3056)
supersedes https://github.com/woodpecker-ci/woodpecker/pull/874 --------- Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
qwerty287
GitHub
306
server/pipeline/stepbuilder/stepBuilder.go
Normal file
306
server/pipeline/stepbuilder/stepBuilder.go
Normal file
@@ -0,0 +1,306 @@
|
||||
// Copyright 2022 Woodpecker Authors
|
||||
// Copyright 2018 Drone.IO Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package stepbuilder
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/oklog/ulid/v2"
|
||||
"github.com/rs/zerolog/log"
|
||||
"go.uber.org/multierr"
|
||||
|
||||
backend_types "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
|
||||
"go.woodpecker-ci.org/woodpecker/v2/pipeline/errors"
|
||||
pipeline_errors "go.woodpecker-ci.org/woodpecker/v2/pipeline/errors"
|
||||
yaml_types "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/types"
|
||||
forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types"
|
||||
|
||||
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
|
||||
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml"
|
||||
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/compiler"
|
||||
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/linter"
|
||||
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/matrix"
|
||||
"go.woodpecker-ci.org/woodpecker/v2/server"
|
||||
"go.woodpecker-ci.org/woodpecker/v2/server/model"
|
||||
)
|
||||
|
||||
// StepBuilder Takes the hook data and the yaml and returns in internal data model
|
||||
type StepBuilder struct {
|
||||
Repo *model.Repo
|
||||
Curr *model.Pipeline
|
||||
Last *model.Pipeline
|
||||
Netrc *model.Netrc
|
||||
Secs []*model.Secret
|
||||
Regs []*model.Registry
|
||||
Host string
|
||||
Yamls []*forge_types.FileMeta
|
||||
Envs map[string]string
|
||||
Forge metadata.ServerForge
|
||||
ProxyOpts compiler.ProxyOptions
|
||||
}
|
||||
|
||||
type Item struct {
|
||||
Workflow *model.Workflow
|
||||
Labels map[string]string
|
||||
DependsOn []string
|
||||
RunsOn []string
|
||||
Config *backend_types.Config
|
||||
}
|
||||
|
||||
func (b *StepBuilder) Build() (items []*Item, errorsAndWarnings error) {
|
||||
b.Yamls = forge_types.SortByName(b.Yamls)
|
||||
|
||||
pidSequence := 1
|
||||
|
||||
for _, y := range b.Yamls {
|
||||
// matrix axes
|
||||
axes, err := matrix.ParseString(string(y.Data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(axes) == 0 {
|
||||
axes = append(axes, matrix.Axis{})
|
||||
}
|
||||
|
||||
for i, axis := range axes {
|
||||
workflow := &model.Workflow{
|
||||
PID: pidSequence,
|
||||
State: model.StatusPending,
|
||||
Environ: axis,
|
||||
Name: SanitizePath(y.Name),
|
||||
}
|
||||
if len(axes) > 1 {
|
||||
workflow.AxisID = i + 1
|
||||
}
|
||||
item, err := b.genItemForWorkflow(workflow, axis, string(y.Data))
|
||||
if err != nil && pipeline_errors.HasBlockingErrors(err) {
|
||||
return nil, err
|
||||
} else if err != nil {
|
||||
errorsAndWarnings = multierr.Append(errorsAndWarnings, err)
|
||||
}
|
||||
|
||||
if item == nil {
|
||||
continue
|
||||
}
|
||||
items = append(items, item)
|
||||
pidSequence++
|
||||
}
|
||||
|
||||
// TODO: add summary workflow that send status back based on workflows generated by matrix function
|
||||
// depend on https://github.com/woodpecker-ci/woodpecker/issues/778
|
||||
}
|
||||
|
||||
items = filterItemsWithMissingDependencies(items)
|
||||
|
||||
// check if at least one step can start if slice is not empty
|
||||
if len(items) > 0 && !stepListContainsItemsToRun(items) {
|
||||
return nil, fmt.Errorf("pipeline has no steps to run")
|
||||
}
|
||||
|
||||
return items, errorsAndWarnings
|
||||
}
|
||||
|
||||
func (b *StepBuilder) genItemForWorkflow(workflow *model.Workflow, axis matrix.Axis, data string) (item *Item, errorsAndWarnings error) {
|
||||
workflowMetadata := MetadataFromStruct(b.Forge, b.Repo, b.Curr, b.Last, workflow, b.Host)
|
||||
environ := b.environmentVariables(workflowMetadata, axis)
|
||||
|
||||
// add global environment variables for substituting
|
||||
for k, v := range b.Envs {
|
||||
if _, exists := environ[k]; exists {
|
||||
// don't override existing values
|
||||
continue
|
||||
}
|
||||
environ[k] = v
|
||||
}
|
||||
|
||||
// substitute vars
|
||||
substituted, err := metadata.EnvVarSubst(data, environ)
|
||||
if err != nil {
|
||||
return nil, multierr.Append(errorsAndWarnings, err)
|
||||
}
|
||||
|
||||
// parse yaml pipeline
|
||||
parsed, err := yaml.ParseString(substituted)
|
||||
if err != nil {
|
||||
return nil, &errors.PipelineError{Message: err.Error(), Type: errors.PipelineErrorTypeCompiler}
|
||||
}
|
||||
|
||||
// lint pipeline
|
||||
errorsAndWarnings = multierr.Append(errorsAndWarnings, linter.New(
|
||||
linter.WithTrusted(b.Repo.IsTrusted),
|
||||
).Lint([]*linter.WorkflowConfig{{
|
||||
Workflow: parsed,
|
||||
File: workflow.Name,
|
||||
RawConfig: data,
|
||||
}}))
|
||||
if pipeline_errors.HasBlockingErrors(errorsAndWarnings) {
|
||||
return nil, errorsAndWarnings
|
||||
}
|
||||
|
||||
// checking if filtered.
|
||||
if match, err := parsed.When.Match(workflowMetadata, true, environ); !match && err == nil {
|
||||
log.Debug().Str("pipeline", workflow.Name).Msg(
|
||||
"Marked as skipped, does not match metadata",
|
||||
)
|
||||
return nil, nil
|
||||
} else if err != nil {
|
||||
log.Debug().Str("pipeline", workflow.Name).Msg(
|
||||
"Pipeline config could not be parsed",
|
||||
)
|
||||
return nil, multierr.Append(errorsAndWarnings, err)
|
||||
}
|
||||
|
||||
ir, err := b.toInternalRepresentation(parsed, environ, workflowMetadata, workflow.ID)
|
||||
if err != nil {
|
||||
return nil, multierr.Append(errorsAndWarnings, err)
|
||||
}
|
||||
|
||||
if len(ir.Stages) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
item = &Item{
|
||||
Workflow: workflow,
|
||||
Config: ir,
|
||||
Labels: parsed.Labels,
|
||||
DependsOn: parsed.DependsOn,
|
||||
RunsOn: parsed.RunsOn,
|
||||
}
|
||||
if item.Labels == nil {
|
||||
item.Labels = map[string]string{}
|
||||
}
|
||||
|
||||
return item, errorsAndWarnings
|
||||
}
|
||||
|
||||
func stepListContainsItemsToRun(items []*Item) bool {
|
||||
for i := range items {
|
||||
if items[i].Workflow.State == model.StatusPending {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func filterItemsWithMissingDependencies(items []*Item) []*Item {
|
||||
itemsToRemove := make([]*Item, 0)
|
||||
|
||||
for _, item := range items {
|
||||
for _, dep := range item.DependsOn {
|
||||
if !containsItemWithName(dep, items) {
|
||||
itemsToRemove = append(itemsToRemove, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(itemsToRemove) > 0 {
|
||||
filtered := make([]*Item, 0)
|
||||
for _, item := range items {
|
||||
if !containsItemWithName(item.Workflow.Name, itemsToRemove) {
|
||||
filtered = append(filtered, item)
|
||||
}
|
||||
}
|
||||
// Recursive to handle transitive deps
|
||||
return filterItemsWithMissingDependencies(filtered)
|
||||
}
|
||||
|
||||
return items
|
||||
}
|
||||
|
||||
func containsItemWithName(name string, items []*Item) bool {
|
||||
for _, item := range items {
|
||||
if name == item.Workflow.Name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (b *StepBuilder) environmentVariables(metadata metadata.Metadata, axis matrix.Axis) map[string]string {
|
||||
environ := metadata.Environ()
|
||||
for k, v := range axis {
|
||||
environ[k] = v
|
||||
}
|
||||
return environ
|
||||
}
|
||||
|
||||
func (b *StepBuilder) toInternalRepresentation(parsed *yaml_types.Workflow, environ map[string]string, metadata metadata.Metadata, stepID int64) (*backend_types.Config, error) {
|
||||
var secrets []compiler.Secret
|
||||
for _, sec := range b.Secs {
|
||||
if !sec.Match(b.Curr.Event) {
|
||||
continue
|
||||
}
|
||||
secrets = append(secrets, compiler.Secret{
|
||||
Name: sec.Name,
|
||||
Value: sec.Value,
|
||||
AllowedPlugins: sec.Images,
|
||||
})
|
||||
}
|
||||
|
||||
var registries []compiler.Registry
|
||||
for _, reg := range b.Regs {
|
||||
registries = append(registries, compiler.Registry{
|
||||
Hostname: reg.Address,
|
||||
Username: reg.Username,
|
||||
Password: reg.Password,
|
||||
Email: reg.Email,
|
||||
})
|
||||
}
|
||||
|
||||
return compiler.New(
|
||||
compiler.WithEnviron(environ),
|
||||
compiler.WithEnviron(b.Envs),
|
||||
// TODO: server deps should be moved into StepBuilder fields and set on StepBuilder creation
|
||||
compiler.WithEscalated(server.Config.Pipeline.Privileged...),
|
||||
compiler.WithResourceLimit(server.Config.Pipeline.Limits.MemSwapLimit, server.Config.Pipeline.Limits.MemLimit, server.Config.Pipeline.Limits.ShmSize, server.Config.Pipeline.Limits.CPUQuota, server.Config.Pipeline.Limits.CPUShares, server.Config.Pipeline.Limits.CPUSet),
|
||||
compiler.WithVolumes(server.Config.Pipeline.Volumes...),
|
||||
compiler.WithNetworks(server.Config.Pipeline.Networks...),
|
||||
compiler.WithLocal(false),
|
||||
compiler.WithOption(
|
||||
compiler.WithNetrc(
|
||||
b.Netrc.Login,
|
||||
b.Netrc.Password,
|
||||
b.Netrc.Machine,
|
||||
),
|
||||
b.Repo.IsSCMPrivate || server.Config.Pipeline.AuthenticatePublicRepos,
|
||||
),
|
||||
compiler.WithDefaultCloneImage(server.Config.Pipeline.DefaultCloneImage),
|
||||
compiler.WithRegistry(registries...),
|
||||
compiler.WithSecret(secrets...),
|
||||
compiler.WithPrefix(
|
||||
fmt.Sprintf(
|
||||
"wp_%s_%d",
|
||||
strings.ToLower(ulid.Make().String()),
|
||||
stepID,
|
||||
),
|
||||
),
|
||||
compiler.WithProxy(b.ProxyOpts),
|
||||
compiler.WithWorkspaceFromURL("/woodpecker", b.Repo.ForgeURL),
|
||||
compiler.WithMetadata(metadata),
|
||||
compiler.WithTrusted(b.Repo.IsTrusted),
|
||||
compiler.WithNetrcOnlyTrusted(b.Repo.NetrcOnlyTrusted),
|
||||
).Compile(parsed)
|
||||
}
|
||||
|
||||
func SanitizePath(path string) string {
|
||||
path = filepath.Base(path)
|
||||
path = strings.TrimSuffix(path, ".yml")
|
||||
path = strings.TrimSuffix(path, ".yaml")
|
||||
path = strings.TrimPrefix(path, ".")
|
||||
return path
|
||||
}
|
||||
Reference in New Issue
Block a user