From ea895baf836fc7d9c46e02fd883f432f2d0261e9 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 5 Jun 2023 00:15:07 +0200 Subject: [PATCH] Resolve built-in variables for global when filter (#1790) addresses https://codeberg.org/Epsilon_02/todo-checker/commit/bd461477bd074ad4a572cdda03c0c3ad119235ec close #1244, close #1580 --------- Co-authored-by: Anbraten --- cli/exec/exec.go | 76 ---- cli/exec/flags.go | 21 +- cli/exec/metadata.go | 117 ++++++ docs/docs/20-usage/50-environment.md | 3 +- pipeline/frontend/metadata.go | 383 ++++++------------ pipeline/frontend/metadata/const.go | 32 ++ .../metadata}/drone_compatibility.go | 2 +- .../metadata/drone_compatibility_test.go | 132 ++++++ pipeline/frontend/metadata/environment.go | 161 ++++++++ pipeline/frontend/metadata/types.go | 122 ++++++ pipeline/frontend/metadata_test.go | 132 ++++++ pipeline/frontend/yaml/compiler/compiler.go | 13 +- pipeline/frontend/yaml/compiler/convert.go | 4 +- pipeline/frontend/yaml/compiler/option.go | 4 +- .../frontend/yaml/compiler/option_test.go | 12 +- pipeline/frontend/yaml/config.go | 17 +- pipeline/frontend/yaml/config_test.go | 20 +- .../frontend/yaml/constraint/constraint.go | 45 +- .../yaml/constraint/constraint_test.go | 56 ++- pipeline/pipeline.go | 6 +- pipeline/stepBuilder.go | 118 +----- pipeline/stepBuilder_test.go | 63 ++- server/pipeline/create.go | 2 +- server/pipeline/filter.go | 29 +- server/pipeline/items.go | 1 + 25 files changed, 990 insertions(+), 581 deletions(-) create mode 100644 cli/exec/metadata.go create mode 100644 pipeline/frontend/metadata/const.go rename pipeline/{ => frontend/metadata}/drone_compatibility.go (99%) create mode 100644 pipeline/frontend/metadata/drone_compatibility_test.go create mode 100644 pipeline/frontend/metadata/environment.go create mode 100644 pipeline/frontend/metadata/types.go create mode 100644 pipeline/frontend/metadata_test.go diff --git a/cli/exec/exec.go b/cli/exec/exec.go index 26287d1d8..9fa2531e1 100644 --- a/cli/exec/exec.go +++ b/cli/exec/exec.go @@ -32,7 +32,6 @@ import ( "github.com/woodpecker-ci/woodpecker/pipeline/backend" "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" backendTypes "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" - "github.com/woodpecker-ci/woodpecker/pipeline/frontend" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/linter" @@ -232,81 +231,6 @@ func execWithAxis(c *cli.Context, file, repoPath string, axis matrix.Axis) error ).Run(c.Context) } -// return the metadata from the cli context. -func metadataFromContext(c *cli.Context, axis matrix.Axis) frontend.Metadata { - platform := c.String("system-platform") - if platform == "" { - platform = runtime.GOOS + "/" + runtime.GOARCH - } - - return frontend.Metadata{ - Repo: frontend.Repo{ - Name: c.String("repo-name"), - Link: c.String("repo-link"), - CloneURL: c.String("repo-clone-url"), - Private: c.Bool("repo-private"), - }, - Curr: frontend.Pipeline{ - Number: c.Int64("pipeline-number"), - Parent: c.Int64("pipeline-parent"), - Created: c.Int64("pipeline-created"), - Started: c.Int64("pipeline-started"), - Finished: c.Int64("pipeline-finished"), - Status: c.String("pipeline-status"), - Event: c.String("pipeline-event"), - Link: c.String("pipeline-link"), - Target: c.String("pipeline-target"), - Commit: frontend.Commit{ - Sha: c.String("commit-sha"), - Ref: c.String("commit-ref"), - Refspec: c.String("commit-refspec"), - Branch: c.String("commit-branch"), - Message: c.String("commit-message"), - Author: frontend.Author{ - Name: c.String("commit-author-name"), - Email: c.String("commit-author-email"), - Avatar: c.String("commit-author-avatar"), - }, - }, - }, - Prev: frontend.Pipeline{ - Number: c.Int64("prev-pipeline-number"), - Created: c.Int64("prev-pipeline-created"), - Started: c.Int64("prev-pipeline-started"), - Finished: c.Int64("prev-pipeline-finished"), - Status: c.String("prev-pipeline-status"), - Event: c.String("prev-pipeline-event"), - Link: c.String("prev-pipeline-link"), - Commit: frontend.Commit{ - Sha: c.String("prev-commit-sha"), - Ref: c.String("prev-commit-ref"), - Refspec: c.String("prev-commit-refspec"), - Branch: c.String("prev-commit-branch"), - Message: c.String("prev-commit-message"), - Author: frontend.Author{ - Name: c.String("prev-commit-author-name"), - Email: c.String("prev-commit-author-email"), - Avatar: c.String("prev-commit-author-avatar"), - }, - }, - }, - Workflow: frontend.Workflow{ - Name: c.String("workflow-name"), - Number: c.Int("workflow-number"), - Matrix: axis, - }, - Step: frontend.Step{ - Name: c.String("step-name"), - Number: c.Int("step-number"), - }, - Sys: frontend.System{ - Name: c.String("system-name"), - Link: c.String("system-link"), - Platform: platform, - }, - } -} - func convertPathForWindows(path string) string { base := filepath.VolumeName(path) if len(base) == 2 { diff --git a/cli/exec/flags.go b/cli/exec/flags.go index 12fca119d..0e4fd3792 100644 --- a/cli/exec/flags.go +++ b/cli/exec/flags.go @@ -115,8 +115,13 @@ var flags = []cli.Flag{ Value: "https://github.com/woodpecker-ci/woodpecker", }, &cli.StringFlag{ - EnvVars: []string{"CI_REPO_NAME"}, - Name: "repo-name", + EnvVars: []string{"CI_REPO"}, + Name: "repo", + Usage: "full repo name", + }, + &cli.StringFlag{ + EnvVars: []string{"CI_REPO_REMOTE_ID"}, + Name: "repo-remote-id", }, &cli.StringFlag{ EnvVars: []string{"CI_REPO_URL", "CI_REPO_LINK"}, @@ -130,6 +135,10 @@ var flags = []cli.Flag{ EnvVars: []string{"CI_REPO_PRIVATE"}, Name: "repo-private", }, + &cli.BoolFlag{ + EnvVars: []string{"CI_REPO_TRUSTED"}, + Name: "repo-trusted", + }, &cli.IntFlag{ EnvVars: []string{"CI_PIPELINE_NUMBER"}, Name: "pipeline-number", @@ -275,6 +284,14 @@ var flags = []cli.Flag{ EnvVars: []string{"CI_ENV"}, Name: "env", }, + &cli.StringFlag{ + EnvVars: []string{"CI_FORGE_TYPE"}, + Name: "forge-type", + }, + &cli.StringFlag{ + EnvVars: []string{"CI_FORGE_URL"}, + Name: "forge-url", + }, // backend docker &cli.BoolFlag{ diff --git a/cli/exec/metadata.go b/cli/exec/metadata.go new file mode 100644 index 000000000..c1ef6e487 --- /dev/null +++ b/cli/exec/metadata.go @@ -0,0 +1,117 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package exec + +import ( + "runtime" + "strings" + + "github.com/urfave/cli/v2" + + "github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata" + "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/matrix" + "github.com/woodpecker-ci/woodpecker/version" +) + +// return the metadata from the cli context. +func metadataFromContext(c *cli.Context, axis matrix.Axis) metadata.Metadata { + platform := c.String("system-platform") + if platform == "" { + platform = runtime.GOOS + "/" + runtime.GOARCH + } + + fullRepoName := c.String("repo-name") + repoOwner := "" + repoName := "" + if idx := strings.LastIndex(fullRepoName, "/"); idx != -1 { + repoOwner = fullRepoName[:idx] + repoName = fullRepoName[idx+1:] + } + + return metadata.Metadata{ + Repo: metadata.Repo{ + Name: repoName, + Owner: repoOwner, + RemoteID: c.String("repo-remote-id"), + Link: c.String("repo-link"), + CloneURL: c.String("repo-clone-url"), + Private: c.Bool("repo-private"), + Trusted: c.Bool("repo-trusted"), + }, + Curr: metadata.Pipeline{ + Number: c.Int64("pipeline-number"), + Parent: c.Int64("pipeline-parent"), + Created: c.Int64("pipeline-created"), + Started: c.Int64("pipeline-started"), + Finished: c.Int64("pipeline-finished"), + Status: c.String("pipeline-status"), + Event: c.String("pipeline-event"), + Link: c.String("pipeline-link"), + Target: c.String("pipeline-target"), + Commit: metadata.Commit{ + Sha: c.String("commit-sha"), + Ref: c.String("commit-ref"), + Refspec: c.String("commit-refspec"), + Branch: c.String("commit-branch"), + Message: c.String("commit-message"), + Author: metadata.Author{ + Name: c.String("commit-author-name"), + Email: c.String("commit-author-email"), + Avatar: c.String("commit-author-avatar"), + }, + }, + }, + Prev: metadata.Pipeline{ + Number: c.Int64("prev-pipeline-number"), + Created: c.Int64("prev-pipeline-created"), + Started: c.Int64("prev-pipeline-started"), + Finished: c.Int64("prev-pipeline-finished"), + Status: c.String("prev-pipeline-status"), + Event: c.String("prev-pipeline-event"), + Link: c.String("prev-pipeline-link"), + Commit: metadata.Commit{ + Sha: c.String("prev-commit-sha"), + Ref: c.String("prev-commit-ref"), + Refspec: c.String("prev-commit-refspec"), + Branch: c.String("prev-commit-branch"), + Message: c.String("prev-commit-message"), + Author: metadata.Author{ + Name: c.String("prev-commit-author-name"), + Email: c.String("prev-commit-author-email"), + Avatar: c.String("prev-commit-author-avatar"), + }, + }, + }, + Workflow: metadata.Workflow{ + Name: c.String("workflow-name"), + Number: c.Int("workflow-number"), + Matrix: axis, + }, + Step: metadata.Step{ + Name: c.String("step-name"), + Number: c.Int("step-number"), + }, + Sys: metadata.System{ + Name: c.String("system-name"), + Link: c.String("system-link"), + Platform: platform, + Version: version.Version, + }, + Forge: metadata.Forge{ + Type: c.String("forge-type"), + URL: c.String("forge-url"), + }, + } +} diff --git a/docs/docs/20-usage/50-environment.md b/docs/docs/20-usage/50-environment.md index bb7160dec..5770bee68 100644 --- a/docs/docs/20-usage/50-environment.md +++ b/docs/docs/20-usage/50-environment.md @@ -53,8 +53,9 @@ This is the reference list of all environment variables available to your pipeli | `CI_REPO` | repository full name `/` | | `CI_REPO_OWNER` | repository owner | | `CI_REPO_NAME` | repository name | +| `CI_REPO_REMOTE_ID` | repository remote ID, is the UID it has in the forge | | `CI_REPO_SCM` | repository SCM (git) | -| `CI_REPO_URL` | repository web URL | +| `CI_REPO_URL` | repository web URL | | `CI_REPO_CLONE_URL` | repository clone URL | | `CI_REPO_DEFAULT_BRANCH` | repository default branch (master) | | `CI_REPO_PRIVATE` | repository is private | diff --git a/pipeline/frontend/metadata.go b/pipeline/frontend/metadata.go index 1a77227cb..412005ff2 100644 --- a/pipeline/frontend/metadata.go +++ b/pipeline/frontend/metadata.go @@ -1,4 +1,4 @@ -// Copyright 2022 Woodpecker Authors +// Copyright 2023 Woodpecker Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,274 +15,131 @@ package frontend import ( - "regexp" - "strconv" + "fmt" + "net/url" "strings" + "github.com/drone/envsubst" + + "github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata" + "github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/version" ) -// Event types corresponding to scm hooks. -const ( - EventPush = "push" - EventPull = "pull_request" - EventTag = "tag" - EventDeploy = "deployment" - EventCron = "cron" - EventManual = "manual" -) - -// Different ways to handle failure states -const ( - FailureIgnore = "ignore" - FailureFail = "fail" - // FailureCancel = "cancel" // Not implemented yet -) - -type ( - // Metadata defines runtime m. - Metadata struct { - ID string `json:"id,omitempty"` - Repo Repo `json:"repo,omitempty"` - Curr Pipeline `json:"curr,omitempty"` - Prev Pipeline `json:"prev,omitempty"` - Workflow Workflow `json:"workflow,omitempty"` - Step Step `json:"step,omitempty"` - Sys System `json:"sys,omitempty"` - Forge Forge `json:"forge,omitempty"` - } - - // Repo defines runtime metadata for a repository. - Repo struct { - Name string `json:"name,omitempty"` - Link string `json:"link,omitempty"` - CloneURL string `json:"clone_url,omitempty"` - Private bool `json:"private,omitempty"` - Secrets []Secret `json:"secrets,omitempty"` - Branch string `json:"default_branch,omitempty"` - } - - // Pipeline defines runtime metadata for a pipeline. - Pipeline struct { - Number int64 `json:"number,omitempty"` - Created int64 `json:"created,omitempty"` - Started int64 `json:"started,omitempty"` - Finished int64 `json:"finished,omitempty"` - Timeout int64 `json:"timeout,omitempty"` - Status string `json:"status,omitempty"` - Event string `json:"event,omitempty"` - Link string `json:"link,omitempty"` - Target string `json:"target,omitempty"` - Trusted bool `json:"trusted,omitempty"` - Commit Commit `json:"commit,omitempty"` - Parent int64 `json:"parent,omitempty"` - Cron string `json:"cron,omitempty"` - } - - // Commit defines runtime metadata for a commit. - Commit struct { - Sha string `json:"sha,omitempty"` - Ref string `json:"ref,omitempty"` - Refspec string `json:"refspec,omitempty"` - Branch string `json:"branch,omitempty"` - Message string `json:"message,omitempty"` - Author Author `json:"author,omitempty"` - ChangedFiles []string `json:"changed_files,omitempty"` - PullRequestLabels []string `json:"labels,omitempty"` - } - - // Author defines runtime metadata for a commit author. - Author struct { - Name string `json:"name,omitempty"` - Email string `json:"email,omitempty"` - Avatar string `json:"avatar,omitempty"` - } - - // Workflow defines runtime metadata for a workflow. - Workflow struct { - Name string `json:"name,omitempty"` - Number int `json:"number,omitempty"` - Matrix map[string]string `json:"matrix,omitempty"` - } - - // Step defines runtime metadata for a step. - Step struct { - Name string `json:"name,omitempty"` - Number int `json:"number,omitempty"` - } - - // Secret defines a runtime secret - Secret struct { - Name string `json:"name,omitempty"` - Value string `json:"value,omitempty"` - Mount string `json:"mount,omitempty"` - Mask bool `json:"mask,omitempty"` - } - - // System defines runtime metadata for a ci/cd system. - System struct { - Name string `json:"name,omitempty"` - Host string `json:"host,omitempty"` - Link string `json:"link,omitempty"` - Platform string `json:"arch,omitempty"` - Version string `json:"version,omitempty"` - } - - // Forge defines runtime metadata about the forge that host the repo - Forge struct { - Type string `json:"type,omitempty"` - URL string `json:"url,omitempty"` - } -) - -// Environ returns the metadata as a map of environment variables. -func (m *Metadata) Environ() map[string]string { - var ( - repoOwner string - repoName string - sourceBranch string - targetBranch string - ) - - repoParts := strings.Split(m.Repo.Name, "/") - if len(repoParts) == 2 { - repoOwner = repoParts[0] - repoName = repoParts[1] - } else { - repoName = m.Repo.Name - } - - branchParts := strings.Split(m.Curr.Commit.Refspec, ":") - if len(branchParts) == 2 { - sourceBranch = branchParts[0] - targetBranch = branchParts[1] - } - - params := map[string]string{ - "CI": m.Sys.Name, - "CI_REPO": m.Repo.Name, - "CI_REPO_OWNER": repoOwner, - "CI_REPO_NAME": repoName, - "CI_REPO_SCM": "git", - "CI_REPO_URL": m.Repo.Link, - "CI_REPO_CLONE_URL": m.Repo.CloneURL, - "CI_REPO_DEFAULT_BRANCH": m.Repo.Branch, - "CI_REPO_PRIVATE": strconv.FormatBool(m.Repo.Private), - "CI_REPO_TRUSTED": "false", // TODO should this be added? - - "CI_COMMIT_SHA": m.Curr.Commit.Sha, - "CI_COMMIT_REF": m.Curr.Commit.Ref, - "CI_COMMIT_REFSPEC": m.Curr.Commit.Refspec, - "CI_COMMIT_BRANCH": m.Curr.Commit.Branch, - "CI_COMMIT_SOURCE_BRANCH": sourceBranch, - "CI_COMMIT_TARGET_BRANCH": targetBranch, - "CI_COMMIT_URL": m.Curr.Link, - "CI_COMMIT_MESSAGE": m.Curr.Commit.Message, - "CI_COMMIT_AUTHOR": m.Curr.Commit.Author.Name, - "CI_COMMIT_AUTHOR_EMAIL": m.Curr.Commit.Author.Email, - "CI_COMMIT_AUTHOR_AVATAR": m.Curr.Commit.Author.Avatar, - "CI_COMMIT_TAG": "", // will be set if event is tag - "CI_COMMIT_PULL_REQUEST": "", // will be set if event is pr - "CI_COMMIT_PULL_REQUEST_LABELS": "", // will be set if event is pr - - "CI_PIPELINE_NUMBER": strconv.FormatInt(m.Curr.Number, 10), - "CI_PIPELINE_PARENT": strconv.FormatInt(m.Curr.Parent, 10), - "CI_PIPELINE_EVENT": m.Curr.Event, - "CI_PIPELINE_URL": m.Curr.Link, - "CI_PIPELINE_DEPLOY_TARGET": m.Curr.Target, - "CI_PIPELINE_STATUS": m.Curr.Status, - "CI_PIPELINE_CREATED": strconv.FormatInt(m.Curr.Created, 10), - "CI_PIPELINE_STARTED": strconv.FormatInt(m.Curr.Started, 10), - "CI_PIPELINE_FINISHED": strconv.FormatInt(m.Curr.Finished, 10), - - "CI_WORKFLOW_NAME": m.Workflow.Name, - "CI_WORKFLOW_NUMBER": strconv.Itoa(m.Workflow.Number), - - "CI_STEP_NAME": m.Step.Name, - "CI_STEP_NUMBER": strconv.Itoa(m.Step.Number), - "CI_STEP_STATUS": "", // will be set by agent - "CI_STEP_STARTED": "", // will be set by agent - "CI_STEP_FINISHED": "", // will be set by agent - - "CI_PREV_COMMIT_SHA": m.Prev.Commit.Sha, - "CI_PREV_COMMIT_REF": m.Prev.Commit.Ref, - "CI_PREV_COMMIT_REFSPEC": m.Prev.Commit.Refspec, - "CI_PREV_COMMIT_BRANCH": m.Prev.Commit.Branch, - "CI_PREV_COMMIT_URL": m.Prev.Link, - "CI_PREV_COMMIT_MESSAGE": m.Prev.Commit.Message, - "CI_PREV_COMMIT_AUTHOR": m.Prev.Commit.Author.Name, - "CI_PREV_COMMIT_AUTHOR_EMAIL": m.Prev.Commit.Author.Email, - "CI_PREV_COMMIT_AUTHOR_AVATAR": m.Prev.Commit.Author.Avatar, - - "CI_PREV_PIPELINE_NUMBER": strconv.FormatInt(m.Prev.Number, 10), - "CI_PREV_PIPELINE_PARENT": strconv.FormatInt(m.Prev.Parent, 10), - "CI_PREV_PIPELINE_EVENT": m.Prev.Event, - "CI_PREV_PIPELINE_URL": m.Prev.Link, - "CI_PREV_PIPELINE_DEPLOY_TARGET": m.Prev.Target, - "CI_PREV_PIPELINE_STATUS": m.Prev.Status, - "CI_PREV_PIPELINE_CREATED": strconv.FormatInt(m.Prev.Created, 10), - "CI_PREV_PIPELINE_STARTED": strconv.FormatInt(m.Prev.Started, 10), - "CI_PREV_PIPELINE_FINISHED": strconv.FormatInt(m.Prev.Finished, 10), - - "CI_SYSTEM_NAME": m.Sys.Name, - "CI_SYSTEM_URL": m.Sys.Link, - "CI_SYSTEM_HOST": m.Sys.Host, - "CI_SYSTEM_PLATFORM": m.Sys.Platform, // will be set by pipeline platform option or by agent - "CI_SYSTEM_VERSION": version.Version, - - "CI_FORGE_TYPE": m.Forge.Type, - "CI_FORGE_URL": m.Forge.URL, - - // DEPRECATED - "CI_SYSTEM_ARCH": m.Sys.Platform, // TODO: remove after v1.0.x version - // use CI_PIPELINE_* - "CI_BUILD_NUMBER": strconv.FormatInt(m.Curr.Number, 10), - "CI_BUILD_PARENT": strconv.FormatInt(m.Curr.Parent, 10), - "CI_BUILD_EVENT": m.Curr.Event, - "CI_BUILD_LINK": m.Curr.Link, - "CI_BUILD_DEPLOY_TARGET": m.Curr.Target, - "CI_BUILD_STATUS": m.Curr.Status, - "CI_BUILD_CREATED": strconv.FormatInt(m.Curr.Created, 10), - "CI_BUILD_STARTED": strconv.FormatInt(m.Curr.Started, 10), - "CI_BUILD_FINISHED": strconv.FormatInt(m.Curr.Finished, 10), - // use CI_PREV_PIPELINE_* - "CI_PREV_BUILD_NUMBER": strconv.FormatInt(m.Prev.Number, 10), - "CI_PREV_BUILD_PARENT": strconv.FormatInt(m.Prev.Parent, 10), - "CI_PREV_BUILD_EVENT": m.Prev.Event, - "CI_PREV_BUILD_LINK": m.Prev.Link, - "CI_PREV_BUILD_DEPLOY_TARGET": m.Prev.Target, - "CI_PREV_BUILD_STATUS": m.Prev.Status, - "CI_PREV_BUILD_CREATED": strconv.FormatInt(m.Prev.Created, 10), - "CI_PREV_BUILD_STARTED": strconv.FormatInt(m.Prev.Started, 10), - "CI_PREV_BUILD_FINISHED": strconv.FormatInt(m.Prev.Finished, 10), - // use CI_STEP_* - "CI_JOB_NUMBER": strconv.Itoa(m.Step.Number), - "CI_JOB_STATUS": "", // will be set by agent - "CI_JOB_STARTED": "", // will be set by agent - "CI_JOB_FINISHED": "", // will be set by agent - // CI_REPO_CLONE_URL - "CI_REPO_REMOTE": m.Repo.CloneURL, - // use *_URL - "CI_REPO_LINK": m.Repo.Link, - "CI_COMMIT_LINK": m.Curr.Link, - "CI_PIPELINE_LINK": m.Curr.Link, - "CI_PREV_COMMIT_LINK": m.Prev.Link, - "CI_PREV_PIPELINE_LINK": m.Prev.Link, - "CI_SYSTEM_LINK": m.Sys.Link, - } - if m.Curr.Event == EventTag { - params["CI_COMMIT_TAG"] = strings.TrimPrefix(m.Curr.Commit.Ref, "refs/tags/") - } - if m.Curr.Event == EventPull { - params["CI_COMMIT_PULL_REQUEST"] = pullRegexp.FindString(m.Curr.Commit.Ref) - params["CI_COMMIT_PULL_REQUEST_LABELS"] = strings.Join(m.Curr.Commit.PullRequestLabels, ",") - } - - return params +func EnvVarSubst(yaml string, environ map[string]string) (string, error) { + return envsubst.Eval(yaml, func(name string) string { + env := environ[name] + if strings.Contains(env, "\n") { + env = fmt.Sprintf("%q", env) + } + return env + }) } -var pullRegexp = regexp.MustCompile(`\d+`) +// MetadataFromStruct return the metadata from a pipeline will run with. +func MetadataFromStruct(forge metadata.ServerForge, repo *model.Repo, pipeline, last *model.Pipeline, workflow *model.Step, link string) metadata.Metadata { + host := link + uri, err := url.Parse(link) + if err == nil { + host = uri.Host + } -func (m *Metadata) SetPlatform(platform string) { - m.Sys.Platform = platform + fForge := metadata.Forge{} + if forge != nil { + fForge = metadata.Forge{ + Type: forge.Name(), + URL: forge.URL(), + } + } + + fRepo := metadata.Repo{} + if repo != nil { + fRepo = metadata.Repo{ + Name: repo.Name, + Owner: repo.Owner, + RemoteID: fmt.Sprint(repo.ForgeRemoteID), + Link: repo.Link, + CloneURL: repo.Clone, + Private: repo.IsSCMPrivate, + Branch: repo.Branch, + Trusted: repo.IsTrusted, + } + + if idx := strings.LastIndex(repo.FullName, "/"); idx != -1 { + if fRepo.Name == "" && repo.FullName != "" { + fRepo.Name = repo.FullName[idx+1:] + } + if fRepo.Owner == "" && repo.FullName != "" { + fRepo.Owner = repo.FullName[:idx] + } + } + } + + fWorkflow := metadata.Workflow{} + if workflow != nil { + fWorkflow = metadata.Workflow{ + Name: workflow.Name, + Number: workflow.PID, + Matrix: workflow.Environ, + } + } + + return metadata.Metadata{ + Repo: fRepo, + Curr: metadataPipelineFromModelPipeline(pipeline, true), + Prev: metadataPipelineFromModelPipeline(last, false), + Workflow: fWorkflow, + Step: metadata.Step{}, + Sys: metadata.System{ + Name: "woodpecker", + Link: link, + Host: host, + Platform: "", // will be set by pipeline platform option or by agent + Version: version.Version, + }, + Forge: fForge, + } +} + +func metadataPipelineFromModelPipeline(pipeline *model.Pipeline, includeParent bool) metadata.Pipeline { + if pipeline == nil { + return metadata.Pipeline{} + } + + cron := "" + if pipeline.Event == model.EventCron { + cron = pipeline.Sender + } + + parent := int64(0) + if includeParent { + parent = pipeline.Parent + } + + return metadata.Pipeline{ + Number: pipeline.Number, + Parent: parent, + Created: pipeline.Created, + Started: pipeline.Started, + Finished: pipeline.Finished, + Status: string(pipeline.Status), + Event: string(pipeline.Event), + Link: pipeline.Link, + Target: pipeline.Deploy, + Commit: metadata.Commit{ + Sha: pipeline.Commit, + Ref: pipeline.Ref, + Refspec: pipeline.Refspec, + Branch: pipeline.Branch, + Message: pipeline.Message, + Author: metadata.Author{ + Name: pipeline.Author, + Email: pipeline.Email, + Avatar: pipeline.Avatar, + }, + ChangedFiles: pipeline.ChangedFiles, + PullRequestLabels: pipeline.PullRequestLabels, + }, + Cron: cron, + } } diff --git a/pipeline/frontend/metadata/const.go b/pipeline/frontend/metadata/const.go new file mode 100644 index 000000000..75742cf8d --- /dev/null +++ b/pipeline/frontend/metadata/const.go @@ -0,0 +1,32 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadata + +// Event types corresponding to scm hooks. +const ( + EventPush = "push" + EventPull = "pull_request" + EventTag = "tag" + EventDeploy = "deployment" + EventCron = "cron" + EventManual = "manual" +) + +// Different ways to handle failure states +const ( + FailureIgnore = "ignore" + FailureFail = "fail" + // FailureCancel = "cancel" // Not implemented yet +) diff --git a/pipeline/drone_compatibility.go b/pipeline/frontend/metadata/drone_compatibility.go similarity index 99% rename from pipeline/drone_compatibility.go rename to pipeline/frontend/metadata/drone_compatibility.go index 43507904d..16dae443a 100644 --- a/pipeline/drone_compatibility.go +++ b/pipeline/frontend/metadata/drone_compatibility.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +package metadata // SetDroneEnviron set dedicated to DroneCI environment vars as compatibility // layer. Main purpose is to be compatible with drone plugins. diff --git a/pipeline/frontend/metadata/drone_compatibility_test.go b/pipeline/frontend/metadata/drone_compatibility_test.go new file mode 100644 index 000000000..854e39f3b --- /dev/null +++ b/pipeline/frontend/metadata/drone_compatibility_test.go @@ -0,0 +1,132 @@ +package metadata_test + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata" +) + +func TestSetDroneEnviron(t *testing.T) { + woodpeckerVars := `CI=woodpecker +CI_BUILD_CREATED=1685749339 +CI_BUILD_EVENT=pull_request +CI_BUILD_FINISHED=1685749350 +CI_BUILD_LINK=https://codeberg.org/Epsilon_02/todo-checker/pulls/9 +CI_BUILD_NUMBER=41 +CI_BUILD_STARTED=1685749339 +CI_BUILD_STATUS=success +CI_COMMIT_AUTHOR=6543 +CI_COMMIT_AUTHOR_AVATAR=https://codeberg.org/avatars/09a234c768cb9bca78f6b2f82d6af173 +CI_COMMIT_BRANCH=main +CI_COMMIT_LINK=https://codeberg.org/Epsilon_02/todo-checker/pulls/9 +CI_COMMIT_MESSAGE=fix testscript +CI_COMMIT_PULL_REQUEST=9 +CI_COMMIT_REF=refs/pull/9/head +CI_COMMIT_REFSPEC=fix_fail-on-err:main +CI_COMMIT_SHA=a778b069d9f5992786d2db9be493b43868cfce76 +CI_COMMIT_SOURCE_BRANCH=fix_fail-on-err +CI_COMMIT_TARGET_BRANCH=main +CI_JOB_FINISHED=1685749350 +CI_JOB_STARTED=1685749339 +CI_JOB_STATUS=success +CI_MACHINE=7939910e431b +CI_PIPELINE_CREATED=1685749339 +CI_PIPELINE_EVENT=pull_request +CI_PIPELINE_FINISHED=1685749350 +CI_PIPELINE_LINK=https://codeberg.org/Epsilon_02/todo-checker/pulls/9 +CI_PIPELINE_NUMBER=41 +CI_PIPELINE_STARTED=1685749339 +CI_PIPELINE_STATUS=success +CI_PREV_BUILD_CREATED=1685748680 +CI_PREV_BUILD_EVENT=pull_request +CI_PREV_BUILD_FINISHED=1685748704 +CI_PREV_BUILD_LINK=https://codeberg.org/Epsilon_02/todo-checker/pulls/13 +CI_PREV_BUILD_NUMBER=40 +CI_PREV_BUILD_STARTED=1685748680 +CI_PREV_BUILD_STATUS=success +CI_PREV_COMMIT_AUTHOR=6543 +CI_PREV_COMMIT_AUTHOR_AVATAR=https://codeberg.org/avatars/09a234c768cb9bca78f6b2f82d6af173 +CI_PREV_COMMIT_BRANCH=main +CI_PREV_COMMIT_LINK=https://codeberg.org/Epsilon_02/todo-checker/pulls/13 +CI_PREV_COMMIT_MESSAGE=Print filename and linenuber on fail +CI_PREV_COMMIT_REF=refs/pull/13/head +CI_PREV_COMMIT_REFSPEC=print_file_and_line:main +CI_PREV_COMMIT_SHA=e246aff5a9466df2e522efc9007823a7496d9d41 +CI_PREV_PIPELINE_CREATED=1685748680 +CI_PREV_PIPELINE_EVENT=pull_request +CI_PREV_PIPELINE_FINISHED=1685748704 +CI_PREV_PIPELINE_LINK=https://codeberg.org/Epsilon_02/todo-checker/pulls/13 +CI_PREV_PIPELINE_NUMBER=40 +CI_PREV_PIPELINE_STARTED=1685748680 +CI_PREV_PIPELINE_STATUS=success +CI_REPO=Epsilon_02/todo-checker +CI_REPO_CLONE_URL=https://codeberg.org/Epsilon_02/todo-checker.git +CI_REPO_DEFAULT_BRANCH=main +CI_REPO_LINK=https://codeberg.org/Epsilon_02/todo-checker +CI_REPO_NAME=todo-checker +CI_REPO_OWNER=Epsilon_02 +CI_REPO_REMOTE=https://codeberg.org/Epsilon_02/todo-checker.git +CI_REPO_SCM=git +CI_STEP_FINISHED=1685749350 +CI_STEP_NAME=wp_01h1z7v5d1tskaqjexw0ng6w7d_0_step_3 +CI_STEP_STARTED=1685749339 +CI_STEP_STATUS=success +CI_SYSTEM_ARCH=linux/amd64 +CI_SYSTEM_HOST=ci.codeberg.org +CI_SYSTEM_LINK=https://ci.codeberg.org +CI_SYSTEM_NAME=woodpecker +CI_SYSTEM_VERSION=next-dd644da3 +CI_WORKFLOW_NAME=woodpecker +CI_WORKFLOW_NUMBER=1 +CI_WORKSPACE=/woodpecker/src/codeberg.org/Epsilon_02/todo-checker` + + droneVars := `DRONE_BRANCH=main +DRONE_BUILD_CREATED=1685749339 +DRONE_BUILD_EVENT=pull_request +DRONE_BUILD_FINISHED=1685749350 +DRONE_BUILD_NUMBER=41 +DRONE_BUILD_STARTED=1685749339 +DRONE_BUILD_STATUS=success +DRONE_COMMIT=a778b069d9f5992786d2db9be493b43868cfce76 +DRONE_COMMIT_AUTHOR=6543 +DRONE_COMMIT_AUTHOR_AVATAR=https://codeberg.org/avatars/09a234c768cb9bca78f6b2f82d6af173 +DRONE_COMMIT_AUTHOR_NAME=6543 +DRONE_COMMIT_BEFORE=e246aff5a9466df2e522efc9007823a7496d9d41 +DRONE_COMMIT_BRANCH=main +DRONE_COMMIT_MESSAGE=fix testscript +DRONE_COMMIT_REF=refs/pull/9/head +DRONE_COMMIT_SHA=a778b069d9f5992786d2db9be493b43868cfce76 +DRONE_GIT_HTTP_URL=https://codeberg.org/Epsilon_02/todo-checker.git +DRONE_PULL_REQUEST=9 +DRONE_REMOTE_URL=https://codeberg.org/Epsilon_02/todo-checker.git +DRONE_REPO=Epsilon_02/todo-checker +DRONE_REPO_BRANCH=main +DRONE_REPO_NAME=todo-checker +DRONE_REPO_OWNER=Epsilon_02 +DRONE_REPO_SCM=git +DRONE_SOURCE_BRANCH=fix_fail-on-err +DRONE_SYSTEM_HOST=ci.codeberg.org +DRONE_TARGET_BRANCH=main` + + env := convertListToEnvMap(t, woodpeckerVars) + metadata.SetDroneEnviron(env) + // filter only new added env vars + for k := range convertListToEnvMap(t, woodpeckerVars) { + delete(env, k) + } + assert.EqualValues(t, convertListToEnvMap(t, droneVars), env) +} + +func convertListToEnvMap(t *testing.T, list string) map[string]string { + result := make(map[string]string) + for _, s := range strings.Split(list, "\n") { + ss := strings.SplitN(strings.TrimSpace(s), "=", 2) + if len(ss) != 2 { + t.Fatal("helper function got invalid test data") + } + result[ss[0]] = ss[1] + } + return result +} diff --git a/pipeline/frontend/metadata/environment.go b/pipeline/frontend/metadata/environment.go new file mode 100644 index 000000000..30a80381e --- /dev/null +++ b/pipeline/frontend/metadata/environment.go @@ -0,0 +1,161 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadata + +import ( + "path" + "regexp" + "strconv" + "strings" +) + +var pullRegexp = regexp.MustCompile(`\d+`) + +// Environ returns the metadata as a map of environment variables. +func (m *Metadata) Environ() map[string]string { + var ( + sourceBranch string + targetBranch string + ) + + branchParts := strings.Split(m.Curr.Commit.Refspec, ":") + if len(branchParts) == 2 { + sourceBranch = branchParts[0] + targetBranch = branchParts[1] + } + + params := map[string]string{ + "CI": m.Sys.Name, + "CI_REPO": path.Join(m.Repo.Owner, m.Repo.Name), + "CI_REPO_NAME": m.Repo.Name, + "CI_REPO_OWNER": m.Repo.Owner, + "CI_REPO_REMOTE_ID": m.Repo.RemoteID, + "CI_REPO_SCM": "git", + "CI_REPO_URL": m.Repo.Link, + "CI_REPO_CLONE_URL": m.Repo.CloneURL, + "CI_REPO_DEFAULT_BRANCH": m.Repo.Branch, + "CI_REPO_PRIVATE": strconv.FormatBool(m.Repo.Private), + "CI_REPO_TRUSTED": strconv.FormatBool(m.Repo.Trusted), + + "CI_COMMIT_SHA": m.Curr.Commit.Sha, + "CI_COMMIT_REF": m.Curr.Commit.Ref, + "CI_COMMIT_REFSPEC": m.Curr.Commit.Refspec, + "CI_COMMIT_BRANCH": m.Curr.Commit.Branch, + "CI_COMMIT_SOURCE_BRANCH": sourceBranch, + "CI_COMMIT_TARGET_BRANCH": targetBranch, + "CI_COMMIT_URL": m.Curr.Link, + "CI_COMMIT_MESSAGE": m.Curr.Commit.Message, + "CI_COMMIT_AUTHOR": m.Curr.Commit.Author.Name, + "CI_COMMIT_AUTHOR_EMAIL": m.Curr.Commit.Author.Email, + "CI_COMMIT_AUTHOR_AVATAR": m.Curr.Commit.Author.Avatar, + "CI_COMMIT_TAG": "", // will be set if event is tag + "CI_COMMIT_PULL_REQUEST": "", // will be set if event is pr + "CI_COMMIT_PULL_REQUEST_LABELS": "", // will be set if event is pr + + "CI_PIPELINE_NUMBER": strconv.FormatInt(m.Curr.Number, 10), + "CI_PIPELINE_PARENT": strconv.FormatInt(m.Curr.Parent, 10), + "CI_PIPELINE_EVENT": m.Curr.Event, + "CI_PIPELINE_URL": m.Curr.Link, + "CI_PIPELINE_DEPLOY_TARGET": m.Curr.Target, + "CI_PIPELINE_STATUS": m.Curr.Status, + "CI_PIPELINE_CREATED": strconv.FormatInt(m.Curr.Created, 10), + "CI_PIPELINE_STARTED": strconv.FormatInt(m.Curr.Started, 10), + "CI_PIPELINE_FINISHED": strconv.FormatInt(m.Curr.Finished, 10), + + "CI_WORKFLOW_NAME": m.Workflow.Name, + "CI_WORKFLOW_NUMBER": strconv.Itoa(m.Workflow.Number), + + "CI_STEP_NAME": m.Step.Name, + "CI_STEP_NUMBER": strconv.Itoa(m.Step.Number), + "CI_STEP_STATUS": "", // will be set by agent + "CI_STEP_STARTED": "", // will be set by agent + "CI_STEP_FINISHED": "", // will be set by agent + + "CI_PREV_COMMIT_SHA": m.Prev.Commit.Sha, + "CI_PREV_COMMIT_REF": m.Prev.Commit.Ref, + "CI_PREV_COMMIT_REFSPEC": m.Prev.Commit.Refspec, + "CI_PREV_COMMIT_BRANCH": m.Prev.Commit.Branch, + "CI_PREV_COMMIT_URL": m.Prev.Link, + "CI_PREV_COMMIT_MESSAGE": m.Prev.Commit.Message, + "CI_PREV_COMMIT_AUTHOR": m.Prev.Commit.Author.Name, + "CI_PREV_COMMIT_AUTHOR_EMAIL": m.Prev.Commit.Author.Email, + "CI_PREV_COMMIT_AUTHOR_AVATAR": m.Prev.Commit.Author.Avatar, + + "CI_PREV_PIPELINE_NUMBER": strconv.FormatInt(m.Prev.Number, 10), + "CI_PREV_PIPELINE_PARENT": strconv.FormatInt(m.Prev.Parent, 10), + "CI_PREV_PIPELINE_EVENT": m.Prev.Event, + "CI_PREV_PIPELINE_URL": m.Prev.Link, + "CI_PREV_PIPELINE_DEPLOY_TARGET": m.Prev.Target, + "CI_PREV_PIPELINE_STATUS": m.Prev.Status, + "CI_PREV_PIPELINE_CREATED": strconv.FormatInt(m.Prev.Created, 10), + "CI_PREV_PIPELINE_STARTED": strconv.FormatInt(m.Prev.Started, 10), + "CI_PREV_PIPELINE_FINISHED": strconv.FormatInt(m.Prev.Finished, 10), + + "CI_SYSTEM_NAME": m.Sys.Name, + "CI_SYSTEM_URL": m.Sys.Link, + "CI_SYSTEM_HOST": m.Sys.Host, + "CI_SYSTEM_PLATFORM": m.Sys.Platform, // will be set by pipeline platform option or by agent + "CI_SYSTEM_VERSION": m.Sys.Version, + + "CI_FORGE_TYPE": m.Forge.Type, + "CI_FORGE_URL": m.Forge.URL, + + // DEPRECATED + "CI_SYSTEM_ARCH": m.Sys.Platform, // TODO: remove after v1.0.x version + // use CI_PIPELINE_* + "CI_BUILD_NUMBER": strconv.FormatInt(m.Curr.Number, 10), + "CI_BUILD_PARENT": strconv.FormatInt(m.Curr.Parent, 10), + "CI_BUILD_EVENT": m.Curr.Event, + "CI_BUILD_LINK": m.Curr.Link, + "CI_BUILD_DEPLOY_TARGET": m.Curr.Target, + "CI_BUILD_STATUS": m.Curr.Status, + "CI_BUILD_CREATED": strconv.FormatInt(m.Curr.Created, 10), + "CI_BUILD_STARTED": strconv.FormatInt(m.Curr.Started, 10), + "CI_BUILD_FINISHED": strconv.FormatInt(m.Curr.Finished, 10), + // use CI_PREV_PIPELINE_* + "CI_PREV_BUILD_NUMBER": strconv.FormatInt(m.Prev.Number, 10), + "CI_PREV_BUILD_PARENT": strconv.FormatInt(m.Prev.Parent, 10), + "CI_PREV_BUILD_EVENT": m.Prev.Event, + "CI_PREV_BUILD_LINK": m.Prev.Link, + "CI_PREV_BUILD_DEPLOY_TARGET": m.Prev.Target, + "CI_PREV_BUILD_STATUS": m.Prev.Status, + "CI_PREV_BUILD_CREATED": strconv.FormatInt(m.Prev.Created, 10), + "CI_PREV_BUILD_STARTED": strconv.FormatInt(m.Prev.Started, 10), + "CI_PREV_BUILD_FINISHED": strconv.FormatInt(m.Prev.Finished, 10), + // use CI_STEP_* + "CI_JOB_NUMBER": strconv.Itoa(m.Step.Number), + "CI_JOB_STATUS": "", // will be set by agent + "CI_JOB_STARTED": "", // will be set by agent + "CI_JOB_FINISHED": "", // will be set by agent + // CI_REPO_CLONE_URL + "CI_REPO_REMOTE": m.Repo.CloneURL, + // use *_URL + "CI_REPO_LINK": m.Repo.Link, + "CI_COMMIT_LINK": m.Curr.Link, + "CI_PIPELINE_LINK": m.Curr.Link, + "CI_PREV_COMMIT_LINK": m.Prev.Link, + "CI_PREV_PIPELINE_LINK": m.Prev.Link, + "CI_SYSTEM_LINK": m.Sys.Link, + } + if m.Curr.Event == EventTag { + params["CI_COMMIT_TAG"] = strings.TrimPrefix(m.Curr.Commit.Ref, "refs/tags/") + } + if m.Curr.Event == EventPull { + params["CI_COMMIT_PULL_REQUEST"] = pullRegexp.FindString(m.Curr.Commit.Ref) + params["CI_COMMIT_PULL_REQUEST_LABELS"] = strings.Join(m.Curr.Commit.PullRequestLabels, ",") + } + + return params +} diff --git a/pipeline/frontend/metadata/types.go b/pipeline/frontend/metadata/types.go new file mode 100644 index 000000000..43f086917 --- /dev/null +++ b/pipeline/frontend/metadata/types.go @@ -0,0 +1,122 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadata + +type ( + // Metadata defines runtime m. + Metadata struct { + ID string `json:"id,omitempty"` + Repo Repo `json:"repo,omitempty"` + Curr Pipeline `json:"curr,omitempty"` + Prev Pipeline `json:"prev,omitempty"` + Workflow Workflow `json:"workflow,omitempty"` + Step Step `json:"step,omitempty"` + Sys System `json:"sys,omitempty"` + Forge Forge `json:"forge,omitempty"` + } + + // Repo defines runtime metadata for a repository. + Repo struct { + Name string `json:"name,omitempty"` + Owner string `json:"owner,omitempty"` + RemoteID string `json:"remote_id,omitempty"` + Link string `json:"link,omitempty"` + CloneURL string `json:"clone_url,omitempty"` + Private bool `json:"private,omitempty"` + Secrets []Secret `json:"secrets,omitempty"` + Branch string `json:"default_branch,omitempty"` + Trusted bool `json:"trusted,omitempty"` + } + + // Pipeline defines runtime metadata for a pipeline. + Pipeline struct { + Number int64 `json:"number,omitempty"` + Created int64 `json:"created,omitempty"` + Started int64 `json:"started,omitempty"` + Finished int64 `json:"finished,omitempty"` + Timeout int64 `json:"timeout,omitempty"` + Status string `json:"status,omitempty"` + Event string `json:"event,omitempty"` + Link string `json:"link,omitempty"` + Target string `json:"target,omitempty"` + Trusted bool `json:"trusted,omitempty"` + Commit Commit `json:"commit,omitempty"` + Parent int64 `json:"parent,omitempty"` + Cron string `json:"cron,omitempty"` + } + + // Commit defines runtime metadata for a commit. + Commit struct { + Sha string `json:"sha,omitempty"` + Ref string `json:"ref,omitempty"` + Refspec string `json:"refspec,omitempty"` + Branch string `json:"branch,omitempty"` + Message string `json:"message,omitempty"` + Author Author `json:"author,omitempty"` + ChangedFiles []string `json:"changed_files,omitempty"` + PullRequestLabels []string `json:"labels,omitempty"` + } + + // Author defines runtime metadata for a commit author. + Author struct { + Name string `json:"name,omitempty"` + Email string `json:"email,omitempty"` + Avatar string `json:"avatar,omitempty"` + } + + // Workflow defines runtime metadata for a workflow. + Workflow struct { + Name string `json:"name,omitempty"` + Number int `json:"number,omitempty"` + Matrix map[string]string `json:"matrix,omitempty"` + } + + // Step defines runtime metadata for a step. + Step struct { + Name string `json:"name,omitempty"` + Number int `json:"number,omitempty"` + } + + // Secret defines a runtime secret + Secret struct { + Name string `json:"name,omitempty"` + Value string `json:"value,omitempty"` + Mount string `json:"mount,omitempty"` + Mask bool `json:"mask,omitempty"` + } + + // System defines runtime metadata for a ci/cd system. + System struct { + Name string `json:"name,omitempty"` + Host string `json:"host,omitempty"` + Link string `json:"link,omitempty"` + Platform string `json:"arch,omitempty"` + Version string `json:"version,omitempty"` + } + + // Forge defines runtime metadata about the forge that host the repo + Forge struct { + Type string `json:"type,omitempty"` + URL string `json:"url,omitempty"` + } + + // ServerForge represent the needed func of a server forge to get its metadata + ServerForge interface { + // Name returns the string name of this driver + Name() string + // URL returns the root url of a configured forge + URL() string + } +) diff --git a/pipeline/frontend/metadata_test.go b/pipeline/frontend/metadata_test.go new file mode 100644 index 000000000..3b5860742 --- /dev/null +++ b/pipeline/frontend/metadata_test.go @@ -0,0 +1,132 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package frontend_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/woodpecker-ci/woodpecker/pipeline/frontend" + "github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata" + "github.com/woodpecker-ci/woodpecker/server/forge/mocks" + "github.com/woodpecker-ci/woodpecker/server/model" +) + +func TestEnvVarSubst(t *testing.T) { + testCases := []struct { + name string + yaml string + environ map[string]string + want string + }{{ + name: "simple substitution", + yaml: `pipeline: + step1: + image: ${HELLO_IMAGE}`, + environ: map[string]string{"HELLO_IMAGE": "hello-world"}, + want: `pipeline: + step1: + image: hello-world`, + }} + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + result, err := frontend.EnvVarSubst(testCase.yaml, testCase.environ) + assert.NoError(t, err) + assert.EqualValues(t, testCase.want, result) + }) + } +} + +func TestMetadataFromStruct(t *testing.T) { + forge := mocks.NewForge(t) + forge.On("Name").Return("gitea") + forge.On("URL").Return("https://gitea.com") + + testCases := []struct { + name string + forge metadata.ServerForge + repo *model.Repo + pipeline, last *model.Pipeline + workflow *model.Step + link string + expectedMetadata metadata.Metadata + expectedEnviron map[string]string + }{ + { + name: "Test with empty info", + expectedMetadata: metadata.Metadata{Sys: metadata.System{Name: "woodpecker"}}, + expectedEnviron: map[string]string{ + "CI": "woodpecker", "CI_BUILD_CREATED": "0", "CI_BUILD_DEPLOY_TARGET": "", "CI_BUILD_EVENT": "", "CI_BUILD_FINISHED": "0", "CI_BUILD_LINK": "", "CI_BUILD_NUMBER": "0", "CI_BUILD_PARENT": "0", + "CI_BUILD_STARTED": "0", "CI_BUILD_STATUS": "", "CI_COMMIT_AUTHOR": "", "CI_COMMIT_AUTHOR_AVATAR": "", "CI_COMMIT_AUTHOR_EMAIL": "", "CI_COMMIT_BRANCH": "", "CI_COMMIT_LINK": "", + "CI_COMMIT_MESSAGE": "", "CI_COMMIT_PULL_REQUEST": "", "CI_COMMIT_PULL_REQUEST_LABELS": "", "CI_COMMIT_REF": "", "CI_COMMIT_REFSPEC": "", "CI_COMMIT_SHA": "", "CI_COMMIT_SOURCE_BRANCH": "", + "CI_COMMIT_TAG": "", "CI_COMMIT_TARGET_BRANCH": "", "CI_COMMIT_URL": "", "CI_FORGE_TYPE": "", "CI_FORGE_URL": "", "CI_JOB_FINISHED": "", "CI_JOB_NUMBER": "0", "CI_JOB_STARTED": "", + "CI_JOB_STATUS": "", "CI_PIPELINE_CREATED": "0", "CI_PIPELINE_DEPLOY_TARGET": "", "CI_PIPELINE_EVENT": "", "CI_PIPELINE_FINISHED": "0", "CI_PIPELINE_LINK": "", "CI_PIPELINE_NUMBER": "0", + "CI_PIPELINE_PARENT": "0", "CI_PIPELINE_STARTED": "0", "CI_PIPELINE_STATUS": "", "CI_PIPELINE_URL": "", "CI_PREV_BUILD_CREATED": "0", "CI_PREV_BUILD_DEPLOY_TARGET": "", + "CI_PREV_BUILD_EVENT": "", "CI_PREV_BUILD_FINISHED": "0", "CI_PREV_BUILD_LINK": "", "CI_PREV_BUILD_NUMBER": "0", "CI_PREV_BUILD_PARENT": "0", "CI_PREV_BUILD_STARTED": "0", + "CI_PREV_BUILD_STATUS": "", "CI_PREV_COMMIT_AUTHOR": "", "CI_PREV_COMMIT_AUTHOR_AVATAR": "", "CI_PREV_COMMIT_AUTHOR_EMAIL": "", "CI_PREV_COMMIT_BRANCH": "", "CI_PREV_COMMIT_LINK": "", + "CI_PREV_COMMIT_MESSAGE": "", "CI_PREV_COMMIT_REF": "", "CI_PREV_COMMIT_REFSPEC": "", "CI_PREV_COMMIT_SHA": "", "CI_PREV_COMMIT_URL": "", "CI_PREV_PIPELINE_CREATED": "0", + "CI_PREV_PIPELINE_DEPLOY_TARGET": "", "CI_PREV_PIPELINE_EVENT": "", "CI_PREV_PIPELINE_FINISHED": "0", "CI_PREV_PIPELINE_LINK": "", "CI_PREV_PIPELINE_NUMBER": "0", "CI_PREV_PIPELINE_PARENT": "0", + "CI_PREV_PIPELINE_STARTED": "0", "CI_PREV_PIPELINE_STATUS": "", "CI_PREV_PIPELINE_URL": "", "CI_REPO": "", "CI_REPO_CLONE_URL": "", "CI_REPO_DEFAULT_BRANCH": "", "CI_REPO_LINK": "", "CI_REPO_REMOTE_ID": "", + "CI_REPO_NAME": "", "CI_REPO_OWNER": "", "CI_REPO_PRIVATE": "false", "CI_REPO_REMOTE": "", "CI_REPO_SCM": "git", "CI_REPO_TRUSTED": "false", "CI_REPO_URL": "", "CI_STEP_FINISHED": "", + "CI_STEP_NAME": "", "CI_STEP_NUMBER": "0", "CI_STEP_STARTED": "", "CI_STEP_STATUS": "", "CI_SYSTEM_ARCH": "", "CI_SYSTEM_HOST": "", "CI_SYSTEM_LINK": "", "CI_SYSTEM_NAME": "woodpecker", + "CI_SYSTEM_PLATFORM": "", "CI_SYSTEM_URL": "", "CI_SYSTEM_VERSION": "", "CI_WORKFLOW_NAME": "", "CI_WORKFLOW_NUMBER": "0", + }, + }, + { + name: "Test with forge", + forge: forge, + repo: &model.Repo{FullName: "testUser/testRepo", Link: "https://gitea.com/testUser/testRepo", Clone: "https://gitea.com/testUser/testRepo.git", Branch: "main", IsSCMPrivate: true}, + pipeline: &model.Pipeline{Number: 3}, + last: &model.Pipeline{Number: 2}, + workflow: &model.Step{Name: "hello"}, + link: "https://example.com", + expectedMetadata: metadata.Metadata{ + Forge: metadata.Forge{Type: "gitea", URL: "https://gitea.com"}, + Sys: metadata.System{Name: "woodpecker", Host: "example.com", Link: "https://example.com"}, + Repo: metadata.Repo{Owner: "testUser", Name: "testRepo", Link: "https://gitea.com/testUser/testRepo", CloneURL: "https://gitea.com/testUser/testRepo.git", Branch: "main", Private: true}, + Curr: metadata.Pipeline{Number: 3}, + Prev: metadata.Pipeline{Number: 2}, + Workflow: metadata.Workflow{Name: "hello"}, + }, + expectedEnviron: map[string]string{ + "CI": "woodpecker", "CI_BUILD_CREATED": "0", "CI_BUILD_DEPLOY_TARGET": "", "CI_BUILD_EVENT": "", "CI_BUILD_FINISHED": "0", "CI_BUILD_LINK": "", "CI_BUILD_NUMBER": "3", "CI_BUILD_PARENT": "0", + "CI_BUILD_STARTED": "0", "CI_BUILD_STATUS": "", "CI_COMMIT_AUTHOR": "", "CI_COMMIT_AUTHOR_AVATAR": "", "CI_COMMIT_AUTHOR_EMAIL": "", "CI_COMMIT_BRANCH": "", "CI_COMMIT_LINK": "", + "CI_COMMIT_MESSAGE": "", "CI_COMMIT_PULL_REQUEST": "", "CI_COMMIT_PULL_REQUEST_LABELS": "", "CI_COMMIT_REF": "", "CI_COMMIT_REFSPEC": "", "CI_COMMIT_SHA": "", "CI_COMMIT_SOURCE_BRANCH": "", + "CI_COMMIT_TAG": "", "CI_COMMIT_TARGET_BRANCH": "", "CI_COMMIT_URL": "", "CI_FORGE_TYPE": "gitea", "CI_FORGE_URL": "https://gitea.com", "CI_JOB_FINISHED": "", "CI_JOB_NUMBER": "0", + "CI_JOB_STARTED": "", "CI_JOB_STATUS": "", "CI_PIPELINE_CREATED": "0", "CI_PIPELINE_DEPLOY_TARGET": "", "CI_PIPELINE_EVENT": "", "CI_PIPELINE_FINISHED": "0", "CI_PIPELINE_LINK": "", + "CI_PIPELINE_NUMBER": "3", "CI_PIPELINE_PARENT": "0", "CI_PIPELINE_STARTED": "0", "CI_PIPELINE_STATUS": "", "CI_PIPELINE_URL": "", "CI_PREV_BUILD_CREATED": "0", "CI_PREV_BUILD_DEPLOY_TARGET": "", + "CI_PREV_BUILD_EVENT": "", "CI_PREV_BUILD_FINISHED": "0", "CI_PREV_BUILD_LINK": "", "CI_PREV_BUILD_NUMBER": "2", "CI_PREV_BUILD_PARENT": "0", "CI_PREV_BUILD_STARTED": "0", + "CI_PREV_BUILD_STATUS": "", "CI_PREV_COMMIT_AUTHOR": "", "CI_PREV_COMMIT_AUTHOR_AVATAR": "", "CI_PREV_COMMIT_AUTHOR_EMAIL": "", "CI_PREV_COMMIT_BRANCH": "", "CI_PREV_COMMIT_LINK": "", + "CI_PREV_COMMIT_MESSAGE": "", "CI_PREV_COMMIT_REF": "", "CI_PREV_COMMIT_REFSPEC": "", "CI_PREV_COMMIT_SHA": "", "CI_PREV_COMMIT_URL": "", "CI_PREV_PIPELINE_CREATED": "0", + "CI_PREV_PIPELINE_DEPLOY_TARGET": "", "CI_PREV_PIPELINE_EVENT": "", "CI_PREV_PIPELINE_FINISHED": "0", "CI_PREV_PIPELINE_LINK": "", "CI_PREV_PIPELINE_NUMBER": "2", "CI_PREV_PIPELINE_PARENT": "0", + "CI_PREV_PIPELINE_STARTED": "0", "CI_PREV_PIPELINE_STATUS": "", "CI_PREV_PIPELINE_URL": "", "CI_REPO": "testUser/testRepo", "CI_REPO_CLONE_URL": "https://gitea.com/testUser/testRepo.git", + "CI_REPO_DEFAULT_BRANCH": "main", "CI_REPO_LINK": "https://gitea.com/testUser/testRepo", "CI_REPO_NAME": "testRepo", "CI_REPO_OWNER": "testUser", "CI_REPO_PRIVATE": "true", "CI_REPO_REMOTE_ID": "", + "CI_REPO_REMOTE": "https://gitea.com/testUser/testRepo.git", "CI_REPO_SCM": "git", "CI_REPO_TRUSTED": "false", "CI_REPO_URL": "https://gitea.com/testUser/testRepo", "CI_STEP_FINISHED": "", + "CI_STEP_NAME": "", "CI_STEP_NUMBER": "0", "CI_STEP_STARTED": "", "CI_STEP_STATUS": "", "CI_SYSTEM_ARCH": "", "CI_SYSTEM_HOST": "example.com", "CI_SYSTEM_LINK": "https://example.com", + "CI_SYSTEM_NAME": "woodpecker", "CI_SYSTEM_PLATFORM": "", "CI_SYSTEM_URL": "https://example.com", "CI_SYSTEM_VERSION": "", "CI_WORKFLOW_NAME": "hello", "CI_WORKFLOW_NUMBER": "0", + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + result := frontend.MetadataFromStruct(testCase.forge, testCase.repo, testCase.pipeline, testCase.last, testCase.workflow, testCase.link) + assert.EqualValues(t, testCase.expectedMetadata, result) + assert.EqualValues(t, testCase.expectedEnviron, result.Environ()) + }) + } +} diff --git a/pipeline/frontend/yaml/compiler/compiler.go b/pipeline/frontend/yaml/compiler/compiler.go index 96539909c..58126daf2 100644 --- a/pipeline/frontend/yaml/compiler/compiler.go +++ b/pipeline/frontend/yaml/compiler/compiler.go @@ -2,10 +2,11 @@ package compiler import ( "fmt" + "path" "strings" backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" - "github.com/woodpecker-ci/woodpecker/pipeline/frontend" + "github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml" "github.com/woodpecker-ci/woodpecker/shared/constant" ) @@ -75,7 +76,7 @@ type Compiler struct { cloneEnv map[string]string base string path string - metadata frontend.Metadata + metadata metadata.Metadata registries []Registry secrets secretMap cacher Cacher @@ -156,7 +157,7 @@ func (c *Compiler) Compile(conf *yaml.Config) (*backend.Config, error) { // add default clone step if !c.local && len(conf.Clone.Containers) == 0 && !conf.SkipClone { cloneSettings := map[string]interface{}{"depth": "0"} - if c.metadata.Curr.Event == frontend.EventTag { + if c.metadata.Curr.Event == metadata.EventTag { cloneSettings["tags"] = "true" } container := &yaml.Container{ @@ -263,7 +264,7 @@ func (c *Compiler) setupCache(conf *yaml.Config, ir *backend.Config) { return } - container := c.cacher.Restore(c.metadata.Repo.Name, c.metadata.Curr.Commit.Branch, conf.Cache) + container := c.cacher.Restore(path.Join(c.metadata.Repo.Owner, c.metadata.Repo.Name), c.metadata.Curr.Commit.Branch, conf.Cache) name := fmt.Sprintf("%s_restore_cache", c.prefix) step := c.createProcess(name, container, "cache") @@ -276,10 +277,10 @@ func (c *Compiler) setupCache(conf *yaml.Config, ir *backend.Config) { } func (c *Compiler) setupCacheRebuild(conf *yaml.Config, ir *backend.Config) { - if c.local || len(conf.Cache) == 0 || c.metadata.Curr.Event != frontend.EventPush || c.cacher == nil { + if c.local || len(conf.Cache) == 0 || c.metadata.Curr.Event != metadata.EventPush || c.cacher == nil { return } - container := c.cacher.Rebuild(c.metadata.Repo.Name, c.metadata.Curr.Commit.Branch, conf.Cache) + container := c.cacher.Rebuild(path.Join(c.metadata.Repo.Owner, c.metadata.Repo.Name), c.metadata.Curr.Commit.Branch, conf.Cache) name := fmt.Sprintf("%s_rebuild_cache", c.prefix) step := c.createProcess(name, container, "cache") diff --git a/pipeline/frontend/yaml/compiler/convert.go b/pipeline/frontend/yaml/compiler/convert.go index a661f05a4..ba52d10e4 100644 --- a/pipeline/frontend/yaml/compiler/convert.go +++ b/pipeline/frontend/yaml/compiler/convert.go @@ -9,7 +9,7 @@ import ( "github.com/rs/zerolog/log" backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" - "github.com/woodpecker-ci/woodpecker/pipeline/frontend" + "github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler/settings" ) @@ -152,7 +152,7 @@ func (c *Compiler) createProcess(name string, container *yaml.Container, section failure := container.Failure if container.Failure == "" { - failure = frontend.FailureFail + failure = metadata.FailureFail } return &backend.Step{ diff --git a/pipeline/frontend/yaml/compiler/option.go b/pipeline/frontend/yaml/compiler/option.go index 74bdbb2ec..055b38e3b 100644 --- a/pipeline/frontend/yaml/compiler/option.go +++ b/pipeline/frontend/yaml/compiler/option.go @@ -20,7 +20,7 @@ import ( "path/filepath" "strings" - "github.com/woodpecker-ci/woodpecker/pipeline/frontend" + "github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata" ) // Option configures a compiler option. @@ -67,7 +67,7 @@ func WithSecret(secrets ...Secret) Option { // and system metadata. The metadata is used to remove steps from // the compiled pipeline configuration that should be skipped. The // metadata is also added to each container as environment variables. -func WithMetadata(metadata frontend.Metadata) Option { +func WithMetadata(metadata metadata.Metadata) Option { return func(compiler *Compiler) { compiler.metadata = metadata diff --git a/pipeline/frontend/yaml/compiler/option_test.go b/pipeline/frontend/yaml/compiler/option_test.go index 38c7bb545..3af0acddb 100644 --- a/pipeline/frontend/yaml/compiler/option_test.go +++ b/pipeline/frontend/yaml/compiler/option_test.go @@ -3,10 +3,9 @@ package compiler import ( "os" "reflect" - "strings" "testing" - "github.com/woodpecker-ci/woodpecker/pipeline/frontend" + "github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata" ) func TestWithWorkspace(t *testing.T) { @@ -98,9 +97,10 @@ func TestWithPrefix(t *testing.T) { } func TestWithMetadata(t *testing.T) { - metadata := frontend.Metadata{ - Repo: frontend.Repo{ - Name: "octocat/hello-world", + metadata := metadata.Metadata{ + Repo: metadata.Repo{ + Owner: "octacat", + Name: "hello-world", Private: true, Link: "https://github.com/octocat/hello-world", CloneURL: "https://github.com/octocat/hello-world.git", @@ -113,7 +113,7 @@ func TestWithMetadata(t *testing.T) { t.Errorf("WithMetadata must set compiler the metadata") } - if compiler.env["CI_REPO_NAME"] != strings.Split(metadata.Repo.Name, "/")[1] { + if compiler.env["CI_REPO_NAME"] != metadata.Repo.Name { t.Errorf("WithMetadata must set CI_REPO_NAME") } if compiler.env["CI_REPO_URL"] != metadata.Repo.Link { diff --git a/pipeline/frontend/yaml/config.go b/pipeline/frontend/yaml/config.go index 25e30ed0d..214d0a6f3 100644 --- a/pipeline/frontend/yaml/config.go +++ b/pipeline/frontend/yaml/config.go @@ -1,7 +1,10 @@ package yaml import ( + "fmt" + "codeberg.org/6543/xyaml" + "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/constraint" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types" ) @@ -23,7 +26,7 @@ type ( RunsOn []string `yaml:"runs_on,omitempty"` SkipClone bool `yaml:"skip_clone"` // Deprecated use When.Branch - Branches constraint.List + BranchesDontUseIt *constraint.List `yaml:"branches,omitempty"` } // Workspace defines a pipeline workspace. @@ -41,6 +44,18 @@ func ParseBytes(b []byte) (*Config, error) { return nil, err } + // support deprecated branch filter + if out.BranchesDontUseIt != nil { + if out.When.Constraints == nil { + out.When.Constraints = []constraint.Constraint{{Branch: *out.BranchesDontUseIt}} + } else if len(out.When.Constraints) == 1 && out.When.Constraints[0].Branch.IsEmpty() { + out.When.Constraints[0].Branch = *out.BranchesDontUseIt + } else { + return nil, fmt.Errorf("could not apply deprecated branches filter into global when filter") + } + out.BranchesDontUseIt = nil + } + return out, nil } diff --git a/pipeline/frontend/yaml/config_test.go b/pipeline/frontend/yaml/config_test.go index ce8f9d11f..4f6698de5 100644 --- a/pipeline/frontend/yaml/config_test.go +++ b/pipeline/frontend/yaml/config_test.go @@ -5,7 +5,7 @@ import ( "github.com/franela/goblin" - "github.com/woodpecker-ci/woodpecker/pipeline/frontend" + "github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types" ) @@ -79,8 +79,8 @@ func TestParse(t *testing.T) { } g.It("Should match event tester", func() { - match, err := matchConfig.When.Match(frontend.Metadata{ - Curr: frontend.Pipeline{ + match, err := matchConfig.When.Match(metadata.Metadata{ + Curr: metadata.Pipeline{ Event: "tester", }, }, false) @@ -89,8 +89,8 @@ func TestParse(t *testing.T) { }) g.It("Should match event tester2", func() { - match, err := matchConfig.When.Match(frontend.Metadata{ - Curr: frontend.Pipeline{ + match, err := matchConfig.When.Match(metadata.Metadata{ + Curr: metadata.Pipeline{ Event: "tester2", }, }, false) @@ -99,9 +99,9 @@ func TestParse(t *testing.T) { }) g.It("Should match branch tester", func() { - match, err := matchConfig.When.Match(frontend.Metadata{ - Curr: frontend.Pipeline{ - Commit: frontend.Commit{ + match, err := matchConfig.When.Match(metadata.Metadata{ + Curr: metadata.Pipeline{ + Commit: metadata.Commit{ Branch: "tester", }, }, @@ -111,8 +111,8 @@ func TestParse(t *testing.T) { }) g.It("Should not match event push", func() { - match, err := matchConfig.When.Match(frontend.Metadata{ - Curr: frontend.Pipeline{ + match, err := matchConfig.When.Match(metadata.Metadata{ + Curr: metadata.Pipeline{ Event: "push", }, }, false) diff --git a/pipeline/frontend/yaml/constraint/constraint.go b/pipeline/frontend/yaml/constraint/constraint.go index 41b189070..91e971446 100644 --- a/pipeline/frontend/yaml/constraint/constraint.go +++ b/pipeline/frontend/yaml/constraint/constraint.go @@ -3,13 +3,14 @@ package constraint import ( "errors" "fmt" + "path" "strings" "github.com/antonmedv/expr" "github.com/bmatcuk/doublestar/v4" "gopkg.in/yaml.v3" - "github.com/woodpecker-ci/woodpecker/pipeline/frontend" + "github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types" ) @@ -61,7 +62,7 @@ func (when *When) IsEmpty() bool { } // Returns true if at least one of the internal constraints is true. -func (when *When) Match(metadata frontend.Metadata, global bool) (bool, error) { +func (when *When) Match(metadata metadata.Metadata, global bool) (bool, error) { for _, c := range when.Constraints { match, err := c.Match(metadata, global) if err != nil { @@ -138,37 +139,37 @@ func (when *When) UnmarshalYAML(value *yaml.Node) error { // Match returns true if all constraints match the given input. If a single // constraint fails a false value is returned. -func (c *Constraint) Match(metadata frontend.Metadata, global bool) (bool, error) { +func (c *Constraint) Match(m metadata.Metadata, global bool) (bool, error) { match := true if !global { c.SetDefaultEventFilter() // apply step only filters - match = c.Matrix.Match(metadata.Workflow.Matrix) + match = c.Matrix.Match(m.Workflow.Matrix) } - match = match && c.Platform.Match(metadata.Sys.Platform) && - c.Environment.Match(metadata.Curr.Target) && - c.Event.Match(metadata.Curr.Event) && - c.Repo.Match(metadata.Repo.Name) && - c.Ref.Match(metadata.Curr.Commit.Ref) && - c.Instance.Match(metadata.Sys.Host) + match = match && c.Platform.Match(m.Sys.Platform) && + c.Environment.Match(m.Curr.Target) && + c.Event.Match(m.Curr.Event) && + c.Repo.Match(path.Join(m.Repo.Owner, m.Repo.Name)) && + c.Ref.Match(m.Curr.Commit.Ref) && + c.Instance.Match(m.Sys.Host) // changed files filter apply only for pull-request and push events - if metadata.Curr.Event == frontend.EventPull || metadata.Curr.Event == frontend.EventPush { - match = match && c.Path.Match(metadata.Curr.Commit.ChangedFiles, metadata.Curr.Commit.Message) + if m.Curr.Event == metadata.EventPull || m.Curr.Event == metadata.EventPush { + match = match && c.Path.Match(m.Curr.Commit.ChangedFiles, m.Curr.Commit.Message) } - if metadata.Curr.Event != frontend.EventTag { - match = match && c.Branch.Match(metadata.Curr.Commit.Branch) + if m.Curr.Event != metadata.EventTag { + match = match && c.Branch.Match(m.Curr.Commit.Branch) } - if metadata.Curr.Event == frontend.EventCron { - match = match && c.Cron.Match(metadata.Curr.Cron) + if m.Curr.Event == metadata.EventCron { + match = match && c.Cron.Match(m.Curr.Cron) } if c.Evaluate != "" { - env := metadata.Environ() + env := m.Environ() out, err := expr.Compile(c.Evaluate, expr.Env(env), expr.AsBool()) if err != nil { return false, err @@ -187,11 +188,11 @@ func (c *Constraint) Match(metadata frontend.Metadata, global bool) (bool, error func (c *Constraint) SetDefaultEventFilter() { if c.Event.IsEmpty() { c.Event.Include = []string{ - frontend.EventPush, - frontend.EventPull, - frontend.EventTag, - frontend.EventDeploy, - frontend.EventManual, + metadata.EventPush, + metadata.EventPull, + metadata.EventTag, + metadata.EventDeploy, + metadata.EventManual, } } } diff --git a/pipeline/frontend/yaml/constraint/constraint_test.go b/pipeline/frontend/yaml/constraint/constraint_test.go index 3c72f59b0..227ee5e95 100644 --- a/pipeline/frontend/yaml/constraint/constraint_test.go +++ b/pipeline/frontend/yaml/constraint/constraint_test.go @@ -7,6 +7,7 @@ import ( "gopkg.in/yaml.v3" "github.com/woodpecker-ci/woodpecker/pipeline/frontend" + "github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata" ) func TestConstraint(t *testing.T) { @@ -403,15 +404,15 @@ func TestConstraints(t *testing.T) { testdata := []struct { desc string conf string - with frontend.Metadata + with metadata.Metadata want bool }{ { desc: "no constraints, must match on default events", conf: "", - with: frontend.Metadata{ - Curr: frontend.Pipeline{ - Event: frontend.EventPush, + with: metadata.Metadata{ + Curr: metadata.Pipeline{ + Event: metadata.EventPush, }, }, want: true, @@ -419,106 +420,117 @@ func TestConstraints(t *testing.T) { { desc: "global branch filter", conf: "{ branch: develop }", - with: frontend.Metadata{Curr: frontend.Pipeline{Event: frontend.EventPush, Commit: frontend.Commit{Branch: "master"}}}, + with: metadata.Metadata{Curr: metadata.Pipeline{Event: metadata.EventPush, Commit: metadata.Commit{Branch: "master"}}}, want: false, }, { desc: "global branch filter", conf: "{ branch: master }", - with: frontend.Metadata{Curr: frontend.Pipeline{Event: frontend.EventPush, Commit: frontend.Commit{Branch: "master"}}}, + with: metadata.Metadata{Curr: metadata.Pipeline{Event: metadata.EventPush, Commit: metadata.Commit{Branch: "master"}}}, want: true, }, { desc: "repo constraint", conf: "{ repo: owner/* }", - with: frontend.Metadata{Curr: frontend.Pipeline{Event: frontend.EventPush}, Repo: frontend.Repo{Name: "owner/repo"}}, + with: metadata.Metadata{Curr: metadata.Pipeline{Event: metadata.EventPush}, Repo: metadata.Repo{Owner: "owner", Name: "repo"}}, want: true, }, { desc: "repo constraint", conf: "{ repo: octocat/* }", - with: frontend.Metadata{Curr: frontend.Pipeline{Event: frontend.EventPush}, Repo: frontend.Repo{Name: "owner/repo"}}, + with: metadata.Metadata{Curr: metadata.Pipeline{Event: metadata.EventPush}, Repo: metadata.Repo{Owner: "owner", Name: "repo"}}, want: false, }, { desc: "ref constraint", conf: "{ ref: refs/tags/* }", - with: frontend.Metadata{Curr: frontend.Pipeline{Commit: frontend.Commit{Ref: "refs/tags/v1.0.0"}, Event: frontend.EventPush}}, + with: metadata.Metadata{Curr: metadata.Pipeline{Commit: metadata.Commit{Ref: "refs/tags/v1.0.0"}, Event: metadata.EventPush}}, want: true, }, { desc: "ref constraint", conf: "{ ref: refs/tags/* }", - with: frontend.Metadata{Curr: frontend.Pipeline{Commit: frontend.Commit{Ref: "refs/heads/master"}, Event: frontend.EventPush}}, + with: metadata.Metadata{Curr: metadata.Pipeline{Commit: metadata.Commit{Ref: "refs/heads/master"}, Event: metadata.EventPush}}, want: false, }, { desc: "platform constraint", conf: "{ platform: linux/amd64 }", - with: frontend.Metadata{Curr: frontend.Pipeline{Event: frontend.EventPush}, Sys: frontend.System{Platform: "linux/amd64"}}, + with: metadata.Metadata{Curr: metadata.Pipeline{Event: metadata.EventPush}, Sys: metadata.System{Platform: "linux/amd64"}}, want: true, }, { desc: "platform constraint", conf: "{ repo: linux/amd64 }", - with: frontend.Metadata{Curr: frontend.Pipeline{Event: frontend.EventPush}, Sys: frontend.System{Platform: "windows/amd64"}}, + with: metadata.Metadata{Curr: metadata.Pipeline{Event: metadata.EventPush}, Sys: metadata.System{Platform: "windows/amd64"}}, want: false, }, { desc: "instance constraint", conf: "{ instance: agent.tld }", - with: frontend.Metadata{Curr: frontend.Pipeline{Event: frontend.EventPush}, Sys: frontend.System{Host: "agent.tld"}}, + with: metadata.Metadata{Curr: metadata.Pipeline{Event: metadata.EventPush}, Sys: metadata.System{Host: "agent.tld"}}, want: true, }, { desc: "instance constraint", conf: "{ instance: agent.tld }", - with: frontend.Metadata{Curr: frontend.Pipeline{Event: frontend.EventPush}, Sys: frontend.System{Host: "beta.agent.tld"}}, + with: metadata.Metadata{Curr: metadata.Pipeline{Event: metadata.EventPush}, Sys: metadata.System{Host: "beta.agent.tld"}}, want: false, }, { desc: "filter cron by default constraint", conf: "{}", - with: frontend.Metadata{Curr: frontend.Pipeline{Event: frontend.EventCron}}, + with: metadata.Metadata{Curr: metadata.Pipeline{Event: metadata.EventCron}}, want: false, }, { desc: "filter cron by matching name", conf: "{ event: cron, cron: job1 }", - with: frontend.Metadata{Curr: frontend.Pipeline{Event: frontend.EventCron, Cron: "job1"}}, + with: metadata.Metadata{Curr: metadata.Pipeline{Event: metadata.EventCron, Cron: "job1"}}, want: true, }, { desc: "filter cron by name", conf: "{ event: cron, cron: job2 }", - with: frontend.Metadata{Curr: frontend.Pipeline{Event: frontend.EventCron, Cron: "job1"}}, + with: metadata.Metadata{Curr: metadata.Pipeline{Event: metadata.EventCron, Cron: "job1"}}, want: false, }, { desc: "no constraints, event gets filtered by default event filter", conf: "", - with: frontend.Metadata{ - Curr: frontend.Pipeline{Event: "non-default"}, + with: metadata.Metadata{ + Curr: metadata.Pipeline{Event: "non-default"}, }, want: false, }, + { + desc: "filter with build-in env passes", + conf: "{ branch: ${CI_REPO_DEFAULT_BRANCH} }", + with: metadata.Metadata{ + Curr: metadata.Pipeline{Event: metadata.EventPush, Commit: metadata.Commit{Branch: "stable"}}, + Repo: metadata.Repo{Branch: "stable"}, + }, + want: true, + }, { desc: "filter by eval based on event", conf: `{ evaluate: 'CI_PIPELINE_EVENT == "push"' }`, - with: frontend.Metadata{Curr: frontend.Pipeline{Event: frontend.EventPush}}, + with: metadata.Metadata{Curr: metadata.Pipeline{Event: metadata.EventPush}}, want: true, }, { desc: "filter by eval based on event and repo", conf: `{ evaluate: 'CI_PIPELINE_EVENT == "push" && CI_REPO == "owner/repo"' }`, - with: frontend.Metadata{Curr: frontend.Pipeline{Event: frontend.EventPush}, Repo: frontend.Repo{Name: "owner/repo"}}, + with: metadata.Metadata{Curr: metadata.Pipeline{Event: metadata.EventPush}, Repo: metadata.Repo{Owner: "owner", Name: "repo"}}, want: true, }, } for _, test := range testdata { t.Run(test.desc, func(t *testing.T) { - c := parseConstraints(t, test.conf) + conf, err := frontend.EnvVarSubst(test.conf, test.with.Environ()) + assert.NoError(t, err) + c := parseConstraints(t, conf) got, err := c.Match(test.with, false) if err != nil { t.Errorf("Match returned error: %v", err) diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index f1584f5e3..bab76520c 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -12,7 +12,7 @@ import ( "golang.org/x/sync/errgroup" backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" - "github.com/woodpecker-ci/woodpecker/pipeline/frontend" + "github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata" "github.com/woodpecker-ci/woodpecker/pipeline/multipart" ) @@ -177,7 +177,7 @@ func (r *Runtime) execAll(steps []*backend.Step) <-chan error { } // add compatibility for drone-ci plugins - SetDroneEnviron(step.Environment) + metadata.SetDroneEnviron(step.Environment) logger.Debug(). Str("Step", step.Name). @@ -199,7 +199,7 @@ func (r *Runtime) execAll(steps []*backend.Step) <-chan error { // Return the error after tracing it. err = r.traceStep(processState, err, step) - if err != nil && step.Failure == frontend.FailureIgnore { + if err != nil && step.Failure == metadata.FailureIgnore { return nil } return err diff --git a/pipeline/stepBuilder.go b/pipeline/stepBuilder.go index 4a55d5a96..048fdb705 100644 --- a/pipeline/stepBuilder.go +++ b/pipeline/stepBuilder.go @@ -17,16 +17,15 @@ package pipeline import ( "fmt" - "net/url" "path/filepath" "strings" - "github.com/drone/envsubst" "github.com/oklog/ulid/v2" "github.com/rs/zerolog/log" backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" "github.com/woodpecker-ci/woodpecker/pipeline/frontend" + "github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/linter" @@ -47,6 +46,7 @@ type StepBuilder struct { Link string Yamls []*forge_types.FileMeta Envs map[string]string + Forge metadata.ServerForge } type Item struct { @@ -85,8 +85,8 @@ func (b *StepBuilder) Build() ([]*Item, error) { Name: SanitizePath(y.Name), } - metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, workflow, b.Link) - environ := b.environmentVariables(metadata, axis) + workflowMetadata := frontend.MetadataFromStruct(b.Forge, b.Repo, b.Curr, b.Last, workflow, b.Link) + environ := b.environmentVariables(workflowMetadata, axis) // add global environment variables for substituting for k, v := range b.Envs { @@ -98,7 +98,7 @@ func (b *StepBuilder) Build() ([]*Item, error) { } // substitute vars - substituted, err := b.envsubst(string(y.Data), environ) + substituted, err := frontend.EnvVarSubst(string(y.Data), environ) if err != nil { return nil, err } @@ -117,7 +117,7 @@ func (b *StepBuilder) Build() ([]*Item, error) { } // checking if filtered. - if match, err := parsed.When.Match(metadata, true); !match && err == nil { + if match, err := parsed.When.Match(workflowMetadata, true); !match && err == nil { log.Debug().Str("pipeline", workflow.Name).Msg( "Marked as skipped, dose not match metadata", ) @@ -129,15 +129,7 @@ func (b *StepBuilder) Build() ([]*Item, error) { return nil, err } - // TODO: deprecated branches filter => remove after some time - if !parsed.Branches.Match(b.Curr.Branch) && (b.Curr.Event != model.EventDeploy && b.Curr.Event != model.EventTag) { - log.Debug().Str("pipeline", workflow.Name).Msg( - "Marked as skipped, dose not match branch", - ) - workflow.State = model.StatusSkipped - } - - ir, err := b.toInternalRepresentation(parsed, environ, metadata, workflow.ID) + ir, err := b.toInternalRepresentation(parsed, environ, workflowMetadata, workflow.ID) if err != nil { return nil, err } @@ -216,17 +208,7 @@ func containsItemWithName(name string, items []*Item) bool { return false } -func (b *StepBuilder) envsubst(y string, environ map[string]string) (string, error) { - return envsubst.Eval(y, func(name string) string { - env := environ[name] - if strings.Contains(env, "\n") { - env = fmt.Sprintf("%q", env) - } - return env - }) -} - -func (b *StepBuilder) environmentVariables(metadata frontend.Metadata, axis matrix.Axis) map[string]string { +func (b *StepBuilder) environmentVariables(metadata metadata.Metadata, axis matrix.Axis) map[string]string { environ := metadata.Environ() for k, v := range axis { environ[k] = v @@ -234,7 +216,7 @@ func (b *StepBuilder) environmentVariables(metadata frontend.Metadata, axis matr return environ } -func (b *StepBuilder) toInternalRepresentation(parsed *yaml.Config, environ map[string]string, metadata frontend.Metadata, stepID int64) (*backend.Config, error) { +func (b *StepBuilder) toInternalRepresentation(parsed *yaml.Config, environ map[string]string, metadata metadata.Metadata, stepID int64) (*backend.Config, error) { var secrets []compiler.Secret for _, sec := range b.Secs { if !sec.Match(b.Curr.Event) { @@ -261,6 +243,7 @@ func (b *StepBuilder) toInternalRepresentation(parsed *yaml.Config, environ map[ return compiler.New( compiler.WithEnviron(environ), compiler.WithEnviron(b.Envs), + // TODO: server deps should be moved into StepBuilder fields and set on StepBuilder creation compiler.WithEscalated(server.Config.Pipeline.Privileged...), compiler.WithResourceLimit(server.Config.Pipeline.Limits.MemSwapLimit, server.Config.Pipeline.Limits.MemLimit, server.Config.Pipeline.Limits.ShmSize, server.Config.Pipeline.Limits.CPUQuota, server.Config.Pipeline.Limits.CPUShares, server.Config.Pipeline.Limits.CPUSet), compiler.WithVolumes(server.Config.Pipeline.Volumes...), @@ -328,87 +311,6 @@ func SetPipelineStepsOnPipeline(pipeline *model.Pipeline, pipelineItems []*Item) return pipeline } -// return the metadata from the cli context. -func metadataFromStruct(repo *model.Repo, pipeline, last *model.Pipeline, workflow *model.Step, link string) frontend.Metadata { - host := link - uri, err := url.Parse(link) - if err == nil { - host = uri.Host - } - - forge := frontend.Forge{} - if server.Config.Services.Forge != nil { - forge = frontend.Forge{ - Type: server.Config.Services.Forge.Name(), - URL: server.Config.Services.Forge.URL(), - } - } - - return frontend.Metadata{ - Repo: frontend.Repo{ - Name: repo.FullName, - Link: repo.Link, - CloneURL: repo.Clone, - Private: repo.IsSCMPrivate, - Branch: repo.Branch, - }, - Curr: metadataPipelineFromModelPipeline(pipeline, true), - Prev: metadataPipelineFromModelPipeline(last, false), - Workflow: frontend.Workflow{ - Name: workflow.Name, - Number: workflow.PID, - Matrix: workflow.Environ, - }, - Step: frontend.Step{}, - Sys: frontend.System{ - Name: "woodpecker", - Link: link, - Host: host, - Platform: "", // will be set by pipeline platform option or by agent - }, - Forge: forge, - } -} - -func metadataPipelineFromModelPipeline(pipeline *model.Pipeline, includeParent bool) frontend.Pipeline { - cron := "" - if pipeline.Event == model.EventCron { - cron = pipeline.Sender - } - - parent := int64(0) - if includeParent { - parent = pipeline.Parent - } - - return frontend.Pipeline{ - Number: pipeline.Number, - Parent: parent, - Created: pipeline.Created, - Started: pipeline.Started, - Finished: pipeline.Finished, - Status: string(pipeline.Status), - Event: string(pipeline.Event), - Link: pipeline.Link, - Target: pipeline.Deploy, - Commit: frontend.Commit{ - Sha: pipeline.Commit, - Ref: pipeline.Ref, - Refspec: pipeline.Refspec, - Branch: pipeline.Branch, - Message: pipeline.Message, - Author: frontend.Author{ - Name: pipeline.Author, - Email: pipeline.Email, - Avatar: pipeline.Avatar, - }, - ChangedFiles: pipeline.ChangedFiles, - PullRequestLabels: pipeline.PullRequestLabels, - }, - Cron: cron, - } -} - func SanitizePath(path string) string { path = filepath.Base(path) path = strings.TrimSuffix(path, ".yml") diff --git a/pipeline/stepBuilder_test.go b/pipeline/stepBuilder_test.go index 789464d42..dd9df6121 100644 --- a/pipeline/stepBuilder_test.go +++ b/pipeline/stepBuilder_test.go @@ -17,10 +17,11 @@ package pipeline import ( "fmt" - "sync" "testing" - "github.com/woodpecker-ci/woodpecker/server" + "github.com/stretchr/testify/assert" + + "github.com/woodpecker-ci/woodpecker/server/forge" "github.com/woodpecker-ci/woodpecker/server/forge/mocks" forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types" "github.com/woodpecker-ci/woodpecker/server/model" @@ -28,9 +29,9 @@ import ( func TestGlobalEnvsubst(t *testing.T) { t.Parallel() - setupMockForge(t) b := StepBuilder{ + Forge: getMockForge(t), Envs: map[string]string{ "KEY_K": "VALUE_V", "IMAGE": "scratch", @@ -63,9 +64,9 @@ pipeline: func TestMissingGlobalEnvsubst(t *testing.T) { t.Parallel() - setupMockForge(t) b := StepBuilder{ + Forge: getMockForge(t), Envs: map[string]string{ "KEY_K": "VALUE_V", "NO_IMAGE": "scratch", @@ -98,10 +99,10 @@ pipeline: func TestMultilineEnvsubst(t *testing.T) { t.Parallel() - setupMockForge(t) b := StepBuilder{ - Repo: &model.Repo{}, + Forge: getMockForge(t), + Repo: &model.Repo{}, Curr: &model.Pipeline{ Message: `aaa bbb`, @@ -136,9 +137,9 @@ pipeline: func TestMultiPipeline(t *testing.T) { t.Parallel() - setupMockForge(t) b := StepBuilder{ + Forge: getMockForge(t), Repo: &model.Repo{}, Curr: &model.Pipeline{}, Last: &model.Pipeline{}, @@ -171,9 +172,9 @@ pipeline: func TestDependsOn(t *testing.T) { t.Parallel() - setupMockForge(t) b := StepBuilder{ + Forge: getMockForge(t), Repo: &model.Repo{}, Curr: &model.Pipeline{}, Last: &model.Pipeline{}, @@ -218,9 +219,9 @@ depends_on: func TestRunsOn(t *testing.T) { t.Parallel() - setupMockForge(t) b := StepBuilder{ + Forge: getMockForge(t), Repo: &model.Repo{}, Curr: &model.Pipeline{}, Last: &model.Pipeline{}, @@ -255,9 +256,9 @@ runs_on: func TestPipelineName(t *testing.T) { t.Parallel() - setupMockForge(t) b := StepBuilder{ + Forge: getMockForge(t), Repo: &model.Repo{Config: ".woodpecker"}, Curr: &model.Pipeline{}, Last: &model.Pipeline{}, @@ -291,9 +292,9 @@ pipeline: func TestBranchFilter(t *testing.T) { t.Parallel() - setupMockForge(t) b := StepBuilder{ + Forge: getMockForge(t), Repo: &model.Repo{}, Curr: &model.Pipeline{Branch: "dev"}, Last: &model.Pipeline{}, @@ -320,27 +321,19 @@ pipeline: if err != nil { t.Fatal(err) } - if len(pipelineItems) != 2 { - t.Fatal("Should have generated 2 pipeline") + if !assert.Len(t, pipelineItems, 1) { + t.Fatal("Should have generated 1 pipeline") } - if pipelineItems[0].Workflow.State != model.StatusSkipped { - t.Fatal("Should not run on dev branch") - } - for _, child := range pipelineItems[0].Workflow.Children { - if child.State != model.StatusSkipped { - t.Fatal("Children should skipped status too") - } - } - if pipelineItems[1].Workflow.State != model.StatusPending { + if pipelineItems[0].Workflow.State != model.StatusPending { t.Fatal("Should run on dev branch") } } func TestRootWhenFilter(t *testing.T) { t.Parallel() - setupMockForge(t) b := StepBuilder{ + Forge: getMockForge(t), Repo: &model.Repo{}, Curr: &model.Pipeline{Event: "tester"}, Last: &model.Pipeline{}, @@ -385,11 +378,11 @@ pipeline: func TestZeroSteps(t *testing.T) { t.Parallel() - setupMockForge(t) pipeline := &model.Pipeline{Branch: "dev"} b := StepBuilder{ + Forge: getMockForge(t), Repo: &model.Repo{}, Curr: pipeline, Last: &model.Pipeline{}, @@ -420,11 +413,11 @@ pipeline: func TestZeroStepsAsMultiPipelineDeps(t *testing.T) { t.Parallel() - setupMockForge(t) pipeline := &model.Pipeline{Branch: "dev"} b := StepBuilder{ + Forge: getMockForge(t), Repo: &model.Repo{}, Curr: pipeline, Last: &model.Pipeline{}, @@ -469,11 +462,11 @@ depends_on: [ zerostep ] func TestZeroStepsAsMultiPipelineTransitiveDeps(t *testing.T) { t.Parallel() - setupMockForge(t) pipeline := &model.Pipeline{Branch: "dev"} b := StepBuilder{ + Forge: getMockForge(t), Repo: &model.Repo{}, Curr: pipeline, Last: &model.Pipeline{}, @@ -524,13 +517,13 @@ depends_on: [ shouldbefiltered ] func TestTree(t *testing.T) { t.Parallel() - setupMockForge(t) pipeline := &model.Pipeline{ Event: model.EventPush, } b := StepBuilder{ + Forge: getMockForge(t), Repo: &model.Repo{}, Curr: pipeline, Last: &model.Pipeline{}, @@ -565,7 +558,6 @@ pipeline: func TestSanitizePath(t *testing.T) { t.Parallel() - setupMockForge(t) testTable := []struct { path string @@ -604,14 +596,9 @@ func TestSanitizePath(t *testing.T) { } } -var setupMockForgeLock = sync.Once{} - -func setupMockForge(t *testing.T) { - setupMockForgeLock.Do(func() { - forge := mocks.NewForge(t) - forge.On("Name").Return("mock") - forge.On("URL").Return("https://codeberg.org") - - server.Config.Services.Forge = forge - }) +func getMockForge(t *testing.T) forge.Forge { + forge := mocks.NewForge(t) + forge.On("Name").Return("mock") + forge.On("URL").Return("https://codeberg.org") + return forge } diff --git a/server/pipeline/create.go b/server/pipeline/create.go index 477e22e9f..17eb45002 100644 --- a/server/pipeline/create.go +++ b/server/pipeline/create.go @@ -63,7 +63,7 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline configFetcher := forge.NewConfigFetcher(server.Config.Services.Forge, server.Config.Services.Timeout, server.Config.Services.ConfigService, repoUser, repo, pipeline) forgeYamlConfigs, configFetchErr = configFetcher.Fetch(ctx) if configFetchErr == nil { - filtered, parseErr = checkIfFiltered(pipeline, forgeYamlConfigs) + filtered, parseErr = checkIfFiltered(repo, pipeline, forgeYamlConfigs) if parseErr == nil { if filtered { err := ErrFiltered{Msg: "branch does not match restrictions defined in yaml"} diff --git a/server/pipeline/filter.go b/server/pipeline/filter.go index 56458ba93..5f4e0a91f 100644 --- a/server/pipeline/filter.go +++ b/server/pipeline/filter.go @@ -22,6 +22,7 @@ import ( "github.com/woodpecker-ci/woodpecker/pipeline" "github.com/woodpecker-ci/woodpecker/pipeline/frontend" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml" + "github.com/woodpecker-ci/woodpecker/server" forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types" "github.com/woodpecker-ci/woodpecker/server/model" ) @@ -36,6 +37,7 @@ func zeroSteps(currentPipeline *model.Pipeline, forgeYamlConfigs []*forge_types. Regs: []*model.Registry{}, Link: "", Yamls: forgeYamlConfigs, + Forge: server.Config.Services.Forge, } pipelineItems, err := b.Build() @@ -51,22 +53,20 @@ func zeroSteps(currentPipeline *model.Pipeline, forgeYamlConfigs []*forge_types. // TODO: parse yaml once and not for each filter function // Check if at least one pipeline step will be execute otherwise we will just ignore this webhook -func checkIfFiltered(pipeline *model.Pipeline, forgeYamlConfigs []*forge_types.FileMeta) (bool, error) { - log.Trace().Msgf("hook.branchFiltered(): pipeline branch: '%s' pipeline event: '%s' config count: %d", pipeline.Branch, pipeline.Event, len(forgeYamlConfigs)) +func checkIfFiltered(repo *model.Repo, p *model.Pipeline, forgeYamlConfigs []*forge_types.FileMeta) (bool, error) { + log.Trace().Msgf("hook.branchFiltered(): pipeline branch: '%s' pipeline event: '%s' config count: %d", p.Branch, p.Event, len(forgeYamlConfigs)) - matchMetadata := frontend.Metadata{ - Curr: frontend.Pipeline{ - Event: string(pipeline.Event), - Commit: frontend.Commit{ - Branch: pipeline.Branch, - }, - }, - } + matchMetadata := frontend.MetadataFromStruct(server.Config.Services.Forge, repo, p, nil, nil, "") for _, forgeYamlConfig := range forgeYamlConfigs { - parsedPipelineConfig, err := yaml.ParseBytes(forgeYamlConfig.Data) + substitutedConfigData, err := frontend.EnvVarSubst(string(forgeYamlConfig.Data), matchMetadata.Environ()) if err != nil { - log.Trace().Msgf("parse config '%s': %s", forgeYamlConfig.Name, err) + log.Trace().Err(err).Msgf("failed to substitute config '%s'", forgeYamlConfig.Name) + return false, err + } + parsedPipelineConfig, err := yaml.ParseString(substitutedConfigData) + if err != nil { + log.Trace().Err(err).Msgf("failed to parse config '%s'", forgeYamlConfig.Name) return false, err } log.Trace().Msgf("config '%s': %#v", forgeYamlConfig.Name, parsedPipelineConfig) @@ -78,11 +78,6 @@ func checkIfFiltered(pipeline *model.Pipeline, forgeYamlConfigs []*forge_types.F return false, err } - // ignore if the pipeline was filtered by the branch (legacy) - if !parsedPipelineConfig.Branches.Match(pipeline.Branch) { - continue - } - // at least one config yielded in a valid run. return false, nil } diff --git a/server/pipeline/items.go b/server/pipeline/items.go index ddf19f67a..6c0b67cc5 100644 --- a/server/pipeline/items.go +++ b/server/pipeline/items.go @@ -77,6 +77,7 @@ func createPipelineItems(c context.Context, store store.Store, Envs: envs, Link: server.Config.Server.Host, Yamls: yamls, + Forge: server.Config.Services.Forge, } pipelineItems, err := b.Build() if err != nil {