1
0
mirror of https://github.com/goreleaser/goreleaser.git synced 2025-10-30 23:58:09 +02:00

feat: Add support for custom publishers (#1481)

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
This commit is contained in:
Radek Simko
2020-05-10 17:03:49 +01:00
committed by GitHub
parent 31fedc4e67
commit 8749030d3b
14 changed files with 710 additions and 3 deletions

View File

@@ -261,6 +261,10 @@ func And(filters ...Filter) Filter {
// is accepted.
// You can compose filters by using the And and Or filters.
func (artifacts *Artifacts) Filter(filter Filter) Artifacts {
if filter == nil {
return *artifacts
}
var result = New()
for _, a := range artifacts.items {
if filter(a) {

View File

@@ -86,6 +86,8 @@ func TestFilter(t *testing.T) {
assert.Len(t, artifacts.Filter(ByType(Checksum)).items, 2)
assert.Len(t, artifacts.Filter(ByType(Binary)).items, 0)
assert.Len(t, artifacts.Filter(nil).items, 5)
assert.Len(t, artifacts.Filter(
And(
ByType(Checksum),

167
internal/exec/exec.go Normal file
View File

@@ -0,0 +1,167 @@
package exec
import (
"fmt"
"os/exec"
"github.com/apex/log"
"github.com/goreleaser/goreleaser/internal/artifact"
"github.com/goreleaser/goreleaser/internal/logext"
"github.com/goreleaser/goreleaser/internal/pipe"
"github.com/goreleaser/goreleaser/internal/semerrgroup"
"github.com/goreleaser/goreleaser/internal/tmpl"
"github.com/goreleaser/goreleaser/pkg/config"
"github.com/goreleaser/goreleaser/pkg/context"
"github.com/mattn/go-shellwords"
)
func Execute(ctx *context.Context, publishers []config.Publisher) error {
if ctx.SkipPublish {
return pipe.ErrSkipPublishEnabled
}
for _, p := range publishers {
log.WithField("name", p.Name).Debug("executing custom publisher")
err := executePublisher(ctx, p)
if err != nil {
return err
}
}
return nil
}
func executePublisher(ctx *context.Context, publisher config.Publisher) error {
log.Debugf("filtering %d artifacts", len(ctx.Artifacts.List()))
artifacts := filterArtifacts(ctx.Artifacts, publisher)
log.Debugf("will execute custom publisher with %d artifacts", len(artifacts))
var g = semerrgroup.New(ctx.Parallelism)
for _, artifact := range artifacts {
artifact := artifact
g.Go(func() error {
c, err := resolveCommand(ctx, publisher, artifact)
if err != nil {
return err
}
return executeCommand(c)
})
}
return g.Wait()
}
func executeCommand(c *command) error {
log.WithField("args", c.Args).
WithField("env", c.Env).
Debug("executing command")
// nolint: gosec
var cmd = exec.CommandContext(c.Ctx, c.Args[0], c.Args[1:]...)
cmd.Env = c.Env
if c.Dir != "" {
cmd.Dir = c.Dir
}
entry := log.WithField("cmd", c.Args[0])
cmd.Stderr = logext.NewErrWriter(entry)
cmd.Stdout = logext.NewWriter(entry)
log.WithField("cmd", cmd.Args).Info("publishing")
if err := cmd.Run(); err != nil {
return fmt.Errorf("publishing: %s failed: %w",
c.Args[0], err)
}
log.Debugf("command %s finished successfully", c.Args[0])
return nil
}
func filterArtifacts(artifacts artifact.Artifacts, publisher config.Publisher) []*artifact.Artifact {
filters := []artifact.Filter{
artifact.ByType(artifact.UploadableArchive),
artifact.ByType(artifact.UploadableFile),
artifact.ByType(artifact.LinuxPackage),
artifact.ByType(artifact.UploadableBinary),
}
if publisher.Checksum {
filters = append(filters, artifact.ByType(artifact.Checksum))
}
if publisher.Signature {
filters = append(filters, artifact.ByType(artifact.Signature))
}
var filter = artifact.Or(filters...)
if len(publisher.IDs) > 0 {
filter = artifact.And(filter, artifact.ByIDs(publisher.IDs...))
}
return artifacts.Filter(filter).List()
}
type command struct {
Ctx *context.Context
Dir string
Env []string
Args []string
}
// resolveCommand returns the a command based on publisher template with replaced variables
// Those variables can be replaced by the given context, goos, goarch, goarm and more
func resolveCommand(ctx *context.Context, publisher config.Publisher, artifact *artifact.Artifact) (*command, error) {
var err error
replacements := make(map[string]string)
// TODO: Replacements should be associated only with relevant artifacts/archives
archives := ctx.Config.Archives
if len(archives) > 0 {
replacements = archives[0].Replacements
}
dir := publisher.Dir
if dir != "" {
dir, err = tmpl.New(ctx).
WithArtifact(artifact, replacements).
Apply(dir)
if err != nil {
return nil, err
}
}
cmd := publisher.Cmd
if cmd != "" {
cmd, err = tmpl.New(ctx).
WithArtifact(artifact, replacements).
Apply(cmd)
if err != nil {
return nil, err
}
}
args, err := shellwords.Parse(cmd)
if err != nil {
return nil, err
}
env := make([]string, len(publisher.Env))
for i, e := range publisher.Env {
e, err = tmpl.New(ctx).
WithArtifact(artifact, replacements).
Apply(e)
if err != nil {
return nil, err
}
env[i] = e
}
return &command{
Ctx: ctx,
Dir: dir,
Env: env,
Args: args,
}, nil
}

104
internal/exec/exec_mock.go Normal file
View File

@@ -0,0 +1,104 @@
package exec
import (
"encoding/json"
"fmt"
"os"
"reflect"
"strings"
)
// nolint: gochecknoglobals
var (
MockEnvVar = "GORELEASER_MOCK_DATA"
MockCmd = os.Args[0]
)
type MockData struct {
AnyOf []MockCall `json:"any_of,omitempty"`
}
type MockCall struct {
Stdout string `json:"stdout,omitempty"`
Stderr string `json:"stderr,omitempty"`
ExpectedArgs []string `json:"args"`
ExpectedEnv []string `json:"env"`
ExitCode int `json:"exit_code"`
}
func (m *MockData) MarshalJSON() ([]byte, error) {
type t MockData
return json.Marshal((*t)(m))
}
func (m *MockData) UnmarshalJSON(b []byte) error {
type t MockData
return json.Unmarshal(b, (*t)(m))
}
// nolint: interfacer
func MarshalMockEnv(data *MockData) string {
b, err := data.MarshalJSON()
if err != nil {
errData := &MockData{
AnyOf: []MockCall{
{
Stderr: fmt.Sprintf("unable to marshal mock data: %s", err),
ExitCode: 1,
},
},
}
b, _ = errData.MarshalJSON()
}
return MockEnvVar + "=" + string(b)
}
func ExecuteMockData(jsonData string) int {
md := &MockData{}
err := md.UnmarshalJSON([]byte(jsonData))
if err != nil {
fmt.Fprintf(os.Stderr, "unable to unmarshal mock data: %s", err)
return 1
}
givenArgs := os.Args[1:]
givenEnv := filterEnv(os.Environ())
if len(md.AnyOf) == 0 {
fmt.Fprintf(os.Stderr, "no mock calls expected. args: %q, env: %q",
givenArgs, givenEnv)
return 1
}
for _, item := range md.AnyOf {
if item.ExpectedArgs == nil {
item.ExpectedArgs = []string{}
}
if item.ExpectedEnv == nil {
item.ExpectedEnv = []string{}
}
if reflect.DeepEqual(item.ExpectedArgs, givenArgs) &&
reflect.DeepEqual(item.ExpectedEnv, givenEnv) {
fmt.Fprint(os.Stdout, item.Stdout)
fmt.Fprint(os.Stderr, item.Stderr)
return item.ExitCode
}
}
fmt.Fprintf(os.Stderr, "no mock calls matched. args: %q, env: %q",
givenArgs, givenEnv)
return 1
}
func filterEnv(vars []string) []string {
for i, env := range vars {
if strings.HasPrefix(env, MockEnvVar+"=") {
return append(vars[:i], vars[i+1:]...)
}
}
return vars
}

View File

@@ -0,0 +1,15 @@
package exec
import (
"os"
"testing"
)
func TestMain(m *testing.M) {
if v := os.Getenv(MockEnvVar); v != "" {
os.Exit(ExecuteMockData(v))
return
}
os.Exit(m.Run())
}

233
internal/exec/exec_test.go Normal file
View File

@@ -0,0 +1,233 @@
package exec
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/goreleaser/goreleaser/internal/artifact"
"github.com/goreleaser/goreleaser/pkg/config"
"github.com/goreleaser/goreleaser/pkg/context"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestExecute(t *testing.T) {
ctx := context.New(config.Project{
ProjectName: "blah",
Archives: []config.Archive{
{
Replacements: map[string]string{
"linux": "Linux",
},
},
},
})
ctx.Env["TEST_A_SECRET"] = "x"
ctx.Env["TEST_A_USERNAME"] = "u2"
ctx.Version = "2.1.0"
// Preload artifacts
ctx.Artifacts = artifact.New()
folder, err := ioutil.TempDir("", "goreleasertest")
require.NoError(t, err)
defer os.RemoveAll(folder)
for _, a := range []struct {
id string
ext string
typ artifact.Type
}{
{"docker", "---", artifact.DockerImage},
{"debpkg", "deb", artifact.LinuxPackage},
{"binary", "bin", artifact.Binary},
{"archive", "tar", artifact.UploadableArchive},
{"ubinary", "ubi", artifact.UploadableBinary},
{"checksum", "sum", artifact.Checksum},
{"signature", "sig", artifact.Signature},
} {
var file = filepath.Join(folder, "a."+a.ext)
require.NoError(t, ioutil.WriteFile(file, []byte("lorem ipsum"), 0644))
ctx.Artifacts.Add(&artifact.Artifact{
Name: "a." + a.ext,
Goos: "linux",
Goarch: "amd64",
Path: file,
Type: a.typ,
Extra: map[string]interface{}{
"ID": a.id,
},
})
}
testCases := []struct {
name string
publishers []config.Publisher
expectErr error
}{
{
"filter by IDs",
[]config.Publisher{
{
Name: "test",
IDs: []string{"archive"},
Cmd: MockCmd + " {{ .ArtifactName }}",
Env: []string{
MarshalMockEnv(&MockData{
AnyOf: []MockCall{
{ExpectedArgs: []string{"a.tar"}, ExitCode: 0},
},
}),
},
},
},
nil,
},
{
"no filter",
[]config.Publisher{
{
Name: "test",
Cmd: MockCmd + " {{ .ArtifactName }}",
Env: []string{
MarshalMockEnv(&MockData{
AnyOf: []MockCall{
{ExpectedArgs: []string{"a.deb"}, ExitCode: 0},
{ExpectedArgs: []string{"a.ubi"}, ExitCode: 0},
{ExpectedArgs: []string{"a.tar"}, ExitCode: 0},
},
}),
},
},
},
nil,
},
{
"include checksum",
[]config.Publisher{
{
Name: "test",
Checksum: true,
Cmd: MockCmd + " {{ .ArtifactName }}",
Env: []string{
MarshalMockEnv(&MockData{
AnyOf: []MockCall{
{ExpectedArgs: []string{"a.deb"}, ExitCode: 0},
{ExpectedArgs: []string{"a.ubi"}, ExitCode: 0},
{ExpectedArgs: []string{"a.tar"}, ExitCode: 0},
{ExpectedArgs: []string{"a.sum"}, ExitCode: 0},
},
}),
},
},
},
nil,
},
{
"include signatures",
[]config.Publisher{
{
Name: "test",
Signature: true,
Cmd: MockCmd + " {{ .ArtifactName }}",
Env: []string{
MarshalMockEnv(&MockData{
AnyOf: []MockCall{
{ExpectedArgs: []string{"a.deb"}, ExitCode: 0},
{ExpectedArgs: []string{"a.ubi"}, ExitCode: 0},
{ExpectedArgs: []string{"a.tar"}, ExitCode: 0},
{ExpectedArgs: []string{"a.sig"}, ExitCode: 0},
},
}),
},
},
},
nil,
},
{
"try dir templating",
[]config.Publisher{
{
Name: "test",
Signature: true,
IDs: []string{"debpkg"},
Dir: "{{ dir .ArtifactPath }}",
Cmd: MockCmd + " {{ .ArtifactName }}",
Env: []string{
MarshalMockEnv(&MockData{
AnyOf: []MockCall{
{ExpectedArgs: []string{"a.deb"}, ExitCode: 0},
},
}),
},
},
},
nil,
},
{
"check env templating",
[]config.Publisher{
{
Name: "test",
IDs: []string{"debpkg"},
Cmd: MockCmd,
Env: []string{
"PROJECT={{.ProjectName}}",
"ARTIFACT={{.ArtifactName}}",
"SECRET={{.Env.TEST_A_SECRET}}",
MarshalMockEnv(&MockData{
AnyOf: []MockCall{
{
ExpectedEnv: []string{
"PROJECT=blah",
"ARTIFACT=a.deb",
"SECRET=x",
},
ExitCode: 0,
},
},
}),
},
},
},
nil,
},
{
"command error",
[]config.Publisher{
{
Name: "test",
IDs: []string{"debpkg"},
Cmd: MockCmd + " {{.ArtifactName}}",
Env: []string{
MarshalMockEnv(&MockData{
AnyOf: []MockCall{
{
ExpectedArgs: []string{"a.deb"},
Stderr: "test error",
ExitCode: 1,
},
},
}),
},
},
},
// stderr is sent to output via logger
fmt.Errorf(`publishing: %s failed: exit status 1`, MockCmd),
},
}
for i, tc := range testCases {
t.Run(fmt.Sprintf("%d-%s", i, tc.name), func(t *testing.T) {
err := Execute(ctx, tc.publishers)
if tc.expectErr == nil {
require.NoError(t, err)
return
}
if assert.Error(t, err) {
assert.Equal(t, tc.expectErr.Error(), err.Error())
}
})
}
}

View File

@@ -16,3 +16,18 @@ func (t Writer) Write(p []byte) (n int, err error) {
t.ctx.Info(string(p))
return len(p), nil
}
// Writer writes with log.Error
type ErrorWriter struct {
ctx *log.Entry
}
// NewWriter creates a new log writer
func NewErrWriter(ctx *log.Entry) ErrorWriter {
return ErrorWriter{ctx: ctx}
}
func (w ErrorWriter) Write(p []byte) (n int, err error) {
w.ctx.Error(string(p))
return len(p), nil
}

View File

@@ -0,0 +1,25 @@
// Package custompublishers provides a Pipe that executes a custom publisher
package custompublishers
import (
"github.com/goreleaser/goreleaser/internal/exec"
"github.com/goreleaser/goreleaser/internal/pipe"
"github.com/goreleaser/goreleaser/pkg/context"
)
// Pipe for custom publisher
type Pipe struct{}
// String returns the description of the pipe
func (Pipe) String() string {
return "custom publisher"
}
// Publish artifacts
func (Pipe) Publish(ctx *context.Context) error {
if len(ctx.Config.Publishers) == 0 {
return pipe.Skip("publishers section is not configured")
}
return exec.Execute(ctx, ctx.Config.Publishers)
}

View File

@@ -8,6 +8,7 @@ import (
"github.com/goreleaser/goreleaser/internal/pipe/artifactory"
"github.com/goreleaser/goreleaser/internal/pipe/blob"
"github.com/goreleaser/goreleaser/internal/pipe/brew"
"github.com/goreleaser/goreleaser/internal/pipe/custompublishers"
"github.com/goreleaser/goreleaser/internal/pipe/docker"
"github.com/goreleaser/goreleaser/internal/pipe/release"
"github.com/goreleaser/goreleaser/internal/pipe/scoop"
@@ -36,6 +37,7 @@ type Publisher interface {
var publishers = []Publisher{
blob.Pipe{},
upload.Pipe{},
custompublishers.Pipe{},
artifactory.Pipe{},
docker.Pipe{},
snapcraft.Pipe{},

View File

@@ -38,12 +38,13 @@ const (
timestamp = "Timestamp"
// artifact-only keys
os = "Os"
osKey = "Os"
arch = "Arch"
arm = "Arm"
mips = "Mips"
binary = "Binary"
artifactName = "ArtifactName"
artifactPath = "ArtifactPath"
// gitlab only
artifactUploadHash = "ArtifactUploadHash"
@@ -109,12 +110,13 @@ func (t *Template) WithArtifact(a *artifact.Artifact, replacements map[string]st
if bin == nil {
bin = t.fields[projectName]
}
t.fields[os] = replace(replacements, a.Goos)
t.fields[osKey] = replace(replacements, a.Goos)
t.fields[arch] = replace(replacements, a.Goarch)
t.fields[arm] = replace(replacements, a.Goarm)
t.fields[mips] = replace(replacements, a.Gomips)
t.fields[binary] = bin.(string)
t.fields[artifactName] = a.Name
t.fields[artifactPath] = a.Path
if val, ok := a.Extra["ArtifactUploadHash"]; ok {
t.fields[artifactUploadHash] = val
} else {
@@ -150,6 +152,7 @@ func (t *Template) Apply(s string) (string, error) {
"toupper": strings.ToUpper,
"trim": strings.TrimSpace,
"dir": filepath.Dir,
"abs": filepath.Abs,
}).
Parse(s)
if err != nil {

View File

@@ -1,12 +1,15 @@
package tmpl
import (
"os"
"path/filepath"
"testing"
"github.com/goreleaser/goreleaser/internal/artifact"
"github.com/goreleaser/goreleaser/pkg/config"
"github.com/goreleaser/goreleaser/pkg/context"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestWithArtifact(t *testing.T) {
@@ -152,6 +155,9 @@ func TestFuncMap(t *testing.T) {
var ctx = context.New(config.Project{
ProjectName: "proj",
})
wd, err := os.Getwd()
require.NoError(t, err)
ctx.Git.CurrentTag = "v1.2.4"
for _, tc := range []struct {
Template string
@@ -190,6 +196,11 @@ func TestFuncMap(t *testing.T) {
Name: "trim",
Expected: "test",
},
{
Template: `{{ abs "file" }}`,
Name: "abs",
Expected: filepath.Join(wd, "file"),
},
} {
out, err := New(ctx).Apply(tc.Template)
assert.NoError(t, err)

View File

@@ -399,6 +399,17 @@ type Upload struct {
CustomArtifactName bool `yaml:"custom_artifact_name,omitempty"`
}
// Publisher configuration
type Publisher struct {
Name string `yaml:",omitempty"`
IDs []string `yaml:"ids,omitempty"`
Checksum bool `yaml:",omitempty"`
Signature bool `yaml:",omitempty"`
Dir string `yaml:",omitempty"`
Cmd string `yaml:",omitempty"`
Env []string `yaml:",omitempty"`
}
// Source configuration
type Source struct {
NameTemplate string `yaml:"name_template,omitempty"`
@@ -423,6 +434,7 @@ type Project struct {
Artifactories []Upload `yaml:",omitempty"`
Uploads []Upload `yaml:",omitempty"`
Blobs []Blob `yaml:"blobs,omitempty"`
Publishers []Publisher `yaml:"publishers,omitempty"`
Changelog Changelog `yaml:",omitempty"`
Dist string `yaml:",omitempty"`
Signs []Sign `yaml:",omitempty"`

112
www/content/publishers.md Normal file
View File

@@ -0,0 +1,112 @@
---
title: Custom Publishers
series: customization
hideFromIndex: true
weight: 130
---
GoReleaser supports publishing artifacts by executing a custom publisher.
## How it works
You can declare multiple `publishers` instances. Each publisher will be
executed for each (filtered) artifact. For example, there will be a total of
6 executions for 2 publishers with 3 artifacts.
Publishers run sequentially in the order they're defined
and executions are parallelised between all artifacts.
In other words the publisher is expected to be safe to run
in multiple instances in parallel.
If you have only one `publishers` instance, the configuration is as easy as adding
the command to your `.goreleaser.yml` file:
```yaml
publishers:
- name: my-publisher
cmd: custom-publisher -version={{ .Version }} {{ abs .ArtifactPath }}
```
### Environment
Commands which are executed as custom publishers do not inherit any environment variables
(unlike existing hooks) as a precaution to avoid leaking sensitive data accidentally
and provide better control of the environment for each individual process
where variable names may overlap unintentionally.
You can however use `.Env.NAME` templating syntax which enables
more explicit inheritance.
```yaml
- cmd: custom-publisher
env:
- SECRET_TOKEN={{ .Env.SECRET_TOKEN }}
```
### Variables
Command (`cmd`), workdir (`dir`) and environment variables (`env`) support templating
```yaml
publishers:
- name: production
cmd: |
custom-publisher \
-product={{ .ProjectName }} \
-version={{ .Version }} \
{{ .ArtifactName }}
dir: "{{ dir .ArtifactPath }}"
env:
- TOKEN={{ .Env.CUSTOM_PUBLISHER_TOKEN }}
```
so the above example will execute `custom-publisher -product=goreleaser -version=1.0.0 goreleaser_1.0.0_linux_amd64.zip` in `/path/to/dist` with `TOKEN=token`, assuming that GoReleaser is executed with `CUSTOM_PUBLISHER_TOKEN=token`.
Supported variables:
- `Version`
- `Tag`
- `ProjectName`
- `ArtifactName`
- `ArtifactPath`
- `Os`
- `Arch`
- `Arm`
## Customization
Of course, you can customize a lot of things:
```yaml
# .goreleaser.yml
publishers:
-
# Unique name of your publisher. Used for identification
name: "custom"
# IDs of the artifacts you want to publish
ids:
- foo
- bar
# Publish checksums (defaults to false)
checksum: true
# Publish signatures (defaults to false)
signature: true
# Working directory in which to execute the command
dir: "/utils"
# Command to be executed
cmd: custom-publisher -product={{ .ProjectName }} -version={{ .Version }} {{ .ArtifactPath }}
# Environment variables
env:
- API_TOKEN=secret-token
```
These settings should allow you to push your artifacts to any number of endpoints
which may require non-trivial authentication or has otherwise complex requirements.
> Learn more about the [name template engine](/templates).

View File

@@ -40,6 +40,7 @@ may have some extra fields:
| `.Mips` | `GOMIPS` (usually allow replacements) |
| `.Binary` | Binary name |
| `.ArtifactName` | Archive name |
| `.ArtifactPath` | Relative path to artifact |
On the NFPM name template field, you can use those extra fields as well:
@@ -57,7 +58,8 @@ On all fields, you have these available functions:
| `tolower "V1.2"` | makes input string lowercase. See [ToLower](https://golang.org/pkg/strings/#ToLower) |
| `toupper "v1.2"` | makes input string uppercase. See [ToUpper](https://golang.org/pkg/strings/#ToUpper) |
| `trim " v1.2 "` | removes all leading and trailing white space. See [TrimSpace](https://golang.org/pkg/strings/#TrimSpace) |
| `dir .Path` | returns all but the last element of path, typically the path's directory. See [Dir](https://golang.org/pkg/path/filepath/#Dir)
| `dir .Path` | returns all but the last element of path, typically the path's directory. See [Dir](https://golang.org/pkg/path/filepath/#Dir) |
| `abs .ArtifactPath` | returns an absolute representation of path. See [Abs](https://golang.org/pkg/path/filepath/#Abs) |
With all those fields, you may be able to compose the name of your artifacts
pretty much the way you want: