go/goreleaser
go/goreleaser
1
0
Fork 0
mirror of https://github.com/goreleaser/goreleaser.git synced 2025-03-11 14:39:28 +02:00
Code Issues Releases Activity

feat: auto-refresh checksums (#2573)

Browse Source 
* fix: refresh checksums

Signed-off-by: Carlos A Becker <caarlos0@gmail.com>

* fix: better logs

Signed-off-by: Carlos A Becker <caarlos0@gmail.com>

* fix: keep no art behavior, add more tests

Signed-off-by: Carlos A Becker <caarlos0@gmail.com>

* fix: refresh in more places

Signed-off-by: Carlos A Becker <caarlos0@gmail.com>

* chore: fmt

Signed-off-by: Carlos A Becker <caarlos0@gmail.com>

* fix: refresh in the end

Signed-off-by: Carlos A Becker <caarlos0@gmail.com>

* fix: marshal extra with refresh

Signed-off-by: Carlos A Becker <caarlos0@gmail.com>

* fix: signature

Signed-off-by: Carlos A Becker <caarlos0@gmail.com>
This commit is contained in:
Carlos Alexandro Becker 2021-12-05 22:42:13 -03:00 committed by GitHub
parent 73867736a5
commit cbcdd41f97
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 283 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
internal/artifact/artifact.go
View File

@ -8,6 +8,7 @@ import (
"crypto/sha256"
"crypto/sha512"
"encoding/hex"
"encoding/json"
"fmt"
"hash"
"hash/crc32"
@ -111,18 +112,35 @@ const (
ExtraFormat = "Format"
ExtraWrappedIn = "WrappedIn"
ExtraBinaries = "Binaries"
ExtraRefresh = "Refresh"
)
// Extras represents the extra fields in an artifact.
type Extras map[string]interface{}
func (e Extras) MarshalJSON() ([]byte, error) {
m := map[string]interface{}{}
for k, v := range e {
if k == ExtraRefresh {
// refresh is a func, so we can't serialize it.
// set v to a string representation of the function signature instead.
v = "func() error"
}
m[k] = v
}
return json.Marshal(m)
}
// Artifact represents an artifact and its relevant info.
type Artifact struct {
Name string `json:"name,omitempty"`
Path string `json:"path,omitempty"`
Goos string `json:"goos,omitempty"`
Goarch string `json:"goarch,omitempty"`
Goarm string `json:"goarm,omitempty"`
Gomips string `json:"gomips,omitempty"`
Type Type `json:"type,omitempty"`
Extra map[string]interface{} `json:"extra,omitempty"`
Name string `json:"name,omitempty"`
Path string `json:"path,omitempty"`
Goos string `json:"goos,omitempty"`
Goarch string `json:"goarch,omitempty"`
Goarm string `json:"goarm,omitempty"`
Gomips string `json:"gomips,omitempty"`
Type Type `json:"type,omitempty"`
Extra Extras `json:"extra,omitempty"`
}
// ExtraOr returns the Extra field with the given key or the or value specified
@ -169,6 +187,25 @@ func (a Artifact) Checksum(algorithm string) (string, error) {
return hex.EncodeToString(h.Sum(nil)), nil
}
var noRefresh = func() error { return nil }
// Refresh executes a Refresh extra function on artifacts, if it exists.
func (a Artifact) Refresh() error {
// for now lets only do it for checksums, as we know for a fact that
// they are the only ones that support this right now.
if a.Type != Checksum {
return nil
}
fn, ok := a.ExtraOr(ExtraRefresh, noRefresh).(func() error)
if !ok {
return nil
}
if err := fn(); err != nil {
return fmt.Errorf("failed to refresh %q: %w", a.Name, err)
}
return nil
}
// ID returns the artifact ID if it exists, empty otherwise.
func (a Artifact) ID() string {
return a.ExtraOr(ExtraID, "").(string)
@ -355,3 +392,16 @@ func (artifacts Artifacts) Paths() []string {
}
return result
}
// VisitFn is a function that can be executed against each artifact in a list.
type VisitFn func(a *Artifact) error
// Visit executes the given function for each artifact in the list.
func (artifacts Artifacts) Visit(fn VisitFn) error {
for _, artifact := range artifacts.List() {
if err := fn(artifact); err != nil {
return err
}
}
return nil
}

122
internal/artifact/artifact_test.go
View File

@ -1,11 +1,13 @@
package artifact
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"testing"
"github.com/goreleaser/goreleaser/internal/golden"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
)
@ -390,3 +392,123 @@ func TestPaths(t *testing.T) {
}
require.ElementsMatch(t, paths, artifacts.Paths())
}
func TestRefresher(t *testing.T) {
t.Run("ok", func(t *testing.T) {
artifacts := New()
path := filepath.Join(t.TempDir(), "f")
artifacts.Add(&Artifact{
Name: "f",
Path: path,
Type: Checksum,
Extra: map[string]interface{}{
"Refresh": func() error {
return os.WriteFile(path, []byte("hello"), 0o765)
},
},
})
artifacts.Add(&Artifact{
Name: "invalid",
Type: Checksum,
Extra: map[string]interface{}{
"Refresh": func() {
t.Fatalf("should not have been called")
},
},
})
artifacts.Add(&Artifact{
Name: "no refresh",
Type: Checksum,
})
for _, item := range artifacts.List() {
require.NoError(t, item.Refresh())
}
bts, err := os.ReadFile(path)
require.NoError(t, err)
require.Equal(t, "hello", string(bts))
})
t.Run("nok", func(t *testing.T) {
artifacts := New()
artifacts.Add(&Artifact{
Name: "fail",
Type: Checksum,
Extra: map[string]interface{}{
"ID": "nok",
"Refresh": func() error {
return fmt.Errorf("fake err")
},
},
})
for _, item := range artifacts.List() {
require.EqualError(t, item.Refresh(), `failed to refresh "fail": fake err`)
}
})
t.Run("not a checksum", func(t *testing.T) {
artifacts := New()
artifacts.Add(&Artifact{
Name: "will be ignored",
Type: Binary,
Extra: map[string]interface{}{
"ID": "ignored",
"Refresh": func() error {
return fmt.Errorf("err that should not happen")
},
},
})
for _, item := range artifacts.List() {
require.NoError(t, item.Refresh())
}
})
}
func TestVisit(t *testing.T) {
artifacts := New()
artifacts.Add(&Artifact{
Name: "foo",
Type: Checksum,
})
artifacts.Add(&Artifact{
Name: "foo",
Type: Binary,
})
t.Run("ok", func(t *testing.T) {
require.NoError(t, artifacts.Visit(func(a *Artifact) error {
require.Equal(t, "foo", a.Name)
return nil
}))
})
t.Run("nok", func(t *testing.T) {
require.EqualError(t, artifacts.Visit(func(a *Artifact) error {
return fmt.Errorf("fake err")
}), `fake err`)
})
}
func TestMarshalJSON(t *testing.T) {
artifacts := New()
artifacts.Add(&Artifact{
Name: "foo",
Type: Binary,
Extra: map[string]interface{}{
ExtraID: "adsad",
},
})
artifacts.Add(&Artifact{
Name: "foo",
Type: Checksum,
Extra: map[string]interface{}{
ExtraRefresh: func() error { return nil },
},
})
bts, err := json.Marshal(artifacts.List())
require.NoError(t, err)
golden.RequireEqualJSON(t, bts)
}

1
internal/artifact/testdata/TestMarshalJSON.json.golden vendored Normal file
View File

@ -0,0 +1 @@
[{"name":"foo","type":"Binary","extra":{"ID":"adsad"}},{"name":"foo","type":"Checksum","extra":{"Refresh":"func() error"}}]

4
internal/pipe/blob/upload.go
View File

@ -102,9 +102,7 @@ func doUpload(ctx *context.Context, conf config.Blob) error {
dataFile := artifact.Path
uploadFile := path.Join(folder, artifact.Name)
err := uploadData(ctx, conf, up, dataFile, uploadFile, bucketURL)
return err
return uploadData(ctx, conf, up, dataFile, uploadFile, bucketURL)
})
}

45
internal/pipe/checksums/checksums.go
View File

@ -3,6 +3,7 @@
package checksums
import (
"errors"
"fmt"
"os"
"path/filepath"
@ -17,6 +18,8 @@ import (
"github.com/goreleaser/goreleaser/pkg/context"
)
var errNoArtifacts = errors.New("there are no artifacts to sign")
// Pipe for checksums.
type Pipe struct{}
@ -35,7 +38,33 @@ func (Pipe) Default(ctx *context.Context) error {
}
// Run the pipe.
func (Pipe) Run(ctx *context.Context) (err error) {
func (Pipe) Run(ctx *context.Context) error {
filename, err := tmpl.New(ctx).Apply(ctx.Config.Checksum.NameTemplate)
if err != nil {
return err
}
filepath := filepath.Join(ctx.Config.Dist, filename)
if err := refresh(ctx, filepath); err != nil {
if errors.Is(err, errNoArtifacts) {
return nil
}
return err
}
ctx.Artifacts.Add(&artifact.Artifact{
Type: artifact.Checksum,
Path: filepath,
Name: filename,
Extra: map[string]interface{}{
artifact.ExtraRefresh: func() error {
log.WithField("file", filename).Info("refreshing checksums")
return refresh(ctx, filepath)
},
},
})
return nil
}
func refresh(ctx *context.Context, filepath string) error {
filter := artifact.Or(
artifact.ByType(artifact.UploadableArchive),
artifact.ByType(artifact.UploadableBinary),
@ -62,7 +91,7 @@ func (Pipe) Run(ctx *context.Context) (err error) {
}
if len(artifactList) == 0 {
return nil
return errNoArtifacts
}
g := semerrgroup.New(ctx.Parallelism)
@ -85,12 +114,8 @@ func (Pipe) Run(ctx *context.Context) (err error) {
return err
}
filename, err := tmpl.New(ctx).Apply(ctx.Config.Checksum.NameTemplate)
if err != nil {
return err
}
file, err := os.OpenFile(
filepath.Join(ctx.Config.Dist, filename),
filepath,
os.O_APPEND|os.O_WRONLY|os.O_CREATE|os.O_TRUNC,
0o644,
)
@ -99,12 +124,6 @@ func (Pipe) Run(ctx *context.Context) (err error) {
}
defer file.Close()
ctx.Artifacts.Add(&artifact.Artifact{
Type: artifact.Checksum,
Path: file.Name(),
Name: filename,
})
// sort to ensure the signature is deterministic downstream
sort.Strings(sumLines)
_, err = file.WriteString(strings.Join(sumLines, ""))

65
internal/pipe/checksums/checksums_test.go
View File

@ -3,6 +3,7 @@ package checksums
import (
"os"
"path/filepath"
"strings"
"testing"
"github.com/goreleaser/goreleaser/internal/artifact"
@ -20,27 +21,28 @@ func TestPipe(t *testing.T) {
const archive = binary + ".tar.gz"
const linuxPackage = binary + ".rpm"
const checksums = binary + "_bar_checksums.txt"
const sum = "61d034473102d7dac305902770471fd50f4c5b26f6831a56dd90b5184b3c30fc "
tests := map[string]struct {
ids []string
want []string
want string
}{
"default": {
want: []string{
binary,
archive,
linuxPackage,
},
want: strings.Join([]string{
sum + binary,
sum + linuxPackage,
sum + archive,
}, "\n") + "\n",
},
"select ids": {
ids: []string{
"id-1",
"id-2",
},
want: []string{
binary,
archive,
},
want: strings.Join([]string{
sum + binary,
sum + archive,
}, "\n") + "\n",
},
}
for name, tt := range tests {
@ -89,17 +91,50 @@ func TestPipe(t *testing.T) {
var artifacts []string
for _, a := range ctx.Artifacts.List() {
artifacts = append(artifacts, a.Name)
require.NoError(t, a.Refresh(), "refresh should not fail and yield same results as nothing changed")
}
require.Contains(t, artifacts, checksums, binary)
bts, err := os.ReadFile(filepath.Join(folder, checksums))
require.NoError(t, err)
for _, want := range tt.want {
require.Contains(t, string(bts), "61d034473102d7dac305902770471fd50f4c5b26f6831a56dd90b5184b3c30fc "+want)
}
require.Contains(t, tt.want, string(bts))
})
}
}
func TestRefreshModifying(t *testing.T) {
const binary = "binary"
folder := t.TempDir()
file := filepath.Join(folder, binary)
require.NoError(t, os.WriteFile(file, []byte("some string"), 0o644))
ctx := context.New(
config.Project{
Dist: folder,
ProjectName: binary,
Checksum: config.Checksum{
NameTemplate: "{{ .ProjectName }}_{{ .Env.FOO }}_checksums.txt",
Algorithm: "sha256",
},
},
)
ctx.Git.CurrentTag = "1.2.3"
ctx.Env = map[string]string{"FOO": "bar"}
ctx.Artifacts.Add(&artifact.Artifact{
Name: binary,
Path: file,
Type: artifact.UploadableBinary,
})
require.NoError(t, Pipe{}.Run(ctx))
checks := ctx.Artifacts.Filter(artifact.ByType(artifact.Checksum)).List()
require.Len(t, checks, 1)
previous, err := os.ReadFile(checks[0].Path)
require.NoError(t, err)
require.NoError(t, os.WriteFile(file, []byte("some other string"), 0o644))
require.NoError(t, checks[0].Refresh())
current, err := os.ReadFile(checks[0].Path)
require.NoError(t, err)
require.NotEqual(t, string(previous), string(current))
}
func TestPipeFileNotExist(t *testing.T) {
folder := t.TempDir()
ctx := context.New(
@ -188,7 +223,9 @@ func TestPipeCouldNotOpenChecksumsTxt(t *testing.T) {
}
func TestPipeWhenNoArtifacts(t *testing.T) {
ctx := &context.Context{}
ctx := &context.Context{
Artifacts: artifact.New(),
}
require.NoError(t, Pipe{}.Run(ctx))
require.Len(t, ctx.Artifacts.List(), 0)
}

13
internal/pipe/sign/sign.go
View File

@ -95,11 +95,22 @@ func (Pipe) Run(ctx *context.Context) error {
return sign(ctx, cfg, ctx.Artifacts.Filter(artifact.And(filters...)).List())
})
}
return g.Wait()
if err := g.Wait(); err != nil {
return err
}
return ctx.Artifacts.
Filter(artifact.ByType(artifact.Checksum)).
Visit(func(a *artifact.Artifact) error {
return a.Refresh()
})
}
func sign(ctx *context.Context, cfg config.Sign, artifacts []*artifact.Artifact) error {
for _, a := range artifacts {
if err := a.Refresh(); err != nil {
return err
}
artifacts, err := signone(ctx, cfg, a)
if err != nil {
return err

6
internal/pipe/sign/sign_test.go
View File

@ -612,6 +612,12 @@ func testSign(tb testing.TB, ctx *context.Context, certificateNames, signaturePa
Name: "checksum2",
Path: filepath.Join(tmpdir, "checksum2"),
Type: artifact.Checksum,
Extra: map[string]interface{}{
"Refresh": func() error {
file := filepath.Join(tmpdir, "checksum2")
return os.WriteFile(file, []byte("foo"), 0o644)
},
},
})
ctx.Artifacts.Add(&artifact.Artifact{
Name: "artifact4_1.0.0_linux_amd64",