fix: source archive add subfolders (#3343)

`--add-file` adds only the `--prefix` joined with the basename of the added file, so, adding a folder like `vendor` would break havoc.

this basically writes our own source archives (which are more compressed) and allows to add an entire folder easily, as well as other mappings as `archives` already supports. 

refs #3102 #2911

Signed-off-by: Carlos A Becker <caarlos0@users.noreply.github.com>

internal/pipe/sourcearchive/source.go

@@ -2,13 +2,17 @@
 package sourcearchive
 
 import (
+	"fmt"
+	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/caarlos0/log"
 	"github.com/goreleaser/goreleaser/internal/archivefiles"
 	"github.com/goreleaser/goreleaser/internal/artifact"
 	"github.com/goreleaser/goreleaser/internal/git"
 	"github.com/goreleaser/goreleaser/internal/tmpl"
+	"github.com/goreleaser/goreleaser/pkg/archive"
 	"github.com/goreleaser/goreleaser/pkg/config"
 	"github.com/goreleaser/goreleaser/pkg/context"
 )
@@ -33,31 +37,54 @@ func (Pipe) Run(ctx *context.Context) (err error) {
 	filename := name + "." + ctx.Config.Source.Format
 	path := filepath.Join(ctx.Config.Dist, filename)
 	log.WithField("file", filename).Info("creating source archive")
-	args := []string{
+
-		"archive",
+	out, err := git.Run(ctx, "ls-files")
-		"-o", path,
+	if err != nil {
-		"--format", ctx.Config.Source.Format,
+		return fmt.Errorf("could not list source files: %w", err)
 	}
 
-	if ctx.Config.Source.PrefixTemplate != "" {
+	prefix, err := tmpl.New(ctx).Apply(ctx.Config.Source.PrefixTemplate)
-		prefix, err := tmpl.New(ctx).Apply(ctx.Config.Source.PrefixTemplate)
+	if err != nil {
-		if err != nil {
+		return err
-			return err
+	}
+
+	af, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("could not create archive: %w", err)
+	}
+	defer af.Close() //nolint:errcheck
+
+	arch, err := archive.New(af, ctx.Config.Source.Format)
+	if err != nil {
+		return err
+	}
+
+	var ff []config.File
+	for _, f := range strings.Split(out, "\n") {
+		if strings.TrimSpace(f) == "" {
+			continue
 		}
-		args = append(args, "--prefix", prefix)
+		ff = append(ff, config.File{
+			Source: f,
+		})
 	}
-
+	files, err := archivefiles.Eval(tmpl.New(ctx), append(ff, ctx.Config.Source.Files...))
-	files, err := evalFiles(ctx)
 	if err != nil {
 		return err
 	}
 	for _, f := range files {
-		args = append(args, "--add-file", f)
+		f.Destination = filepath.Join(prefix, f.Destination)
+		if err := arch.Add(f); err != nil {
+			return fmt.Errorf("could not add %q to archive: %w", f.Source, err)
+		}
 	}
 
-	args = append(args, ctx.Git.FullCommit)
+	if err := arch.Close(); err != nil {
-	out, err := git.Clean(git.Run(ctx, args...))
+		return fmt.Errorf("could not close archive file: %w", err)
-	log.Debug(out)
+	}
+	if err := af.Close(); err != nil {
+		return fmt.Errorf("could not close archive file: %w", err)
+	}
 
 	ctx.Artifacts.Add(&artifact.Artifact{
 		Type: artifact.UploadableSourceArchive,
@@ -70,41 +97,6 @@ func (Pipe) Run(ctx *context.Context) (err error) {
 	return err
 }
 
-// to reuse the archivefiles packages, we do something funky:
-// - convert the []string to []config.File
-// - eval it in archivefiles
-// - convert it back to []string
-//
-// we also handle files already tracked, as if we add them again,
-// they'll get duplicated in the archive.
-func evalFiles(ctx *context.Context) ([]string, error) {
-	var files []config.File
-	for _, f := range ctx.Config.Source.Files {
-		files = append(files, config.File{
-			Source: f,
-		})
-	}
-	addFiles, err := archivefiles.Eval(tmpl.New(ctx), files)
-	if err != nil {
-		return nil, err
-	}
-
-	var result []string
-	for _, f := range addFiles {
-		if isTracked(ctx, f.Source) {
-			continue
-		}
-		result = append(result, f.Source)
-	}
-	return result, nil
-}
-
-// check if file is tracked, and, if it is we should not add it to the archive again.
-func isTracked(ctx *context.Context, path string) bool {
-	_, err := git.Run(ctx, "ls-files", "--error-unmatch", path)
-	return err == nil
-}
-
 // Default sets the pipe defaults.
 func (Pipe) Default(ctx *context.Context) error {
 	archive := &ctx.Config.Source

internal/pipe/sourcearchive/source_test.go

@@ -27,6 +27,8 @@ func TestArchive(t *testing.T) {
 			testlib.GitCommit(t, "feat: first")
 			require.NoError(t, os.WriteFile("added-later.txt", []byte("this file was added later"), 0o655))
 			require.NoError(t, os.WriteFile("ignored.md", []byte("never added"), 0o655))
+			require.NoError(t, os.MkdirAll("subfolder", 0o755))
+			require.NoError(t, os.WriteFile("subfolder/file.md", []byte("a file within a folder, added later"), 0o655))
 
 			ctx := context.New(config.Project{
 				ProjectName: "foo",
@@ -35,8 +37,9 @@ func TestArchive(t *testing.T) {
 					Format:         format,
 					Enabled:        true,
 					PrefixTemplate: "{{ .ProjectName }}-{{ .Version }}/",
-					Files: []string{
+					Files: []config.File{
-						"*.txt",
+						{Source: "*.txt"},
+						{Source: "subfolder/*"},
 					},
 				},
 			})
@@ -65,22 +68,13 @@ func TestArchive(t *testing.T) {
 				return
 			}
 
-			f, err := os.Open(path)
+			require.ElementsMatch(t, []string{
-			require.NoError(t, err)
-			z, err := zip.NewReader(f, stat.Size())
-			require.NoError(t, err)
-
-			var paths []string
-			for _, zf := range z.File {
-				paths = append(paths, zf.Name)
-			}
-			require.Equal(t, []string{
-				"foo-1.0.0/",
 				"foo-1.0.0/README.md",
 				"foo-1.0.0/code.py",
 				"foo-1.0.0/code.txt",
 				"foo-1.0.0/added-later.txt",
-			}, paths)
+				"foo-1.0.0/subfolder/file.md",
+			}, lsZip(t, path))
 		})
 	}
 }
@@ -96,7 +90,7 @@ func TestInvalidFormat(t *testing.T) {
 		},
 	})
 	require.NoError(t, Pipe{}.Default(ctx))
-	require.EqualError(t, Pipe{}.Run(ctx), "fatal: Unknown archive format '7z'")
+	require.EqualError(t, Pipe{}.Run(ctx), "invalid archive format: 7z")
 }
 
 func TestDefault(t *testing.T) {
@@ -115,7 +109,45 @@ func TestInvalidNameTemplate(t *testing.T) {
 			NameTemplate: "{{ .foo }-asdda",
 		},
 	})
-	require.EqualError(t, Pipe{}.Run(ctx), "template: tmpl:1: unexpected \"}\" in operand")
+	testlib.RequireTemplateError(t, Pipe{}.Run(ctx))
+}
+
+func TestInvalidInvalidFileTemplate(t *testing.T) {
+	testlib.Mktmp(t)
+	require.NoError(t, os.Mkdir("dist", 0o744))
+
+	testlib.GitInit(t)
+	require.NoError(t, os.WriteFile("code.txt", []byte("not really code"), 0o655))
+	testlib.GitAdd(t)
+	testlib.GitCommit(t, "feat: first")
+
+	ctx := context.New(config.Project{
+		ProjectName: "foo",
+		Dist:        "dist",
+		Source: config.Source{
+			Format:  "tar.gz",
+			Enabled: true,
+			Files: []config.File{
+				{Source: "{{.Test}"},
+			},
+		},
+	})
+	ctx.Git.FullCommit = "HEAD"
+	ctx.Version = "1.0.0"
+	require.NoError(t, Pipe{}.Default(ctx))
+	testlib.RequireTemplateError(t, Pipe{}.Run(ctx))
+}
+
+func TestInvalidPrefixTemplate(t *testing.T) {
+	ctx := context.New(config.Project{
+		Dist: t.TempDir(),
+		Source: config.Source{
+			Enabled:        true,
+			PrefixTemplate: "{{ .ProjectName }/",
+		},
+	})
+	require.NoError(t, Pipe{}.Default(ctx))
+	testlib.RequireTemplateError(t, Pipe{}.Run(ctx))
 }
 
 func TestDisabled(t *testing.T) {
@@ -136,3 +168,24 @@ func TestSkip(t *testing.T) {
 		require.False(t, Pipe{}.Skip(ctx))
 	})
 }
+
+func TestString(t *testing.T) {
+	require.NotEmpty(t, Pipe{}.String())
+}
+
+func lsZip(tb testing.TB, path string) []string {
+	tb.Helper()
+
+	stat, err := os.Stat(path)
+	require.NoError(tb, err)
+	f, err := os.Open(path)
+	require.NoError(tb, err)
+	z, err := zip.NewReader(f, stat.Size())
+	require.NoError(tb, err)
+
+	var paths []string
+	for _, zf := range z.File {
+		paths = append(paths, zf.Name)
+	}
+	return paths
+}

pkg/config/config.go

@@ -876,11 +876,11 @@ type Publisher struct {
 
 // Source configuration.
 type Source struct {
-	NameTemplate   string   `yaml:"name_template,omitempty" json:"name_template,omitempty"`
+	NameTemplate   string `yaml:"name_template,omitempty" json:"name_template,omitempty"`
-	Format         string   `yaml:"format,omitempty" json:"format,omitempty"`
+	Format         string `yaml:"format,omitempty" json:"format,omitempty"`
-	Enabled        bool     `yaml:"enabled,omitempty" json:"enabled,omitempty"`
+	Enabled        bool   `yaml:"enabled,omitempty" json:"enabled,omitempty"`
-	PrefixTemplate string   `yaml:"prefix_template,omitempty" json:"prefix_template,omitempty"`
+	PrefixTemplate string `yaml:"prefix_template,omitempty" json:"prefix_template,omitempty"`
-	Files          []string `yaml:"files,omitempty" json:"files,omitempty"`
+	Files          []File `yaml:"files,omitempty" json:"files,omitempty"`
 }
 
 // Project includes all project configuration.

www/docs/customization/source.md

@@ -25,7 +25,6 @@ source:
   prefix_template: '{{ .ProjectName }}-{{ .Version }}/'
 
   # Additional files/template/globs you want to add to the source archive.
-  # Will use --add-file of git-archive.
   # Defaults to empty.
   files:
     - LICENSE.txt
@@ -34,6 +33,22 @@ source:
     - docs/*
     - design/*.png
     - templates/**/*
+    # a more complete example, check the globbing deep dive below
+    - src: '*.md'
+      dst: docs
+      # Strip parent folders when adding files to the archive.
+      # Default: false
+      strip_parent: true
+      # File info.
+      # Not all fields are supported by all formats available formats.
+      # Defaults to the file info of the actual file if not provided.
+      info:
+        owner: root
+        group: root
+        mode: 0644
+        # format is `time.RFC3339Nano`
+        mtime: 2008-01-02T15:04:05Z
+
 ```
 
 !!! tip

www/docs/static/schema.json

@@ -2382,7 +2382,7 @@
 					},
 					"files": {
 						"items": {
-							"type": "string"
+							"$ref": "#/$defs/File"
 						},
 						"type": "array"
 					}