diff --git a/internal/gio/copy.go b/internal/gio/copy.go new file mode 100644 index 000000000..365e855f3 --- /dev/null +++ b/internal/gio/copy.go @@ -0,0 +1,62 @@ +package gio + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/apex/log" +) + +// Copy recursively copies src into dst with src's file modes. +func Copy(src, dst string) error { + return CopyWithMode(src, dst, 0) +} + +// CopyWithMode recursively copies src into dst with the given mode. +// The given mode applies only to files. Their parent dirs will have the same mode as their src counterparts. +func CopyWithMode(src, dst string, mode os.FileMode) error { + return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("failed to copy %s to %s: %w", src, dst, err) + } + // We have the following: + // - src = "a/b" + // - dst = "dist/linuxamd64/b" + // - path = "a/b/c.txt" + // So we join "a/b" with "c.txt" and use it as the destination. + dst := filepath.Join(dst, strings.Replace(path, src, "", 1)) + log.WithFields(log.Fields{ + "src": path, + "dst": dst, + }).Debug("copying file") + if info.IsDir() { + return os.MkdirAll(dst, info.Mode()) + } + if mode != 0 { + return copyFile(path, dst, mode) + } + return copyFile(path, dst, info.Mode()) + }) +} + +func copyFile(src, dst string, mode os.FileMode) error { + original, err := os.Open(src) + if err != nil { + return fmt.Errorf("failed to open '%s': %w", src, err) + } + defer original.Close() + + new, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) + if err != nil { + return fmt.Errorf("failed to open '%s': %w", dst, err) + } + defer new.Close() + + if _, err := io.Copy(new, original); err != nil { + return fmt.Errorf("failed to copy: %w", err) + } + return nil +} diff --git a/internal/gio/copy_test.go b/internal/gio/copy_test.go new file mode 100644 index 000000000..f9beb12e4 --- /dev/null +++ b/internal/gio/copy_test.go @@ -0,0 +1,94 @@ +package gio + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCopy(t *testing.T) { + tmp := t.TempDir() + a := "testdata/somefile.txt" + b := tmp + "/somefile.txt" + require.NoError(t, Copy(a, b)) + requireEqualFiles(t, a, b) +} + +func TestEqualFilesModeChanged(t *testing.T) { + tmp := t.TempDir() + a := "testdata/somefile.txt" + b := tmp + "/somefile.txt" + require.NoError(t, CopyWithMode(a, b, 0o755)) + requireNotEqualFiles(t, a, b) +} + +func TestEqualFilesContentsChanged(t *testing.T) { + tmp := t.TempDir() + a := "testdata/somefile.txt" + b := tmp + "/somefile.txt" + require.NoError(t, Copy(a, b)) + require.NoError(t, os.WriteFile(b, []byte("hello world"), 0o644)) + requireNotEqualFiles(t, a, b) +} + +func TestEqualFilesDontExist(t *testing.T) { + a := "testdata/nope.txt" + b := "testdata/somefile.txt" + c := "testdata/notadir/lala" + require.Error(t, Copy(a, b)) + require.Error(t, CopyWithMode(a, b, 0o644)) + require.Error(t, Copy(b, c)) +} + +func TestCopyFile(t *testing.T) { + dir := t.TempDir() + src, err := ioutil.TempFile(dir, "src") + require.NoError(t, err) + require.NoError(t, src.Close()) + dst := filepath.Join(dir, "dst") + require.NoError(t, os.WriteFile(src.Name(), []byte("foo"), 0o644)) + require.NoError(t, Copy(src.Name(), dst)) + requireEqualFiles(t, src.Name(), dst) +} + +func TestCopyDirectory(t *testing.T) { + srcDir := t.TempDir() + dstDir := t.TempDir() + const testFile = "test" + require.NoError(t, os.WriteFile(filepath.Join(srcDir, testFile), []byte("foo"), 0o644)) + require.NoError(t, Copy(srcDir, dstDir)) + requireEqualFiles(t, filepath.Join(srcDir, testFile), filepath.Join(dstDir, testFile)) +} + +func TestCopyTwoLevelDirectory(t *testing.T) { + srcDir := t.TempDir() + dstDir := t.TempDir() + srcLevel2 := filepath.Join(srcDir, "level2") + const testFile = "test" + + require.NoError(t, os.Mkdir(srcLevel2, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(srcDir, testFile), []byte("foo"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(srcLevel2, testFile), []byte("foo"), 0o644)) + + require.NoError(t, Copy(srcDir, dstDir)) + + requireEqualFiles(t, filepath.Join(srcDir, testFile), filepath.Join(dstDir, testFile)) + requireEqualFiles(t, filepath.Join(srcLevel2, testFile), filepath.Join(dstDir, "level2", testFile)) +} + +func requireEqualFiles(tb testing.TB, a, b string) { + tb.Helper() + eq, err := EqualFiles(a, b) + require.NoError(tb, err) + require.True(tb, eq, "%s != %s", a, b) +} + +func requireNotEqualFiles(tb testing.TB, a, b string) { + tb.Helper() + eq, err := EqualFiles(a, b) + require.NoError(tb, err) + require.False(tb, eq, "%s == %s", a, b) +} diff --git a/internal/gio/hash.go b/internal/gio/hash.go new file mode 100644 index 000000000..ff70f95d7 --- /dev/null +++ b/internal/gio/hash.go @@ -0,0 +1,43 @@ +package gio + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/fs" + "os" +) + +// EqualFiles returns true if both files sha256sums and their modes are equal. +func EqualFiles(a, b string) (bool, error) { + am, as, err := sha256sum(a) + if err != nil { + return false, fmt.Errorf("could not hash %s: %w", a, err) + } + bm, bs, err := sha256sum(b) + if err != nil { + return false, fmt.Errorf("could not hash %s: %w", b, err) + } + return as == bs && am == bm, nil +} + +func sha256sum(path string) (fs.FileMode, string, error) { + f, err := os.Open(path) + if err != nil { + return 0, "", err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return 0, "", err + } + + st, err := f.Stat() + if err != nil { + return 0, "", err + } + + return st.Mode(), hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/gio/testdata/somefile.txt b/internal/gio/testdata/somefile.txt new file mode 100644 index 000000000..d9013191c --- /dev/null +++ b/internal/gio/testdata/somefile.txt @@ -0,0 +1 @@ +adasasd diff --git a/internal/pipe/docker/docker.go b/internal/pipe/docker/docker.go index 7d2ce80d4..0a218340e 100644 --- a/internal/pipe/docker/docker.go +++ b/internal/pipe/docker/docker.go @@ -12,6 +12,7 @@ import ( "github.com/apex/log" "github.com/goreleaser/goreleaser/internal/artifact" "github.com/goreleaser/goreleaser/internal/deprecate" + "github.com/goreleaser/goreleaser/internal/gio" "github.com/goreleaser/goreleaser/internal/pipe" "github.com/goreleaser/goreleaser/internal/semerrgroup" "github.com/goreleaser/goreleaser/internal/tmpl" @@ -151,20 +152,20 @@ func process(ctx *context.Context, docker config.Docker, artifacts []*artifact.A log := log.WithField("image", images[0]) log.Debug("tempdir: " + tmp) - if err := os.Link(docker.Dockerfile, filepath.Join(tmp, "Dockerfile")); err != nil { - return fmt.Errorf("failed to link dockerfile: %w", err) + if err := gio.Copy(docker.Dockerfile, filepath.Join(tmp, "Dockerfile")); err != nil { + return fmt.Errorf("failed to copy dockerfile: %w", err) } for _, file := range docker.Files { if err := os.MkdirAll(filepath.Join(tmp, filepath.Dir(file)), 0o755); err != nil { - return fmt.Errorf("failed to link extra file '%s': %w", file, err) + return fmt.Errorf("failed to copy extra file '%s': %w", file, err) } - if err := link(file, filepath.Join(tmp, file)); err != nil { - return fmt.Errorf("failed to link extra file '%s': %w", file, err) + if err := gio.Copy(file, filepath.Join(tmp, file)); err != nil { + return fmt.Errorf("failed to copy extra file '%s': %w", file, err) } } for _, art := range artifacts { - if err := os.Link(art.Path, filepath.Join(tmp, filepath.Base(art.Path))); err != nil { - return fmt.Errorf("failed to link artifact: %w", err) + if err := gio.Copy(art.Path, filepath.Join(tmp, filepath.Base(art.Path))); err != nil { + return fmt.Errorf("failed to copy artifact: %w", err) } } @@ -238,29 +239,6 @@ func processBuildFlagTemplates(ctx *context.Context, docker config.Docker) ([]st return buildFlags, nil } -// walks the src, recreating dirs and hard-linking files. -func link(src, dest string) error { - return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - // We have the following: - // - src = "a/b" - // - dest = "dist/linuxamd64/b" - // - path = "a/b/c.txt" - // So we join "a/b" with "c.txt" and use it as the destination. - dst := filepath.Join(dest, strings.Replace(path, src, "", 1)) - log.WithFields(log.Fields{ - "src": path, - "dst": dst, - }).Debug("extra file") - if info.IsDir() { - return os.MkdirAll(dst, info.Mode()) - } - return os.Link(path, dst) - }) -} - func dockerPush(ctx *context.Context, image *artifact.Artifact) error { log.WithField("image", image.Name).Info("pushing docker image") docker := image.Extra[dockerConfigExtra].(config.Docker) diff --git a/internal/pipe/docker/docker_test.go b/internal/pipe/docker/docker_test.go index dd0f680de..0e3bd8428 100644 --- a/internal/pipe/docker/docker_test.go +++ b/internal/pipe/docker/docker_test.go @@ -3,12 +3,10 @@ package docker import ( "flag" "fmt" - "io/ioutil" "os" "os/exec" "path/filepath" "strings" - "syscall" "testing" "github.com/apex/log" @@ -826,7 +824,7 @@ func TestRunPipe(t *testing.T) { }, }, assertImageLabels: noLabels, - assertError: shouldErr(`failed to link dockerfile`), + assertError: shouldErr(`failed to copy dockerfile`), }, "extra_file_doesnt_exist": { dockers: []config.Docker{ @@ -841,7 +839,7 @@ func TestRunPipe(t *testing.T) { }, }, assertImageLabels: noLabels, - assertError: shouldErr(`failed to link extra file 'testdata/nope.txt'`), + assertError: shouldErr(`failed to copy extra file 'testdata/nope.txt'`), }, "binary doesnt exist": { dockers: []config.Docker{ @@ -854,7 +852,7 @@ func TestRunPipe(t *testing.T) { }, }, assertImageLabels: noLabels, - assertError: shouldErr(`/wont-exist: no such file or directory`), + assertError: shouldErr(`failed to copy wont-exist`), extraPrepare: func(t *testing.T, ctx *context.Context) { t.Helper() ctx.Artifacts.Add(&artifact.Artifact{ @@ -1304,50 +1302,3 @@ func Test_processImageTemplates(t *testing.T) { "gcr.io/image:v1.0", }, images) } - -func TestLinkFile(t *testing.T) { - dir := t.TempDir() - src, err := ioutil.TempFile(dir, "src") - require.NoError(t, err) - require.NoError(t, src.Close()) - dst := filepath.Join(dir, "dst") - fmt.Println("src:", src.Name()) - fmt.Println("dst:", dst) - require.NoError(t, os.WriteFile(src.Name(), []byte("foo"), 0o644)) - require.NoError(t, link(src.Name(), dst)) - require.Equal(t, inode(src.Name()), inode(dst)) -} - -func TestLinkDirectory(t *testing.T) { - srcDir := t.TempDir() - dstDir := t.TempDir() - const testFile = "test" - require.NoError(t, os.WriteFile(filepath.Join(srcDir, testFile), []byte("foo"), 0o644)) - require.NoError(t, link(srcDir, dstDir)) - require.Equal(t, inode(filepath.Join(srcDir, testFile)), inode(filepath.Join(dstDir, testFile))) -} - -func TestLinkTwoLevelDirectory(t *testing.T) { - srcDir := t.TempDir() - dstDir := t.TempDir() - srcLevel2 := filepath.Join(srcDir, "level2") - const testFile = "test" - - require.NoError(t, os.Mkdir(srcLevel2, 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(srcDir, testFile), []byte("foo"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(srcLevel2, testFile), []byte("foo"), 0o644)) - - require.NoError(t, link(srcDir, dstDir)) - - require.Equal(t, inode(filepath.Join(srcDir, testFile)), inode(filepath.Join(dstDir, testFile))) - require.Equal(t, inode(filepath.Join(srcLevel2, testFile)), inode(filepath.Join(dstDir, "level2", testFile))) -} - -func inode(file string) uint64 { - fileInfo, err := os.Stat(file) - if err != nil { - return 0 - } - stat := fileInfo.Sys().(*syscall.Stat_t) - return stat.Ino -} diff --git a/internal/pipe/snapcraft/snapcraft.go b/internal/pipe/snapcraft/snapcraft.go index a6fd32563..5480caf7b 100644 --- a/internal/pipe/snapcraft/snapcraft.go +++ b/internal/pipe/snapcraft/snapcraft.go @@ -4,7 +4,6 @@ package snapcraft import ( "errors" "fmt" - "io" "os" "os/exec" "path/filepath" @@ -14,6 +13,7 @@ import ( "gopkg.in/yaml.v2" "github.com/goreleaser/goreleaser/internal/artifact" + "github.com/goreleaser/goreleaser/internal/gio" "github.com/goreleaser/goreleaser/internal/ids" "github.com/goreleaser/goreleaser/internal/linux" "github.com/goreleaser/goreleaser/internal/pipe" @@ -208,7 +208,7 @@ func create(ctx *context.Context, snap config.Snapcraft, arch string, binaries [ if err := os.MkdirAll(destinationDir, 0o755); err != nil { return fmt.Errorf("failed to create directory '%s': %w", destinationDir, err) } - if err := link(file.Source, filepath.Join(primeDir, file.Destination), os.FileMode(file.Mode)); err != nil { + if err := gio.CopyWithMode(file.Source, filepath.Join(primeDir, file.Destination), os.FileMode(file.Mode)); err != nil { return fmt.Errorf("failed to link extra file '%s': %w", file.Source, err) } } @@ -266,10 +266,10 @@ func create(ctx *context.Context, snap config.Snapcraft, arch string, binaries [ destBinaryPath := filepath.Join(primeDir, filepath.Base(binary.Path)) log.WithField("src", binary.Path). WithField("dst", destBinaryPath). - Debug("linking") + Debug("copying") - if err = copyFile(binary.Path, destBinaryPath, 0o555); err != nil { - return fmt.Errorf("failed to link binary: %w", err) + if err = gio.CopyWithMode(binary.Path, destBinaryPath, 0o555); err != nil { + return fmt.Errorf("failed to copy binary: %w", err) } } @@ -301,7 +301,7 @@ func create(ctx *context.Context, snap config.Snapcraft, arch string, binaries [ WithField("dst", destCompleterPath). Debug("copy") - if err := copyFile(config.Completer, destCompleterPath, 0o644); err != nil { + if err := gio.CopyWithMode(config.Completer, destCompleterPath, 0o644); err != nil { return fmt.Errorf("failed to copy completer: %w", err) } @@ -370,54 +370,6 @@ func push(ctx *context.Context, snap *artifact.Artifact) error { return nil } -// walks the src, recreating dirs and copying files. -func link(src, dest string, mode os.FileMode) error { - return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - // We have the following: - // - src = "a/b" - // - dest = "dist/linuxamd64/b" - // - path = "a/b/c.txt" - // So we join "a/b" with "c.txt" and use it as the destination. - dst := filepath.Join(dest, strings.Replace(path, src, "", 1)) - log.WithFields(log.Fields{ - "src": path, - "dst": dst, - }).Debug("extra file") - if info.IsDir() { - return os.MkdirAll(dst, info.Mode()) - } - if err := copyFile(path, dst, mode); err != nil { - return fmt.Errorf("fail copy file '%s': %w", path, err) - } - return nil - }) -} - -func copyFile(src, dst string, mode os.FileMode) error { - // Open original file - original, err := os.Open(src) - if err != nil { - return fmt.Errorf("fail to open '%s': %w", src, err) - } - defer original.Close() - - // Create new file - new, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) - if err != nil { - return fmt.Errorf("fail to open '%s': %w", dst, err) - } - defer new.Close() - - // This will copy - if _, err := io.Copy(new, original); err != nil { - return fmt.Errorf("fail to copy: %w", err) - } - return nil -} - func processChannelsTemplates(ctx *context.Context, snap config.Snapcraft) ([]string, error) { // nolint:prealloc var channels []string @@ -433,9 +385,5 @@ func processChannelsTemplates(ctx *context.Context, snap config.Snapcraft) ([]st channels = append(channels, channel) } - if len(channels) == 0 { - return channels, errors.New("no image templates found") - } - return channels, nil } diff --git a/internal/pipe/snapcraft/snapcraft_test.go b/internal/pipe/snapcraft/snapcraft_test.go index e365e5dda..ce21d00d2 100644 --- a/internal/pipe/snapcraft/snapcraft_test.go +++ b/internal/pipe/snapcraft/snapcraft_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/goreleaser/goreleaser/internal/artifact" + "github.com/goreleaser/goreleaser/internal/gio" "github.com/goreleaser/goreleaser/internal/pipe" "github.com/goreleaser/goreleaser/internal/testlib" "github.com/goreleaser/goreleaser/pkg/config" @@ -355,19 +356,8 @@ func TestExtraFile(t *testing.T) { addBinaries(t, ctx, "foo", dist) require.NoError(t, Pipe{}.Run(ctx)) - srcFile, err := os.Stat("testdata/extra-file.txt") - require.NoError(t, err) - destFile, err := os.Stat(filepath.Join(dist, "foo_amd64", "prime", "a", "b", "c", "extra-file.txt")) - require.NoError(t, err) - require.Equal(t, srcFile.Size(), destFile.Size()) - require.Equal(t, destFile.Mode(), os.FileMode(0o755)) - - srcFile, err = os.Stat("testdata/extra-file-2.txt") - require.NoError(t, err) - destFileWithDefaults, err := os.Stat(filepath.Join(dist, "foo_amd64", "prime", "testdata", "extra-file-2.txt")) - require.NoError(t, err) - require.Equal(t, destFileWithDefaults.Mode(), os.FileMode(0o644)) - require.Equal(t, srcFile.Size(), destFileWithDefaults.Size()) + requireEqualFiles(t, "testdata/extra-file.txt", filepath.Join(dist, "foo_amd64", "prime", "a", "b", "c", "extra-file.txt")) + requireEqualFiles(t, "testdata/extra-file-2.txt", filepath.Join(dist, "foo_amd64", "prime", "testdata", "extra-file-2.txt")) } func TestDefault(t *testing.T) { @@ -547,3 +537,10 @@ func Test_isValidArch(t *testing.T) { }) } } + +func requireEqualFiles(tb testing.TB, a, b string) { + tb.Helper() + eq, err := gio.EqualFiles(a, b) + require.NoError(tb, err) + require.True(tb, eq, "%s != %s", a, b) +}