From 2dc3ae3010a4cb58f81178901b9bb32d994bf4ab Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Wed, 20 Nov 2019 13:08:25 -0300 Subject: [PATCH] feat: fully support all s3 pipe feats on blob (#1253) * feat: fully support all s3 pipe feats on blob Signed-off-by: Carlos Alexandro Becker * fix: tidy deps Signed-off-by: Carlos Alexandro Becker * docs: document endpoint option Signed-off-by: Carlos Alexandro Becker * fix: test Signed-off-by: Carlos Alexandro Becker --- go.mod | 2 +- go.sum | 16 +- internal/pipe/blob/blob.go | 9 +- internal/pipe/blob/blob_minio_test.go | 205 ++++++++++++++++++ internal/pipe/blob/blob_test.go | 3 +- internal/pipe/blob/doc.go | 4 + internal/pipe/blob/openbucket.go | 20 +- internal/pipe/blob/testdata/data/.gitignore | 3 + .../pipe/blob/testdata/data/test/.gitignore | 1 - .../pipe/blob/testdata/data/test/.gitkeep | 0 pkg/config/config.go | 1 + www/content/blob.md | 5 + 12 files changed, 246 insertions(+), 23 deletions(-) create mode 100644 internal/pipe/blob/blob_minio_test.go create mode 100644 internal/pipe/blob/doc.go create mode 100644 internal/pipe/blob/testdata/data/.gitignore delete mode 100644 internal/pipe/blob/testdata/data/test/.gitignore create mode 100644 internal/pipe/blob/testdata/data/test/.gitkeep diff --git a/go.mod b/go.mod index 5fc2d023b..005642d03 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/pkg/errors v0.8.1 github.com/stretchr/testify v1.4.0 github.com/xanzy/go-gitlab v0.21.0 - gocloud.dev v0.17.0 + gocloud.dev v0.18.0 golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271 // indirect golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e diff --git a/go.sum b/go.sum index 728fb666e..2357b2db1 100644 --- a/go.sum +++ b/go.sum @@ -12,15 +12,14 @@ contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= github.com/Azure/azure-amqp-common-go/v2 v2.1.0/go.mod h1:R8rea+gJRuJR6QxTir/XuEd+YuKoUiazDC/N96FiDEU= -github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= -github.com/Azure/azure-pipeline-go v0.1.9 h1:u7JFb9fFTE6Y/j8ae2VK33ePrRqJqoCM/IWkQdAZ+rg= -github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= +github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo= +github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-sdk-for-go v29.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v30.1.0+incompatible h1:HyYPft8wXpxMd0kfLtXo6etWcO+XuPbLkcgx9g2cqxU= github.com/Azure/azure-sdk-for-go v30.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-service-bus-go v0.9.1/go.mod h1:yzBx6/BUGfjfeqbRZny9AQIbIe3AcV9WZbAdpkoXOa0= -github.com/Azure/azure-storage-blob-go v0.6.0 h1:SEATKb3LIHcaSIX+E6/K4kJpwfuozFEsmt5rS56N6CE= -github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= +github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o= +github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= github.com/Azure/go-autorest v12.0.0+incompatible h1:N+VqClcomLGD/sHb3smbSYYtNMgKpVV3Cd5r5i8z6bQ= github.com/Azure/go-autorest v12.0.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -141,6 +140,8 @@ github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljmERptWhwa8XzP7H3T+Z2N26gTsaDaA= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -188,8 +189,8 @@ go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -gocloud.dev v0.17.0 h1:UuDiCphYsiNhRNLtgHVL/eZheQeCt00hL3XjDfbt820= -gocloud.dev v0.17.0/go.mod h1:tIHTRdR1V5dlD8sTkzYdTGizBJ314BDykJ8KmadEXwo= +gocloud.dev v0.18.0 h1:HX6uFZYZs9tUP87jzoWgB8dl4ihsRpiAsBDKTthiApY= +gocloud.dev v0.18.0/go.mod h1:lhLOb91+9tKB8RnNlsx+weJGEd0AHM94huK1bmrhPwM= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -287,7 +288,6 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= diff --git a/internal/pipe/blob/blob.go b/internal/pipe/blob/blob.go index daf7ae4c4..d3c2c81bf 100644 --- a/internal/pipe/blob/blob.go +++ b/internal/pipe/blob/blob.go @@ -1,4 +1,3 @@ -// Package blob provides a Pipe that push artifacts to blob supported by Go CDK package blob import ( @@ -8,7 +7,6 @@ import ( "github.com/goreleaser/goreleaser/internal/deprecate" "github.com/goreleaser/goreleaser/internal/pipe" "github.com/goreleaser/goreleaser/internal/semerrgroup" - "github.com/goreleaser/goreleaser/internal/tmpl" "github.com/goreleaser/goreleaser/pkg/context" ) @@ -49,13 +47,8 @@ func (Pipe) Publish(ctx *context.Context) error { var g = semerrgroup.New(ctx.Parallelism) for _, conf := range ctx.Config.Blobs { conf := conf - template := tmpl.New(ctx) - folder, err := template.Apply(conf.Folder) - if err != nil { - return err - } g.Go(func() error { - return o.Upload(ctx, conf, folder) + return o.Upload(ctx, conf) }) } return g.Wait() diff --git a/internal/pipe/blob/blob_minio_test.go b/internal/pipe/blob/blob_minio_test.go new file mode 100644 index 000000000..1a664ee43 --- /dev/null +++ b/internal/pipe/blob/blob_minio_test.go @@ -0,0 +1,205 @@ +package blob + +// this is pretty much copied from the s3 pipe to ensure both work the same way +// only differences are that it sets `blobs` instead of `s3` on test cases and +// the test setup and teardown + +import ( + "io/ioutil" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/goreleaser/goreleaser/internal/artifact" + "github.com/goreleaser/goreleaser/pkg/config" + "github.com/goreleaser/goreleaser/pkg/context" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMinioUpload(t *testing.T) { + var listen = randomListen(t) + folder, err := ioutil.TempDir("", "goreleasertest") + assert.NoError(t, err) + tgzpath := filepath.Join(folder, "bin.tar.gz") + debpath := filepath.Join(folder, "bin.deb") + assert.NoError(t, ioutil.WriteFile(tgzpath, []byte("fake\ntargz"), 0744)) + assert.NoError(t, ioutil.WriteFile(debpath, []byte("fake\ndeb"), 0744)) + var ctx = context.New(config.Project{ + Dist: folder, + ProjectName: "testupload", + Blobs: []config.Blob{ + { + Provider: "s3", + Bucket: "test", + Endpoint: "http://" + listen, + IDs: []string{"foo", "bar"}, + }, + }, + }) + ctx.Git = context.GitInfo{CurrentTag: "v1.0.0"} + ctx.Artifacts.Add(&artifact.Artifact{ + Type: artifact.UploadableArchive, + Name: "bin.tar.gz", + Path: tgzpath, + Extra: map[string]interface{}{ + "ID": "foo", + }, + }) + ctx.Artifacts.Add(&artifact.Artifact{ + Type: artifact.LinuxPackage, + Name: "bin.deb", + Path: debpath, + Extra: map[string]interface{}{ + "ID": "bar", + }, + }) + var name = "test_upload" + defer stop(t, name) + start(t, name, listen) + prepareEnv(t, listen) + assert.NoError(t, Pipe{}.Default(ctx)) + assert.NoError(t, Pipe{}.Publish(ctx)) +} + +func TestMinioUploadCustomBucketID(t *testing.T) { + var listen = randomListen(t) + folder, err := ioutil.TempDir("", "goreleasertest") + assert.NoError(t, err) + tgzpath := filepath.Join(folder, "bin.tar.gz") + debpath := filepath.Join(folder, "bin.deb") + assert.NoError(t, ioutil.WriteFile(tgzpath, []byte("fake\ntargz"), 0744)) + assert.NoError(t, ioutil.WriteFile(debpath, []byte("fake\ndeb"), 0744)) + // Set custom BUCKET_ID env variable. + err = os.Setenv("BUCKET_ID", "test") + assert.NoError(t, err) + var ctx = context.New(config.Project{ + Dist: folder, + ProjectName: "testupload", + Blobs: []config.Blob{ + { + Provider: "s3", + Bucket: "{{.Env.BUCKET_ID}}", + Endpoint: "http://" + listen, + }, + }, + }) + ctx.Git = context.GitInfo{CurrentTag: "v1.0.0"} + ctx.Artifacts.Add(&artifact.Artifact{ + Type: artifact.UploadableArchive, + Name: "bin.tar.gz", + Path: tgzpath, + }) + ctx.Artifacts.Add(&artifact.Artifact{ + Type: artifact.LinuxPackage, + Name: "bin.deb", + Path: debpath, + }) + var name = "custom_bucket_id" + defer stop(t, name) + start(t, name, listen) + prepareEnv(t, listen) + assert.NoError(t, Pipe{}.Default(ctx)) + assert.NoError(t, Pipe{}.Publish(ctx)) +} + +func TestMinioUploadInvalidCustomBucketID(t *testing.T) { + var listen = randomListen(t) + folder, err := ioutil.TempDir("", "goreleasertest") + assert.NoError(t, err) + tgzpath := filepath.Join(folder, "bin.tar.gz") + debpath := filepath.Join(folder, "bin.deb") + assert.NoError(t, ioutil.WriteFile(tgzpath, []byte("fake\ntargz"), 0744)) + assert.NoError(t, ioutil.WriteFile(debpath, []byte("fake\ndeb"), 0744)) + // Set custom BUCKET_ID env variable. + assert.NoError(t, err) + var ctx = context.New(config.Project{ + Dist: folder, + ProjectName: "testupload", + Blobs: []config.Blob{ + { + Provider: "s3", + Bucket: "{{.Bad}}", + Endpoint: "http://" + listen, + }, + }, + }) + ctx.Git = context.GitInfo{CurrentTag: "v1.0.0"} + ctx.Artifacts.Add(&artifact.Artifact{ + Type: artifact.UploadableArchive, + Name: "bin.tar.gz", + Path: tgzpath, + }) + ctx.Artifacts.Add(&artifact.Artifact{ + Type: artifact.LinuxPackage, + Name: "bin.deb", + Path: debpath, + }) + var name = "invalid_bucket_id" + defer stop(t, name) + start(t, name, listen) + prepareEnv(t, listen) + assert.NoError(t, Pipe{}.Default(ctx)) + assert.Error(t, Pipe{}.Publish(ctx)) +} + +func randomListen(t *testing.T) string { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + listener.Close() + return listener.Addr().String() +} + +func prepareEnv(t *testing.T, listen string) { + os.Setenv("AWS_ACCESS_KEY_ID", "minio") + os.Setenv("AWS_SECRET_ACCESS_KEY", "miniostorage") + os.Setenv("AWS_REGION", "us-east-1") +} + +func start(t *testing.T, name, listen string) { + wd, err := os.Getwd() + require.NoError(t, err) + + removeTestData(t) + + if out, err := exec.Command( + "docker", "run", "-d", "--rm", + "-v", filepath.Join(wd, "testdata/data")+":/data", + "--name", name, + "-p", listen+":9000", + "-e", "MINIO_ACCESS_KEY=minio", + "-e", "MINIO_SECRET_KEY=miniostorage", + "--health-interval", "1s", + "minio/minio:RELEASE.2019-05-14T23-57-45Z", + "server", "/data", + ).CombinedOutput(); err != nil { + t.Fatalf("failed to start minio: %s", string(out)) + } + + for range time.Tick(time.Second) { + out, err := exec.Command("docker", "inspect", "--format='{{json .State.Health}}'", name).CombinedOutput() + if err != nil { + t.Fatalf("failed to check minio status: %s", string(out)) + } + if strings.Contains(string(out), `"Status":"healthy"`) { + t.Log("minio is healthy") + break + } + t.Log("waiting for minio to be healthy") + } +} + +func stop(t *testing.T, name string) { + if out, err := exec.Command("docker", "stop", name).CombinedOutput(); err != nil { + t.Fatalf("failed to stop minio: %s", string(out)) + } + removeTestData(t) +} + +func removeTestData(t *testing.T) { + _ = os.RemoveAll("./testdata/data/test/testupload") // dont care if it fails +} diff --git a/internal/pipe/blob/blob_test.go b/internal/pipe/blob/blob_test.go index 8eedbaafd..2ae59fab3 100644 --- a/internal/pipe/blob/blob_test.go +++ b/internal/pipe/blob/blob_test.go @@ -7,13 +7,12 @@ import ( "strings" "testing" - "github.com/stretchr/testify/require" - "github.com/goreleaser/goreleaser/internal/artifact" "github.com/goreleaser/goreleaser/internal/testlib" "github.com/goreleaser/goreleaser/pkg/config" "github.com/goreleaser/goreleaser/pkg/context" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestDescription(t *testing.T) { diff --git a/internal/pipe/blob/doc.go b/internal/pipe/blob/doc.go new file mode 100644 index 000000000..d9fd107af --- /dev/null +++ b/internal/pipe/blob/doc.go @@ -0,0 +1,4 @@ +// Package blob provides a Pipe that push artifacts to blob supported by Go CDK +// +// TODO: replace current s3 implementation with this one. +package blob diff --git a/internal/pipe/blob/openbucket.go b/internal/pipe/blob/openbucket.go index 7a9d1dedc..0103f7bc6 100644 --- a/internal/pipe/blob/openbucket.go +++ b/internal/pipe/blob/openbucket.go @@ -8,6 +8,7 @@ import ( "github.com/apex/log" "github.com/goreleaser/goreleaser/internal/artifact" "github.com/goreleaser/goreleaser/internal/semerrgroup" + "github.com/goreleaser/goreleaser/internal/tmpl" "github.com/goreleaser/goreleaser/pkg/config" "github.com/goreleaser/goreleaser/pkg/context" "github.com/pkg/errors" @@ -28,7 +29,7 @@ import ( // OpenBucket is the interface that wraps the BucketConnect and UploadBucket method type OpenBucket interface { Connect(ctx *context.Context, bucketURL string) (*blob.Bucket, error) - Upload(ctx *context.Context, conf config.Blob, folder string) error + Upload(ctx *context.Context, conf config.Blob) error } // Bucket is object which holds connection for Go Bucker Provider @@ -52,8 +53,21 @@ func (b Bucket) Connect(ctx *context.Context, bucketURL string) (*blob.Bucket, e // Upload takes connection initilized from newOpenBucket to upload goreleaser artifacts // Takes goreleaser context(which includes artificats) and bucketURL for upload destination (gs://gorelease-bucket) -func (b Bucket) Upload(ctx *context.Context, conf config.Blob, folder string) error { - var bucketURL = fmt.Sprintf("%s://%s", conf.Provider, conf.Bucket) +func (b Bucket) Upload(ctx *context.Context, conf config.Blob) error { + bucket, err := tmpl.New(ctx).Apply(conf.Bucket) + if err != nil { + return err + } + + folder, err := tmpl.New(ctx).Apply(conf.Folder) + if err != nil { + return err + } + + var bucketURL = fmt.Sprintf("%s://%s", conf.Provider, bucket) + if conf.Endpoint != "" && conf.Provider == "s3" { + bucketURL = fmt.Sprintf("%s?endpoint=%s&s3ForcePathStyle=true", bucketURL, conf.Endpoint) + } // Get the openbucket connection for specific provider conn, err := b.Connect(ctx, bucketURL) diff --git a/internal/pipe/blob/testdata/data/.gitignore b/internal/pipe/blob/testdata/data/.gitignore new file mode 100644 index 000000000..63ebdc9de --- /dev/null +++ b/internal/pipe/blob/testdata/data/.gitignore @@ -0,0 +1,3 @@ +.minio.sys +test/* +!test/.gitkeep diff --git a/internal/pipe/blob/testdata/data/test/.gitignore b/internal/pipe/blob/testdata/data/test/.gitignore deleted file mode 100644 index 6c504cff0..000000000 --- a/internal/pipe/blob/testdata/data/test/.gitignore +++ /dev/null @@ -1 +0,0 @@ -testupload \ No newline at end of file diff --git a/internal/pipe/blob/testdata/data/test/.gitkeep b/internal/pipe/blob/testdata/data/test/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/pkg/config/config.go b/pkg/config/config.go index 25a219149..da0323bdc 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -329,6 +329,7 @@ type Blob struct { Folder string `yaml:",omitempty"` KMSKey string `yaml:",omitempty"` IDs []string `yaml:"ids,omitempty"` + Endpoint string `yaml:",omitempty"` // used for minio for example } // Upload configuration diff --git a/www/content/blob.md b/www/content/blob.md index 871502ccb..b6a639611 100644 --- a/www/content/blob.md +++ b/www/content/blob.md @@ -18,6 +18,11 @@ blobs: # gs for Google Cloud Storage provider: azblob + # Set a custom endpoint, useful if you're using a minio backend or + # other s3-compatible backends. + # Implies s3ForcePathStyle and requires provider to be `s3` + endpoint: https://minio.foo.bar + # Template for the bucket name bucket: goreleaser-bucket