feat: id filter on s3/blob pipes (#1042)

* docs: deprecate s3 in favor of blob

* feat: id filter on s3/blob pipes

* fix: close res.body

internal/http/http.go

@@ -218,7 +218,7 @@ func uploadAsset(ctx *context.Context, put *config.Put, artifact artifact.Artifa
 		headers[put.ChecksumHeader] = sum
 	}
 
-	_, err = uploadAssetToServer(ctx, put, targetURL, username, secret, headers, asset, check)
+	res, err := uploadAssetToServer(ctx, put, targetURL, username, secret, headers, asset, check)
 	if err != nil {
 		msg := fmt.Sprintf("%s: upload failed", kind)
 		log.WithError(err).WithFields(log.Fields{
@@ -227,6 +227,9 @@ func uploadAsset(ctx *context.Context, put *config.Put, artifact artifact.Artifa
 		}).Error(msg)
 		return errors.Wrap(err, msg)
 	}
+	if err := res.Body.Close(); err != nil {
+		log.WithError(err).Warn("failed to close response body")
+	}
 
 	log.WithFields(log.Fields{
 		"instance": put.Name,

internal/pipe/blob/blob.go

@@ -61,7 +61,7 @@ func (Pipe) Publish(ctx *context.Context) error {
 			return err
 		}
 		g.Go(func() error {
-			return o.Upload(ctx, fmt.Sprintf("%s://%s", conf.Provider, conf.Bucket), folder)
+			return o.Upload(ctx, conf, folder)
 		})
 	}
 	return g.Wait()

internal/pipe/blob/blob_test.go

@@ -176,30 +176,6 @@ func TestPipe_Publish(t *testing.T) {
 		Path: debpath,
 	})
 
-	// Azure Blob Context Without ENV
-	var azblobctx1 = context.New(config.Project{
-		Dist:        folder,
-		ProjectName: "testupload1",
-		Blobs: []config.Blob{
-			{
-				Bucket:   "foobar",
-				Provider: "azblob",
-			},
-		},
-	})
-
-	azblobctx1.Git = context.GitInfo{CurrentTag: "v1.0.0"}
-	azblobctx1.Artifacts.Add(artifact.Artifact{
-		Type: artifact.UploadableArchive,
-		Name: "bin.tar.gz",
-		Path: tgzpath,
-	})
-	azblobctx1.Artifacts.Add(artifact.Artifact{
-		Type: artifact.LinuxPackage,
-		Name: "bin.deb",
-		Path: debpath,
-	})
-
 	// Google Cloud Storage Context
 	var gsctx = context.New(config.Project{
 		Dist:        folder,
@@ -219,7 +195,7 @@ func TestPipe_Publish(t *testing.T) {
 		Name: "bin.tar.gz",
 		Path: tgzpath,
 	})
-	azblobctx.Artifacts.Add(artifact.Artifact{
+	gsctx.Artifacts.Add(artifact.Artifact{
 		Type: artifact.LinuxPackage,
 		Name: "bin.deb",
 		Path: debpath,
@@ -236,10 +212,8 @@ func TestPipe_Publish(t *testing.T) {
 			},
 		},
 	})
-
 	s3ctx.Git = context.GitInfo{CurrentTag: "v1.0.0"}
-
-	gsctx.Artifacts.Add(artifact.Artifact{
+	s3ctx.Artifacts.Add(artifact.Artifact{
 		Type: artifact.UploadableArchive,
 		Name: "bin.tar.gz",
 		Path: tgzpath,
@@ -309,9 +283,7 @@ func setEnvVariables() {
 }
 
 func setEnv(env map[string]string) {
-
 	for k, v := range env {
 		os.Setenv(k, v)
-
 	}
 }

internal/pipe/blob/openbucket.go

@@ -8,6 +8,7 @@ import (
 	"github.com/apex/log"
 	"github.com/goreleaser/goreleaser/internal/artifact"
 	"github.com/goreleaser/goreleaser/internal/semerrgroup"
+	"github.com/goreleaser/goreleaser/pkg/config"
 	"github.com/goreleaser/goreleaser/pkg/context"
 	gocdk "gocloud.dev/blob"
 
@@ -20,7 +21,7 @@ import (
 // OpenBucket is the interface that wraps the BucketConnect and UploadBucket method
 type OpenBucket interface {
 	Connect(ctx *context.Context, bucketURL string) (*gocdk.Bucket, error)
-	Upload(ctx *context.Context, bucketURL string, folder string) error
+	Upload(ctx *context.Context, conf config.Blob, folder string) error
 }
 
 // Bucket is object which holds connection for Go Bucker Provider
@@ -33,9 +34,8 @@ func newOpenBucket() OpenBucket {
 	return Bucket{}
 }
 
-// BucketConnect makes connection with provider
+// Connect makes connection with provider
 func (b Bucket) Connect(ctx *context.Context, bucketURL string) (*gocdk.Bucket, error) {
-
 	bucketConnection, err := gocdk.OpenBucket(ctx, bucketURL)
 	if err != nil {
 		return nil, err
@@ -43,9 +43,10 @@ func (b Bucket) Connect(ctx *context.Context, bucketURL string) (*gocdk.Bucket,
 	return bucketConnection, nil
 }
 
-// UploadBucket takes connection initilized from newOpenBucket to upload goreleaser artifacts
+// Upload takes connection initilized from newOpenBucket to upload goreleaser artifacts
 // Takes goreleaser context(which includes artificats) and bucketURL for upload destination (gs://gorelease-bucket)
-func (b Bucket) Upload(ctx *context.Context, bucketURL string, folder string) error {
+func (b Bucket) Upload(ctx *context.Context, conf config.Blob, folder string) error {
+	var bucketURL = fmt.Sprintf("%s://%s", conf.Provider, conf.Bucket)
 
 	// Get the openbucket connection for specific provider
 	openbucketConn, err := b.Connect(ctx, bucketURL)
@@ -54,16 +55,19 @@ func (b Bucket) Upload(ctx *context.Context, bucketURL string, folder string) er
 	}
 	defer openbucketConn.Close()
 
+	var filter = artifact.Or(
+		artifact.ByType(artifact.UploadableArchive),
+		artifact.ByType(artifact.UploadableBinary),
+		artifact.ByType(artifact.Checksum),
+		artifact.ByType(artifact.Signature),
+		artifact.ByType(artifact.LinuxPackage),
+	)
+	if len(conf.IDs) > 0 {
+		filter = artifact.And(filter, artifact.ByIDs(conf.IDs...))
+	}
+
 	var g = semerrgroup.New(ctx.Parallelism)
-	for _, artifact := range ctx.Artifacts.Filter(
-		artifact.Or(
-			artifact.ByType(artifact.UploadableArchive),
-			artifact.ByType(artifact.UploadableBinary),
-			artifact.ByType(artifact.Checksum),
-			artifact.ByType(artifact.Signature),
-			artifact.ByType(artifact.LinuxPackage),
-		),
-	).List() {
+	for _, artifact := range ctx.Artifacts.Filter(filter).List() {
 		artifact := artifact
 		g.Go(func() error {
 			// Prepare artifact for upload.
@@ -89,7 +93,6 @@ func (b Bucket) Upload(ctx *context.Context, bucketURL string, folder string) er
 			}
 			if err = w.Close(); err != nil {
 				switch {
-
 				case errorContains(err, "InvalidAccessKeyId"):
 					return fmt.Errorf("aws access key id you provided does not exist in our records")
 				case errorContains(err, "AuthenticationFailed"):

internal/pipe/blob/util.go

@@ -8,7 +8,6 @@ import (
 
 // Check required ENV variables based on Blob Provider
 func checkProvider(provider string) error {
-
 	switch provider {
 	case "azblob":
 		return checkEnv("AZURE_STORAGE_ACCOUNT", "AZURE_STORAGE_KEY")
@@ -19,11 +18,9 @@ func checkProvider(provider string) error {
 	default:
 		return fmt.Errorf("unknown provider [%v],currently supported providers: [azblob, gs, s3]", provider)
 	}
-
 }
 
 func checkEnv(envs ...string) error {
-
 	var missingEnv []string
 
 	for _, env := range envs {
@@ -41,7 +38,6 @@ func checkEnv(envs ...string) error {
 
 // Check if error contains specific string
 func errorContains(err error, subs ...string) bool {
-
 	for _, sub := range subs {
 		if strings.Contains(err.Error(), sub) {
 			return true

internal/pipe/s3/s3.go

@@ -9,6 +9,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/goreleaser/goreleaser/internal/artifact"
+	"github.com/goreleaser/goreleaser/internal/deprecate"
 	"github.com/goreleaser/goreleaser/internal/pipe"
 	"github.com/goreleaser/goreleaser/internal/semerrgroup"
 	"github.com/goreleaser/goreleaser/internal/tmpl"
@@ -31,6 +32,7 @@ func (Pipe) Default(ctx *context.Context) error {
 		if s3.Bucket == "" {
 			continue
 		}
+		deprecate.Notice("s3")
 		if s3.Folder == "" {
 			s3.Folder = "{{ .ProjectName }}/{{ .Tag }}"
 		}
@@ -87,16 +89,19 @@ func upload(ctx *context.Context, conf config.S3) error {
 		return err
 	}
 
+	var filter = artifact.Or(
+		artifact.ByType(artifact.UploadableArchive),
+		artifact.ByType(artifact.UploadableBinary),
+		artifact.ByType(artifact.Checksum),
+		artifact.ByType(artifact.Signature),
+		artifact.ByType(artifact.LinuxPackage),
+	)
+	if len(conf.IDs) > 0 {
+		filter = artifact.And(filter, artifact.ByIDs(conf.IDs...))
+	}
+
 	var g = semerrgroup.New(ctx.Parallelism)
-	for _, artifact := range ctx.Artifacts.Filter(
-		artifact.Or(
-			artifact.ByType(artifact.UploadableArchive),
-			artifact.ByType(artifact.UploadableBinary),
-			artifact.ByType(artifact.Checksum),
-			artifact.ByType(artifact.Signature),
-			artifact.ByType(artifact.LinuxPackage),
-		),
-	).List() {
+	for _, artifact := range ctx.Artifacts.Filter(filter).List() {
 		artifact := artifact
 		g.Go(func() error {
 			f, err := os.Open(artifact.Path)

internal/pipe/s3/s3_test.go

@@ -72,6 +72,7 @@ func TestUpload(t *testing.T) {
 			{
 				Bucket:   "test",
 				Endpoint: "http://" + listen,
+				IDs:      []string{"foo", "bar"},
 			},
 		},
 	})
@@ -80,11 +81,17 @@ func TestUpload(t *testing.T) {
 		Type: artifact.UploadableArchive,
 		Name: "bin.tar.gz",
 		Path: tgzpath,
+		Extra: map[string]interface{}{
+			"ID": "foo",
+		},
 	})
 	ctx.Artifacts.Add(artifact.Artifact{
 		Type: artifact.LinuxPackage,
 		Name: "bin.deb",
 		Path: debpath,
+		Extra: map[string]interface{}{
+			"ID": "bar",
+		},
 	})
 	var name = "test_upload"
 	defer stop(t, name)

pkg/config/config.go

@@ -297,6 +297,7 @@ type S3 struct {
 	Profile  string
 	Endpoint string // used for minio for example
 	ACL      string
+	IDs      []string `yaml:"ids,omitempty"`
 }
 
 // Blob contains config for GO CDK blob
@@ -304,6 +305,7 @@ type Blob struct {
 	Bucket   string
 	Provider string
 	Folder   string
+	IDs      []string `yaml:"ids,omitempty"`
 }
 
 // Put HTTP upload configuration

www/content/blob.md

@@ -15,8 +15,15 @@ blob:
     # azblob for Azure Blob Storage
     # gs for Google Cloud Storage
     provider: azblob
+
     # Template for the bucket name
     bucket: goreleaser-bucket
+
+    # IDs of the artifacts you want to upload.
+    ids:
+    - foo
+    - bar
+
     # Template for the path/name inside the bucket.
     # Default is `{{ .ProjectName }}/{{ .Tag }}`
     folder: "foo/bar/{{.Version}}"

www/content/deprecations.md

@@ -33,6 +33,30 @@ to this:
 
 -->
 
+### s3
+
+> since 2019-06-09
+
+S3 was deprecated in favor of the new `blob`, which supports S3, Azure Blob and
+GCS.
+
+Change this:
+
+```yaml
+s3:
+-
+  # etc
+```
+
+to this:
+
+```yaml
+blobs:
+-
+  provider: s3
+  # etc
+```
+
 ### snapcraft
 
 > since 2019-05-39

www/content/s3.md

@@ -26,21 +26,31 @@ s3:
     # Template for the bucket name(without the s3:// prefix)
     # Default is empty.
     bucket: my-bucket
+
+    # IDs of the artifacts you want to upload.
+    ids:
+    - foo
+    - bar
+
     # AWS Region to use.
     # Defaults is us-east-1
     region: us-east-1
+
     # Template for the path/name inside the bucket.
     # Default is `{{ .ProjectName }}/{{ .Tag }}`
     folder: "foo/bar/{{.Version}}"
+
     # Set a custom profile to use for this s3 config. If you have multiple
     # profiles setup in you ~/.aws config, this shall help defining which
     # profile to use in which s3 bucket.
     # Default is empty.
     profile: my-profile
+
     # Endpoint allows you to set a custom endpoint, which is useful if you
     # want to push your artifacts to a minio server for example.
     # Default is AWS S3 URL.
     endpoint: "http://minio.foo.com"
+
     # Sets the ACL of the object using the specified canned ACL.
     # Default is private.
     acl: public-read