fix: blob provider authentication issues for default aws mechanism (#1061)

* fix: blob provider authentication issues for default aws mechanism

* remove commented code

* docs: update the authentication doc

* fix: linter issues

go.mod

@@ -16,8 +16,11 @@ require (
 	github.com/pkg/errors v0.8.1
 	github.com/stretchr/testify v1.3.0
 	gocloud.dev v0.13.0
+	golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect
 	golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421
-	golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58
+	golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect
+	golang.org/x/text v0.3.2 // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/yaml.v2 v2.2.2
 )

go.sum

@@ -517,6 +517,8 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190328230028-74de082e2cca h1:hyA6yiAgbUwuWqtscNvWAI7U1CtlaD1KilQ6iudt1aI=
 golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180603041954-1e0a3fa8ba9a/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -529,6 +531,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180828065106-d99a578cf41b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -546,13 +550,18 @@ golang.org/x/sys v0.0.0-20181218192612-074acd46bca6/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190402142545-baf5eb976a8c h1:3xiKTkef8QqBJ8q+4fVUDMRoxnI0H/MVNFswa+aExbo=
 golang.org/x/sys v0.0.0-20190402142545-baf5eb976a8c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k=
+golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181017214349-06f26fdaaa28/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181219222714-6e267b5cc78e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

internal/pipe/blob/blob.go

@@ -3,6 +3,7 @@ package blob
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/goreleaser/goreleaser/internal/pipe"
 	"github.com/goreleaser/goreleaser/internal/semerrgroup"
@@ -34,13 +35,6 @@ func (Pipe) Default(ctx *context.Context) error {
 		if blob.Folder == "" {
 			blob.Folder = "{{ .ProjectName }}/{{ .Tag }}"
 		}
-		// Validation before opening connection to bucket
-		// gocdk also does this validation but doing it in advance for better error handling
-		// as currently, go cdk does not throw error if AZURE_STORAGE_KEY is missing.
-		err := checkProvider(blob.Provider)
-		if err != nil {
-			return err
-		}
 	}
 	return nil
 }
@@ -66,3 +60,13 @@ func (Pipe) Publish(ctx *context.Context) error {
 	}
 	return g.Wait()
 }
+
+// errorContains check if error contains specific string
+func errorContains(err error, subs ...string) bool {
+	for _, sub := range subs {
+		if strings.Contains(err.Error(), sub) {
+			return true
+		}
+	}
+	return false
+}

internal/pipe/blob/blob_test.go

@@ -1,7 +1,6 @@
 package blob
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -12,9 +11,6 @@ import (
 	"github.com/goreleaser/goreleaser/pkg/config"
 	"github.com/goreleaser/goreleaser/pkg/context"
 	"github.com/stretchr/testify/assert"
-	_ "gocloud.dev/blob/azureblob"
-	_ "gocloud.dev/blob/gcsblob"
-	_ "gocloud.dev/blob/s3blob"
 )
 
 func TestDescription(t *testing.T) {
@@ -69,15 +65,16 @@ func TestDefaults(t *testing.T) {
 			{
 				Bucket:   "foo",
 				Provider: "azblob",
+				IDs:      []string{"foo", "bar"},
 			},
 		},
 	})
-	setEnvVariables()
 	assert.NoError(Pipe{}.Default(ctx))
 	assert.Equal([]config.Blob{{
 		Bucket:   "foo",
 		Provider: "azblob",
 		Folder:   "{{ .ProjectName }}/{{ .Tag }}",
+		IDs:      []string{"foo", "bar"},
 	}}, ctx.Config.Blobs)
 }
 
@@ -99,49 +96,9 @@ func TestDefaultsWithProvider(t *testing.T) {
 			},
 		},
 	})
-
-	setEnvVariables()
 	assert.Nil(Pipe{}.Default(ctx))
 }
 
-func TestDefaultsWithInvalidProvider(t *testing.T) {
-	var assert = assert.New(t)
-
-	// This is invalid provider, meaning not registred with GO CDK
-	invalidProvider := "bar"
-	errorString := fmt.Sprintf("unknown provider [%v],currently supported providers: [azblob, gs, s3]", invalidProvider)
-	var ctx = context.New(config.Project{
-		Blobs: []config.Blob{
-			{
-				Bucket:   "foo",
-				Provider: invalidProvider,
-			},
-		},
-	})
-
-	setEnvVariables()
-	assert.EqualError(Pipe{}.Default(ctx), errorString)
-}
-
-func TestDefaultsWithMissingEnv(t *testing.T) {
-	var assert = assert.New(t)
-
-	errorString := "missing AZURE_STORAGE_ACCOUNT,AZURE_STORAGE_KEY"
-	var ctx = context.New(config.Project{
-		Blobs: []config.Blob{
-			{
-				Bucket:   "foo",
-				Provider: "azblob",
-			},
-		},
-	})
-
-	os.Unsetenv("AZURE_STORAGE_ACCOUNT")
-	os.Unsetenv("AZURE_STORAGE_KEY")
-
-	assert.EqualError(Pipe{}.Default(ctx), errorString)
-}
-
 func TestPipe_Publish(t *testing.T) {
 	gcloudCredentials, _ := filepath.Abs("./testdata/credentials.json")
 
@@ -152,7 +109,7 @@ func TestPipe_Publish(t *testing.T) {
 	assert.NoError(t, ioutil.WriteFile(tgzpath, []byte("fake\ntargz"), 0744))
 	assert.NoError(t, ioutil.WriteFile(debpath, []byte("fake\ndeb"), 0744))
 
-	// Azure Blob Context Without ENV
+	// Azure Blob Context
 	var azblobctx = context.New(config.Project{
 		Dist:        folder,
 		ProjectName: "testupload",
@@ -235,7 +192,7 @@ func TestPipe_Publish(t *testing.T) {
 		wantErrString string
 	}{
 		{
-			name:          "Azure Blob Bucket test Publish(StorageAccount)",
+			name:          "Azure Blob Bucket Test Publish",
 			args:          args{azblobctx},
 			env:           map[string]string{"AZURE_STORAGE_ACCOUNT": "hjsdhjsdhs", "AZURE_STORAGE_KEY": "eHCSajxLvl94l36gIMlzZ/oW2O0rYYK+cVn5jNT2"},
 			wantErr:       false,
@@ -260,30 +217,24 @@ func TestPipe_Publish(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			p := Pipe{}
 			setEnv(tt.env)
-
+			defer unsetEnv(tt.env)
 			if err := p.Publish(tt.args.ctx); (err != nil) != tt.wantErr {
 				if err.Error() != tt.wantErrString {
 					t.Errorf("Pipe.Publish() error = %v, wantErr %v", err, tt.wantErrString)
 				}
 			}
-			os.Clearenv()
 		})
 	}
 }
 
-// Fake secret ENV VARIABLES use to authenticate against cloud provider
-func setEnvVariables() {
-	os.Setenv("AWS_ACCESS_KEY", "WPXKJC7CZQCFPKY5727N")
-	os.Setenv("AWS_SECRET_KEY", "eHCSajxLvl94l36gIMlzZ/oW2O0rYYK+cVn5jNT2")
-	os.Setenv("AWS_REGION", "us-east-1")
-	os.Setenv("AZURE_STORAGE_ACCOUNT", "goreleaser")
-	os.Setenv("AZURE_STORAGE_KEY", "eHCSajxLvl94l36gIMlzZ/oW2O0rYYK+cVn5jNT2")
-	gcloudCredentials, _ := filepath.Abs("./testdata/credentials.json")
-	os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", gcloudCredentials)
-}
-
 func setEnv(env map[string]string) {
 	for k, v := range env {
 		os.Setenv(k, v)
 	}
 }
+
+func unsetEnv(env map[string]string) {
+	for k, _ := range env {
+		os.Unsetenv(k)
+	}
+}

internal/pipe/blob/openbucket.go

@@ -86,11 +86,15 @@ func (b Bucket) Upload(ctx *context.Context, conf config.Blob, folder string) er
 			}
 			_, err = w.Write(data)
 			if err != nil {
-				if errorContains(err, "NoSuchBucket", "ContainerNotFound", "notFound") {
+				switch {
+				case errorContains(err, "NoSuchBucket", "ContainerNotFound", "notFound"):
 					return fmt.Errorf("(%v) provided bucket does not exist", bucketURL)
-				}
+				case errorContains(err, "NoCredentialProviders"):
+					return fmt.Errorf("check credentials and access to bucket %s", bucketURL)
+				default:
 					return fmt.Errorf("failed to write to bucket : %s", err)
 				}
+			}
 			if err = w.Close(); err != nil {
 				switch {
 				case errorContains(err, "InvalidAccessKeyId"):
@@ -103,6 +107,10 @@ func (b Bucket) Upload(ctx *context.Context, conf config.Blob, folder string) er
 					return fmt.Errorf("azure storage account you provided is not valid")
 				case errorContains(err, "NoSuchBucket", "ContainerNotFound", "notFound"):
 					return fmt.Errorf("(%v) provided bucket does not exist", bucketURL)
+				case errorContains(err, "NoCredentialProviders"):
+					return fmt.Errorf("check credentials and access to bucket %s", bucketURL)
+				case errorContains(err, "ServiceCode=ResourceNotFound"):
+					return fmt.Errorf("missing azure storage key for provided bucket %s", bucketURL)
 				default:
 					return fmt.Errorf("failed to close Bucket writer: %s", err)
 				}

internal/pipe/blob/util.go

@@ -1,47 +0,0 @@
-package blob
-
-import (
-	"fmt"
-	"os"
-	"strings"
-)
-
-// Check required ENV variables based on Blob Provider
-func checkProvider(provider string) error {
-	switch provider {
-	case "azblob":
-		return checkEnv("AZURE_STORAGE_ACCOUNT", "AZURE_STORAGE_KEY")
-	case "gs":
-		return checkEnv("GOOGLE_APPLICATION_CREDENTIALS")
-	case "s3":
-		return checkEnv("AWS_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_REGION")
-	default:
-		return fmt.Errorf("unknown provider [%v],currently supported providers: [azblob, gs, s3]", provider)
-	}
-}
-
-func checkEnv(envs ...string) error {
-	var missingEnv []string
-
-	for _, env := range envs {
-		s := os.Getenv(env)
-		if s == "" {
-			missingEnv = append(missingEnv, env)
-		}
-	}
-
-	if len(missingEnv) != 0 {
-		return fmt.Errorf("missing %v", strings.Join(missingEnv, ","))
-	}
-	return nil
-}
-
-// Check if error contains specific string
-func errorContains(err error, subs ...string) bool {
-	for _, sub := range subs {
-		if strings.Contains(err.Error(), sub) {
-			return true
-		}
-	}
-	return false
-}

www/content/blob.md

@@ -41,19 +41,29 @@ blob:
 
 ## Authentication
 
-Currently it supports authentication only with Environment Variable, Below is the list of ENV variable required:
+Goreleaser's blob pipe athentication varies depends on the blob provider as mentioned below:
 
-### [S3 Provider](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html)
+### S3 Provider
 
-- AWS_ACCESS_KEY
+S3 provider support AWS [default credential provider](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials) chain in the following order:
-- AWS_SECRET_KEY
-- AWS_DEFAULT_REGION
 
-### [Azure Blob Provider](https://docs.microsoft.com/en-us/azure/storage/common/storage-azure-cli#set-default-azure-storage-account-environment-variables)
+- Environment variables.
+
+- Shared credentials file.
+
+- If your application is running on an Amazon EC2 instance, IAM role for Amazon EC2. 
+
+### Azure Blob Provider
+
+Currently it supports authentication only with [environment variables](https://docs.microsoft.com/en-us/azure/storage/common/storage-azure-cli#set-default-azure-storage-account-environment-variables):
 
 - AZURE_STORAGE_ACCOUNT
-- AZURE_STORAGE_KEY
+- AZURE_STORAGE_KEY or AZURE_STORAGE_SAS_TOKEN
 
 ### [GCS Provider](https://cloud.google.com/docs/authentication/production)
 
-- GOOGLE_APPLICATION_CREDENTIALS
+GCS provider uses [Application Default Credentials](https://cloud.google.com/docs/authentication/production) in the following order:
+
+- Environment Variable (GOOGLE_APPLICATION_CREDENTIALS)
+- Default Service Account from the compute instance(Compute Engine, Kubernetes Engine, Cloud function etc).
+