upgraded to aws-sdk-go-v2 and added a special middleware for GCP

tools/filesystem/filesystem.go

@@ -15,9 +15,10 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go-v2/config"
-	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/disintegration/imaging"
 	"github.com/gabriel-vasile/mimetype"
 	"github.com/pocketbase/pocketbase/tools/list"
@@ -26,6 +27,8 @@ import (
 	"gocloud.dev/blob/s3blob"
 )
 
+var gcpIgnoreHeaders = []string{"Accept-Encoding"}
+
 type System struct {
 	ctx    context.Context
 	bucket *blob.Bucket
@@ -44,19 +47,38 @@ func NewS3(
 ) (*System, error) {
 	ctx := context.Background() // default context
 
-	cred := credentials.NewStaticCredentials(accessKey, secretKey, "")
+	cred := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")
 
-	sess, err := session.NewSession(&aws.Config{
+	cfg, err := config.LoadDefaultConfig(ctx,
-		Region:           aws.String(region),
+		config.WithCredentialsProvider(cred),
-		Endpoint:         aws.String(endpoint),
+		config.WithRegion(region),
-		Credentials:      cred,
+		config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
-		S3ForcePathStyle: aws.Bool(s3ForcePathStyle),
+			// ensure that the endpoint has url scheme for
-	})
+			// backward compatibility with v1 of the aws sdk
+			prefixedEndpoint := endpoint
+			if !strings.Contains(endpoint, "://") {
+				prefixedEndpoint = "https://" + endpoint
+			}
+
+			return aws.Endpoint{URL: prefixedEndpoint, SigningRegion: region}, nil
+		})),
+	)
 	if err != nil {
 		return nil, err
 	}
 
-	bucket, err := s3blob.OpenBucket(ctx, sess, bucketName, nil)
+	client := s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = s3ForcePathStyle
+
+		// Google Cloud Storage alters the Accept-Encoding header,
+		// which breaks the v2 request signature
+		// (https://github.com/aws/aws-sdk-go-v2/issues/1816)
+		if strings.Contains(endpoint, "storage.googleapis.com") {
+			ignoreSigningHeaders(o, gcpIgnoreHeaders)
+		}
+	})
+
+	bucket, err := s3blob.OpenBucketV2(ctx, client, bucketName, nil)
 	if err != nil {
 		return nil, err
 	}

tools/filesystem/ignores_signing_headers.go

@@ -0,0 +1,72 @@
+package filesystem
+
+import (
+	"context"
+	"fmt"
+
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/smithy-go/middleware"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+)
+
+// ignoreSigningHeaders excludes the listed headers
+// from the request signing because some providers may alter them.
+//
+// See https://github.com/aws/aws-sdk-go-v2/issues/1816.
+func ignoreSigningHeaders(o *s3.Options, headers []string) {
+	o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error {
+		if err := stack.Finalize.Insert(ignoreHeaders(headers), "Signing", middleware.Before); err != nil {
+			return err
+		}
+
+		if err := stack.Finalize.Insert(restoreIgnored(), "Signing", middleware.After); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+type ignoredHeadersKey struct{}
+
+func ignoreHeaders(headers []string) middleware.FinalizeMiddleware {
+	return middleware.FinalizeMiddlewareFunc(
+		"IgnoreHeaders",
+		func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
+			req, ok := in.Request.(*smithyhttp.Request)
+			if !ok {
+				return out, metadata, &v4.SigningError{Err: fmt.Errorf("(ignoreHeaders) unexpected request middleware type %T", in.Request)}
+			}
+
+			ignored := make(map[string]string, len(headers))
+			for _, h := range headers {
+				ignored[h] = req.Header.Get(h)
+				req.Header.Del(h)
+			}
+
+			ctx = middleware.WithStackValue(ctx, ignoredHeadersKey{}, ignored)
+
+			return next.HandleFinalize(ctx, in)
+		},
+	)
+}
+
+func restoreIgnored() middleware.FinalizeMiddleware {
+	return middleware.FinalizeMiddlewareFunc(
+		"RestoreIgnored",
+		func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
+			req, ok := in.Request.(*smithyhttp.Request)
+			if !ok {
+				return out, metadata, &v4.SigningError{Err: fmt.Errorf("(restoreIgnored) unexpected request middleware type %T", in.Request)}
+			}
+
+			ignored, _ := middleware.GetStackValue(ctx, ignoredHeadersKey{}).(map[string]string)
+			for k, v := range ignored {
+				req.Header.Set(k, v)
+			}
+
+			return next.HandleFinalize(ctx, in)
+		},
+	)
+}