s3: add --s3-use-data-integrity-protections to fix BadDigest error in Alibaba, Tencent

Since aws/aws-sdk-go-v2#2960, aws-go-sdk-v2 changes its default integrity
behavior. This breaks some s3 providers (eg Tencent, Alibaba)

https://github.com/aws/aws-sdk-go-v2/discussions/2960

This introduces `use_data_integrity_protections` option to disable it.

Defaults to false with it set to true for AWS.

Fixes #8432
Fixes #8483

backend/s3/provider/AWS.yaml

@@ -137,3 +137,4 @@ use_accelerate_endpoint: true
 quirks:
   might_gzip: false # Never auto gzips objects
   use_unsigned_payload: false # AWS has trailer support which means it adds checksums in the trailer without seeking
+  use_data_integrity_protections: true

backend/s3/providers.go

@@ -26,6 +26,7 @@ type Quirks struct {
 	UseMultipartEtag            *bool  `yaml:"use_multipart_etag,omitempty"`
 	UseAlreadyExists            *bool  `yaml:"use_already_exists,omitempty"`
 	UseAcceptEncodingGzip       *bool  `yaml:"use_accept_encoding_gzip,omitempty"`
+	UseDataIntegrityProtections *bool  `yaml:"use_data_integrity_protections,omitempty"`
 	MightGzip                   *bool  `yaml:"might_gzip,omitempty"`
 	UseMultipartUploads         *bool  `yaml:"use_multipart_uploads,omitempty"`
 	UseUnsignedPayload          *bool  `yaml:"use_unsigned_payload,omitempty"`

backend/s3/s3.go

@@ -39,6 +39,9 @@ import (
 	smithyhttp "github.com/aws/smithy-go/transport/http"
 	"github.com/ncw/swift/v2"
 
+	"golang.org/x/net/http/httpguts"
+	"golang.org/x/sync/errgroup"
+
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/chunksize"
@@ -59,8 +62,6 @@ import (
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
 	"github.com/rclone/rclone/lib/version"
-	"golang.org/x/net/http/httpguts"
-	"golang.org/x/sync/errgroup"
 )
 
 // Register with Fs
@@ -574,6 +575,13 @@ circumstances or for testing.
 `,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "use_data_integrity_protections",
+			Help: `If true use AWS S3 data integrity protections.
+
+See [AWS Docs on Data Integrity Protections](https://docs.aws.amazon.com/sdkref/latest/guide/feature-dataintegrity.html)`,
+			Default:  fs.Tristate{},
+			Advanced: true,
 		}, {
 			Name:     "versions",
 			Help:     "Include old versions in directory listings.",
@@ -937,6 +945,7 @@ type Options struct {
 	DirectoryMarkers            bool                 `config:"directory_markers"`
 	UseMultipartEtag            fs.Tristate          `config:"use_multipart_etag"`
 	UsePresignedRequest         bool                 `config:"use_presigned_request"`
+	UseDataIntegrityProtections fs.Tristate          `config:"use_data_integrity_protections"`
 	Versions                    bool                 `config:"versions"`
 	VersionAt                   fs.Time              `config:"version_at"`
 	VersionDeleted              bool                 `config:"version_deleted"`
@@ -1302,6 +1311,10 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli
 		} else {
 			s3Opt.EndpointOptions.UseDualStackEndpoint = aws.DualStackEndpointStateDisabled
 		}
+		if !opt.UseDataIntegrityProtections.Value {
+			s3Opt.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
+			s3Opt.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
+		}
 		// FIXME not ported from SDK v1 - not sure what this does
 		// s3Opt.UsEast1RegionalEndpoint = endpoints.RegionalS3UsEast1Endpoint
 	})
@@ -1497,6 +1510,7 @@ func setQuirks(opt *Options, provider *Provider) {
 	set(&opt.ListURLEncode, true, provider.Quirks.ListURLEncode)
 	set(&opt.UseMultipartEtag, true, provider.Quirks.UseMultipartEtag)
 	set(&opt.UseAcceptEncodingGzip, true, provider.Quirks.UseAcceptEncodingGzip)
+	set(&opt.UseDataIntegrityProtections, false, provider.Quirks.UseDataIntegrityProtections)
 	set(&opt.MightGzip, true, provider.Quirks.MightGzip)
 	set(&opt.UseAlreadyExists, true, provider.Quirks.UseAlreadyExists)
 	set(&opt.UseMultipartUploads, true, provider.Quirks.UseMultipartUploads)