go/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-11-23 21:44:49 +02:00
Code Issues Releases Activity

s3: add --s3-use-data-integrity-protections to fix BadDigest error in Alibaba, Tencent

Browse Source 
Since aws/aws-sdk-go-v2#2960, aws-go-sdk-v2 changes its default integrity
behavior. This breaks some s3 providers (eg Tencent, Alibaba)

https://github.com/aws/aws-sdk-go-v2/discussions/2960

This introduces `use_data_integrity_protections` option to disable it.

Defaults to false with it set to true for AWS.

Fixes #8432
Fixes #8483
This commit is contained in:
hunshcn
2025-11-12 23:15:13 +08:00
committed by GitHub
parent 9f75af38e3
commit 16971ab6b9
3 changed files with 93 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/s3/provider/AWS.yaml
View File

@@ -137,3 +137,4 @@ use_accelerate_endpoint: true
quirks: quirks:
might_gzip: false # Never auto gzips objects might_gzip: false # Never auto gzips objects
use_unsigned_payload: false # AWS has trailer support which means it adds checksums in the trailer without seeking use_unsigned_payload: false # AWS has trailer support which means it adds checksums in the trailer without seeking
use_data_integrity_protections: true

29
backend/s3/providers.go
View File

@@ -20,20 +20,21 @@ var NewYamlMap = orderedmap.New[string, string]
// Quirks defines all the S3 provider quirks // Quirks defines all the S3 provider quirks
type Quirks struct { type Quirks struct {
ListVersion *int `yaml:"list_version,omitempty"` // 1 or 2 ListVersion *int `yaml:"list_version,omitempty"` // 1 or 2
ForcePathStyle *bool `yaml:"force_path_style,omitempty"` // true = path-style ForcePathStyle *bool `yaml:"force_path_style,omitempty"` // true = path-style
ListURLEncode *bool `yaml:"list_url_encode,omitempty"` ListURLEncode *bool `yaml:"list_url_encode,omitempty"`
UseMultipartEtag *bool `yaml:"use_multipart_etag,omitempty"` UseMultipartEtag *bool `yaml:"use_multipart_etag,omitempty"`
UseAlreadyExists *bool `yaml:"use_already_exists,omitempty"` UseAlreadyExists *bool `yaml:"use_already_exists,omitempty"`
UseAcceptEncodingGzip *bool `yaml:"use_accept_encoding_gzip,omitempty"` UseAcceptEncodingGzip *bool `yaml:"use_accept_encoding_gzip,omitempty"`
MightGzip *bool `yaml:"might_gzip,omitempty"` UseDataIntegrityProtections *bool `yaml:"use_data_integrity_protections,omitempty"`
UseMultipartUploads *bool `yaml:"use_multipart_uploads,omitempty"` MightGzip *bool `yaml:"might_gzip,omitempty"`
UseUnsignedPayload *bool `yaml:"use_unsigned_payload,omitempty"` UseMultipartUploads *bool `yaml:"use_multipart_uploads,omitempty"`
UseXID *bool `yaml:"use_x_id,omitempty"` UseUnsignedPayload *bool `yaml:"use_unsigned_payload,omitempty"`
SignAcceptEncoding *bool `yaml:"sign_accept_encoding,omitempty"` UseXID *bool `yaml:"use_x_id,omitempty"`
CopyCutoff *int64 `yaml:"copy_cutoff,omitempty"` SignAcceptEncoding *bool `yaml:"sign_accept_encoding,omitempty"`
MaxUploadParts *int `yaml:"max_upload_parts,omitempty"` CopyCutoff *int64 `yaml:"copy_cutoff,omitempty"`
MinChunkSize *int64 `yaml:"min_chunk_size,omitempty"` MaxUploadParts *int `yaml:"max_upload_parts,omitempty"`
MinChunkSize *int64 `yaml:"min_chunk_size,omitempty"`
} }
// Provider defines the configurable data in each provider.yaml // Provider defines the configurable data in each provider.yaml

140
backend/s3/s3.go
View File

@@ -39,6 +39,9 @@ import (
smithyhttp "github.com/aws/smithy-go/transport/http" smithyhttp "github.com/aws/smithy-go/transport/http"
"github.com/ncw/swift/v2" "github.com/ncw/swift/v2"
"golang.org/x/net/http/httpguts"
"golang.org/x/sync/errgroup"
"github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/chunksize" "github.com/rclone/rclone/fs/chunksize"
@@ -59,8 +62,6 @@ import (
"github.com/rclone/rclone/lib/readers" "github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/rest" "github.com/rclone/rclone/lib/rest"
"github.com/rclone/rclone/lib/version" "github.com/rclone/rclone/lib/version"
"golang.org/x/net/http/httpguts"
"golang.org/x/sync/errgroup"
) )
// Register with Fs // Register with Fs
@@ -574,6 +575,13 @@ circumstances or for testing.
`, `,
Default: false, Default: false,
Advanced: true, Advanced: true,
}, {
Name: "use_data_integrity_protections",
Help: `If true use AWS S3 data integrity protections.
See [AWS Docs on Data Integrity Protections](https://docs.aws.amazon.com/sdkref/latest/guide/feature-dataintegrity.html)`,
Default: fs.Tristate{},
Advanced: true,
}, { }, {
Name: "versions", Name: "versions",
Help: "Include old versions in directory listings.", Help: "Include old versions in directory listings.",
@@ -892,67 +900,68 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
// Options defines the configuration for this backend // Options defines the configuration for this backend
type Options struct { type Options struct {
Provider string `config:"provider"` Provider string `config:"provider"`
EnvAuth bool `config:"env_auth"` EnvAuth bool `config:"env_auth"`
AccessKeyID string `config:"access_key_id"` AccessKeyID string `config:"access_key_id"`
SecretAccessKey string `config:"secret_access_key"` SecretAccessKey string `config:"secret_access_key"`
Region string `config:"region"` Region string `config:"region"`
Endpoint string `config:"endpoint"` Endpoint string `config:"endpoint"`
STSEndpoint string `config:"sts_endpoint"` STSEndpoint string `config:"sts_endpoint"`
UseDualStack bool `config:"use_dual_stack"` UseDualStack bool `config:"use_dual_stack"`
LocationConstraint string `config:"location_constraint"` LocationConstraint string `config:"location_constraint"`
ACL string `config:"acl"` ACL string `config:"acl"`
BucketACL string `config:"bucket_acl"` BucketACL string `config:"bucket_acl"`
RequesterPays bool `config:"requester_pays"` RequesterPays bool `config:"requester_pays"`
ServerSideEncryption string `config:"server_side_encryption"` ServerSideEncryption string `config:"server_side_encryption"`
SSEKMSKeyID string `config:"sse_kms_key_id"` SSEKMSKeyID string `config:"sse_kms_key_id"`
SSECustomerAlgorithm string `config:"sse_customer_algorithm"` SSECustomerAlgorithm string `config:"sse_customer_algorithm"`
SSECustomerKey string `config:"sse_customer_key"` SSECustomerKey string `config:"sse_customer_key"`
SSECustomerKeyBase64 string `config:"sse_customer_key_base64"` SSECustomerKeyBase64 string `config:"sse_customer_key_base64"`
SSECustomerKeyMD5 string `config:"sse_customer_key_md5"` SSECustomerKeyMD5 string `config:"sse_customer_key_md5"`
StorageClass string `config:"storage_class"` StorageClass string `config:"storage_class"`
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
CopyCutoff fs.SizeSuffix `config:"copy_cutoff"` CopyCutoff fs.SizeSuffix `config:"copy_cutoff"`
ChunkSize fs.SizeSuffix `config:"chunk_size"` ChunkSize fs.SizeSuffix `config:"chunk_size"`
MaxUploadParts int `config:"max_upload_parts"` MaxUploadParts int `config:"max_upload_parts"`
DisableChecksum bool `config:"disable_checksum"` DisableChecksum bool `config:"disable_checksum"`
SharedCredentialsFile string `config:"shared_credentials_file"` SharedCredentialsFile string `config:"shared_credentials_file"`
Profile string `config:"profile"` Profile string `config:"profile"`
SessionToken string `config:"session_token"` SessionToken string `config:"session_token"`
UploadConcurrency int `config:"upload_concurrency"` UploadConcurrency int `config:"upload_concurrency"`
ForcePathStyle bool `config:"force_path_style"` ForcePathStyle bool `config:"force_path_style"`
V2Auth bool `config:"v2_auth"` V2Auth bool `config:"v2_auth"`
UseAccelerateEndpoint bool `config:"use_accelerate_endpoint"` UseAccelerateEndpoint bool `config:"use_accelerate_endpoint"`
UseARNRegion bool `config:"use_arn_region"` UseARNRegion bool `config:"use_arn_region"`
LeavePartsOnError bool `config:"leave_parts_on_error"` LeavePartsOnError bool `config:"leave_parts_on_error"`
ListChunk int32 `config:"list_chunk"` ListChunk int32 `config:"list_chunk"`
ListVersion int `config:"list_version"` ListVersion int `config:"list_version"`
ListURLEncode fs.Tristate `config:"list_url_encode"` ListURLEncode fs.Tristate `config:"list_url_encode"`
NoCheckBucket bool `config:"no_check_bucket"` NoCheckBucket bool `config:"no_check_bucket"`
NoHead bool `config:"no_head"` NoHead bool `config:"no_head"`
NoHeadObject bool `config:"no_head_object"` NoHeadObject bool `config:"no_head_object"`
Enc encoder.MultiEncoder `config:"encoding"` Enc encoder.MultiEncoder `config:"encoding"`
DisableHTTP2 bool `config:"disable_http2"` DisableHTTP2 bool `config:"disable_http2"`
DownloadURL string `config:"download_url"` DownloadURL string `config:"download_url"`
DirectoryMarkers bool `config:"directory_markers"` DirectoryMarkers bool `config:"directory_markers"`
UseMultipartEtag fs.Tristate `config:"use_multipart_etag"` UseMultipartEtag fs.Tristate `config:"use_multipart_etag"`
UsePresignedRequest bool `config:"use_presigned_request"` UsePresignedRequest bool `config:"use_presigned_request"`
Versions bool `config:"versions"` UseDataIntegrityProtections fs.Tristate `config:"use_data_integrity_protections"`
VersionAt fs.Time `config:"version_at"` Versions bool `config:"versions"`
VersionDeleted bool `config:"version_deleted"` VersionAt fs.Time `config:"version_at"`
Decompress bool `config:"decompress"` VersionDeleted bool `config:"version_deleted"`
MightGzip fs.Tristate `config:"might_gzip"` Decompress bool `config:"decompress"`
UseAcceptEncodingGzip fs.Tristate `config:"use_accept_encoding_gzip"` MightGzip fs.Tristate `config:"might_gzip"`
NoSystemMetadata bool `config:"no_system_metadata"` UseAcceptEncodingGzip fs.Tristate `config:"use_accept_encoding_gzip"`
UseAlreadyExists fs.Tristate `config:"use_already_exists"` NoSystemMetadata bool `config:"no_system_metadata"`
UseMultipartUploads fs.Tristate `config:"use_multipart_uploads"` UseAlreadyExists fs.Tristate `config:"use_already_exists"`
UseUnsignedPayload fs.Tristate `config:"use_unsigned_payload"` UseMultipartUploads fs.Tristate `config:"use_multipart_uploads"`
SDKLogMode sdkLogMode `config:"sdk_log_mode"` UseUnsignedPayload fs.Tristate `config:"use_unsigned_payload"`
DirectoryBucket bool `config:"directory_bucket"` SDKLogMode sdkLogMode `config:"sdk_log_mode"`
IBMAPIKey string `config:"ibm_api_key"` DirectoryBucket bool `config:"directory_bucket"`
IBMInstanceID string `config:"ibm_resource_instance_id"` IBMAPIKey string `config:"ibm_api_key"`
UseXID fs.Tristate `config:"use_x_id"` IBMInstanceID string `config:"ibm_resource_instance_id"`
SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"` UseXID fs.Tristate `config:"use_x_id"`
SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"`
} }
// Fs represents a remote s3 server // Fs represents a remote s3 server
@@ -1302,6 +1311,10 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli
} else { } else {
s3Opt.EndpointOptions.UseDualStackEndpoint = aws.DualStackEndpointStateDisabled s3Opt.EndpointOptions.UseDualStackEndpoint = aws.DualStackEndpointStateDisabled
} }
if !opt.UseDataIntegrityProtections.Value {
s3Opt.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
s3Opt.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
}
// FIXME not ported from SDK v1 - not sure what this does // FIXME not ported from SDK v1 - not sure what this does
// s3Opt.UsEast1RegionalEndpoint = endpoints.RegionalS3UsEast1Endpoint // s3Opt.UsEast1RegionalEndpoint = endpoints.RegionalS3UsEast1Endpoint
}) })
@@ -1497,6 +1510,7 @@ func setQuirks(opt *Options, provider *Provider) {
set(&opt.ListURLEncode, true, provider.Quirks.ListURLEncode) set(&opt.ListURLEncode, true, provider.Quirks.ListURLEncode)
set(&opt.UseMultipartEtag, true, provider.Quirks.UseMultipartEtag) set(&opt.UseMultipartEtag, true, provider.Quirks.UseMultipartEtag)
set(&opt.UseAcceptEncodingGzip, true, provider.Quirks.UseAcceptEncodingGzip) set(&opt.UseAcceptEncodingGzip, true, provider.Quirks.UseAcceptEncodingGzip)
set(&opt.UseDataIntegrityProtections, false, provider.Quirks.UseDataIntegrityProtections)
set(&opt.MightGzip, true, provider.Quirks.MightGzip) set(&opt.MightGzip, true, provider.Quirks.MightGzip)
set(&opt.UseAlreadyExists, true, provider.Quirks.UseAlreadyExists) set(&opt.UseAlreadyExists, true, provider.Quirks.UseAlreadyExists)
set(&opt.UseMultipartUploads, true, provider.Quirks.UseMultipartUploads) set(&opt.UseMultipartUploads, true, provider.Quirks.UseMultipartUploads)