mirror of
https://github.com/rclone/rclone.git
synced 2025-01-29 21:04:23 +02:00
s3: make v2 signatures work for ceph
This commit is contained in:
parent
0da6f24221
commit
6a8de87116
27
s3/s3.go
27
s3/s3.go
@ -11,18 +11,12 @@ Progress of port to aws-sdk
|
|||||||
What happens if you CTRL-C a multipart upload
|
What happens if you CTRL-C a multipart upload
|
||||||
* get an incomplete upload
|
* get an incomplete upload
|
||||||
* disappears when you delete the bucket
|
* disappears when you delete the bucket
|
||||||
|
|
||||||
Doesn't support v2 signing so can't interface with Ceph
|
|
||||||
* http://tracker.ceph.com/issues/10333
|
|
||||||
* https://github.com/aws/aws-sdk-go/issues/291
|
|
||||||
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
|
||||||
"path"
|
"path"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
@ -31,6 +25,7 @@ import (
|
|||||||
"github.com/aws/aws-sdk-go/aws"
|
"github.com/aws/aws-sdk-go/aws"
|
||||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||||
|
"github.com/aws/aws-sdk-go/aws/service"
|
||||||
"github.com/aws/aws-sdk-go/service/s3"
|
"github.com/aws/aws-sdk-go/service/s3"
|
||||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||||
"github.com/ncw/rclone/fs"
|
"github.com/ncw/rclone/fs"
|
||||||
@ -205,11 +200,27 @@ func s3Connection(name string) (*s3.S3, error) {
|
|||||||
WithMaxRetries(maxRetries).
|
WithMaxRetries(maxRetries).
|
||||||
WithCredentials(auth).
|
WithCredentials(auth).
|
||||||
WithEndpoint(endpoint).
|
WithEndpoint(endpoint).
|
||||||
WithHTTPClient(fs.Config.Client())
|
WithHTTPClient(fs.Config.Client()).
|
||||||
|
WithS3ForcePathStyle(true)
|
||||||
|
// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
|
||||||
c := s3.New(awsConfig)
|
c := s3.New(awsConfig)
|
||||||
if region == "other-v2-signature" {
|
if region == "other-v2-signature" {
|
||||||
log.Fatal("Sorry v2 signatures not supported yet :-(")
|
fs.Debug(name, "Using v2 auth")
|
||||||
|
signer := func(req *service.Request) {
|
||||||
|
// Ignore AnonymousCredentials object
|
||||||
|
if req.Service.Config.Credentials == credentials.AnonymousCredentials {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
sign(accessKeyId, secretAccessKey, req.HTTPRequest)
|
||||||
|
}
|
||||||
|
c.Handlers.Sign.Clear()
|
||||||
|
c.Handlers.Sign.PushBack(service.BuildContentLength)
|
||||||
|
c.Handlers.Sign.PushBack(signer)
|
||||||
}
|
}
|
||||||
|
// Add user agent
|
||||||
|
c.Handlers.Build.PushBack(func(r *service.Request) {
|
||||||
|
r.HTTPRequest.Header.Set("User-Agent", fs.UserAgent)
|
||||||
|
})
|
||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
115
s3/v2sign.go
Normal file
115
s3/v2sign.go
Normal file
@ -0,0 +1,115 @@
|
|||||||
|
// v2 signing
|
||||||
|
|
||||||
|
package s3
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/hmac"
|
||||||
|
"crypto/sha1"
|
||||||
|
"encoding/base64"
|
||||||
|
"net/http"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// URL parameters that need to be added to the signature
|
||||||
|
var s3ParamsToSign = map[string]struct{}{
|
||||||
|
"acl": struct{}{},
|
||||||
|
"location": struct{}{},
|
||||||
|
"logging": struct{}{},
|
||||||
|
"notification": struct{}{},
|
||||||
|
"partNumber": struct{}{},
|
||||||
|
"policy": struct{}{},
|
||||||
|
"requestPayment": struct{}{},
|
||||||
|
"torrent": struct{}{},
|
||||||
|
"uploadId": struct{}{},
|
||||||
|
"uploads": struct{}{},
|
||||||
|
"versionId": struct{}{},
|
||||||
|
"versioning": struct{}{},
|
||||||
|
"versions": struct{}{},
|
||||||
|
"response-content-type": struct{}{},
|
||||||
|
"response-content-language": struct{}{},
|
||||||
|
"response-expires": struct{}{},
|
||||||
|
"response-cache-control": struct{}{},
|
||||||
|
"response-content-disposition": struct{}{},
|
||||||
|
"response-content-encoding": struct{}{},
|
||||||
|
}
|
||||||
|
|
||||||
|
// sign signs requests using v2 auth
|
||||||
|
//
|
||||||
|
// Cobbled together from goamz and aws-sdk-go
|
||||||
|
func sign(AccessKey, SecretKey string, req *http.Request) {
|
||||||
|
// Set date
|
||||||
|
date := time.Now().UTC().Format(time.RFC1123)
|
||||||
|
req.Header.Set("Date", date)
|
||||||
|
|
||||||
|
// Sort out URI
|
||||||
|
uri := req.URL.Opaque
|
||||||
|
if uri != "" {
|
||||||
|
if strings.HasPrefix(uri, "//") {
|
||||||
|
// Strip off //host/uri
|
||||||
|
uri = "/" + strings.Join(strings.Split(uri, "/")[3:], "/")
|
||||||
|
req.URL.Opaque = uri // reset to plain URI otherwise Ceph gets confused
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
uri = req.URL.Path
|
||||||
|
}
|
||||||
|
if uri == "" {
|
||||||
|
uri = "/"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Look through headers of interest
|
||||||
|
var md5 string
|
||||||
|
var contentType string
|
||||||
|
var headersToSign []string
|
||||||
|
for k, v := range req.Header {
|
||||||
|
k = strings.ToLower(k)
|
||||||
|
switch k {
|
||||||
|
case "content-md5":
|
||||||
|
md5 = v[0]
|
||||||
|
case "content-type":
|
||||||
|
contentType = v[0]
|
||||||
|
default:
|
||||||
|
if strings.HasPrefix(k, "x-amz-") {
|
||||||
|
vall := strings.Join(v, ",")
|
||||||
|
headersToSign = append(headersToSign, k+":"+vall)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Make headers of interest into canonical string
|
||||||
|
var joinedHeadersToSign string
|
||||||
|
if len(headersToSign) > 0 {
|
||||||
|
sort.StringSlice(headersToSign).Sort()
|
||||||
|
joinedHeadersToSign = strings.Join(headersToSign, "\n") + "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Look for query parameters which need to be added to the signature
|
||||||
|
params := req.URL.Query()
|
||||||
|
var queriesToSign []string
|
||||||
|
for k, vs := range params {
|
||||||
|
if _, ok := s3ParamsToSign[k]; ok {
|
||||||
|
for _, v := range vs {
|
||||||
|
if v == "" {
|
||||||
|
queriesToSign = append(queriesToSign, k)
|
||||||
|
} else {
|
||||||
|
queriesToSign = append(queriesToSign, k+"="+v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Add query parameters to URI
|
||||||
|
if len(queriesToSign) > 0 {
|
||||||
|
sort.StringSlice(queriesToSign).Sort()
|
||||||
|
uri += "?" + strings.Join(queriesToSign, "&")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Make signature
|
||||||
|
payload := req.Method + "\n" + md5 + "\n" + contentType + "\n" + date + "\n" + joinedHeadersToSign + uri
|
||||||
|
hash := hmac.New(sha1.New, []byte(SecretKey))
|
||||||
|
hash.Write([]byte(payload))
|
||||||
|
signature := make([]byte, base64.StdEncoding.EncodedLen(hash.Size()))
|
||||||
|
base64.StdEncoding.Encode(signature, hash.Sum(nil))
|
||||||
|
|
||||||
|
// Set signature in request
|
||||||
|
req.Header.Set("Authorization", "AWS "+AccessKey+":"+string(signature))
|
||||||
|
}
|
Loading…
x
Reference in New Issue
Block a user