1
0
mirror of https://github.com/pocketbase/pocketbase.git synced 2025-03-18 21:57:50 +02:00

added basic s3blob tests

This commit is contained in:
Gani Georgiev 2025-03-08 22:28:51 +02:00
parent f799083c4f
commit ed36add853
13 changed files with 676 additions and 121 deletions

View File

@ -92,12 +92,13 @@ type ListPage struct {
// including across pages. I.e., all objects returned from a ListPage request
// made using a PageToken from a previous ListPage request's NextPageToken
// should have Key >= the Key for all objects from the previous request.
Objects []*ListObject
Objects []*ListObject `json:"objects"`
// NextPageToken should be left empty unless there are more objects
// to return. The value may be returned as ListOptions.PageToken on a
// subsequent ListPaged call, to fetch the next page of results.
// It can be an arbitrary []byte; it need not be a valid key.
NextPageToken []byte
NextPageToken []byte `json:"nextPageToken"`
}
// ListIterator iterates over List results.
@ -157,22 +158,22 @@ func (i *ListIterator) Next(ctx context.Context) (*ListObject, error) {
// ListObject represents a single blob returned from List.
type ListObject struct {
// Key is the key for this blob.
Key string
Key string `json:"key"`
// ModTime is the time the blob was last modified.
ModTime time.Time
ModTime time.Time `json:"modTime"`
// Size is the size of the blob's content in bytes.
Size int64
Size int64 `json:"size"`
// MD5 is an MD5 hash of the blob contents or nil if not available.
MD5 []byte
MD5 []byte `json:"md5"`
// IsDir indicates that this result represents a "directory" in the
// hierarchical namespace, ending in ListOptions.Delimiter. Key can be
// passed as ListOptions.Prefix to list items in the "directory".
// Fields other than Key and IsDir will not be set if IsDir is true.
IsDir bool
IsDir bool `json:"isDir"`
}
// List returns a ListIterator that can be used to iterate over blobs in a
@ -296,38 +297,48 @@ type Attributes struct {
// CacheControl specifies caching attributes that services may use
// when serving the blob.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control
CacheControl string
CacheControl string `json:"cacheControl"`
// ContentDisposition specifies whether the blob content is expected to be
// displayed inline or as an attachment.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition
ContentDisposition string
ContentDisposition string `json:"contentDisposition"`
// ContentEncoding specifies the encoding used for the blob's content, if any.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding
ContentEncoding string
ContentEncoding string `json:"contentEncoding"`
// ContentLanguage specifies the language used in the blob's content, if any.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Language
ContentLanguage string
ContentLanguage string `json:"contentLanguage"`
// ContentType is the MIME type of the blob. It will not be empty.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type
ContentType string
ContentType string `json:"contentType"`
// Metadata holds key/value pairs associated with the blob.
// Keys are guaranteed to be in lowercase, even if the backend service
// has case-sensitive keys (although note that Metadata written via
// this package will always be lowercased). If there are duplicate
// case-insensitive keys (e.g., "foo" and "FOO"), only one value
// will be kept, and it is undefined which one.
Metadata map[string]string
Metadata map[string]string `json:"metadata"`
// CreateTime is the time the blob was created, if available. If not available,
// CreateTime will be the zero time.
CreateTime time.Time
CreateTime time.Time `json:"createTime"`
// ModTime is the time the blob was last modified.
ModTime time.Time
ModTime time.Time `json:"modTime"`
// Size is the size of the blob's content in bytes.
Size int64
Size int64 `json:"size"`
// MD5 is an MD5 hash of the blob contents or nil if not available.
MD5 []byte
MD5 []byte `json:"md5"`
// ETag for the blob; see https://en.wikipedia.org/wiki/HTTP_ETag.
ETag string
ETag string `json:"etag"`
}
// Attributes returns attributes for the blob stored at key.

View File

@ -11,11 +11,13 @@ import (
type ReaderAttributes struct {
// ContentType is the MIME type of the blob object. It must not be empty.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type
ContentType string
ContentType string `json:"contentType"`
// ModTime is the time the blob object was last modified.
ModTime time.Time
ModTime time.Time `json:"modTime"`
// Size is the size of the object in bytes.
Size int64
Size int64 `json:"size"`
}
// DriverReader reads an object from the blob.

View File

@ -8,17 +8,18 @@ import (
"testing"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3/tests"
)
func TestS3CopyObject(t *testing.T) {
t.Parallel()
httpClient := NewTestClient(
&RequestStub{
httpClient := tests.NewClient(
&tests.RequestStub{
Method: http.MethodPut,
URL: "http://test_bucket.example.com/@dst_test",
Match: func(req *http.Request) bool {
return checkHeaders(req.Header, map[string]string{
return tests.ExpectHeaders(req.Header, map[string]string{
"test_header": "test",
"x-amz-copy-source": "test_bucket%2F@src_test",
"Authorization": "^.+Credential=123/.+$",

View File

@ -6,17 +6,18 @@ import (
"testing"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3/tests"
)
func TestS3DeleteObject(t *testing.T) {
t.Parallel()
httpClient := NewTestClient(
&RequestStub{
httpClient := tests.NewClient(
&tests.RequestStub{
Method: http.MethodDelete,
URL: "http://test_bucket.example.com/test_key",
Match: func(req *http.Request) bool {
return checkHeaders(req.Header, map[string]string{
return tests.ExpectHeaders(req.Header, map[string]string{
"test_header": "test",
"Authorization": "^.+Credential=123/.+$",
})

View File

@ -9,17 +9,18 @@ import (
"testing"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3/tests"
)
func TestS3GetObject(t *testing.T) {
t.Parallel()
httpClient := NewTestClient(
&RequestStub{
httpClient := tests.NewClient(
&tests.RequestStub{
Method: http.MethodGet,
URL: "http://test_bucket.example.com/test_key",
Match: func(req *http.Request) bool {
return checkHeaders(req.Header, map[string]string{
return tests.ExpectHeaders(req.Header, map[string]string{
"test_header": "test",
"Authorization": "^.+Credential=123/.+$",
})

View File

@ -7,17 +7,18 @@ import (
"testing"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3/tests"
)
func TestS3HeadObject(t *testing.T) {
t.Parallel()
httpClient := NewTestClient(
&RequestStub{
httpClient := tests.NewClient(
&tests.RequestStub{
Method: http.MethodHead,
URL: "http://test_bucket.example.com/test_key",
Match: func(req *http.Request) bool {
return checkHeaders(req.Header, map[string]string{
return tests.ExpectHeaders(req.Header, map[string]string{
"test_header": "test",
"Authorization": "^.+Credential=123/.+$",
})

View File

@ -9,6 +9,7 @@ import (
"testing"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3/tests"
)
func TestS3ListParamsEncode(t *testing.T) {
@ -62,12 +63,12 @@ func TestS3ListObjects(t *testing.T) {
FetchOwner: true,
}
httpClient := NewTestClient(
&RequestStub{
httpClient := tests.NewClient(
&tests.RequestStub{
Method: http.MethodGet,
URL: "http://test_bucket.example.com/?" + listParams.Encode(),
Match: func(req *http.Request) bool {
return checkHeaders(req.Header, map[string]string{
return tests.ExpectHeaders(req.Header, map[string]string{
"test_header": "test",
"Authorization": "^.+Credential=123/.+$",
})

View File

@ -7,6 +7,7 @@ import (
"testing"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3/tests"
)
func TestS3URL(t *testing.T) {
@ -111,12 +112,12 @@ func TestS3SignAndSend(t *testing.T) {
Endpoint: "https://example.com/",
AccessKey: "123",
SecretKey: "abc",
Client: NewTestClient(&RequestStub{
Client: tests.NewClient(&tests.RequestStub{
Method: http.MethodGet,
URL: "https://test_bucket.example.com/test",
Response: testResponse(),
Match: func(req *http.Request) bool {
return checkHeaders(req.Header, map[string]string{
return tests.ExpectHeaders(req.Header, map[string]string{
"Authorization": "AWS4-HMAC-SHA256 Credential=123/20250102/test_region/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=ea093662bc1deef08dfb4ac35453dfaad5ea89edf102e9dd3b7156c9a27e4c1f",
"Host": "test_bucket.example.com",
"X-Amz-Content-Sha256": "UNSIGNED-PAYLOAD",
@ -137,12 +138,12 @@ func TestS3SignAndSend(t *testing.T) {
Endpoint: "https://example.com/",
AccessKey: "456",
SecretKey: "def",
Client: NewTestClient(&RequestStub{
Client: tests.NewClient(&tests.RequestStub{
Method: http.MethodGet,
URL: "https://test_bucket.example.com/test",
Response: testResponse(),
Match: func(req *http.Request) bool {
return checkHeaders(req.Header, map[string]string{
return tests.ExpectHeaders(req.Header, map[string]string{
"Authorization": "AWS4-HMAC-SHA256 Credential=456/20250102/test_region/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=17510fa1f724403dd0a563b61c9b31d1d718f877fcbd75455620d17a8afce5fb",
"Host": "test_bucket.example.com",
"X-Amz-Content-Sha256": "UNSIGNED-PAYLOAD",
@ -168,12 +169,12 @@ func TestS3SignAndSend(t *testing.T) {
Endpoint: "https://example.com/",
AccessKey: "123",
SecretKey: "abc",
Client: NewTestClient(&RequestStub{
Client: tests.NewClient(&tests.RequestStub{
Method: http.MethodGet,
URL: "https://test_bucket.example.com/test",
Response: testResponse(),
Match: func(req *http.Request) bool {
return checkHeaders(req.Header, map[string]string{
return tests.ExpectHeaders(req.Header, map[string]string{
"authorization": "AWS4-HMAC-SHA256 Credential=123/20250102/test_region/s3/aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-example;x-amz-meta-a, Signature=86dccbcd012c33073dc99e9d0a9e0b717a4d8c11c37848cfa9a4a02716bc0db3",
"host": "test_bucket.example.com",
"x-amz-date": "20250102T150405Z",
@ -205,7 +206,7 @@ func TestS3SignAndSend(t *testing.T) {
}
defer resp.Body.Close()
err = s.s3Client.Client.(*TestClient).AssertNoRemaining()
err = s.s3Client.Client.(*tests.Client).AssertNoRemaining()
if err != nil {
t.Fatal(err)
}

View File

@ -1,4 +1,6 @@
package s3_test
// Package tests contains various tests helpers and utilities to assist
// with the S3 client testing.
package tests
import (
"errors"
@ -11,26 +13,9 @@ import (
"sync"
)
func checkHeaders(headers http.Header, expectations map[string]string) bool {
for h, expected := range expectations {
v := headers.Get(h)
pattern := expected
if !strings.HasPrefix(pattern, "^") && !strings.HasSuffix(pattern, "$") {
pattern = "^" + regexp.QuoteMeta(pattern) + "$"
}
expectedRegex, err := regexp.Compile(pattern)
if err != nil {
return false
}
if !expectedRegex.MatchString(v) {
return false
}
}
return true
// NewClient creates a new test Client loaded with the specified RequestStubs.
func NewClient(stubs ...*RequestStub) *Client {
return &Client{stubs: stubs}
}
type RequestStub struct {
@ -40,16 +25,13 @@ type RequestStub struct {
Response *http.Response
}
func NewTestClient(stubs ...*RequestStub) *TestClient {
return &TestClient{stubs: stubs}
}
type TestClient struct {
type Client struct {
stubs []*RequestStub
mu sync.Mutex
}
func (c *TestClient) AssertNoRemaining() error {
// AssertNoRemaining asserts that current client has no unprocessed requests remaining.
func (c *Client) AssertNoRemaining() error {
c.mu.Lock()
defer c.mu.Unlock()
@ -66,7 +48,8 @@ func (c *TestClient) AssertNoRemaining() error {
return errors.New(strings.Join(msgParts, "\n"))
}
func (c *TestClient) Do(req *http.Request) (*http.Response, error) {
// Do implements the [s3.HTTPClient] interface.
func (c *Client) Do(req *http.Request) (*http.Response, error) {
c.mu.Lock()
defer c.mu.Unlock()

View File

@ -0,0 +1,33 @@
package tests
import (
"net/http"
"regexp"
"strings"
)
// ExpectHeaders checks whether specified headers match the expectations.
// The expectations map entry key is the header name.
// The expectations map entry value is the first header value. If wrapped with `^...$`
// it is compared as regular expression.
func ExpectHeaders(headers http.Header, expectations map[string]string) bool {
for h, expected := range expectations {
v := headers.Get(h)
pattern := expected
if !strings.HasPrefix(pattern, "^") && !strings.HasSuffix(pattern, "$") {
pattern = "^" + regexp.QuoteMeta(pattern) + "$"
}
expectedRegex, err := regexp.Compile(pattern)
if err != nil {
return false
}
if !expectedRegex.MatchString(v) {
return false
}
}
return true
}

View File

@ -8,13 +8,14 @@ import (
"testing"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3/tests"
)
func TestUploaderRequiredFields(t *testing.T) {
t.Parallel()
s3Client := &s3.S3{
Client: NewTestClient(&RequestStub{Method: "PUT", URL: `^.+$`}), // match every upload
Client: tests.NewClient(&tests.RequestStub{Method: "PUT", URL: `^.+$`}), // match every upload
Region: "test_region",
Bucket: "test_bucket",
Endpoint: "http://example.com",
@ -71,8 +72,8 @@ func TestUploaderRequiredFields(t *testing.T) {
func TestUploaderSingleUpload(t *testing.T) {
t.Parallel()
httpClient := NewTestClient(
&RequestStub{
httpClient := tests.NewClient(
&tests.RequestStub{
Method: http.MethodPut,
URL: "http://test_bucket.example.com/test_key",
Match: func(req *http.Request) bool {
@ -81,7 +82,7 @@ func TestUploaderSingleUpload(t *testing.T) {
return false
}
return string(body) == "abcdefg" && checkHeaders(req.Header, map[string]string{
return string(body) == "abcdefg" && tests.ExpectHeaders(req.Header, map[string]string{
"Content-Length": "7",
"x-amz-meta-a": "123",
"x-amz-meta-b": "456",
@ -123,12 +124,12 @@ func TestUploaderSingleUpload(t *testing.T) {
func TestUploaderMultipartUploadSuccess(t *testing.T) {
t.Parallel()
httpClient := NewTestClient(
&RequestStub{
httpClient := tests.NewClient(
&tests.RequestStub{
Method: http.MethodPost,
URL: "http://test_bucket.example.com/test_key?uploads",
Match: func(req *http.Request) bool {
return checkHeaders(req.Header, map[string]string{
return tests.ExpectHeaders(req.Header, map[string]string{
"x-amz-meta-a": "123",
"x-amz-meta-b": "456",
"test_header": "test",
@ -146,7 +147,7 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) {
`)),
},
},
&RequestStub{
&tests.RequestStub{
Method: http.MethodPut,
URL: "http://test_bucket.example.com/test_key?partNumber=1&uploadId=test_id",
Match: func(req *http.Request) bool {
@ -155,7 +156,7 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) {
return false
}
return string(body) == "abc" && checkHeaders(req.Header, map[string]string{
return string(body) == "abc" && tests.ExpectHeaders(req.Header, map[string]string{
"Content-Length": "3",
"test_header": "test",
"Authorization": "^.+Credential=123/.+$",
@ -165,7 +166,7 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) {
Header: http.Header{"Etag": []string{"etag1"}},
},
},
&RequestStub{
&tests.RequestStub{
Method: http.MethodPut,
URL: "http://test_bucket.example.com/test_key?partNumber=2&uploadId=test_id",
Match: func(req *http.Request) bool {
@ -174,7 +175,7 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) {
return false
}
return string(body) == "def" && checkHeaders(req.Header, map[string]string{
return string(body) == "def" && tests.ExpectHeaders(req.Header, map[string]string{
"Content-Length": "3",
"test_header": "test",
"Authorization": "^.+Credential=123/.+$",
@ -184,7 +185,7 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) {
Header: http.Header{"Etag": []string{"etag2"}},
},
},
&RequestStub{
&tests.RequestStub{
Method: http.MethodPut,
URL: "http://test_bucket.example.com/test_key?partNumber=3&uploadId=test_id",
Match: func(req *http.Request) bool {
@ -192,7 +193,7 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) {
if err != nil {
return false
}
return string(body) == "g" && checkHeaders(req.Header, map[string]string{
return string(body) == "g" && tests.ExpectHeaders(req.Header, map[string]string{
"Content-Length": "1",
"test_header": "test",
"Authorization": "^.+Credential=123/.+$",
@ -202,7 +203,7 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) {
Header: http.Header{"Etag": []string{"etag3"}},
},
},
&RequestStub{
&tests.RequestStub{
Method: http.MethodPost,
URL: "http://test_bucket.example.com/test_key?uploadId=test_id",
Match: func(req *http.Request) bool {
@ -213,7 +214,7 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) {
expected := `<CompleteMultipartUpload><Part><ETag>etag1</ETag><PartNumber>1</PartNumber></Part><Part><ETag>etag2</ETag><PartNumber>2</PartNumber></Part><Part><ETag>etag3</ETag><PartNumber>3</PartNumber></Part></CompleteMultipartUpload>`
return strings.Contains(string(body), expected) && checkHeaders(req.Header, map[string]string{
return strings.Contains(string(body), expected) && tests.ExpectHeaders(req.Header, map[string]string{
"test_header": "test",
"Authorization": "^.+Credential=123/.+$",
})
@ -252,12 +253,12 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) {
func TestUploaderMultipartUploadPartFailure(t *testing.T) {
t.Parallel()
httpClient := NewTestClient(
&RequestStub{
httpClient := tests.NewClient(
&tests.RequestStub{
Method: http.MethodPost,
URL: "http://test_bucket.example.com/test_key?uploads",
Match: func(req *http.Request) bool {
return checkHeaders(req.Header, map[string]string{
return tests.ExpectHeaders(req.Header, map[string]string{
"x-amz-meta-a": "123",
"x-amz-meta-b": "456",
"test_header": "test",
@ -275,7 +276,7 @@ func TestUploaderMultipartUploadPartFailure(t *testing.T) {
`)),
},
},
&RequestStub{
&tests.RequestStub{
Method: http.MethodPut,
URL: "http://test_bucket.example.com/test_key?partNumber=1&uploadId=test_id",
Match: func(req *http.Request) bool {
@ -283,7 +284,7 @@ func TestUploaderMultipartUploadPartFailure(t *testing.T) {
if err != nil {
return false
}
return string(body) == "abc" && checkHeaders(req.Header, map[string]string{
return string(body) == "abc" && tests.ExpectHeaders(req.Header, map[string]string{
"Content-Length": "3",
"test_header": "test",
"Authorization": "^.+Credential=123/.+$",
@ -293,11 +294,11 @@ func TestUploaderMultipartUploadPartFailure(t *testing.T) {
Header: http.Header{"Etag": []string{"etag1"}},
},
},
&RequestStub{
&tests.RequestStub{
Method: http.MethodPut,
URL: "http://test_bucket.example.com/test_key?partNumber=2&uploadId=test_id",
Match: func(req *http.Request) bool {
return checkHeaders(req.Header, map[string]string{
return tests.ExpectHeaders(req.Header, map[string]string{
"test_header": "test",
"Authorization": "^.+Credential=123/.+$",
})
@ -306,11 +307,11 @@ func TestUploaderMultipartUploadPartFailure(t *testing.T) {
StatusCode: 400,
},
},
&RequestStub{
&tests.RequestStub{
Method: http.MethodDelete,
URL: "http://test_bucket.example.com/test_key?uploadId=test_id",
Match: func(req *http.Request) bool {
return checkHeaders(req.Header, map[string]string{
return tests.ExpectHeaders(req.Header, map[string]string{
"test_header": "test",
"Authorization": "^.+Credential=123/.+$",
})
@ -349,12 +350,12 @@ func TestUploaderMultipartUploadPartFailure(t *testing.T) {
func TestUploaderMultipartUploadCompleteFailure(t *testing.T) {
t.Parallel()
httpClient := NewTestClient(
&RequestStub{
httpClient := tests.NewClient(
&tests.RequestStub{
Method: http.MethodPost,
URL: "http://test_bucket.example.com/test_key?uploads",
Match: func(req *http.Request) bool {
return checkHeaders(req.Header, map[string]string{
return tests.ExpectHeaders(req.Header, map[string]string{
"x-amz-meta-a": "123",
"x-amz-meta-b": "456",
"test_header": "test",
@ -372,7 +373,7 @@ func TestUploaderMultipartUploadCompleteFailure(t *testing.T) {
`)),
},
},
&RequestStub{
&tests.RequestStub{
Method: http.MethodPut,
URL: "http://test_bucket.example.com/test_key?partNumber=1&uploadId=test_id",
Match: func(req *http.Request) bool {
@ -380,7 +381,7 @@ func TestUploaderMultipartUploadCompleteFailure(t *testing.T) {
if err != nil {
return false
}
return string(body) == "abc" && checkHeaders(req.Header, map[string]string{
return string(body) == "abc" && tests.ExpectHeaders(req.Header, map[string]string{
"Content-Length": "3",
"test_header": "test",
"Authorization": "^.+Credential=123/.+$",
@ -390,7 +391,7 @@ func TestUploaderMultipartUploadCompleteFailure(t *testing.T) {
Header: http.Header{"Etag": []string{"etag1"}},
},
},
&RequestStub{
&tests.RequestStub{
Method: http.MethodPut,
URL: "http://test_bucket.example.com/test_key?partNumber=2&uploadId=test_id",
Match: func(req *http.Request) bool {
@ -398,7 +399,7 @@ func TestUploaderMultipartUploadCompleteFailure(t *testing.T) {
if err != nil {
return false
}
return string(body) == "def" && checkHeaders(req.Header, map[string]string{
return string(body) == "def" && tests.ExpectHeaders(req.Header, map[string]string{
"Content-Length": "3",
"test_header": "test",
"Authorization": "^.+Credential=123/.+$",
@ -408,11 +409,11 @@ func TestUploaderMultipartUploadCompleteFailure(t *testing.T) {
Header: http.Header{"Etag": []string{"etag2"}},
},
},
&RequestStub{
&tests.RequestStub{
Method: http.MethodPost,
URL: "http://test_bucket.example.com/test_key?uploadId=test_id",
Match: func(req *http.Request) bool {
return checkHeaders(req.Header, map[string]string{
return tests.ExpectHeaders(req.Header, map[string]string{
"test_header": "test",
"Authorization": "^.+Credential=123/.+$",
})
@ -421,11 +422,11 @@ func TestUploaderMultipartUploadCompleteFailure(t *testing.T) {
StatusCode: 400,
},
},
&RequestStub{
&tests.RequestStub{
Method: http.MethodDelete,
URL: "http://test_bucket.example.com/test_key?uploadId=test_id",
Match: func(req *http.Request) bool {
return checkHeaders(req.Header, map[string]string{
return tests.ExpectHeaders(req.Header, map[string]string{
"test_header": "test",
"Authorization": "^.+Credential=123/.+$",
})

View File

@ -69,11 +69,17 @@ type driver struct {
// Close implements [blob/Driver.Close].
func (drv *driver) Close() error {
return nil
return nil // nothing to close
}
// NormalizeError implements [blob/Driver.NormalizeError].
func (drv *driver) NormalizeError(err error) error {
// already normalized
if errors.Is(err, blob.ErrNotFound) {
return err
}
// normalize base on its S3 error code
var ae s3.ResponseError
if errors.As(err, &ae) {
switch ae.Code {
@ -92,22 +98,20 @@ func (drv *driver) ListPaged(ctx context.Context, opts *blob.ListOptions) (*blob
pageSize = defaultPageSize
}
in := s3.ListParams{
listParams := s3.ListParams{
MaxKeys: pageSize,
}
if len(opts.PageToken) > 0 {
in.ContinuationToken = string(opts.PageToken)
listParams.ContinuationToken = string(opts.PageToken)
}
if opts.Prefix != "" {
in.Prefix = escapeKey(opts.Prefix)
listParams.Prefix = escapeKey(opts.Prefix)
}
if opts.Delimiter != "" {
in.Delimiter = escapeKey(opts.Delimiter)
listParams.Delimiter = escapeKey(opts.Delimiter)
}
var reqOptions []func(*http.Request)
resp, err := drv.s3.ListObjects(ctx, in, reqOptions...)
resp, err := drv.s3.ListObjects(ctx, listParams)
if err != nil {
return nil, err
}
@ -157,8 +161,7 @@ func (drv *driver) Attributes(ctx context.Context, key string) (*blob.Attributes
md := make(map[string]string, len(resp.Metadata))
for k, v := range resp.Metadata {
// See the package comments for more details on escaping of metadata
// keys & values.
// See the package comments for more details on escaping of metadata keys & values.
md[blob.HexUnescape(urlUnescape(k))] = urlUnescape(v)
}
@ -192,13 +195,11 @@ func (drv *driver) NewRangeReader(ctx context.Context, key string, offset, lengt
byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)
}
reqOptions := []func(*http.Request){
func(req *http.Request) {
req.Header.Set("Range", byteRange)
},
reqOpt := func(req *http.Request) {
req.Header.Set("Range", byteRange)
}
resp, err := drv.s3.GetObject(ctx, key, reqOptions...)
resp, err := drv.s3.GetObject(ctx, key, reqOpt)
if err != nil {
return nil, err
}

View File

@ -0,0 +1,518 @@
package s3blob_test
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"testing"
"github.com/pocketbase/pocketbase/tools/filesystem/blob"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3/tests"
)
func TestNew(t *testing.T) {
t.Parallel()
scenarios := []struct {
name string
s3Client *s3.S3
expectError bool
}{
{
"blank",
&s3.S3{},
true,
},
{
"no bucket",
&s3.S3{Region: "b", Endpoint: "c"},
true,
},
{
"no endpoint",
&s3.S3{Bucket: "a", Region: "b"},
true,
},
{
"no region",
&s3.S3{Bucket: "a", Endpoint: "c"},
true,
},
{
"with bucket, endpoint and region",
&s3.S3{Bucket: "a", Region: "b", Endpoint: "c"},
false,
},
}
for _, s := range scenarios {
t.Run(s.name, func(t *testing.T) {
drv, err := s3blob.New(s.s3Client)
hasErr := err != nil
if hasErr != s.expectError {
t.Fatalf("Expected hasErr %v, got %v (%v)", s.expectError, hasErr, err)
}
if err == nil && drv == nil {
t.Fatal("Expected non-nil driver instance")
}
})
}
}
func TestDriverClose(t *testing.T) {
t.Parallel()
drv, err := s3blob.New(&s3.S3{Bucket: "a", Region: "b", Endpoint: "c"})
if err != nil {
t.Fatal(err)
}
err = drv.Close()
if err != nil {
t.Fatalf("Expected nil, got error %v", err)
}
}
func TestDriverNormilizeError(t *testing.T) {
t.Parallel()
drv, err := s3blob.New(&s3.S3{Bucket: "a", Region: "b", Endpoint: "c"})
if err != nil {
t.Fatal(err)
}
scenarios := []struct {
name string
err error
expectErrNotFound bool
}{
{
"plain error",
errors.New("test"),
false,
},
{
"response error with custom code",
s3.ResponseError{Code: "test"},
false,
},
{
"response error with NoSuchBucket code",
s3.ResponseError{Code: "NoSuchBucket"},
true,
},
{
"response error with NoSuchKey code",
s3.ResponseError{Code: "NoSuchKey"},
true,
},
{
"response error with NotFound code",
s3.ResponseError{Code: "NotFound"},
true,
},
{
"wrapped response error with NotFound code", // ensures that the entire error's tree is checked
fmt.Errorf("test: %w", s3.ResponseError{Code: "NotFound"}),
true,
},
{
"already normalized error",
fmt.Errorf("test: %w", blob.ErrNotFound),
true,
},
}
for _, s := range scenarios {
t.Run(s.name, func(t *testing.T) {
err := drv.NormalizeError(s.err)
if err == nil {
t.Fatal("Expected non-nil error")
}
isErrNotFound := errors.Is(err, blob.ErrNotFound)
if isErrNotFound != s.expectErrNotFound {
t.Fatalf("Expected isErrNotFound %v, got %v (%v)", s.expectErrNotFound, isErrNotFound, err)
}
})
}
}
func TestDriverDeleteEscaping(t *testing.T) {
t.Parallel()
httpClient := tests.NewClient(&tests.RequestStub{
Method: http.MethodDelete,
URL: "https://test_bucket.example.com/..__0x2f__abc/test/",
})
drv, err := s3blob.New(&s3.S3{
Bucket: "test_bucket",
Region: "test_region",
Endpoint: "https://example.com",
Client: httpClient,
})
if err != nil {
t.Fatal(err)
}
err = drv.Delete(context.Background(), "../abc/test/")
if err != nil {
t.Fatal(err)
}
err = httpClient.AssertNoRemaining()
if err != nil {
t.Fatal(err)
}
}
func TestDriverCopyEscaping(t *testing.T) {
t.Parallel()
httpClient := tests.NewClient(&tests.RequestStub{
Method: http.MethodPut,
URL: "https://test_bucket.example.com/..__0x2f__a/",
Match: func(req *http.Request) bool {
return tests.ExpectHeaders(req.Header, map[string]string{
"x-amz-copy-source": "test_bucket%2F..__0x2f__b%2F",
})
},
Response: &http.Response{
Body: io.NopCloser(strings.NewReader(`<CopyObjectResult></CopyObjectResult>`)),
},
})
drv, err := s3blob.New(&s3.S3{
Bucket: "test_bucket",
Region: "test_region",
Endpoint: "https://example.com",
Client: httpClient,
})
if err != nil {
t.Fatal(err)
}
err = drv.Copy(context.Background(), "../a/", "../b/")
if err != nil {
t.Fatal(err)
}
err = httpClient.AssertNoRemaining()
if err != nil {
t.Fatal(err)
}
}
func TestDriverAttributes(t *testing.T) {
t.Parallel()
httpClient := tests.NewClient(&tests.RequestStub{
Method: http.MethodHead,
URL: "https://test_bucket.example.com/..__0x2f__a/",
Response: &http.Response{
Header: http.Header{
"Last-Modified": []string{"Mon, 01 Feb 2025 03:04:05 GMT"},
"Cache-Control": []string{"test_cache"},
"Content-Disposition": []string{"test_disposition"},
"Content-Encoding": []string{"test_encoding"},
"Content-Language": []string{"test_language"},
"Content-Type": []string{"test_type"},
"Content-Range": []string{"test_range"},
"Etag": []string{`"ce5be8b6f53645c596306c4572ece521"`},
"Content-Length": []string{"100"},
"x-amz-meta-AbC%40": []string{"%40test_meta_a"},
"x-amz-meta-Def": []string{"test_meta_b"},
},
Body: http.NoBody,
},
})
drv, err := s3blob.New(&s3.S3{
Bucket: "test_bucket",
Region: "test_region",
Endpoint: "https://example.com",
Client: httpClient,
})
if err != nil {
t.Fatal(err)
}
attrs, err := drv.Attributes(context.Background(), "../a/")
if err != nil {
t.Fatal(err)
}
raw, err := json.Marshal(attrs)
if err != nil {
t.Fatal(err)
}
expected := `{"cacheControl":"test_cache","contentDisposition":"test_disposition","contentEncoding":"test_encoding","contentLanguage":"test_language","contentType":"test_type","metadata":{"abc@":"@test_meta_a","def":"test_meta_b"},"createTime":"0001-01-01T00:00:00Z","modTime":"2025-02-01T03:04:05Z","size":100,"md5":"zlvotvU2RcWWMGxFcuzlIQ==","etag":"\"ce5be8b6f53645c596306c4572ece521\""}`
if str := string(raw); str != expected {
t.Fatalf("Expected attributes\n%s\ngot\n%s", expected, str)
}
err = httpClient.AssertNoRemaining()
if err != nil {
t.Fatal(err)
}
}
func TestDriverListPaged(t *testing.T) {
t.Parallel()
listResponse := func() *http.Response {
return &http.Response{
Body: io.NopCloser(strings.NewReader(`
<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>example</Name>
<ContinuationToken>ct</ContinuationToken>
<NextContinuationToken>test_next</NextContinuationToken>
<StartAfter>example0.txt</StartAfter>
<KeyCount>1</KeyCount>
<MaxKeys>3</MaxKeys>
<Contents>
<Key>..__0x2f__prefixB/test/example.txt</Key>
<LastModified>2025-01-01T01:02:03.123Z</LastModified>
<ETag>"ce5be8b6f53645c596306c4572ece521"</ETag>
<Size>123</Size>
</Contents>
<Contents>
<Key>prefixA/..__0x2f__escape.txt</Key>
<LastModified>2025-01-02T01:02:03.123Z</LastModified>
<Size>456</Size>
</Contents>
<CommonPrefixes>
<Prefix>prefixA</Prefix>
</CommonPrefixes>
<CommonPrefixes>
<Prefix>..__0x2f__prefixB</Prefix>
</CommonPrefixes>
</ListBucketResult>
`)),
}
}
expectedPage := `{"objects":[{"key":"../prefixB","modTime":"0001-01-01T00:00:00Z","size":0,"md5":null,"isDir":true},{"key":"../prefixB/test/example.txt","modTime":"2025-01-01T01:02:03.123Z","size":123,"md5":"zlvotvU2RcWWMGxFcuzlIQ==","isDir":false},{"key":"prefixA","modTime":"0001-01-01T00:00:00Z","size":0,"md5":null,"isDir":true},{"key":"prefixA/../escape.txt","modTime":"2025-01-02T01:02:03.123Z","size":456,"md5":null,"isDir":false}],"nextPageToken":"dGVzdF9uZXh0"}`
httpClient := tests.NewClient(
&tests.RequestStub{
Method: http.MethodGet,
URL: "https://test_bucket.example.com/?list-type=2&max-keys=1000",
Response: listResponse(),
},
&tests.RequestStub{
Method: http.MethodGet,
URL: "https://test_bucket.example.com/?continuation-token=test_token&delimiter=test_delimiter&list-type=2&max-keys=123&prefix=test_prefix",
Response: listResponse(),
},
)
drv, err := s3blob.New(&s3.S3{
Bucket: "test_bucket",
Region: "test_region",
Endpoint: "https://example.com",
Client: httpClient,
})
if err != nil {
t.Fatal(err)
}
scenarios := []struct {
name string
opts *blob.ListOptions
expected string
}{
{
"empty options",
&blob.ListOptions{},
expectedPage,
},
{
"filled options",
&blob.ListOptions{Prefix: "test_prefix", Delimiter: "test_delimiter", PageSize: 123, PageToken: []byte("test_token")},
expectedPage,
},
}
for _, s := range scenarios {
t.Run(s.name, func(t *testing.T) {
page, err := drv.ListPaged(context.Background(), s.opts)
if err != nil {
t.Fatal(err)
}
raw, err := json.Marshal(page)
if err != nil {
t.Fatal(err)
}
if str := string(raw); s.expected != str {
t.Fatalf("Expected page result\n%s\ngot\n%s", s.expected, str)
}
})
}
err = httpClient.AssertNoRemaining()
if err != nil {
t.Fatal(err)
}
}
func TestDriverNewRangeReader(t *testing.T) {
t.Parallel()
scenarios := []struct {
offset int64
length int64
httpClient *tests.Client
expectedAttrs string
}{
{
0,
0,
tests.NewClient(&tests.RequestStub{
Method: http.MethodGet,
URL: "https://test_bucket.example.com/..__0x2f__abc/test.txt",
Match: func(req *http.Request) bool {
return tests.ExpectHeaders(req.Header, map[string]string{
"Range": "bytes=0-0",
})
},
Response: &http.Response{
Header: http.Header{
"Last-Modified": []string{"Mon, 01 Feb 2025 03:04:05 GMT"},
"Content-Type": []string{"test_ct"},
"Content-Length": []string{"123"},
},
Body: io.NopCloser(strings.NewReader("test")),
},
}),
`{"contentType":"test_ct","modTime":"2025-02-01T03:04:05Z","size":123}`,
},
{
10,
-1,
tests.NewClient(&tests.RequestStub{
Method: http.MethodGet,
URL: "https://test_bucket.example.com/..__0x2f__abc/test.txt",
Match: func(req *http.Request) bool {
return tests.ExpectHeaders(req.Header, map[string]string{
"Range": "bytes=10-",
})
},
Response: &http.Response{
Header: http.Header{
"Last-Modified": []string{"Mon, 01 Feb 2025 03:04:05 GMT"},
"Content-Type": []string{"test_ct"},
"Content-Range": []string{"bytes 1-1/456"}, // should take precedence over content-length
"Content-Length": []string{"123"},
},
Body: io.NopCloser(strings.NewReader("test")),
},
}),
`{"contentType":"test_ct","modTime":"2025-02-01T03:04:05Z","size":456}`,
},
{
10,
0,
tests.NewClient(&tests.RequestStub{
Method: http.MethodGet,
URL: "https://test_bucket.example.com/..__0x2f__abc/test.txt",
Match: func(req *http.Request) bool {
return tests.ExpectHeaders(req.Header, map[string]string{
"Range": "bytes=10-10",
})
},
Response: &http.Response{
Header: http.Header{
"Last-Modified": []string{"Mon, 01 Feb 2025 03:04:05 GMT"},
"Content-Type": []string{"test_ct"},
// no range and length headers
// "Content-Range": []string{"bytes 1-1/456"},
// "Content-Length": []string{"123"},
},
Body: io.NopCloser(strings.NewReader("test")),
},
}),
`{"contentType":"test_ct","modTime":"2025-02-01T03:04:05Z","size":0}`,
},
{
10,
20,
tests.NewClient(&tests.RequestStub{
Method: http.MethodGet,
URL: "https://test_bucket.example.com/..__0x2f__abc/test.txt",
Match: func(req *http.Request) bool {
return tests.ExpectHeaders(req.Header, map[string]string{
"Range": "bytes=10-29",
})
},
Response: &http.Response{
Header: http.Header{
"Last-Modified": []string{"Mon, 01 Feb 2025 03:04:05 GMT"},
"Content-Type": []string{"test_ct"},
// with range header but invalid format -> content-length takes precedence
"Content-Range": []string{"bytes invalid-456"},
"Content-Length": []string{"123"},
},
Body: io.NopCloser(strings.NewReader("test")),
},
}),
`{"contentType":"test_ct","modTime":"2025-02-01T03:04:05Z","size":123}`,
},
}
for _, s := range scenarios {
t.Run(fmt.Sprintf("offset_%d_length_%d", s.offset, s.length), func(t *testing.T) {
drv, err := s3blob.New(&s3.S3{
Bucket: "test_bucket",
Region: "tesst_region",
Endpoint: "https://example.com",
Client: s.httpClient,
})
if err != nil {
t.Fatal(err)
}
r, err := drv.NewRangeReader(context.Background(), "../abc/test.txt", s.offset, s.length)
if err != nil {
t.Fatal(err)
}
defer r.Close()
// the response body should be always replaced with http.NoBody
if s.length == 0 {
body := make([]byte, 1)
n, err := r.Read(body)
if n != 0 || !errors.Is(err, io.EOF) {
t.Fatalf("Expected body to be http.NoBody, got %v (%v)", body, err)
}
}
rawAttrs, err := json.Marshal(r.Attributes())
if err != nil {
t.Fatal(err)
}
if str := string(rawAttrs); str != s.expectedAttrs {
t.Fatalf("Expected attributes\n%s\ngot\n%s", s.expectedAttrs, str)
}
err = s.httpClient.AssertNoRemaining()
if err != nil {
t.Fatal(err)
}
})
}
}