diff --git a/tools/filesystem/blob/bucket.go b/tools/filesystem/blob/bucket.go index 3ab3f251..68369221 100644 --- a/tools/filesystem/blob/bucket.go +++ b/tools/filesystem/blob/bucket.go @@ -92,12 +92,13 @@ type ListPage struct { // including across pages. I.e., all objects returned from a ListPage request // made using a PageToken from a previous ListPage request's NextPageToken // should have Key >= the Key for all objects from the previous request. - Objects []*ListObject + Objects []*ListObject `json:"objects"` + // NextPageToken should be left empty unless there are more objects // to return. The value may be returned as ListOptions.PageToken on a // subsequent ListPaged call, to fetch the next page of results. // It can be an arbitrary []byte; it need not be a valid key. - NextPageToken []byte + NextPageToken []byte `json:"nextPageToken"` } // ListIterator iterates over List results. @@ -157,22 +158,22 @@ func (i *ListIterator) Next(ctx context.Context) (*ListObject, error) { // ListObject represents a single blob returned from List. type ListObject struct { // Key is the key for this blob. - Key string + Key string `json:"key"` // ModTime is the time the blob was last modified. - ModTime time.Time + ModTime time.Time `json:"modTime"` // Size is the size of the blob's content in bytes. - Size int64 + Size int64 `json:"size"` // MD5 is an MD5 hash of the blob contents or nil if not available. - MD5 []byte + MD5 []byte `json:"md5"` // IsDir indicates that this result represents a "directory" in the // hierarchical namespace, ending in ListOptions.Delimiter. Key can be // passed as ListOptions.Prefix to list items in the "directory". // Fields other than Key and IsDir will not be set if IsDir is true. - IsDir bool + IsDir bool `json:"isDir"` } // List returns a ListIterator that can be used to iterate over blobs in a @@ -296,38 +297,48 @@ type Attributes struct { // CacheControl specifies caching attributes that services may use // when serving the blob. // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control - CacheControl string + CacheControl string `json:"cacheControl"` + // ContentDisposition specifies whether the blob content is expected to be // displayed inline or as an attachment. // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition - ContentDisposition string + ContentDisposition string `json:"contentDisposition"` + // ContentEncoding specifies the encoding used for the blob's content, if any. // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding - ContentEncoding string + ContentEncoding string `json:"contentEncoding"` + // ContentLanguage specifies the language used in the blob's content, if any. // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Language - ContentLanguage string + ContentLanguage string `json:"contentLanguage"` + // ContentType is the MIME type of the blob. It will not be empty. // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type - ContentType string + ContentType string `json:"contentType"` + // Metadata holds key/value pairs associated with the blob. // Keys are guaranteed to be in lowercase, even if the backend service // has case-sensitive keys (although note that Metadata written via // this package will always be lowercased). If there are duplicate // case-insensitive keys (e.g., "foo" and "FOO"), only one value // will be kept, and it is undefined which one. - Metadata map[string]string + Metadata map[string]string `json:"metadata"` + // CreateTime is the time the blob was created, if available. If not available, // CreateTime will be the zero time. - CreateTime time.Time + CreateTime time.Time `json:"createTime"` + // ModTime is the time the blob was last modified. - ModTime time.Time + ModTime time.Time `json:"modTime"` + // Size is the size of the blob's content in bytes. - Size int64 + Size int64 `json:"size"` + // MD5 is an MD5 hash of the blob contents or nil if not available. - MD5 []byte + MD5 []byte `json:"md5"` + // ETag for the blob; see https://en.wikipedia.org/wiki/HTTP_ETag. - ETag string + ETag string `json:"etag"` } // Attributes returns attributes for the blob stored at key. diff --git a/tools/filesystem/blob/driver.go b/tools/filesystem/blob/driver.go index 6cafefa0..fb71730f 100644 --- a/tools/filesystem/blob/driver.go +++ b/tools/filesystem/blob/driver.go @@ -11,11 +11,13 @@ import ( type ReaderAttributes struct { // ContentType is the MIME type of the blob object. It must not be empty. // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type - ContentType string + ContentType string `json:"contentType"` + // ModTime is the time the blob object was last modified. - ModTime time.Time + ModTime time.Time `json:"modTime"` + // Size is the size of the object in bytes. - Size int64 + Size int64 `json:"size"` } // DriverReader reads an object from the blob. diff --git a/tools/filesystem/internal/s3blob/s3/copy_object_test.go b/tools/filesystem/internal/s3blob/s3/copy_object_test.go index ce2d7abf..968f818e 100644 --- a/tools/filesystem/internal/s3blob/s3/copy_object_test.go +++ b/tools/filesystem/internal/s3blob/s3/copy_object_test.go @@ -8,17 +8,18 @@ import ( "testing" "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3" + "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3/tests" ) func TestS3CopyObject(t *testing.T) { t.Parallel() - httpClient := NewTestClient( - &RequestStub{ + httpClient := tests.NewClient( + &tests.RequestStub{ Method: http.MethodPut, URL: "http://test_bucket.example.com/@dst_test", Match: func(req *http.Request) bool { - return checkHeaders(req.Header, map[string]string{ + return tests.ExpectHeaders(req.Header, map[string]string{ "test_header": "test", "x-amz-copy-source": "test_bucket%2F@src_test", "Authorization": "^.+Credential=123/.+$", diff --git a/tools/filesystem/internal/s3blob/s3/delete_object_test.go b/tools/filesystem/internal/s3blob/s3/delete_object_test.go index 7db6c572..48d245cc 100644 --- a/tools/filesystem/internal/s3blob/s3/delete_object_test.go +++ b/tools/filesystem/internal/s3blob/s3/delete_object_test.go @@ -6,17 +6,18 @@ import ( "testing" "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3" + "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3/tests" ) func TestS3DeleteObject(t *testing.T) { t.Parallel() - httpClient := NewTestClient( - &RequestStub{ + httpClient := tests.NewClient( + &tests.RequestStub{ Method: http.MethodDelete, URL: "http://test_bucket.example.com/test_key", Match: func(req *http.Request) bool { - return checkHeaders(req.Header, map[string]string{ + return tests.ExpectHeaders(req.Header, map[string]string{ "test_header": "test", "Authorization": "^.+Credential=123/.+$", }) diff --git a/tools/filesystem/internal/s3blob/s3/get_object_test.go b/tools/filesystem/internal/s3blob/s3/get_object_test.go index 802b14ab..3dde3b05 100644 --- a/tools/filesystem/internal/s3blob/s3/get_object_test.go +++ b/tools/filesystem/internal/s3blob/s3/get_object_test.go @@ -9,17 +9,18 @@ import ( "testing" "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3" + "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3/tests" ) func TestS3GetObject(t *testing.T) { t.Parallel() - httpClient := NewTestClient( - &RequestStub{ + httpClient := tests.NewClient( + &tests.RequestStub{ Method: http.MethodGet, URL: "http://test_bucket.example.com/test_key", Match: func(req *http.Request) bool { - return checkHeaders(req.Header, map[string]string{ + return tests.ExpectHeaders(req.Header, map[string]string{ "test_header": "test", "Authorization": "^.+Credential=123/.+$", }) diff --git a/tools/filesystem/internal/s3blob/s3/head_object_test.go b/tools/filesystem/internal/s3blob/s3/head_object_test.go index d7a8c965..e2bf9796 100644 --- a/tools/filesystem/internal/s3blob/s3/head_object_test.go +++ b/tools/filesystem/internal/s3blob/s3/head_object_test.go @@ -7,17 +7,18 @@ import ( "testing" "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3" + "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3/tests" ) func TestS3HeadObject(t *testing.T) { t.Parallel() - httpClient := NewTestClient( - &RequestStub{ + httpClient := tests.NewClient( + &tests.RequestStub{ Method: http.MethodHead, URL: "http://test_bucket.example.com/test_key", Match: func(req *http.Request) bool { - return checkHeaders(req.Header, map[string]string{ + return tests.ExpectHeaders(req.Header, map[string]string{ "test_header": "test", "Authorization": "^.+Credential=123/.+$", }) diff --git a/tools/filesystem/internal/s3blob/s3/list_objects_test.go b/tools/filesystem/internal/s3blob/s3/list_objects_test.go index e6d9b728..cd0b7f7c 100644 --- a/tools/filesystem/internal/s3blob/s3/list_objects_test.go +++ b/tools/filesystem/internal/s3blob/s3/list_objects_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3" + "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3/tests" ) func TestS3ListParamsEncode(t *testing.T) { @@ -62,12 +63,12 @@ func TestS3ListObjects(t *testing.T) { FetchOwner: true, } - httpClient := NewTestClient( - &RequestStub{ + httpClient := tests.NewClient( + &tests.RequestStub{ Method: http.MethodGet, URL: "http://test_bucket.example.com/?" + listParams.Encode(), Match: func(req *http.Request) bool { - return checkHeaders(req.Header, map[string]string{ + return tests.ExpectHeaders(req.Header, map[string]string{ "test_header": "test", "Authorization": "^.+Credential=123/.+$", }) diff --git a/tools/filesystem/internal/s3blob/s3/s3_test.go b/tools/filesystem/internal/s3blob/s3/s3_test.go index 32f2f927..b17a788f 100644 --- a/tools/filesystem/internal/s3blob/s3/s3_test.go +++ b/tools/filesystem/internal/s3blob/s3/s3_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3" + "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3/tests" ) func TestS3URL(t *testing.T) { @@ -111,12 +112,12 @@ func TestS3SignAndSend(t *testing.T) { Endpoint: "https://example.com/", AccessKey: "123", SecretKey: "abc", - Client: NewTestClient(&RequestStub{ + Client: tests.NewClient(&tests.RequestStub{ Method: http.MethodGet, URL: "https://test_bucket.example.com/test", Response: testResponse(), Match: func(req *http.Request) bool { - return checkHeaders(req.Header, map[string]string{ + return tests.ExpectHeaders(req.Header, map[string]string{ "Authorization": "AWS4-HMAC-SHA256 Credential=123/20250102/test_region/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=ea093662bc1deef08dfb4ac35453dfaad5ea89edf102e9dd3b7156c9a27e4c1f", "Host": "test_bucket.example.com", "X-Amz-Content-Sha256": "UNSIGNED-PAYLOAD", @@ -137,12 +138,12 @@ func TestS3SignAndSend(t *testing.T) { Endpoint: "https://example.com/", AccessKey: "456", SecretKey: "def", - Client: NewTestClient(&RequestStub{ + Client: tests.NewClient(&tests.RequestStub{ Method: http.MethodGet, URL: "https://test_bucket.example.com/test", Response: testResponse(), Match: func(req *http.Request) bool { - return checkHeaders(req.Header, map[string]string{ + return tests.ExpectHeaders(req.Header, map[string]string{ "Authorization": "AWS4-HMAC-SHA256 Credential=456/20250102/test_region/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=17510fa1f724403dd0a563b61c9b31d1d718f877fcbd75455620d17a8afce5fb", "Host": "test_bucket.example.com", "X-Amz-Content-Sha256": "UNSIGNED-PAYLOAD", @@ -168,12 +169,12 @@ func TestS3SignAndSend(t *testing.T) { Endpoint: "https://example.com/", AccessKey: "123", SecretKey: "abc", - Client: NewTestClient(&RequestStub{ + Client: tests.NewClient(&tests.RequestStub{ Method: http.MethodGet, URL: "https://test_bucket.example.com/test", Response: testResponse(), Match: func(req *http.Request) bool { - return checkHeaders(req.Header, map[string]string{ + return tests.ExpectHeaders(req.Header, map[string]string{ "authorization": "AWS4-HMAC-SHA256 Credential=123/20250102/test_region/s3/aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-example;x-amz-meta-a, Signature=86dccbcd012c33073dc99e9d0a9e0b717a4d8c11c37848cfa9a4a02716bc0db3", "host": "test_bucket.example.com", "x-amz-date": "20250102T150405Z", @@ -205,7 +206,7 @@ func TestS3SignAndSend(t *testing.T) { } defer resp.Body.Close() - err = s.s3Client.Client.(*TestClient).AssertNoRemaining() + err = s.s3Client.Client.(*tests.Client).AssertNoRemaining() if err != nil { t.Fatal(err) } diff --git a/tools/filesystem/internal/s3blob/s3/client_test.go b/tools/filesystem/internal/s3blob/s3/tests/client.go similarity index 71% rename from tools/filesystem/internal/s3blob/s3/client_test.go rename to tools/filesystem/internal/s3blob/s3/tests/client.go index ebdf321e..b06fb22a 100644 --- a/tools/filesystem/internal/s3blob/s3/client_test.go +++ b/tools/filesystem/internal/s3blob/s3/tests/client.go @@ -1,4 +1,6 @@ -package s3_test +// Package tests contains various tests helpers and utilities to assist +// with the S3 client testing. +package tests import ( "errors" @@ -11,26 +13,9 @@ import ( "sync" ) -func checkHeaders(headers http.Header, expectations map[string]string) bool { - for h, expected := range expectations { - v := headers.Get(h) - - pattern := expected - if !strings.HasPrefix(pattern, "^") && !strings.HasSuffix(pattern, "$") { - pattern = "^" + regexp.QuoteMeta(pattern) + "$" - } - - expectedRegex, err := regexp.Compile(pattern) - if err != nil { - return false - } - - if !expectedRegex.MatchString(v) { - return false - } - } - - return true +// NewClient creates a new test Client loaded with the specified RequestStubs. +func NewClient(stubs ...*RequestStub) *Client { + return &Client{stubs: stubs} } type RequestStub struct { @@ -40,16 +25,13 @@ type RequestStub struct { Response *http.Response } -func NewTestClient(stubs ...*RequestStub) *TestClient { - return &TestClient{stubs: stubs} -} - -type TestClient struct { +type Client struct { stubs []*RequestStub mu sync.Mutex } -func (c *TestClient) AssertNoRemaining() error { +// AssertNoRemaining asserts that current client has no unprocessed requests remaining. +func (c *Client) AssertNoRemaining() error { c.mu.Lock() defer c.mu.Unlock() @@ -66,7 +48,8 @@ func (c *TestClient) AssertNoRemaining() error { return errors.New(strings.Join(msgParts, "\n")) } -func (c *TestClient) Do(req *http.Request) (*http.Response, error) { +// Do implements the [s3.HTTPClient] interface. +func (c *Client) Do(req *http.Request) (*http.Response, error) { c.mu.Lock() defer c.mu.Unlock() diff --git a/tools/filesystem/internal/s3blob/s3/tests/headers.go b/tools/filesystem/internal/s3blob/s3/tests/headers.go new file mode 100644 index 00000000..21464c7d --- /dev/null +++ b/tools/filesystem/internal/s3blob/s3/tests/headers.go @@ -0,0 +1,33 @@ +package tests + +import ( + "net/http" + "regexp" + "strings" +) + +// ExpectHeaders checks whether specified headers match the expectations. +// The expectations map entry key is the header name. +// The expectations map entry value is the first header value. If wrapped with `^...$` +// it is compared as regular expression. +func ExpectHeaders(headers http.Header, expectations map[string]string) bool { + for h, expected := range expectations { + v := headers.Get(h) + + pattern := expected + if !strings.HasPrefix(pattern, "^") && !strings.HasSuffix(pattern, "$") { + pattern = "^" + regexp.QuoteMeta(pattern) + "$" + } + + expectedRegex, err := regexp.Compile(pattern) + if err != nil { + return false + } + + if !expectedRegex.MatchString(v) { + return false + } + } + + return true +} diff --git a/tools/filesystem/internal/s3blob/s3/uploader_test.go b/tools/filesystem/internal/s3blob/s3/uploader_test.go index 83eeb7c4..92314855 100644 --- a/tools/filesystem/internal/s3blob/s3/uploader_test.go +++ b/tools/filesystem/internal/s3blob/s3/uploader_test.go @@ -8,13 +8,14 @@ import ( "testing" "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3" + "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3/tests" ) func TestUploaderRequiredFields(t *testing.T) { t.Parallel() s3Client := &s3.S3{ - Client: NewTestClient(&RequestStub{Method: "PUT", URL: `^.+$`}), // match every upload + Client: tests.NewClient(&tests.RequestStub{Method: "PUT", URL: `^.+$`}), // match every upload Region: "test_region", Bucket: "test_bucket", Endpoint: "http://example.com", @@ -71,8 +72,8 @@ func TestUploaderRequiredFields(t *testing.T) { func TestUploaderSingleUpload(t *testing.T) { t.Parallel() - httpClient := NewTestClient( - &RequestStub{ + httpClient := tests.NewClient( + &tests.RequestStub{ Method: http.MethodPut, URL: "http://test_bucket.example.com/test_key", Match: func(req *http.Request) bool { @@ -81,7 +82,7 @@ func TestUploaderSingleUpload(t *testing.T) { return false } - return string(body) == "abcdefg" && checkHeaders(req.Header, map[string]string{ + return string(body) == "abcdefg" && tests.ExpectHeaders(req.Header, map[string]string{ "Content-Length": "7", "x-amz-meta-a": "123", "x-amz-meta-b": "456", @@ -123,12 +124,12 @@ func TestUploaderSingleUpload(t *testing.T) { func TestUploaderMultipartUploadSuccess(t *testing.T) { t.Parallel() - httpClient := NewTestClient( - &RequestStub{ + httpClient := tests.NewClient( + &tests.RequestStub{ Method: http.MethodPost, URL: "http://test_bucket.example.com/test_key?uploads", Match: func(req *http.Request) bool { - return checkHeaders(req.Header, map[string]string{ + return tests.ExpectHeaders(req.Header, map[string]string{ "x-amz-meta-a": "123", "x-amz-meta-b": "456", "test_header": "test", @@ -146,7 +147,7 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) { `)), }, }, - &RequestStub{ + &tests.RequestStub{ Method: http.MethodPut, URL: "http://test_bucket.example.com/test_key?partNumber=1&uploadId=test_id", Match: func(req *http.Request) bool { @@ -155,7 +156,7 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) { return false } - return string(body) == "abc" && checkHeaders(req.Header, map[string]string{ + return string(body) == "abc" && tests.ExpectHeaders(req.Header, map[string]string{ "Content-Length": "3", "test_header": "test", "Authorization": "^.+Credential=123/.+$", @@ -165,7 +166,7 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) { Header: http.Header{"Etag": []string{"etag1"}}, }, }, - &RequestStub{ + &tests.RequestStub{ Method: http.MethodPut, URL: "http://test_bucket.example.com/test_key?partNumber=2&uploadId=test_id", Match: func(req *http.Request) bool { @@ -174,7 +175,7 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) { return false } - return string(body) == "def" && checkHeaders(req.Header, map[string]string{ + return string(body) == "def" && tests.ExpectHeaders(req.Header, map[string]string{ "Content-Length": "3", "test_header": "test", "Authorization": "^.+Credential=123/.+$", @@ -184,7 +185,7 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) { Header: http.Header{"Etag": []string{"etag2"}}, }, }, - &RequestStub{ + &tests.RequestStub{ Method: http.MethodPut, URL: "http://test_bucket.example.com/test_key?partNumber=3&uploadId=test_id", Match: func(req *http.Request) bool { @@ -192,7 +193,7 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) { if err != nil { return false } - return string(body) == "g" && checkHeaders(req.Header, map[string]string{ + return string(body) == "g" && tests.ExpectHeaders(req.Header, map[string]string{ "Content-Length": "1", "test_header": "test", "Authorization": "^.+Credential=123/.+$", @@ -202,7 +203,7 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) { Header: http.Header{"Etag": []string{"etag3"}}, }, }, - &RequestStub{ + &tests.RequestStub{ Method: http.MethodPost, URL: "http://test_bucket.example.com/test_key?uploadId=test_id", Match: func(req *http.Request) bool { @@ -213,7 +214,7 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) { expected := `etag11etag22etag33` - return strings.Contains(string(body), expected) && checkHeaders(req.Header, map[string]string{ + return strings.Contains(string(body), expected) && tests.ExpectHeaders(req.Header, map[string]string{ "test_header": "test", "Authorization": "^.+Credential=123/.+$", }) @@ -252,12 +253,12 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) { func TestUploaderMultipartUploadPartFailure(t *testing.T) { t.Parallel() - httpClient := NewTestClient( - &RequestStub{ + httpClient := tests.NewClient( + &tests.RequestStub{ Method: http.MethodPost, URL: "http://test_bucket.example.com/test_key?uploads", Match: func(req *http.Request) bool { - return checkHeaders(req.Header, map[string]string{ + return tests.ExpectHeaders(req.Header, map[string]string{ "x-amz-meta-a": "123", "x-amz-meta-b": "456", "test_header": "test", @@ -275,7 +276,7 @@ func TestUploaderMultipartUploadPartFailure(t *testing.T) { `)), }, }, - &RequestStub{ + &tests.RequestStub{ Method: http.MethodPut, URL: "http://test_bucket.example.com/test_key?partNumber=1&uploadId=test_id", Match: func(req *http.Request) bool { @@ -283,7 +284,7 @@ func TestUploaderMultipartUploadPartFailure(t *testing.T) { if err != nil { return false } - return string(body) == "abc" && checkHeaders(req.Header, map[string]string{ + return string(body) == "abc" && tests.ExpectHeaders(req.Header, map[string]string{ "Content-Length": "3", "test_header": "test", "Authorization": "^.+Credential=123/.+$", @@ -293,11 +294,11 @@ func TestUploaderMultipartUploadPartFailure(t *testing.T) { Header: http.Header{"Etag": []string{"etag1"}}, }, }, - &RequestStub{ + &tests.RequestStub{ Method: http.MethodPut, URL: "http://test_bucket.example.com/test_key?partNumber=2&uploadId=test_id", Match: func(req *http.Request) bool { - return checkHeaders(req.Header, map[string]string{ + return tests.ExpectHeaders(req.Header, map[string]string{ "test_header": "test", "Authorization": "^.+Credential=123/.+$", }) @@ -306,11 +307,11 @@ func TestUploaderMultipartUploadPartFailure(t *testing.T) { StatusCode: 400, }, }, - &RequestStub{ + &tests.RequestStub{ Method: http.MethodDelete, URL: "http://test_bucket.example.com/test_key?uploadId=test_id", Match: func(req *http.Request) bool { - return checkHeaders(req.Header, map[string]string{ + return tests.ExpectHeaders(req.Header, map[string]string{ "test_header": "test", "Authorization": "^.+Credential=123/.+$", }) @@ -349,12 +350,12 @@ func TestUploaderMultipartUploadPartFailure(t *testing.T) { func TestUploaderMultipartUploadCompleteFailure(t *testing.T) { t.Parallel() - httpClient := NewTestClient( - &RequestStub{ + httpClient := tests.NewClient( + &tests.RequestStub{ Method: http.MethodPost, URL: "http://test_bucket.example.com/test_key?uploads", Match: func(req *http.Request) bool { - return checkHeaders(req.Header, map[string]string{ + return tests.ExpectHeaders(req.Header, map[string]string{ "x-amz-meta-a": "123", "x-amz-meta-b": "456", "test_header": "test", @@ -372,7 +373,7 @@ func TestUploaderMultipartUploadCompleteFailure(t *testing.T) { `)), }, }, - &RequestStub{ + &tests.RequestStub{ Method: http.MethodPut, URL: "http://test_bucket.example.com/test_key?partNumber=1&uploadId=test_id", Match: func(req *http.Request) bool { @@ -380,7 +381,7 @@ func TestUploaderMultipartUploadCompleteFailure(t *testing.T) { if err != nil { return false } - return string(body) == "abc" && checkHeaders(req.Header, map[string]string{ + return string(body) == "abc" && tests.ExpectHeaders(req.Header, map[string]string{ "Content-Length": "3", "test_header": "test", "Authorization": "^.+Credential=123/.+$", @@ -390,7 +391,7 @@ func TestUploaderMultipartUploadCompleteFailure(t *testing.T) { Header: http.Header{"Etag": []string{"etag1"}}, }, }, - &RequestStub{ + &tests.RequestStub{ Method: http.MethodPut, URL: "http://test_bucket.example.com/test_key?partNumber=2&uploadId=test_id", Match: func(req *http.Request) bool { @@ -398,7 +399,7 @@ func TestUploaderMultipartUploadCompleteFailure(t *testing.T) { if err != nil { return false } - return string(body) == "def" && checkHeaders(req.Header, map[string]string{ + return string(body) == "def" && tests.ExpectHeaders(req.Header, map[string]string{ "Content-Length": "3", "test_header": "test", "Authorization": "^.+Credential=123/.+$", @@ -408,11 +409,11 @@ func TestUploaderMultipartUploadCompleteFailure(t *testing.T) { Header: http.Header{"Etag": []string{"etag2"}}, }, }, - &RequestStub{ + &tests.RequestStub{ Method: http.MethodPost, URL: "http://test_bucket.example.com/test_key?uploadId=test_id", Match: func(req *http.Request) bool { - return checkHeaders(req.Header, map[string]string{ + return tests.ExpectHeaders(req.Header, map[string]string{ "test_header": "test", "Authorization": "^.+Credential=123/.+$", }) @@ -421,11 +422,11 @@ func TestUploaderMultipartUploadCompleteFailure(t *testing.T) { StatusCode: 400, }, }, - &RequestStub{ + &tests.RequestStub{ Method: http.MethodDelete, URL: "http://test_bucket.example.com/test_key?uploadId=test_id", Match: func(req *http.Request) bool { - return checkHeaders(req.Header, map[string]string{ + return tests.ExpectHeaders(req.Header, map[string]string{ "test_header": "test", "Authorization": "^.+Credential=123/.+$", }) diff --git a/tools/filesystem/internal/s3blob/driver.go b/tools/filesystem/internal/s3blob/s3blob.go similarity index 95% rename from tools/filesystem/internal/s3blob/driver.go rename to tools/filesystem/internal/s3blob/s3blob.go index 25746b77..756bae27 100644 --- a/tools/filesystem/internal/s3blob/driver.go +++ b/tools/filesystem/internal/s3blob/s3blob.go @@ -69,11 +69,17 @@ type driver struct { // Close implements [blob/Driver.Close]. func (drv *driver) Close() error { - return nil + return nil // nothing to close } // NormalizeError implements [blob/Driver.NormalizeError]. func (drv *driver) NormalizeError(err error) error { + // already normalized + if errors.Is(err, blob.ErrNotFound) { + return err + } + + // normalize base on its S3 error code var ae s3.ResponseError if errors.As(err, &ae) { switch ae.Code { @@ -92,22 +98,20 @@ func (drv *driver) ListPaged(ctx context.Context, opts *blob.ListOptions) (*blob pageSize = defaultPageSize } - in := s3.ListParams{ + listParams := s3.ListParams{ MaxKeys: pageSize, } if len(opts.PageToken) > 0 { - in.ContinuationToken = string(opts.PageToken) + listParams.ContinuationToken = string(opts.PageToken) } if opts.Prefix != "" { - in.Prefix = escapeKey(opts.Prefix) + listParams.Prefix = escapeKey(opts.Prefix) } if opts.Delimiter != "" { - in.Delimiter = escapeKey(opts.Delimiter) + listParams.Delimiter = escapeKey(opts.Delimiter) } - var reqOptions []func(*http.Request) - - resp, err := drv.s3.ListObjects(ctx, in, reqOptions...) + resp, err := drv.s3.ListObjects(ctx, listParams) if err != nil { return nil, err } @@ -157,8 +161,7 @@ func (drv *driver) Attributes(ctx context.Context, key string) (*blob.Attributes md := make(map[string]string, len(resp.Metadata)) for k, v := range resp.Metadata { - // See the package comments for more details on escaping of metadata - // keys & values. + // See the package comments for more details on escaping of metadata keys & values. md[blob.HexUnescape(urlUnescape(k))] = urlUnescape(v) } @@ -192,13 +195,11 @@ func (drv *driver) NewRangeReader(ctx context.Context, key string, offset, lengt byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+length-1) } - reqOptions := []func(*http.Request){ - func(req *http.Request) { - req.Header.Set("Range", byteRange) - }, + reqOpt := func(req *http.Request) { + req.Header.Set("Range", byteRange) } - resp, err := drv.s3.GetObject(ctx, key, reqOptions...) + resp, err := drv.s3.GetObject(ctx, key, reqOpt) if err != nil { return nil, err } diff --git a/tools/filesystem/internal/s3blob/s3blob_test.go b/tools/filesystem/internal/s3blob/s3blob_test.go new file mode 100644 index 00000000..72a05c9a --- /dev/null +++ b/tools/filesystem/internal/s3blob/s3blob_test.go @@ -0,0 +1,518 @@ +package s3blob_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "testing" + + "github.com/pocketbase/pocketbase/tools/filesystem/blob" + "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob" + "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3" + "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3/tests" +) + +func TestNew(t *testing.T) { + t.Parallel() + + scenarios := []struct { + name string + s3Client *s3.S3 + expectError bool + }{ + { + "blank", + &s3.S3{}, + true, + }, + { + "no bucket", + &s3.S3{Region: "b", Endpoint: "c"}, + true, + }, + { + "no endpoint", + &s3.S3{Bucket: "a", Region: "b"}, + true, + }, + { + "no region", + &s3.S3{Bucket: "a", Endpoint: "c"}, + true, + }, + { + "with bucket, endpoint and region", + &s3.S3{Bucket: "a", Region: "b", Endpoint: "c"}, + false, + }, + } + + for _, s := range scenarios { + t.Run(s.name, func(t *testing.T) { + drv, err := s3blob.New(s.s3Client) + + hasErr := err != nil + if hasErr != s.expectError { + t.Fatalf("Expected hasErr %v, got %v (%v)", s.expectError, hasErr, err) + } + + if err == nil && drv == nil { + t.Fatal("Expected non-nil driver instance") + } + }) + } +} + +func TestDriverClose(t *testing.T) { + t.Parallel() + + drv, err := s3blob.New(&s3.S3{Bucket: "a", Region: "b", Endpoint: "c"}) + if err != nil { + t.Fatal(err) + } + + err = drv.Close() + if err != nil { + t.Fatalf("Expected nil, got error %v", err) + } +} + +func TestDriverNormilizeError(t *testing.T) { + t.Parallel() + + drv, err := s3blob.New(&s3.S3{Bucket: "a", Region: "b", Endpoint: "c"}) + if err != nil { + t.Fatal(err) + } + + scenarios := []struct { + name string + err error + expectErrNotFound bool + }{ + { + "plain error", + errors.New("test"), + false, + }, + { + "response error with custom code", + s3.ResponseError{Code: "test"}, + false, + }, + { + "response error with NoSuchBucket code", + s3.ResponseError{Code: "NoSuchBucket"}, + true, + }, + { + "response error with NoSuchKey code", + s3.ResponseError{Code: "NoSuchKey"}, + true, + }, + { + "response error with NotFound code", + s3.ResponseError{Code: "NotFound"}, + true, + }, + { + "wrapped response error with NotFound code", // ensures that the entire error's tree is checked + fmt.Errorf("test: %w", s3.ResponseError{Code: "NotFound"}), + true, + }, + { + "already normalized error", + fmt.Errorf("test: %w", blob.ErrNotFound), + true, + }, + } + + for _, s := range scenarios { + t.Run(s.name, func(t *testing.T) { + err := drv.NormalizeError(s.err) + if err == nil { + t.Fatal("Expected non-nil error") + } + + isErrNotFound := errors.Is(err, blob.ErrNotFound) + if isErrNotFound != s.expectErrNotFound { + t.Fatalf("Expected isErrNotFound %v, got %v (%v)", s.expectErrNotFound, isErrNotFound, err) + } + }) + } +} + +func TestDriverDeleteEscaping(t *testing.T) { + t.Parallel() + + httpClient := tests.NewClient(&tests.RequestStub{ + Method: http.MethodDelete, + URL: "https://test_bucket.example.com/..__0x2f__abc/test/", + }) + + drv, err := s3blob.New(&s3.S3{ + Bucket: "test_bucket", + Region: "test_region", + Endpoint: "https://example.com", + Client: httpClient, + }) + if err != nil { + t.Fatal(err) + } + + err = drv.Delete(context.Background(), "../abc/test/") + if err != nil { + t.Fatal(err) + } + + err = httpClient.AssertNoRemaining() + if err != nil { + t.Fatal(err) + } +} + +func TestDriverCopyEscaping(t *testing.T) { + t.Parallel() + + httpClient := tests.NewClient(&tests.RequestStub{ + Method: http.MethodPut, + URL: "https://test_bucket.example.com/..__0x2f__a/", + Match: func(req *http.Request) bool { + return tests.ExpectHeaders(req.Header, map[string]string{ + "x-amz-copy-source": "test_bucket%2F..__0x2f__b%2F", + }) + }, + Response: &http.Response{ + Body: io.NopCloser(strings.NewReader(``)), + }, + }) + + drv, err := s3blob.New(&s3.S3{ + Bucket: "test_bucket", + Region: "test_region", + Endpoint: "https://example.com", + Client: httpClient, + }) + if err != nil { + t.Fatal(err) + } + + err = drv.Copy(context.Background(), "../a/", "../b/") + if err != nil { + t.Fatal(err) + } + + err = httpClient.AssertNoRemaining() + if err != nil { + t.Fatal(err) + } +} + +func TestDriverAttributes(t *testing.T) { + t.Parallel() + + httpClient := tests.NewClient(&tests.RequestStub{ + Method: http.MethodHead, + URL: "https://test_bucket.example.com/..__0x2f__a/", + Response: &http.Response{ + Header: http.Header{ + "Last-Modified": []string{"Mon, 01 Feb 2025 03:04:05 GMT"}, + "Cache-Control": []string{"test_cache"}, + "Content-Disposition": []string{"test_disposition"}, + "Content-Encoding": []string{"test_encoding"}, + "Content-Language": []string{"test_language"}, + "Content-Type": []string{"test_type"}, + "Content-Range": []string{"test_range"}, + "Etag": []string{`"ce5be8b6f53645c596306c4572ece521"`}, + "Content-Length": []string{"100"}, + "x-amz-meta-AbC%40": []string{"%40test_meta_a"}, + "x-amz-meta-Def": []string{"test_meta_b"}, + }, + Body: http.NoBody, + }, + }) + + drv, err := s3blob.New(&s3.S3{ + Bucket: "test_bucket", + Region: "test_region", + Endpoint: "https://example.com", + Client: httpClient, + }) + if err != nil { + t.Fatal(err) + } + + attrs, err := drv.Attributes(context.Background(), "../a/") + if err != nil { + t.Fatal(err) + } + + raw, err := json.Marshal(attrs) + if err != nil { + t.Fatal(err) + } + + expected := `{"cacheControl":"test_cache","contentDisposition":"test_disposition","contentEncoding":"test_encoding","contentLanguage":"test_language","contentType":"test_type","metadata":{"abc@":"@test_meta_a","def":"test_meta_b"},"createTime":"0001-01-01T00:00:00Z","modTime":"2025-02-01T03:04:05Z","size":100,"md5":"zlvotvU2RcWWMGxFcuzlIQ==","etag":"\"ce5be8b6f53645c596306c4572ece521\""}` + if str := string(raw); str != expected { + t.Fatalf("Expected attributes\n%s\ngot\n%s", expected, str) + } + + err = httpClient.AssertNoRemaining() + if err != nil { + t.Fatal(err) + } +} + +func TestDriverListPaged(t *testing.T) { + t.Parallel() + + listResponse := func() *http.Response { + return &http.Response{ + Body: io.NopCloser(strings.NewReader(` + + + example + ct + test_next + example0.txt + 1 + 3 + + ..__0x2f__prefixB/test/example.txt + 2025-01-01T01:02:03.123Z + "ce5be8b6f53645c596306c4572ece521" + 123 + + + prefixA/..__0x2f__escape.txt + 2025-01-02T01:02:03.123Z + 456 + + + prefixA + + + ..__0x2f__prefixB + + + `)), + } + } + + expectedPage := `{"objects":[{"key":"../prefixB","modTime":"0001-01-01T00:00:00Z","size":0,"md5":null,"isDir":true},{"key":"../prefixB/test/example.txt","modTime":"2025-01-01T01:02:03.123Z","size":123,"md5":"zlvotvU2RcWWMGxFcuzlIQ==","isDir":false},{"key":"prefixA","modTime":"0001-01-01T00:00:00Z","size":0,"md5":null,"isDir":true},{"key":"prefixA/../escape.txt","modTime":"2025-01-02T01:02:03.123Z","size":456,"md5":null,"isDir":false}],"nextPageToken":"dGVzdF9uZXh0"}` + + httpClient := tests.NewClient( + &tests.RequestStub{ + Method: http.MethodGet, + URL: "https://test_bucket.example.com/?list-type=2&max-keys=1000", + Response: listResponse(), + }, + &tests.RequestStub{ + Method: http.MethodGet, + URL: "https://test_bucket.example.com/?continuation-token=test_token&delimiter=test_delimiter&list-type=2&max-keys=123&prefix=test_prefix", + Response: listResponse(), + }, + ) + + drv, err := s3blob.New(&s3.S3{ + Bucket: "test_bucket", + Region: "test_region", + Endpoint: "https://example.com", + Client: httpClient, + }) + if err != nil { + t.Fatal(err) + } + + scenarios := []struct { + name string + opts *blob.ListOptions + expected string + }{ + { + "empty options", + &blob.ListOptions{}, + expectedPage, + }, + { + "filled options", + &blob.ListOptions{Prefix: "test_prefix", Delimiter: "test_delimiter", PageSize: 123, PageToken: []byte("test_token")}, + expectedPage, + }, + } + + for _, s := range scenarios { + t.Run(s.name, func(t *testing.T) { + page, err := drv.ListPaged(context.Background(), s.opts) + if err != nil { + t.Fatal(err) + } + + raw, err := json.Marshal(page) + if err != nil { + t.Fatal(err) + } + + if str := string(raw); s.expected != str { + t.Fatalf("Expected page result\n%s\ngot\n%s", s.expected, str) + } + }) + } + + err = httpClient.AssertNoRemaining() + if err != nil { + t.Fatal(err) + } +} + +func TestDriverNewRangeReader(t *testing.T) { + t.Parallel() + + scenarios := []struct { + offset int64 + length int64 + httpClient *tests.Client + expectedAttrs string + }{ + { + 0, + 0, + tests.NewClient(&tests.RequestStub{ + Method: http.MethodGet, + URL: "https://test_bucket.example.com/..__0x2f__abc/test.txt", + Match: func(req *http.Request) bool { + return tests.ExpectHeaders(req.Header, map[string]string{ + "Range": "bytes=0-0", + }) + }, + Response: &http.Response{ + Header: http.Header{ + "Last-Modified": []string{"Mon, 01 Feb 2025 03:04:05 GMT"}, + "Content-Type": []string{"test_ct"}, + "Content-Length": []string{"123"}, + }, + Body: io.NopCloser(strings.NewReader("test")), + }, + }), + `{"contentType":"test_ct","modTime":"2025-02-01T03:04:05Z","size":123}`, + }, + { + 10, + -1, + tests.NewClient(&tests.RequestStub{ + Method: http.MethodGet, + URL: "https://test_bucket.example.com/..__0x2f__abc/test.txt", + Match: func(req *http.Request) bool { + return tests.ExpectHeaders(req.Header, map[string]string{ + "Range": "bytes=10-", + }) + }, + Response: &http.Response{ + Header: http.Header{ + "Last-Modified": []string{"Mon, 01 Feb 2025 03:04:05 GMT"}, + "Content-Type": []string{"test_ct"}, + "Content-Range": []string{"bytes 1-1/456"}, // should take precedence over content-length + "Content-Length": []string{"123"}, + }, + Body: io.NopCloser(strings.NewReader("test")), + }, + }), + `{"contentType":"test_ct","modTime":"2025-02-01T03:04:05Z","size":456}`, + }, + { + 10, + 0, + tests.NewClient(&tests.RequestStub{ + Method: http.MethodGet, + URL: "https://test_bucket.example.com/..__0x2f__abc/test.txt", + Match: func(req *http.Request) bool { + return tests.ExpectHeaders(req.Header, map[string]string{ + "Range": "bytes=10-10", + }) + }, + Response: &http.Response{ + Header: http.Header{ + "Last-Modified": []string{"Mon, 01 Feb 2025 03:04:05 GMT"}, + "Content-Type": []string{"test_ct"}, + // no range and length headers + // "Content-Range": []string{"bytes 1-1/456"}, + // "Content-Length": []string{"123"}, + }, + Body: io.NopCloser(strings.NewReader("test")), + }, + }), + `{"contentType":"test_ct","modTime":"2025-02-01T03:04:05Z","size":0}`, + }, + { + 10, + 20, + tests.NewClient(&tests.RequestStub{ + Method: http.MethodGet, + URL: "https://test_bucket.example.com/..__0x2f__abc/test.txt", + Match: func(req *http.Request) bool { + return tests.ExpectHeaders(req.Header, map[string]string{ + "Range": "bytes=10-29", + }) + }, + Response: &http.Response{ + Header: http.Header{ + "Last-Modified": []string{"Mon, 01 Feb 2025 03:04:05 GMT"}, + "Content-Type": []string{"test_ct"}, + // with range header but invalid format -> content-length takes precedence + "Content-Range": []string{"bytes invalid-456"}, + "Content-Length": []string{"123"}, + }, + Body: io.NopCloser(strings.NewReader("test")), + }, + }), + `{"contentType":"test_ct","modTime":"2025-02-01T03:04:05Z","size":123}`, + }, + } + + for _, s := range scenarios { + t.Run(fmt.Sprintf("offset_%d_length_%d", s.offset, s.length), func(t *testing.T) { + drv, err := s3blob.New(&s3.S3{ + Bucket: "test_bucket", + Region: "tesst_region", + Endpoint: "https://example.com", + Client: s.httpClient, + }) + if err != nil { + t.Fatal(err) + } + + r, err := drv.NewRangeReader(context.Background(), "../abc/test.txt", s.offset, s.length) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + // the response body should be always replaced with http.NoBody + if s.length == 0 { + body := make([]byte, 1) + n, err := r.Read(body) + if n != 0 || !errors.Is(err, io.EOF) { + t.Fatalf("Expected body to be http.NoBody, got %v (%v)", body, err) + } + } + + rawAttrs, err := json.Marshal(r.Attributes()) + if err != nil { + t.Fatal(err) + } + + if str := string(rawAttrs); str != s.expectedAttrs { + t.Fatalf("Expected attributes\n%s\ngot\n%s", s.expectedAttrs, str) + } + + err = s.httpClient.AssertNoRemaining() + if err != nil { + t.Fatal(err) + } + }) + } +}