go/pocketbase
1
0
Fork 0
mirror of https://github.com/pocketbase/pocketbase.git synced 2025-03-17 21:27:52 +02:00
Code Issues Releases Activity

abstract rest.UploadedFile to allow loading local files

Browse Source
This commit is contained in:
Gani Georgiev 2022-12-10 16:47:45 +02:00
parent aa6eaa7319
commit 37bac5cc50
12 changed files with 322 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
forms/record_upsert.go
View File

@ -18,6 +18,7 @@ import (
"github.com/pocketbase/pocketbase/forms/validators"
"github.com/pocketbase/pocketbase/models"
"github.com/pocketbase/pocketbase/models/schema"
"github.com/pocketbase/pocketbase/tools/filesystem"
"github.com/pocketbase/pocketbase/tools/list"
"github.com/pocketbase/pocketbase/tools/rest"
"github.com/pocketbase/pocketbase/tools/security"
@ -34,7 +35,7 @@ type RecordUpsert struct {
manageAccess bool
record *models.Record
filesToUpload map[string][]*rest.UploadedFile
filesToUpload map[string][]*filesystem.File
filesToDelete []string // names list
// base model fields
@ -66,7 +67,7 @@ func NewRecordUpsert(app core.App, record *models.Record) *RecordUpsert {
dao: app.Dao(),
record: record,
filesToDelete: []string{},
filesToUpload: map[string][]*rest.UploadedFile{},
filesToUpload: map[string][]*filesystem.File{},
}
form.loadFormDefaults()
@ -153,7 +154,7 @@ func (form *RecordUpsert) extractMultipartFormData(r *http.Request, keyPrefix st
arrayValueSupportTypes := schema.ArraybleFieldTypes()
form.filesToUpload = map[string][]*rest.UploadedFile{}
form.filesToUpload = map[string][]*filesystem.File{}
for fullKey, values := range r.PostForm {
key := fullKey
@ -202,7 +203,7 @@ func (form *RecordUpsert) extractMultipartFormData(r *http.Request, keyPrefix st
}
if form.filesToUpload[key] == nil {
form.filesToUpload[key] = []*rest.UploadedFile{}
form.filesToUpload[key] = []*filesystem.File{}
}
if options.MaxSelect == 1 {
@ -363,11 +364,11 @@ func (form *RecordUpsert) LoadData(requestData map[string]any) error {
if len(oldNames) > 0 {
form.filesToDelete = list.ToUniqueStringSlice(append(form.filesToDelete, oldNames...))
}
form.Data[key] = form.filesToUpload[key][0].Name()
form.Data[key] = form.filesToUpload[key][0].Name
} else if options.MaxSelect > 1 {
// append the id of each uploaded file instance
for _, file := range form.filesToUpload[key] {
oldNames = append(oldNames, file.Name())
oldNames = append(oldNames, file.Name)
}
form.Data[key] = oldNames
}
@ -685,7 +686,7 @@ func (form *RecordUpsert) getFilesToUploadNames() []string {
for fieldKey := range form.filesToUpload {
for _, file := range form.filesToUpload[fieldKey] {
names = append(names, file.Name())
names = append(names, file.Name)
}
}
@ -712,8 +713,8 @@ func (form *RecordUpsert) processFilesToUpload() error {
for fieldKey := range form.filesToUpload {
for i, file := range form.filesToUpload[fieldKey] {
path := form.record.BaseFilesPath() + "/" + file.Name()
if err := fs.UploadMultipart(file.Header(), path); err == nil {
path := form.record.BaseFilesPath() + "/" + file.Name
if err := fs.UploadFile(file, path); err == nil {
// keep track of the already uploaded file
uploaded = append(uploaded, path)
} else {

10
forms/validators/file.go
View File

@ -6,7 +6,7 @@ import (
"github.com/gabriel-vasile/mimetype"
validation "github.com/go-ozzo/ozzo-validation/v4"
"github.com/pocketbase/pocketbase/tools/rest"
"github.com/pocketbase/pocketbase/tools/filesystem"
)
// UploadedFileSize checks whether the validated `rest.UploadedFile`
@ -16,12 +16,12 @@ import (
// validation.Field(&form.File, validation.By(validators.UploadedFileSize(1000)))
func UploadedFileSize(maxBytes int) validation.RuleFunc {
return func(value any) error {
v, _ := value.(*rest.UploadedFile)
v, _ := value.(*filesystem.File)
if v == nil {
return nil // nothing to validate
}
if int(v.Header().Size) > maxBytes {
if int(v.Size) > maxBytes {
return validation.NewError("validation_file_size_limit", fmt.Sprintf("Maximum allowed file size is %v bytes.", maxBytes))
}
@ -37,7 +37,7 @@ func UploadedFileSize(maxBytes int) validation.RuleFunc {
// validation.Field(&form.File, validation.By(validators.UploadedFileMimeType(validMimeTypes)))
func UploadedFileMimeType(validTypes []string) validation.RuleFunc {
return func(value any) error {
v, _ := value.(*rest.UploadedFile)
v, _ := value.(*filesystem.File)
if v == nil {
return nil // nothing to validate
}
@ -46,7 +46,7 @@ func UploadedFileMimeType(validTypes []string) validation.RuleFunc {
return validation.NewError("validation_invalid_mime_type", "Unsupported file type.")
}
f, err := v.Header().Open()
f, err := v.Reader.Open()
if err != nil {
return validation.NewError("validation_invalid_mime_type", "Unsupported file type.")
}

5
forms/validators/file_test.go
View File

@ -7,6 +7,7 @@ import (
"github.com/pocketbase/pocketbase/forms/validators"
"github.com/pocketbase/pocketbase/tests"
"github.com/pocketbase/pocketbase/tools/filesystem"
"github.com/pocketbase/pocketbase/tools/rest"
)
@ -30,7 +31,7 @@ func TestUploadedFileSize(t *testing.T) {
scenarios := []struct {
maxBytes int
file *rest.UploadedFile
file *filesystem.File
expectError bool
}{
{0, nil, false},
@ -70,7 +71,7 @@ func TestUploadedFileMimeType(t *testing.T) {
scenarios := []struct {
types []string
file *rest.UploadedFile
file *filesystem.File
expectError bool
}{
{nil, nil, false},

10
forms/validators/record_data.go
View File

@ -12,8 +12,8 @@ import (
"github.com/pocketbase/pocketbase/daos"
"github.com/pocketbase/pocketbase/models"
"github.com/pocketbase/pocketbase/models/schema"
"github.com/pocketbase/pocketbase/tools/filesystem"
"github.com/pocketbase/pocketbase/tools/list"
"github.com/pocketbase/pocketbase/tools/rest"
"github.com/pocketbase/pocketbase/tools/types"
)
@ -28,7 +28,7 @@ var requiredErr = validation.NewError("validation_required", "Missing required v
func NewRecordDataValidator(
dao *daos.Dao,
record *models.Record,
uploadedFiles map[string][]*rest.UploadedFile,
uploadedFiles map[string][]*filesystem.File,
) *RecordDataValidator {
return &RecordDataValidator{
dao: dao,
@ -42,7 +42,7 @@ func NewRecordDataValidator(
type RecordDataValidator struct {
dao *daos.Dao
record *models.Record
uploadedFiles map[string][]*rest.UploadedFile
uploadedFiles map[string][]*filesystem.File
}
// Validate validates the provided `data` by checking it against
@ -314,9 +314,9 @@ func (validator *RecordDataValidator) checkFileValue(field *schema.SchemaField,
}
// extract the uploaded files
files := make([]*rest.UploadedFile, 0, len(validator.uploadedFiles[field.Name]))
files := make([]*filesystem.File, 0, len(validator.uploadedFiles[field.Name]))
for _, file := range validator.uploadedFiles[field.Name] {
if list.ExistInSlice(file.Name(), names) {
if list.ExistInSlice(file.Name, names) {
files = append(files, file)
}
}

27
forms/validators/record_data_test.go
View File

@ -13,6 +13,7 @@ import (
"github.com/pocketbase/pocketbase/models"
"github.com/pocketbase/pocketbase/models/schema"
"github.com/pocketbase/pocketbase/tests"
"github.com/pocketbase/pocketbase/tools/filesystem"
"github.com/pocketbase/pocketbase/tools/rest"
"github.com/pocketbase/pocketbase/tools/types"
)
@ -20,7 +21,7 @@ import (
type testDataFieldScenario struct {
name string
data map[string]any
files map[string][]*rest.UploadedFile
files map[string][]*filesystem.File
expectedErrors []string
}
@ -1086,10 +1087,10 @@ func TestRecordDataValidatorValidateFile(t *testing.T) {
"check MaxSelect constraint",
map[string]any{
"field1": "test1",
"field2": []string{"test1", testFiles[0].Name(), testFiles[3].Name()},
"field2": []string{"test1", testFiles[0].Name, testFiles[3].Name},
"field3": []string{"test1", "test2", "test3", "test4"},
},
map[string][]*rest.UploadedFile{
map[string][]*filesystem.File{
"field2": {testFiles[0], testFiles[3]},
},
[]string{"field2", "field3"},
@ -1097,11 +1098,11 @@ func TestRecordDataValidatorValidateFile(t *testing.T) {
{
"check MaxSize constraint",
map[string]any{
"field1": testFiles[0].Name(),
"field2": []string{"test1", testFiles[0].Name()},
"field1": testFiles[0].Name,
"field2": []string{"test1", testFiles[0].Name},
"field3": []string{"test1", "test2", "test3"},
},
map[string][]*rest.UploadedFile{
map[string][]*filesystem.File{
"field1": {testFiles[0]},
"field2": {testFiles[0]},
},
@ -1111,10 +1112,10 @@ func TestRecordDataValidatorValidateFile(t *testing.T) {
"check MimeTypes constraint",
map[string]any{
"field1": "test1",
"field2": []string{"test1", testFiles[0].Name()},
"field3": []string{testFiles[1].Name(), testFiles[2].Name()},
"field2": []string{"test1", testFiles[0].Name},
"field3": []string{testFiles[1].Name, testFiles[2].Name},
},
map[string][]*rest.UploadedFile{
map[string][]*filesystem.File{
"field2": {testFiles[0], testFiles[1], testFiles[2]},
"field3": {testFiles[1], testFiles[2]},
},
@ -1134,10 +1135,10 @@ func TestRecordDataValidatorValidateFile(t *testing.T) {
"valid data - just new files",
map[string]any{
"field1": nil,
"field2": []string{testFiles[0].Name(), testFiles[1].Name()},
"field2": []string{testFiles[0].Name, testFiles[1].Name},
"field3": nil,
},
map[string][]*rest.UploadedFile{
map[string][]*filesystem.File{
"field2": {testFiles[0], testFiles[1]},
},
[]string{},
@ -1146,10 +1147,10 @@ func TestRecordDataValidatorValidateFile(t *testing.T) {
"valid data - mixed existing and new files",
map[string]any{
"field1": "test1",
"field2": []string{"test1", testFiles[0].Name()},
"field2": []string{"test1", testFiles[0].Name},
"field3": "test1", // will be casted
},
map[string][]*rest.UploadedFile{
map[string][]*filesystem.File{
"field2": {testFiles[0], testFiles[1], testFiles[2]},
},
[]string{},

135
tools/filesystem/file.go Normal file
View File

@ -0,0 +1,135 @@
package filesystem
import (
"fmt"
"io"
"mime/multipart"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/gabriel-vasile/mimetype"
"github.com/pocketbase/pocketbase/tools/inflector"
"github.com/pocketbase/pocketbase/tools/security"
)
// FileReader defines an interface for a file resource reader.
type FileReader interface {
Open() (io.ReadSeekCloser, error)
}
// File defines a single file [io.ReadSeekCloser] resource.
//
// The file could be from a local path, multipipart/formdata header, etc.
type File struct {
Name string
OriginalName string
Size int64
Reader FileReader
}
// NewFileFromPath creates a new File instance from the provided local file path.
func NewFileFromPath(path string) (*File, error) {
f := &File{}
info, err := os.Stat(path)
if err != nil {
return nil, err
}
f.Reader = &PathReader{Path: path}
f.Size = info.Size()
f.OriginalName = info.Name()
f.Name = normalizeName(f.Reader, f.OriginalName)
return f, nil
}
// NewFileFromMultipart creates a new File instace from the provided multipart header.
func NewFileFromMultipart(mh *multipart.FileHeader) (*File, error) {
f := &File{}
f.Reader = &MultipartReader{Header: mh}
f.Size = mh.Size
f.OriginalName = mh.Filename
f.Name = normalizeName(f.Reader, f.OriginalName)
return f, nil
}
// -------------------------------------------------------------------
var _ FileReader = (*MultipartReader)(nil)
// MultipartReader defines a [multipart.FileHeader] reader.
type MultipartReader struct {
Header *multipart.FileHeader
}
// Open implements the [filesystem.FileReader] interface.
func (r *MultipartReader) Open() (io.ReadSeekCloser, error) {
return r.Header.Open()
}
// -------------------------------------------------------------------
var _ FileReader = (*PathReader)(nil)
type PathReader struct {
Path string
}
// Open implements the [filesystem.FileReader] interface.
func (r *PathReader) Open() (io.ReadSeekCloser, error) {
return os.Open(r.Path)
}
// -------------------------------------------------------------------
var extInvalidCharsRegex = regexp.MustCompile(`[^\w\.\*\-\+\=\#]+`)
func normalizeName(fr FileReader, name string) string {
// extension
// ---
originalExt := filepath.Ext(name)
cleanExt := extInvalidCharsRegex.ReplaceAllString(originalExt, "")
if cleanExt == "" {
// try to detect the extension from the file content
cleanExt, _ = detectExtension(fr)
}
// name
// ---
cleanName := inflector.Snakecase(strings.TrimSuffix(name, originalExt))
if length := len(cleanName); length < 3 {
// the name is too short so we concatenate an additional random part
cleanName += security.RandomString(10)
} else if length > 100 {
// keep only the first 100 characters (it is multibyte safe after Snakecase)
cleanName = cleanName[:100]
}
return fmt.Sprintf(
"%s_%s%s",
cleanName,
security.RandomString(10), // ensure that there is always a random part
cleanExt,
)
}
func detectExtension(fr FileReader) (string, error) {
// try to detect the extension from the mime type
r, err := fr.Open()
if err != nil {
return "", err
}
defer r.Close()
mt, _ := mimetype.DetectReader(r)
if err != nil {
return "", err
}
return mt.Extension(), nil
}

79
tools/filesystem/file_test.go Normal file
View File

@ -0,0 +1,79 @@
package filesystem_test
import (
"net/http/httptest"
"os"
"path/filepath"
"regexp"
"testing"
"github.com/labstack/echo/v5"
"github.com/pocketbase/pocketbase/tests"
"github.com/pocketbase/pocketbase/tools/filesystem"
)
func TestNewFileFromFromPath(t *testing.T) {
testDir := createTestDir(t)
defer os.RemoveAll(testDir)
// missing file
_, err := filesystem.NewFileFromPath("missing")
if err == nil {
t.Fatal("Expected error, got nil")
}
// existing file
originalName := "image_! noext"
normalizedNamePattern := regexp.QuoteMeta("image_noext_") + `\w{10}` + regexp.QuoteMeta(".png")
f, err := filesystem.NewFileFromPath(filepath.Join(testDir, originalName))
if err != nil {
t.Fatalf("Expected nil error, got %v", err)
}
if f.OriginalName != originalName {
t.Fatalf("Expected originalName %q, got %q", originalName, f.OriginalName)
}
if match, _ := regexp.Match(normalizedNamePattern, []byte(f.Name)); !match {
t.Fatalf("Expected Name to match %v, got %q (%v)", normalizedNamePattern, f.Name, err)
}
if f.Size != 73 {
t.Fatalf("Expected Size %v, got %v", 73, f.Size)
}
if _, ok := f.Reader.(*filesystem.PathReader); !ok {
t.Fatalf("Expected Reader to be PathReader, got %v", f.Reader)
}
}
func TestNewFileFromMultipart(t *testing.T) {
formData, mp, err := tests.MockMultipartData(nil, "test")
req := httptest.NewRequest("", "/", formData)
req.Header.Set(echo.HeaderContentType, mp.FormDataContentType())
req.ParseMultipartForm(32 << 20)
_, mh, err := req.FormFile("test")
if err != nil {
t.Fatal(err)
}
f, err := filesystem.NewFileFromMultipart(mh)
if err != nil {
t.Fatal(err)
}
originalNamePattern := regexp.QuoteMeta("tmpfile-") + `\w+` + regexp.QuoteMeta(".txt")
if match, _ := regexp.Match(originalNamePattern, []byte(f.OriginalName)); !match {
t.Fatalf("Expected OriginalName to match %v, got %q (%v)", originalNamePattern, f.OriginalName, err)
}
normalizedNamePattern := regexp.QuoteMeta("tmpfile_") + `\w+\_\w{10}` + regexp.QuoteMeta(".txt")
if match, _ := regexp.Match(normalizedNamePattern, []byte(f.Name)); !match {
t.Fatalf("Expected Name to match %v, got %q (%v)", normalizedNamePattern, f.Name, err)
}
if f.Size != 4 {
t.Fatalf("Expected Size %v, got %v", 4, f.Size)
}
if _, ok := f.Reader.(*filesystem.MultipartReader); !ok {
t.Fatalf("Expected Reader to be MultipartReader, got %v", f.Reader)
}
}

44
tools/filesystem/filesystem.go
View File

@ -117,7 +117,49 @@ func (s *System) Upload(content []byte, fileKey string) error {
return w.Close()
}
// UploadMultipart upload the provided multipart file to the fileKey location.
// UploadFile uploads the provided multipart file to the fileKey location.
func (s *System) UploadFile(file *File, fileKey string) error {
f, err := file.Reader.Open()
if err != nil {
return err
}
defer f.Close()
mt, err := mimetype.DetectReader(f)
if err != nil {
return err
}
// rewind
f.Seek(0, io.SeekStart)
originalName := file.OriginalName
if len(originalName) > 255 {
// keep only the first 255 chars as a very rudimentary measure
// to prevent the metadata to grow too big in size
originalName = originalName[:255]
}
opts := &blob.WriterOptions{
ContentType: mt.String(),
Metadata: map[string]string{
"original_filename": originalName,
},
}
w, err := s.bucket.NewWriter(s.ctx, fileKey, opts)
if err != nil {
return err
}
if _, err := w.ReadFrom(f); err != nil {
w.Close()
return err
}
return w.Close()
}
// UploadMultipart uploads the provided multipart file to the fileKey location.
func (s *System) UploadMultipart(fh *multipart.FileHeader, fileKey string) error {
f, err := fh.Open()
if err != nil {

10
tools/filesystem/filesystem_test.go
View File

@ -448,8 +448,7 @@ func createTestDir(t *testing.T) string {
if err != nil {
t.Fatal(err)
}
// tiny 1x1 png
imgRect := image.Rect(0, 0, 1, 1)
imgRect := image.Rect(0, 0, 1, 1) // tiny 1x1 png
png.Encode(file3, imgRect)
file3.Close()
err2 := os.WriteFile(filepath.Join(dir, "image.png.attrs"), []byte(`{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null}`), 0644)
@ -469,5 +468,12 @@ func createTestDir(t *testing.T) string {
}
file5.Close()
file6, err := os.OpenFile(filepath.Join(dir, "image_! noext"), os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
t.Fatal(err)
}
png.Encode(file6, image.Rect(0, 0, 1, 1)) // tiny 1x1 png
file6.Close()
return dir
}

75
tools/rest/uploaded_file.go
View File

@ -1,43 +1,18 @@
package rest
import (
"fmt"
"mime/multipart"
"net/http"
"path/filepath"
"regexp"
"strings"
"github.com/gabriel-vasile/mimetype"
"github.com/pocketbase/pocketbase/tools/inflector"
"github.com/pocketbase/pocketbase/tools/security"
"github.com/pocketbase/pocketbase/tools/filesystem"
)
// DefaultMaxMemory defines the default max memory bytes that
// will be used when parsing a form request body.
const DefaultMaxMemory = 32 << 20 // 32mb
var extensionInvalidCharsRegex = regexp.MustCompile(`[^\w\.\*\-\+\=\#]+`)
// UploadedFile defines a single multipart uploaded file instance.
type UploadedFile struct {
name string
header *multipart.FileHeader
}
// Name returns an assigned unique name to the uploaded file.
func (f *UploadedFile) Name() string {
return f.name
}
// Header returns the file header that comes with the multipart request.
func (f *UploadedFile) Header() *multipart.FileHeader {
return f.header
}
// FindUploadedFiles extracts all form files of `key` from a http request
// and returns a slice with `UploadedFile` instances (if any).
func FindUploadedFiles(r *http.Request, key string) ([]*UploadedFile, error) {
// FindUploadedFiles extracts all form files of "key" from a http request
// and returns a slice with filesystem.File instances (if any).
func FindUploadedFiles(r *http.Request, key string) ([]*filesystem.File, error) {
if r.MultipartForm == nil {
err := r.ParseMultipartForm(DefaultMaxMemory)
if err != nil {
@ -49,51 +24,15 @@ func FindUploadedFiles(r *http.Request, key string) ([]*UploadedFile, error) {
return nil, http.ErrMissingFile
}
result := make([]*UploadedFile, 0, len(r.MultipartForm.File[key]))
result := make([]*filesystem.File, 0, len(r.MultipartForm.File[key]))
for _, fh := range r.MultipartForm.File[key] {
file, err := fh.Open()
file, err := filesystem.NewFileFromMultipart(fh)
if err != nil {
return nil, err
}
defer file.Close()
// extension
// ---
originalExt := filepath.Ext(fh.Filename)
sanitizedExt := extensionInvalidCharsRegex.ReplaceAllString(originalExt, "")
if sanitizedExt == "" {
// try to detect the extension from the mime type
mt, err := mimetype.DetectReader(file)
if err != nil {
return nil, err
}
sanitizedExt = mt.Extension()
}
// name
// ---
originalName := strings.TrimSuffix(fh.Filename, originalExt)
sanitizedName := inflector.Snakecase(originalName)
if length := len(sanitizedName); length < 3 {
// the name is too short so we concatenate an additional random part
sanitizedName += security.RandomString(10)
} else if length > 100 {
// keep only the first 100 characters (it is multibyte safe after Snakecase)
sanitizedName = sanitizedName[:100]
}
uploadedFilename := fmt.Sprintf(
"%s_%s%s",
sanitizedName,
security.RandomString(10), // ensure that there is always a random part
sanitizedExt,
)
result = append(result, &UploadedFile{
name: uploadedFilename,
header: fh,
})
result = append(result, file)
}
return result, nil

8
tools/rest/uploaded_file_test.go
View File

@ -47,16 +47,16 @@ func TestFindUploadedFiles(t *testing.T) {
t.Errorf("[%d] Expected 1 file, got %d", i, len(result))
}
if result[0].Header().Size != 4 {
t.Errorf("[%d] Expected the file size to be 4 bytes, got %d", i, result[0].Header().Size)
if result[0].Size != 4 {
t.Errorf("[%d] Expected the file size to be 4 bytes, got %d", i, result[0].Size)
}
pattern, err := regexp.Compile(s.expectedPattern)
if err != nil {
t.Errorf("[%d] Invalid filename pattern %q: %v", i, s.expectedPattern, err)
}
if !pattern.MatchString(result[0].Name()) {
t.Fatalf("Expected filename to match %s, got filename %s", s.expectedPattern, result[0].Name())
if !pattern.MatchString(result[0].Name) {
t.Fatalf("Expected filename to match %s, got filename %s", s.expectedPattern, result[0].Name)
}
}
}

14
tools/types/datetime.go
View File

@ -81,12 +81,16 @@ func (d *DateTime) Scan(value any) error {
case int:
d.t = cast.ToTime(v)
case string:
t, err := time.Parse(DefaultDateLayout, v)
if err != nil {
// check for other common date layouts
t = cast.ToTime(v)
if v == "" {
d.t = time.Time{}
} else {
t, err := time.Parse(DefaultDateLayout, v)
if err != nil {
// check for other common date layouts
t = cast.ToTime(v)
}
d.t = t
}
d.t = t
default:
str := cast.ToString(v)
if str == "" {