added support for :excerpt(max, withEllipsis?) fields modifier

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 - Added Patreon OAuth2 provider ([#3323](https://github.com/pocketbase/pocketbase/pull/3323); thanks @ghostdevv).
 
+- (@todo docs) Added support for `:excerpt(max, withEllipsis?)` `fields` modifier that will return a short plain text version of any string value (html tags are stripped).
+    This could be used to minimize the downloaded json data when listing records with large `editor` html values.
+    ```js
+    await pb.collection("example").getList(1, 20, {
+      "fields": "*,description:excerpt(100)"
+    })
+    ```
+
 
 ## v0.18.6
 

tools/rest/excerpt_modifier.go

@@ -0,0 +1,131 @@
+package rest
+
+import (
+	"errors"
+	"regexp"
+	"strings"
+
+	"github.com/pocketbase/pocketbase/tools/list"
+	"github.com/spf13/cast"
+	"golang.org/x/net/html"
+)
+
+var whitespaceRegex = regexp.MustCompile(`\s+`)
+
+var excludeTags = []string{
+	"head", "style", "script", "iframe", "embed", "applet", "object",
+	"svg", "img", "picture", "dialog", "template", "button", "form",
+	"textarea", "input", "select", "option",
+}
+
+var inlineTags = []string{
+	"a", "abbr", "acronym", "b", "bdo", "big", "br", "button",
+	"cite", "code", "em", "i", "label", "q", "small", "span",
+	"strong", "strike", "sub", "sup", "time",
+}
+
+var _ FieldModifier = (*excerptModifier)(nil)
+
+type excerptModifier struct {
+	max          int  // approximate max excerpt length
+	withEllipsis bool // if enabled will add ellipsis when the plain text length > max
+}
+
+// newExcerptModifier validates the specified raw string arguments and
+// initializes a new excerptModifier.
+//
+// This method is usually invoked in initModifer().
+func newExcerptModifier(args ...string) (*excerptModifier, error) {
+	totalArgs := len(args)
+
+	if totalArgs == 0 {
+		return nil, errors.New("max argument is required - expected (max, withEllipsis?)")
+	}
+
+	if totalArgs > 2 {
+		return nil, errors.New("too many arguments - expected (max, withEllipsis?)")
+	}
+
+	max := cast.ToInt(args[0])
+	if max == 0 {
+		return nil, errors.New("max argument must be > 0")
+	}
+
+	var withEllipsis bool
+	if totalArgs > 1 {
+		withEllipsis = cast.ToBool(args[1])
+	}
+
+	return &excerptModifier{max, withEllipsis}, nil
+}
+
+// Modify implements the [FieldModifier.Modify] interface method.
+//
+// It returns a plain text excerpt/short-description from a formatted
+// html string (non-string values are kept untouched).
+func (m *excerptModifier) Modify(value any) (any, error) {
+	strValue, ok := value.(string)
+	if !ok {
+		// not a string -> return as it is without applying the modifier
+		// (we don't throw an error because the modifier could be applied for a missing expand field)
+		return value, nil
+	}
+
+	var builder strings.Builder
+
+	doc, err := html.Parse(strings.NewReader(strValue))
+	if err != nil {
+		return "", err
+	}
+
+	var isNotEmpty bool
+	var needSpace bool
+
+	// for all node types and more details check
+	// https://pkg.go.dev/golang.org/x/net/html#Parse
+	var stripTags func(*html.Node)
+	stripTags = func(n *html.Node) {
+		switch n.Type {
+		case html.TextNode:
+			if txt := strings.TrimSpace(whitespaceRegex.ReplaceAllString(n.Data, " ")); txt != "" {
+				if isNotEmpty && needSpace {
+					needSpace = false
+					builder.WriteString(" ")
+				}
+
+				builder.WriteString(txt)
+
+				if !isNotEmpty {
+					isNotEmpty = true
+				}
+			}
+		case html.ElementNode:
+			if !needSpace && !list.ExistInSlice(n.Data, inlineTags) {
+				needSpace = true
+			}
+		}
+
+		if builder.Len() > m.max {
+			return
+		}
+
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			if c.Type != html.ElementNode || !list.ExistInSlice(c.Data, excludeTags) {
+				stripTags(c)
+			}
+		}
+	}
+	stripTags(doc)
+
+	result := builder.String()
+
+	if len(result) > m.max {
+		result = strings.TrimSpace(result[:m.max])
+
+		if m.withEllipsis {
+			result += "..."
+		}
+	}
+
+	return result, nil
+}

tools/rest/excerpt_modifier_test.go

@@ -0,0 +1,157 @@
+package rest
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/spf13/cast"
+)
+
+func TestNewExcerptModifier(t *testing.T) {
+	scenarios := []struct {
+		name        string
+		args        []string
+		expectError bool
+	}{
+		{
+			"no arguments",
+			nil,
+			true,
+		},
+		{
+			"too many arguments",
+			[]string{"12", "false", "something"},
+			true,
+		},
+		{
+			"non-numeric max argument",
+			[]string{"something"}, // should fallback to 0 which is not allowed
+			true,
+		},
+		{
+			"numeric max argument",
+			[]string{"12"},
+			false,
+		},
+		{
+			"non-bool withEllipsis argument",
+			[]string{"12", "something"}, // should fallback to false which is allowed
+			false,
+		},
+		{
+			"truthy withEllipsis argument",
+			[]string{"12", "t"},
+			false,
+		},
+	}
+
+	for _, s := range scenarios {
+		t.Run(s.name, func(t *testing.T) {
+			m, err := newExcerptModifier(s.args...)
+
+			hasErr := err != nil
+			if hasErr != s.expectError {
+				t.Fatalf("Expected hasErr %v, got %v (%v)", s.expectError, hasErr, err)
+			}
+
+			if hasErr {
+				if m != nil {
+					t.Fatalf("Expected nil modifier, got %v", m)
+				}
+
+				return
+			}
+
+			var argMax int
+			if len(s.args) > 0 {
+				argMax = cast.ToInt(s.args[0])
+			}
+
+			var argWithEllipsis bool
+			if len(s.args) > 1 {
+				argWithEllipsis = cast.ToBool(s.args[1])
+			}
+
+			if m.max != argMax {
+				t.Fatalf("Expected max %d, got %d", argMax, m.max)
+			}
+
+			if m.withEllipsis != argWithEllipsis {
+				t.Fatalf("Expected withEllipsis %v, got %v", argWithEllipsis, m.withEllipsis)
+			}
+		})
+	}
+}
+
+func TestExcerptModifierModify(t *testing.T) {
+	// plain text value: "Hello t est12 3 word"
+	html := ` <script>var a = 123;</script>   <p>Hello</p><div id="test_id">t   est<b>12
+	3</b></div> <h1>word  </h1> `
+
+	plainText := "Hello t est12 3 word"
+
+	scenarios := []struct {
+		name     string
+		args     []string
+		value    string
+		expected string
+	}{
+		// without ellipsis
+		{
+			"only max < len(plainText)",
+			[]string{"2"},
+			html,
+			plainText[:2],
+		},
+		{
+			"only max = len(plainText)",
+			[]string{fmt.Sprint(len(plainText))},
+			html,
+			plainText,
+		},
+		{
+			"only max > len(plainText)",
+			[]string{fmt.Sprint(len(plainText) + 5)},
+			html,
+			plainText,
+		},
+
+		// with ellipsis
+		{
+			"with ellipsis and max < len(plainText)",
+			[]string{"2", "t"},
+			html,
+			plainText[:2] + "...",
+		},
+		{
+			"with ellipsis and max = len(plainText)",
+			[]string{fmt.Sprint(len(plainText)), "t"},
+			html,
+			plainText,
+		},
+		{
+			"with ellipsis and max > len(plainText)",
+			[]string{fmt.Sprint(len(plainText) + 5), "t"},
+			html,
+			plainText,
+		},
+	}
+
+	for _, s := range scenarios {
+		t.Run(s.name, func(t *testing.T) {
+			m, err := newExcerptModifier(s.args...)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			raw, err := m.Modify(s.value)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if v := cast.ToString(raw); v != s.expected {
+				t.Fatalf("Expected %q, got %q", s.expected, v)
+			}
+		})
+	}
+}

tools/rest/json_serializer.go

@@ -2,13 +2,19 @@ package rest
 
 import (
 	"encoding/json"
+	"fmt"
 	"strings"
 
 	"github.com/labstack/echo/v5"
-	"github.com/pocketbase/pocketbase/tools/list"
 	"github.com/pocketbase/pocketbase/tools/search"
+	"github.com/pocketbase/pocketbase/tools/tokenizer"
 )
 
+type FieldModifier interface {
+	// Modify executes the modifier and returns a new modified value.
+	Modify(value any) (any, error)
+}
+
 // Serializer represents custom REST JSON serializer based on echo.DefaultJSONSerializer,
 // with support for additional generic response data transformation (eg. fields picker).
 type Serializer struct {
@@ -28,14 +34,14 @@ func (s *Serializer) Serialize(c echo.Context, i any, indent string) error {
 
 	statusCode := c.Response().Status
 
-	param := c.QueryParam(fieldsParam)
-	if param == "" || statusCode < 200 || statusCode > 299 {
+	rawFields := c.QueryParam(fieldsParam)
+	if rawFields == "" || statusCode < 200 || statusCode > 299 {
 		return s.DefaultJSONSerializer.Serialize(c, i, indent)
 	}
 
-	fields := strings.Split(param, ",")
-	for i, f := range fields {
-		fields[i] = strings.TrimSpace(f)
+	parsedFields, err := parseFields(rawFields)
+	if err != nil {
+		return err
 	}
 
 	encoded, err := json.Marshal(i)
@@ -44,13 +50,11 @@ func (s *Serializer) Serialize(c echo.Context, i any, indent string) error {
 	}
 
 	var decoded any
-
 	if err := json.Unmarshal(encoded, &decoded); err != nil {
 		return err
 	}
 
 	var isSearchResult bool
-
 	switch i.(type) {
 	case search.Result, *search.Result:
 		isSearchResult = true
@@ -58,49 +62,111 @@ func (s *Serializer) Serialize(c echo.Context, i any, indent string) error {
 
 	if isSearchResult {
 		if decodedMap, ok := decoded.(map[string]any); ok {
-			pickFields(decodedMap["items"], fields)
+			pickFields(decodedMap["items"], parsedFields)
 		}
 	} else {
-		pickFields(decoded, fields)
+		pickFields(decoded, parsedFields)
 	}
 
 	return s.DefaultJSONSerializer.Serialize(c, decoded, indent)
 }
 
-func pickFields(data any, fields []string) {
+func parseFields(rawFields string) (map[string]FieldModifier, error) {
+	t := tokenizer.NewFromString(rawFields)
+
+	fields, err := t.ScanAll()
+	if err != nil {
+		return nil, err
+	}
+
+	result := make(map[string]FieldModifier, len(fields))
+
+	for _, f := range fields {
+		parts := strings.SplitN(strings.TrimSpace(f), ":", 2)
+
+		if len(parts) > 1 {
+			m, err := initModifer(parts[1])
+			if err != nil {
+				return nil, err
+			}
+			result[parts[0]] = m
+		} else {
+			result[parts[0]] = nil
+		}
+	}
+
+	return result, nil
+}
+
+func initModifer(rawModifier string) (FieldModifier, error) {
+	t := tokenizer.NewFromString(rawModifier)
+	t.Separators('(', ')', ',', ' ')
+	t.IgnoreParenthesis(true)
+
+	parts, err := t.ScanAll()
+	if err != nil {
+		return nil, err
+	}
+
+	if len(parts) == 0 {
+		return nil, fmt.Errorf("invalid or empty modifier expression %q", rawModifier)
+	}
+
+	name := parts[0]
+	args := parts[1:]
+
+	switch name {
+	case "excerpt":
+		m, err := newExcerptModifier(args...)
+		if err != nil {
+			return nil, fmt.Errorf("invalid excerpt modifier: %w", err)
+		}
+		return m, nil
+	}
+
+	return nil, fmt.Errorf("missing or invalid modifier %q", name)
+}
+
+func pickFields(data any, fields map[string]FieldModifier) error {
 	switch v := data.(type) {
 	case map[string]any:
 		pickMapFields(v, fields)
 	case []map[string]any:
 		for _, item := range v {
-			pickMapFields(item, fields)
+			if err := pickMapFields(item, fields); err != nil {
+				return err
+			}
 		}
 	case []any:
 		if len(v) == 0 {
-			return // nothing to pick
+			return nil // nothing to pick
 		}
 
 		if _, ok := v[0].(map[string]any); !ok {
-			return // for now ignore non-map values
+			return nil // for now ignore non-map values
 		}
 
 		for _, item := range v {
-			pickMapFields(item.(map[string]any), fields)
+			if err := pickMapFields(item.(map[string]any), fields); err != nil {
+				return nil
 			}
 		}
 	}
 
-func pickMapFields(data map[string]any, fields []string) {
+	return nil
+}
+
+func pickMapFields(data map[string]any, fields map[string]FieldModifier) error {
 	if len(fields) == 0 {
-		return // nothing to pick
+		return nil // nothing to pick
 	}
 
-	if list.ExistInSlice("*", fields) {
+	if m, ok := fields["*"]; ok {
 		// append all missing root level data keys
 		for k := range data {
 			var exists bool
 
-			for _, f := range fields {
+			for f := range fields {
 				if strings.HasPrefix(f+".", k+".") {
 					exists = true
 					break
@@ -108,17 +174,17 @@ func pickMapFields(data map[string]any, fields []string) {
 			}
 
 			if !exists {
-				fields = append(fields, k)
+				fields[k] = m
 			}
 		}
 	}
 
 DataLoop:
 	for k := range data {
-		matchingFields := make([]string, 0, len(fields))
-		for _, f := range fields {
+		matchingFields := make(map[string]FieldModifier, len(fields))
+		for f, m := range fields {
 			if strings.HasPrefix(f+".", k+".") {
-				matchingFields = append(matchingFields, f)
+				matchingFields[f] = m
 				continue
 			}
 		}
@@ -128,15 +194,31 @@ DataLoop:
 			continue DataLoop
 		}
 
-		// trim the key from the fields
-		for i, v := range matchingFields {
-			trimmed := strings.TrimSuffix(strings.TrimPrefix(v+".", k+"."), ".")
-			if trimmed == "" {
+		// remove the current key from the matching fields path
+		for f, m := range matchingFields {
+			remains := strings.TrimSuffix(strings.TrimPrefix(f+".", k+"."), ".")
+
+			// final key
+			if remains == "" {
+				if m != nil {
+					var err error
+					data[k], err = m.Modify(data[k])
+					if err != nil {
+						return err
+					}
+				}
 				continue DataLoop
 			}
-			matchingFields[i] = trimmed
+
+			// cleanup the old field key and continue with the rest of the field path
+			delete(matchingFields, f)
+			matchingFields[remains] = m
 		}
 
-		pickFields(data[k], matchingFields)
+		if err := pickFields(data[k], matchingFields); err != nil {
+			return err
 		}
 	}
+
+	return nil
+}

tools/rest/json_serializer_test.go

@@ -274,6 +274,33 @@ func TestSerialize(t *testing.T) {
 			"fields=id,rel.*,rel.sub.id",
 			`{"id":"123","rel":{"id":"456","sub":{"id":"789"},"title":"rel_title"}}`,
 		},
+		{
+			"invalid excerpt modifier",
+			rest.Serializer{},
+			400,
+			map[string]any{"a": 1, "b": 2, "c": "test"},
+			"fields=*:excerpt",
+			`{"a":1,"b":2,"c":"test"}`,
+		},
+		{
+			"valid excerpt modifier",
+			rest.Serializer{},
+			200,
+			map[string]any{
+				"id":    "123",
+				"title": "lorem",
+				"rel": map[string]any{
+					"id":    "456",
+					"title": "<p>rel_title</p>",
+					"sub": map[string]any{
+						"id":    "789",
+						"title": "sub_title",
+					},
+				},
+			},
+			"fields=*:excerpt(2),rel.title:excerpt(3, true)",
+			`{"id":"12","rel":{"title":"rel..."},"title":"lo"}`,
+		},
 	}
 
 	for _, s := range scenarios {