synced with master

CHANGELOG.md

@@ -76,6 +76,12 @@
 - Allowed `0` as `RelationOptions.MinSelect` value to avoid the ambiguity between 0 and non-filled input value ([#2817](https://github.com/pocketbase/pocketbase/discussions/2817)).
 
 
+## v0.16.7
+
+- Minor optimization for the list/search queries to use `rowid` with the `COUNT` statement when available.
+  _This eliminates the temp B-TREE step when executing the query and for large datasets (eg. 150k) it could have 10x improvement (from ~580ms to ~60ms)._
+
+
 ## v0.16.6
 
 - Fixed collection index column sort normalization in the Admin UI ([#2681](https://github.com/pocketbase/pocketbase/pull/2681); thanks @SimonLoir).

apis/record_crud.go

@@ -68,6 +68,11 @@ func (api *recordApi) list(c echo.Context) error {
 	searchProvider := search.NewProvider(fieldsResolver).
 		Query(api.app.Dao().RecordQuery(collection))
 
+	// views don't have "rowid" so we fallback to "id"
+	if collection.IsView() {
+		searchProvider.CountCol("id")
+	}
+
 	if requestData.Admin == nil && collection.ListRule != nil {
 		searchProvider.AddFilter(search.FilterData(*collection.ListRule))
 	}

models/schema/schema_field.go

@@ -190,7 +190,7 @@ func (f SchemaField) Validate() error {
 
 	excludeNames := BaseModelFieldNames()
 	// exclude special filter literals
-	excludeNames = append(excludeNames, "null", "true", "false")
+	excludeNames = append(excludeNames, "null", "true", "false", "_rowid_")
 	// exclude system literals
 	excludeNames = append(excludeNames, SystemFieldNames()...)
 

models/schema/schema_field_test.go

@@ -313,6 +313,15 @@ func TestSchemaFieldValidate(t *testing.T) {
 			},
 			[]string{"name"},
 		},
+		{
+			"reserved name (_rowid_)",
+			schema.SchemaField{
+				Type: schema.FieldTypeText,
+				Id:   "1234567890",
+				Name: "_rowid_",
+			},
+			[]string{"name"},
+		},
 		{
 			"reserved name (id)",
 			schema.SchemaField{

tools/search/provider.go

@@ -36,6 +36,7 @@ type Result struct {
 type Provider struct {
 	fieldResolver FieldResolver
 	query         *dbx.SelectQuery
+	countCol      string
 	page          int
 	perPage       int
 	sort          []SortField
@@ -56,6 +57,7 @@ type Provider struct {
 func NewProvider(fieldResolver FieldResolver) *Provider {
 	return &Provider{
 		fieldResolver: fieldResolver,
+		countCol:      "_rowid_",
 		page:          1,
 		perPage:       DefaultPerPage,
 		sort:          []SortField{},
@@ -69,6 +71,13 @@ func (s *Provider) Query(query *dbx.SelectQuery) *Provider {
 	return s
 }
 
+// CountCol allows changing the default column (_rowid_) that is used
+// to generated the COUNT SQL query statement.
+func (s *Provider) CountCol(name string) *Provider {
+	s.countCol = name
+	return s
+}
+
 // Page sets the `page` field of the current search provider.
 //
 // Normalization on the `page` value is done during `Exec()`.
@@ -198,7 +207,7 @@ func (s *Provider) Exec(items any) (*Result, error) {
 		baseTable = queryInfo.From[0]
 	}
 	clone := modelsQuery
-	countQuery := clone.Select("COUNT(DISTINCT [[" + baseTable + ".id]])").OrderBy()
+	countQuery := clone.Distinct(false).Select("COUNT(DISTINCT [[" + baseTable + "." + s.countCol + "]])").OrderBy()
 	if err := countQuery.Row(&totalCount); err != nil {
 		return nil, err
 	}

tools/search/provider_test.go

@@ -42,6 +42,20 @@ func TestProviderQuery(t *testing.T) {
 	}
 }
 
+func TestProviderCountCol(t *testing.T) {
+	p := NewProvider(&testFieldResolver{})
+
+	if p.countCol != "_rowid_" {
+		t.Fatalf("Expected the default countCol to be %s, got %s", "_rowid_", p.countCol)
+	}
+
+	p.CountCol("test")
+
+	if p.countCol != "test" {
+		t.Fatalf("Expected colCount to change to %s, got %s", "test", p.countCol)
+	}
+}
+
 func TestProviderPage(t *testing.T) {
 	r := &testFieldResolver{}
 	p := NewProvider(r).Page(10)
@@ -228,7 +242,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
 			false,
 			`{"page":1,"perPage":10,"totalItems":2,"totalPages":1,"items":[{"test1":1,"test2":"test2.1","test3":""},{"test1":2,"test2":"test2.2","test3":""}]}`,
 			[]string{
-				"SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE NOT (`test1` IS NULL)",
+				"SELECT COUNT(DISTINCT [[test._rowid_]]) FROM `test` WHERE NOT (`test1` IS NULL)",
 				"SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 10",
 			},
 		},
@@ -241,7 +255,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
 			false,
 			`{"page":1,"perPage":30,"totalItems":2,"totalPages":1,"items":[{"test1":1,"test2":"test2.1","test3":""},{"test1":2,"test2":"test2.2","test3":""}]}`,
 			[]string{
-				"SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE NOT (`test1` IS NULL)",
+				"SELECT COUNT(DISTINCT [[test._rowid_]]) FROM `test` WHERE NOT (`test1` IS NULL)",
 				"SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 30",
 			},
 		},
@@ -274,7 +288,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
 			false,
 			`{"page":1,"perPage":` + fmt.Sprint(MaxPerPage) + `,"totalItems":1,"totalPages":1,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`,
 			[]string{
-				"SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE ((NOT (`test1` IS NULL)) AND (((test2 != '' AND test2 IS NOT NULL)))) AND (test1 >= 2)",
+				"SELECT COUNT(DISTINCT [[test._rowid_]]) FROM `test` WHERE ((NOT (`test1` IS NULL)) AND (((test2 != '' AND test2 IS NOT NULL)))) AND (test1 >= 2)",
 				"SELECT * FROM `test` WHERE ((NOT (`test1` IS NULL)) AND (((test2 != '' AND test2 IS NOT NULL)))) AND (test1 >= 2) ORDER BY `test1` ASC, `test2` DESC LIMIT 500",
 			},
 		},
@@ -287,7 +301,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
 			false,
 			`{"page":1,"perPage":10,"totalItems":0,"totalPages":0,"items":[]}`,
 			[]string{
-				"SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE (NOT (`test1` IS NULL)) AND (((test3 != '' AND test3 IS NOT NULL)))",
+				"SELECT COUNT(DISTINCT [[test._rowid_]]) FROM `test` WHERE (NOT (`test1` IS NULL)) AND (((test3 != '' AND test3 IS NOT NULL)))",
 				"SELECT * FROM `test` WHERE (NOT (`test1` IS NULL)) AND (((test3 != '' AND test3 IS NOT NULL))) ORDER BY `test1` ASC, `test3` ASC LIMIT 10",
 			},
 		},
@@ -300,7 +314,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
 			false,
 			`{"page":2,"perPage":1,"totalItems":2,"totalPages":2,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`,
 			[]string{
-				"SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE NOT (`test1` IS NULL)",
+				"SELECT COUNT(DISTINCT [[test._rowid_]]) FROM `test` WHERE NOT (`test1` IS NULL)",
 				"SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 1 OFFSET 1",
 			},
 		},