diff --git a/CHANGELOG.md b/CHANGELOG.md index df551ee0..ca2dddee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,12 @@ - Allowed `0` as `RelationOptions.MinSelect` value to avoid the ambiguity between 0 and non-filled input value ([#2817](https://github.com/pocketbase/pocketbase/discussions/2817)). +## v0.16.7 + +- Minor optimization for the list/search queries to use `rowid` with the `COUNT` statement when available. + _This eliminates the temp B-TREE step when executing the query and for large datasets (eg. 150k) it could have 10x improvement (from ~580ms to ~60ms)._ + + ## v0.16.6 - Fixed collection index column sort normalization in the Admin UI ([#2681](https://github.com/pocketbase/pocketbase/pull/2681); thanks @SimonLoir). diff --git a/apis/record_crud.go b/apis/record_crud.go index 8adb2021..e54098ae 100644 --- a/apis/record_crud.go +++ b/apis/record_crud.go @@ -68,6 +68,11 @@ func (api *recordApi) list(c echo.Context) error { searchProvider := search.NewProvider(fieldsResolver). Query(api.app.Dao().RecordQuery(collection)) + // views don't have "rowid" so we fallback to "id" + if collection.IsView() { + searchProvider.CountCol("id") + } + if requestData.Admin == nil && collection.ListRule != nil { searchProvider.AddFilter(search.FilterData(*collection.ListRule)) } diff --git a/models/schema/schema_field.go b/models/schema/schema_field.go index 17cde507..085f2602 100644 --- a/models/schema/schema_field.go +++ b/models/schema/schema_field.go @@ -190,7 +190,7 @@ func (f SchemaField) Validate() error { excludeNames := BaseModelFieldNames() // exclude special filter literals - excludeNames = append(excludeNames, "null", "true", "false") + excludeNames = append(excludeNames, "null", "true", "false", "_rowid_") // exclude system literals excludeNames = append(excludeNames, SystemFieldNames()...) diff --git a/models/schema/schema_field_test.go b/models/schema/schema_field_test.go index 750906c1..2881896b 100644 --- a/models/schema/schema_field_test.go +++ b/models/schema/schema_field_test.go @@ -313,6 +313,15 @@ func TestSchemaFieldValidate(t *testing.T) { }, []string{"name"}, }, + { + "reserved name (_rowid_)", + schema.SchemaField{ + Type: schema.FieldTypeText, + Id: "1234567890", + Name: "_rowid_", + }, + []string{"name"}, + }, { "reserved name (id)", schema.SchemaField{ diff --git a/tools/search/provider.go b/tools/search/provider.go index 01c0679b..9c6a482a 100644 --- a/tools/search/provider.go +++ b/tools/search/provider.go @@ -36,6 +36,7 @@ type Result struct { type Provider struct { fieldResolver FieldResolver query *dbx.SelectQuery + countCol string page int perPage int sort []SortField @@ -56,6 +57,7 @@ type Provider struct { func NewProvider(fieldResolver FieldResolver) *Provider { return &Provider{ fieldResolver: fieldResolver, + countCol: "_rowid_", page: 1, perPage: DefaultPerPage, sort: []SortField{}, @@ -69,6 +71,13 @@ func (s *Provider) Query(query *dbx.SelectQuery) *Provider { return s } +// CountCol allows changing the default column (_rowid_) that is used +// to generated the COUNT SQL query statement. +func (s *Provider) CountCol(name string) *Provider { + s.countCol = name + return s +} + // Page sets the `page` field of the current search provider. // // Normalization on the `page` value is done during `Exec()`. @@ -198,7 +207,7 @@ func (s *Provider) Exec(items any) (*Result, error) { baseTable = queryInfo.From[0] } clone := modelsQuery - countQuery := clone.Select("COUNT(DISTINCT [[" + baseTable + ".id]])").OrderBy() + countQuery := clone.Distinct(false).Select("COUNT(DISTINCT [[" + baseTable + "." + s.countCol + "]])").OrderBy() if err := countQuery.Row(&totalCount); err != nil { return nil, err } diff --git a/tools/search/provider_test.go b/tools/search/provider_test.go index 002edba2..b037b4d9 100644 --- a/tools/search/provider_test.go +++ b/tools/search/provider_test.go @@ -42,6 +42,20 @@ func TestProviderQuery(t *testing.T) { } } +func TestProviderCountCol(t *testing.T) { + p := NewProvider(&testFieldResolver{}) + + if p.countCol != "_rowid_" { + t.Fatalf("Expected the default countCol to be %s, got %s", "_rowid_", p.countCol) + } + + p.CountCol("test") + + if p.countCol != "test" { + t.Fatalf("Expected colCount to change to %s, got %s", "test", p.countCol) + } +} + func TestProviderPage(t *testing.T) { r := &testFieldResolver{} p := NewProvider(r).Page(10) @@ -228,7 +242,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) { false, `{"page":1,"perPage":10,"totalItems":2,"totalPages":1,"items":[{"test1":1,"test2":"test2.1","test3":""},{"test1":2,"test2":"test2.2","test3":""}]}`, []string{ - "SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE NOT (`test1` IS NULL)", + "SELECT COUNT(DISTINCT [[test._rowid_]]) FROM `test` WHERE NOT (`test1` IS NULL)", "SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 10", }, }, @@ -241,7 +255,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) { false, `{"page":1,"perPage":30,"totalItems":2,"totalPages":1,"items":[{"test1":1,"test2":"test2.1","test3":""},{"test1":2,"test2":"test2.2","test3":""}]}`, []string{ - "SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE NOT (`test1` IS NULL)", + "SELECT COUNT(DISTINCT [[test._rowid_]]) FROM `test` WHERE NOT (`test1` IS NULL)", "SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 30", }, }, @@ -274,7 +288,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) { false, `{"page":1,"perPage":` + fmt.Sprint(MaxPerPage) + `,"totalItems":1,"totalPages":1,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`, []string{ - "SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE ((NOT (`test1` IS NULL)) AND (((test2 != '' AND test2 IS NOT NULL)))) AND (test1 >= 2)", + "SELECT COUNT(DISTINCT [[test._rowid_]]) FROM `test` WHERE ((NOT (`test1` IS NULL)) AND (((test2 != '' AND test2 IS NOT NULL)))) AND (test1 >= 2)", "SELECT * FROM `test` WHERE ((NOT (`test1` IS NULL)) AND (((test2 != '' AND test2 IS NOT NULL)))) AND (test1 >= 2) ORDER BY `test1` ASC, `test2` DESC LIMIT 500", }, }, @@ -287,7 +301,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) { false, `{"page":1,"perPage":10,"totalItems":0,"totalPages":0,"items":[]}`, []string{ - "SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE (NOT (`test1` IS NULL)) AND (((test3 != '' AND test3 IS NOT NULL)))", + "SELECT COUNT(DISTINCT [[test._rowid_]]) FROM `test` WHERE (NOT (`test1` IS NULL)) AND (((test3 != '' AND test3 IS NOT NULL)))", "SELECT * FROM `test` WHERE (NOT (`test1` IS NULL)) AND (((test3 != '' AND test3 IS NOT NULL))) ORDER BY `test1` ASC, `test3` ASC LIMIT 10", }, }, @@ -300,7 +314,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) { false, `{"page":2,"perPage":1,"totalItems":2,"totalPages":2,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`, []string{ - "SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE NOT (`test1` IS NULL)", + "SELECT COUNT(DISTINCT [[test._rowid_]]) FROM `test` WHERE NOT (`test1` IS NULL)", "SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 1 OFFSET 1", }, },