flatten relation joins

apis/realtime.go

@@ -769,7 +769,11 @@ func realtimeCanAccessRecord(
 	}
 
 	q.AndWhere(expr)
-	resolver.UpdateQuery(q)
+
+	err = resolver.UpdateQuery(q)
+	if err != nil {
+		return false
+	}
 
 	err = q.Limit(1).Row(&exists)
 

apis/record_crud.go

@@ -169,12 +169,18 @@ func recordView(e *core.RequestEvent) error {
 	ruleFunc := func(q *dbx.SelectQuery) error {
 		if !requestInfo.HasSuperuserAuth() && collection.ViewRule != nil && *collection.ViewRule != "" {
 			resolver := core.NewRecordFieldResolver(e.App, collection, requestInfo, true)
+
 			expr, err := search.FilterData(*collection.ViewRule).BuildExpr(resolver)
 			if err != nil {
 				return err
 			}
-			resolver.UpdateQuery(q)
+
 			q.AndWhere(expr)
+
+			err = resolver.UpdateQuery(q)
+			if err != nil {
+				return err
+			}
 		}
 		return nil
 	}
@@ -310,7 +316,10 @@ func recordCreate(responseWriteAfterTx bool, optFinalizer func(data any) error)
 				}
 				ruleQuery.AndWhere(expr)
 
-				resolver.UpdateQuery(ruleQuery)
+				err = resolver.UpdateQuery(ruleQuery)
+				if err != nil {
+					return e.BadRequestError("Failed to create record", fmt.Errorf("create rule update query failure: %w", err))
+				}
 
 				var exists int
 				err = ruleQuery.Limit(1).Row(&exists)
@@ -430,12 +439,18 @@ func recordUpdate(responseWriteAfterTx bool, optFinalizer func(data any) error)
 		ruleFunc := func(q *dbx.SelectQuery) error {
 			if !hasSuperuserAuth && collection.UpdateRule != nil && *collection.UpdateRule != "" {
 				resolver := core.NewRecordFieldResolver(e.App, collection, requestInfo, true)
+
 				expr, err := search.FilterData(*collection.UpdateRule).BuildExpr(resolver)
 				if err != nil {
 					return err
 				}
-				resolver.UpdateQuery(q)
+
 				q.AndWhere(expr)
+
+				err = resolver.UpdateQuery(q)
+				if err != nil {
+					return err
+				}
 			}
 			return nil
 		}
@@ -546,12 +561,18 @@ func recordDelete(responseWriteAfterTx bool, optFinalizer func(data any) error)
 		ruleFunc := func(q *dbx.SelectQuery) error {
 			if !requestInfo.HasSuperuserAuth() && collection.DeleteRule != nil && *collection.DeleteRule != "" {
 				resolver := core.NewRecordFieldResolver(e.App, collection, requestInfo, true)
+
 				expr, err := search.FilterData(*collection.DeleteRule).BuildExpr(resolver)
 				if err != nil {
 					return err
 				}
-				resolver.UpdateQuery(q)
+
 				q.AndWhere(expr)
+
+				err = resolver.UpdateQuery(q)
+				if err != nil {
+					return err
+				}
 			}
 			return nil
 		}
@@ -732,7 +753,10 @@ func hasAuthManageAccess(app core.App, requestInfo *core.RequestInfo, collection
 	}
 	query.AndWhere(expr)
 
-	resolver.UpdateQuery(query)
+	err = resolver.UpdateQuery(query)
+	if err != nil {
+		return false
+	}
 
 	var exists int
 

apis/record_helpers.go

@@ -160,7 +160,11 @@ func wantsMFA(e *core.RequestEvent, record *core.Record) (bool, error) {
 	if err != nil {
 		return true, err
 	}
-	resolver.UpdateQuery(query)
+
+	err = resolver.UpdateQuery(query)
+	if err != nil {
+		return true, err
+	}
 
 	err = query.AndWhere(expr).Limit(1).Row(&exists)
 	if err != nil && !errors.Is(err, sql.ErrNoRows) {
@@ -379,12 +383,18 @@ func expandFetch(app core.App, originalRequestInfo *core.RequestInfo) core.Expan
 
 			if *relCollection.ViewRule != "" {
 				resolver := core.NewRecordFieldResolver(app, relCollection, requestInfoPtr, true)
+
 				expr, err := search.FilterData(*(relCollection.ViewRule)).BuildExpr(resolver)
 				if err != nil {
 					return err
 				}
-				resolver.UpdateQuery(q)
+
 				q.AndWhere(expr)
+
+				err = resolver.UpdateQuery(q)
+				if err != nil {
+					return err
+				}
 			}
 
 			return nil
@@ -465,10 +475,16 @@ func autoResolveRecordsFlags(app core.App, records []*core.Record, requestInfo *
 	if err != nil {
 		return err
 	}
-	resolver.UpdateQuery(query)
+
 	query.AndWhere(expr)
 
-	if err := query.Column(&managedIds); err != nil {
+	err = resolver.UpdateQuery(query)
+	if err != nil {
+		return err
+	}
+
+	err = query.Column(&managedIds)
+	if err != nil {
 		return err
 	}
 	// ---

core/db_connect.go

@@ -11,7 +11,7 @@ func DefaultDBConnect(dbPath string) (*dbx.DB, error) {
 	// Note: the busy_timeout pragma must be first because
 	// the connection needs to be set to block on busy before WAL mode
 	// is set in case it hasn't been already set by another connection.
-	pragmas := "?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=journal_size_limit(200000000)&_pragma=synchronous(NORMAL)&_pragma=foreign_keys(ON)&_pragma=temp_store(MEMORY)&_pragma=cache_size(-64000)"
+	pragmas := "?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=journal_size_limit(200000000)&_pragma=synchronous(NORMAL)&_pragma=foreign_keys(ON)&_pragma=temp_store(MEMORY)&_pragma=cache_size(-32000)"
 
 	db, err := dbx.Open("sqlite", dbPath+pragmas)
 	if err != nil {

core/record_field_resolver.go

@@ -48,6 +48,10 @@ type RecordFieldResolver struct {
 	allowedFields     []string
 	joins             []*join
 	allowHiddenFields bool
+	// ---
+	listRuleJoins       map[string]*Collection // tableAlias->collection
+	joinAliasSuffix     string                 // used for uniqueness in the flatten collection list rule join
+	baseCollectionAlias string
 }
 
 // AllowedFields returns a copy of the resolver's allowed fields.
@@ -115,7 +119,6 @@ func NewRecordFieldResolver(
 	return r
 }
 
-// @todo consider removing error return type OR update the existin calls to check the error
 // @todo think of a better a way how to call it automatically after BuildExpr
 //
 // UpdateQuery implements `search.FieldResolver` interface.
@@ -134,6 +137,49 @@ func (r *RecordFieldResolver) UpdateQuery(query *dbx.SelectQuery) error {
 		}
 	}
 
+	// note: for now the joins are not applied for multi-match conditions to avoid excessive checks
+	if len(r.listRuleJoins) > 0 {
+		for alias, c := range r.listRuleJoins {
+			err := r.updateQueryWithCollectionListRule(c, alias, query)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (r *RecordFieldResolver) updateQueryWithCollectionListRule(c *Collection, tableAlias string, query *dbx.SelectQuery) error {
+	if r.allowHiddenFields || c == nil || c.ListRule == nil || *c.ListRule == "" {
+		return nil
+	}
+
+	cloneR := *r
+	cloneR.joins = []*join{}
+	cloneR.baseCollection = c
+	cloneR.baseCollectionAlias = tableAlias
+	cloneR.allowHiddenFields = true
+	cloneR.joinAliasSuffix = security.PseudorandomString(6)
+
+	expr, err := search.FilterData(*c.ListRule).BuildExpr(&cloneR)
+	if err != nil {
+		return fmt.Errorf("to buld %q list rule join subquery filter expression: %w", c.Name, err)
+	}
+
+	query.AndWhere(expr)
+
+	if len(cloneR.joins) > 0 {
+		query.Distinct(true)
+
+		for _, j := range cloneR.joins {
+			query.LeftJoin(
+				(j.tableName + " " + j.tableAlias),
+				j.on,
+			)
+		}
+	}
+
 	return nil
 }
 
@@ -242,80 +288,42 @@ func (r *RecordFieldResolver) loadCollection(collectionNameOrId string) (*Collec
 	return getCollectionByModelOrIdentifier(r.app, collectionNameOrId)
 }
 
-func (r *RecordFieldResolver) registerJoin(tableName string, tableAlias string, on dbx.Expression) {
+func (r *RecordFieldResolver) registerJoin(tableName string, tableAlias string, on dbx.Expression) error {
 	newJoin := &join{
 		tableName:  tableName,
 		tableAlias: tableAlias,
 		on:         on,
 	}
 
+	// (see updateQueryWithCollectionListRule)
 	if !r.allowHiddenFields {
 		c, _ := r.loadCollection(tableName)
-		r.updateCollectionJoinWithListRuleSubquery(c, newJoin)
+
+		// ignore non-collections since the table name could be an expression (e.g. json) or some other subquery
+		if c != nil {
+			// treat all fields as if they are hidden
+			if c.ListRule == nil {
+				return fmt.Errorf("%q fields can be accessed only when allowHiddenFields is enabled or by superusers", c.Name)
+			}
+
+			if r.listRuleJoins == nil {
+				r.listRuleJoins = map[string]*Collection{}
+			}
+			r.listRuleJoins[newJoin.tableAlias] = c
+		}
 	}
 
 	// replace existing join
 	for i, j := range r.joins {
 		if j.tableAlias == newJoin.tableAlias {
 			r.joins[i] = newJoin
-			return
+			return nil
 		}
 	}
 
 	// register new join
 	r.joins = append(r.joins, newJoin)
-}
+	return nil
-
-func (r *RecordFieldResolver) updateCollectionJoinWithListRuleSubquery(c *Collection, j *join) {
-	if c == nil || r.allowHiddenFields {
-		return
-	}
-
-	// resolve to empty set for superusers only collections
-	// (treat all collection fields as "hidden")
-	if c.ListRule == nil {
-		j.on = dbx.NewExp("1=2")
-		return
-	}
-
-	if *c.ListRule == "" {
-		return
-	}
-
-	primaryCol := "_rowid_"
-	if c.IsView() {
-		primaryCol = "id"
-	}
-
-	subquery := r.app.DB().Select("(1)").From(c.Name)
-	subquery.AndWhere(dbx.NewExp("[[" + j.tableAlias + "." + primaryCol + "]]=[[" + c.Name + "." + primaryCol + "]]"))
-
-	cloneR := *r
-	cloneR.joins = []*join{}
-	cloneR.baseCollection = c
-	cloneR.allowHiddenFields = true
-
-	expr, err := search.FilterData(*c.ListRule).BuildExpr(&cloneR)
-	if err != nil {
-		// just log for now and resolve to empty set to minimize breaking changes
-		r.app.Logger().Warn("Failed to buld collection join list rule subquery filter expression", "error", err)
-		j.on = dbx.NewExp("1=2")
-		return
-	}
-
-	subquery.AndWhere(expr)
-
-	err = cloneR.UpdateQuery(subquery)
-	if err != nil {
-		// just log for now and resolve to empty set to minimize breaking changes
-		r.app.Logger().Warn("Failed to update collection join with list rule subquery", "error", err)
-		j.on = dbx.NewExp("1=2")
-		return
-	}
-
-	sb := subquery.Build()
-
-	j.on = dbx.And(j.on, dbx.NewExp("EXISTS ("+sb.SQL()+")", sb.Params()))
 }
 
 type mapExtractor interface {

core/record_field_resolver_runner.go

@@ -132,7 +132,11 @@ func (r *runner) prepare() {
 	r.activeProps = strings.Split(r.fieldName, ".")
 
 	r.activeCollectionName = r.resolver.baseCollection.Name
-	r.activeTableAlias = inflector.Columnify(r.activeCollectionName)
+	if r.resolver.baseCollectionAlias == "" {
+		r.activeTableAlias = inflector.Columnify(r.activeCollectionName)
+	} else {
+		r.activeTableAlias = r.resolver.baseCollectionAlias
+	}
 
 	// enable the ignore flag for missing @request.* fields for backward
 	// compatibility and consistency with all @request.* filter fields and types
@@ -165,18 +169,21 @@ func (r *runner) processCollectionField() (*search.ResolverResult, error) {
 	r.activeCollectionName = collection.Name
 
 	if len(collectionParts) == 2 && collectionParts[1] != "" {
-		r.activeTableAlias = inflector.Columnify("__collection_alias_" + collectionParts[1])
+		r.activeTableAlias = inflector.Columnify("__collection_alias_"+collectionParts[1]) + r.resolver.joinAliasSuffix
 	} else {
-		r.activeTableAlias = inflector.Columnify("__collection_" + r.activeCollectionName)
+		r.activeTableAlias = inflector.Columnify("__collection_"+r.activeCollectionName) + r.resolver.joinAliasSuffix
 	}
 
 	r.withMultiMatch = true
 
 	// join the collection to the main query
-	r.resolver.registerJoin(inflector.Columnify(collection.Name), r.activeTableAlias, nil)
+	err = r.resolver.registerJoin(inflector.Columnify(collection.Name), r.activeTableAlias, nil)
+	if err != nil {
+		return nil, err
+	}
 
 	// join the collection to the multi-match subquery
-	r.multiMatchActiveTableAlias = "__mm" + r.activeTableAlias
+	r.multiMatchActiveTableAlias = "__mm_" + r.activeTableAlias
 	r.multiMatch.joins = append(r.multiMatch.joins, &join{
 		tableName:  inflector.Columnify(collection.Name),
 		tableAlias: r.multiMatchActiveTableAlias,
@@ -205,10 +212,10 @@ func (r *runner) processRequestAuthField() (*search.ResolverResult, error) {
 	collection := r.resolver.requestInfo.Auth.Collection()
 
 	r.activeCollectionName = collection.Name
-	r.activeTableAlias = "__auth_" + inflector.Columnify(r.activeCollectionName)
+	r.activeTableAlias = "__auth_" + inflector.Columnify(r.activeCollectionName) + r.resolver.joinAliasSuffix
 
 	// join the auth collection to the main query
-	r.resolver.registerJoin(
+	err := r.resolver.registerJoin(
 		inflector.Columnify(r.activeCollectionName),
 		r.activeTableAlias,
 		dbx.HashExp{
@@ -216,6 +223,9 @@ func (r *runner) processRequestAuthField() (*search.ResolverResult, error) {
 			(r.activeTableAlias + ".id"): r.resolver.requestInfo.Auth.Id,
 		},
 	)
+	if err != nil {
+		return nil, err
+	}
 
 	// join the auth collection to the multi-match subquery
 	r.multiMatchActiveTableAlias = "__mm_" + r.activeTableAlias
@@ -303,8 +313,12 @@ func (r *runner) processRequestInfoEachModifier(bodyField Field) (*search.Resolv
 	placeholder := "dataEach" + security.PseudorandomString(6)
 	cleanFieldName := inflector.Columnify(bodyField.GetName())
 	jeTable := fmt.Sprintf("json_each({:%s})", placeholder)
-	jeAlias := "__dataEach_" + cleanFieldName + "_je"
+	jeAlias := "__dataEach_je_" + cleanFieldName + r.resolver.joinAliasSuffix
-	r.resolver.registerJoin(jeTable, jeAlias, nil)
+
+	err = r.resolver.registerJoin(jeTable, jeAlias, nil)
+	if err != nil {
+		return nil, err
+	}
 
 	result := &search.ResolverResult{
 		Identifier: fmt.Sprintf("[[%s.value]]", jeAlias),
@@ -318,7 +332,7 @@ func (r *runner) processRequestInfoEachModifier(bodyField Field) (*search.Resolv
 	if r.withMultiMatch {
 		placeholder2 := "mm" + placeholder
 		jeTable2 := fmt.Sprintf("json_each({:%s})", placeholder2)
-		jeAlias2 := "__mm" + jeAlias
+		jeAlias2 := "__mm_" + jeAlias
 
 		r.multiMatch.joins = append(r.multiMatch.joins, &join{
 			tableName:  jeTable2,
@@ -353,10 +367,10 @@ func (r *runner) processRequestInfoRelationField(bodyField Field) (*search.Resol
 	}
 
 	r.activeCollectionName = dataRelCollection.Name
-	r.activeTableAlias = inflector.Columnify("__data_" + dataRelCollection.Name + "_" + relField.Name)
+	r.activeTableAlias = inflector.Columnify("__data_"+dataRelCollection.Name+"_"+relField.Name) + r.resolver.joinAliasSuffix
 
 	// join the data rel collection to the main collection
-	r.resolver.registerJoin(
+	err = r.resolver.registerJoin(
 		r.activeCollectionName,
 		r.activeTableAlias,
 		dbx.In(
@@ -364,13 +378,16 @@ func (r *runner) processRequestInfoRelationField(bodyField Field) (*search.Resol
 			list.ToInterfaceSlice(dataRelIds)...,
 		),
 	)
+	if err != nil {
+		return nil, err
+	}
 
 	if relField.IsMultiple() {
 		r.withMultiMatch = true
 	}
 
 	// join the data rel collection to the multi-match subquery
-	r.multiMatchActiveTableAlias = inflector.Columnify("__data_mm_" + dataRelCollection.Name + "_" + relField.Name)
+	r.multiMatchActiveTableAlias = "__mm_" + r.activeTableAlias
 	r.multiMatch.joins = append(
 		r.multiMatch.joins,
 		&join{
@@ -502,7 +519,7 @@ func (r *runner) processActiveProps() (*search.ResolverResult, error) {
 			// ---
 			cleanProp := inflector.Columnify(prop)
 			cleanBackFieldName := inflector.Columnify(backRelField.Name)
-			newTableAlias := r.activeTableAlias + "_" + cleanProp
+			newTableAlias := r.activeTableAlias + "_" + cleanProp + r.resolver.joinAliasSuffix
 			newCollectionName := inflector.Columnify(backCollection.Name)
 
 			isBackRelMultiple := backRelField.IsMultiple()
@@ -513,14 +530,17 @@ func (r *runner) processActiveProps() (*search.ResolverResult, error) {
 			}
 
 			if !isBackRelMultiple {
-				r.resolver.registerJoin(
+				err := r.resolver.registerJoin(
 					newCollectionName,
 					newTableAlias,
 					dbx.NewExp(fmt.Sprintf("[[%s.%s]] = [[%s.id]]", newTableAlias, cleanBackFieldName, r.activeTableAlias)),
 				)
+				if err != nil {
+					return nil, err
+				}
 			} else {
-				jeAlias := r.activeTableAlias + "_" + cleanProp + "_je"
+				jeAlias := "__je_" + newTableAlias
-				r.resolver.registerJoin(
+				err := r.resolver.registerJoin(
 					newCollectionName,
 					newTableAlias,
 					dbx.NewExp(fmt.Sprintf(
@@ -531,6 +551,9 @@ func (r *runner) processActiveProps() (*search.ResolverResult, error) {
 						jeAlias,
 					)),
 				)
+				if err != nil {
+					return nil, err
+				}
 			}
 
 			r.activeCollectionName = newCollectionName
@@ -543,7 +566,7 @@ func (r *runner) processActiveProps() (*search.ResolverResult, error) {
 				r.withMultiMatch = true // enable multimatch if not already
 			}
 
-			newTableAlias2 := r.multiMatchActiveTableAlias + "_" + cleanProp
+			newTableAlias2 := r.multiMatchActiveTableAlias + "_" + cleanProp + r.resolver.joinAliasSuffix
 
 			if !isBackRelMultiple {
 				r.multiMatch.joins = append(
@@ -555,7 +578,7 @@ func (r *runner) processActiveProps() (*search.ResolverResult, error) {
 					},
 				)
 			} else {
-				jeAlias2 := r.multiMatchActiveTableAlias + "_" + cleanProp + "_je"
+				jeAlias2 := "__je_" + newTableAlias2
 				r.multiMatch.joins = append(
 					r.multiMatch.joins,
 					&join{
@@ -606,23 +629,34 @@ func (r *runner) processActiveProps() (*search.ResolverResult, error) {
 
 		cleanFieldName := inflector.Columnify(relField.Name)
 		prefixedFieldName := r.activeTableAlias + "." + cleanFieldName
-		newTableAlias := r.activeTableAlias + "_" + cleanFieldName
+		newTableAlias := r.activeTableAlias + "_" + cleanFieldName + r.resolver.joinAliasSuffix
 		newCollectionName := relCollection.Name
 
 		if !relField.IsMultiple() {
-			r.resolver.registerJoin(
+			err := r.resolver.registerJoin(
 				inflector.Columnify(newCollectionName),
 				newTableAlias,
 				dbx.NewExp(fmt.Sprintf("[[%s.id]] = [[%s]]", newTableAlias, prefixedFieldName)),
 			)
+			if err != nil {
+				return nil, err
+			}
 		} else {
-			jeAlias := r.activeTableAlias + "_" + cleanFieldName + "_je"
+			jeAlias := "__je_" + newTableAlias
-			r.resolver.registerJoin(dbutils.JSONEach(prefixedFieldName), jeAlias, nil)
+
-			r.resolver.registerJoin(
+			err := r.resolver.registerJoin(dbutils.JSONEach(prefixedFieldName), jeAlias, nil)
+			if err != nil {
+				return nil, err
+			}
+
+			err = r.resolver.registerJoin(
 				inflector.Columnify(newCollectionName),
 				newTableAlias,
 				dbx.NewExp(fmt.Sprintf("[[%s.id]] = [[%s.value]]", newTableAlias, jeAlias)),
 			)
+			if err != nil {
+				return nil, err
+			}
 		}
 
 		r.activeCollectionName = newCollectionName
@@ -714,8 +748,12 @@ func (r *runner) finalizeActivePropsProcessing(collection *Collection, prop stri
 	// -------------------------------------------------------
 	if modifier == eachModifier && isMultivaluer {
 		jePair := r.activeTableAlias + "." + cleanFieldName
-		jeAlias := r.activeTableAlias + "_" + cleanFieldName + "_je"
+		jeAlias := "__je_" + r.activeTableAlias + "_" + cleanFieldName + r.resolver.joinAliasSuffix
-		r.resolver.registerJoin(dbutils.JSONEach(jePair), jeAlias, nil)
+
+		err := r.resolver.registerJoin(dbutils.JSONEach(jePair), jeAlias, nil)
+		if err != nil {
+			return nil, err
+		}
 
 		result := &search.ResolverResult{
 			Identifier: fmt.Sprintf("[[%s.value]]", jeAlias),
@@ -727,7 +765,7 @@ func (r *runner) finalizeActivePropsProcessing(collection *Collection, prop stri
 
 		if r.withMultiMatch {
 			jePair2 := r.multiMatchActiveTableAlias + "." + cleanFieldName
-			jeAlias2 := r.multiMatchActiveTableAlias + "_" + cleanFieldName + "_je"
+			jeAlias2 := "__je_" + r.multiMatchActiveTableAlias + "_" + cleanFieldName + r.resolver.joinAliasSuffix
 
 			r.multiMatch.joins = append(r.multiMatch.joins, &join{
 				tableName:  dbutils.JSONEach(jePair2),

core/record_query.go

@@ -397,7 +397,10 @@ func (app *BaseApp) FindRecordsByFilter(
 		}
 	}
 
-	resolver.UpdateQuery(q) // attaches any adhoc joins and aliases
+	err = resolver.UpdateQuery(q) // attaches any adhoc joins and aliases
+	if err != nil {
+		return nil, err
+	}
 	// ---
 
 	if offset > 0 {
@@ -611,7 +614,11 @@ func (app *BaseApp) CanAccessRecord(record *Record, requestInfo *RequestInfo, ac
 	if err != nil {
 		return false, err
 	}
-	resolver.UpdateQuery(query)
+
+	err = resolver.UpdateQuery(query)
+	if err != nil {
+		return false, err
+	}
 
 	err = query.AndWhere(expr).Limit(1).Row(&exists)
 	if err != nil && !errors.Is(err, sql.ErrNoRows) {