go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2025-08-15 20:02:56 +02:00
Code Issues Releases Activity

Refactor compiler context; rename FuncContext to CompilerContext and update related references for improved clarity

Browse Source
This commit is contained in:
Tim Voronov
2025-06-11 11:42:47 -04:00
parent b0158406da
commit 7d317aca3c
16 changed files with 473 additions and 624 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/compiler/internal/context.go
View File

@@ -2,8 +2,8 @@ package internal
import "github.com/MontFerret/ferret/pkg/compiler/internal/core"
// FuncContext encapsulates the context and state required for compiling and managing functions during code processing.
type FuncContext struct {
// CompilerContext holds the context for the compilation process, including various compilers and allocators.
type CompilerContext struct {
Emitter *core.Emitter
Registers *core.RegisterAllocator
Symbols *core.SymbolTable
@@ -18,9 +18,9 @@ type FuncContext struct {
WaitCompiler *WaitCompiler
}
// NewFuncContext initializes and returns a new instance of FuncContext, setting up all required components for compilation.
func NewFuncContext() *FuncContext {
ctx := &FuncContext{
// NewCompilerContext initializes a new CompilerContext with default values.
func NewCompilerContext() *CompilerContext {
ctx := &CompilerContext{
Emitter: core.NewEmitter(),
Registers: core.NewRegisterAllocator(),
Symbols: nil, // set later

0
pkg/compiler/internal/core/catch_stack.go → pkg/compiler/internal/core/catch.go
View File

63
pkg/compiler/internal/core/constant_pool.go
View File

@@ -1,63 +0,0 @@
package core
import (
"strconv"
"github.com/MontFerret/ferret/pkg/runtime"
"github.com/MontFerret/ferret/pkg/vm"
)
// ConstantPool stores and deduplicates constants
type ConstantPool struct {
values []runtime.Value
index map[uint64]int
}
func NewConstantPool() *ConstantPool {
return &ConstantPool{
values: make([]runtime.Value, 0),
index: make(map[uint64]int),
}
}
func (cp *ConstantPool) Add(val runtime.Value) vm.Operand {
var hash uint64
isNone := val == runtime.None
if runtime.IsScalar(val) {
hash = val.Hash()
}
if hash > 0 || isNone {
if idx, ok := cp.index[hash]; ok {
return vm.NewConstantOperand(idx)
}
}
cp.values = append(cp.values, val)
idx := len(cp.values) - 1
if hash > 0 || isNone {
cp.index[hash] = idx
}
return vm.NewConstantOperand(idx)
}
func (cp *ConstantPool) Get(addr vm.Operand) runtime.Value {
if !addr.IsConstant() {
panic(runtime.Error(ErrInvalidOperandType, strconv.Itoa(int(addr))))
}
idx := addr.Constant()
if idx < 0 || idx >= len(cp.values) {
panic(runtime.Error(ErrConstantNotFound, strconv.Itoa(idx)))
}
return cp.values[idx]
}
func (cp *ConstantPool) All() []runtime.Value {
return cp.values
}

65
pkg/compiler/internal/core/constants.go
View File

@@ -1,8 +1,63 @@
package core
const (
JumpPlaceholder = -1
UndefinedVariable = -1
IgnorePseudoVariable = "_"
PseudoVariable = "CURRENT"
import (
"strconv"
"github.com/MontFerret/ferret/pkg/runtime"
"github.com/MontFerret/ferret/pkg/vm"
)
// ConstantPool stores and deduplicates constants
type ConstantPool struct {
values []runtime.Value
index map[uint64]int
}
func NewConstantPool() *ConstantPool {
return &ConstantPool{
values: make([]runtime.Value, 0),
index: make(map[uint64]int),
}
}
func (cp *ConstantPool) Add(val runtime.Value) vm.Operand {
var hash uint64
isNone := val == runtime.None
if runtime.IsScalar(val) {
hash = val.Hash()
}
if hash > 0 || isNone {
if idx, ok := cp.index[hash]; ok {
return vm.NewConstantOperand(idx)
}
}
cp.values = append(cp.values, val)
idx := len(cp.values) - 1
if hash > 0 || isNone {
cp.index[hash] = idx
}
return vm.NewConstantOperand(idx)
}
func (cp *ConstantPool) Get(addr vm.Operand) runtime.Value {
if !addr.IsConstant() {
panic(runtime.Error(ErrInvalidOperandType, strconv.Itoa(int(addr))))
}
idx := addr.Constant()
if idx < 0 || idx >= len(cp.values) {
panic(runtime.Error(ErrConstantNotFound, strconv.Itoa(idx)))
}
return cp.values[idx]
}
func (cp *ConstantPool) All() []runtime.Value {
return cp.values
}

0
pkg/compiler/internal/core/loop_table.go → pkg/compiler/internal/core/loops.go
View File

0
pkg/compiler/internal/core/register_allocator.go → pkg/compiler/internal/core/registers.go
View File

17
pkg/compiler/internal/core/symbol_table.go → pkg/compiler/internal/core/symbols.go
View File

@@ -7,6 +7,13 @@ import (
"github.com/MontFerret/ferret/pkg/vm"
)
const (
JumpPlaceholder = -1
UndefinedVariable = -1
IgnorePseudoVariable = "_"
PseudoVariable = "CURRENT"
)
type SymbolKind int
const (
@@ -154,7 +161,7 @@ func (st *SymbolTable) Params() []string {
return out
}
func (st *SymbolTable) DebugSymbols() []string {
func (st *SymbolTable) DebugView() []string {
var out []string
for _, v := range st.locals {
@@ -165,5 +172,13 @@ func (st *SymbolTable) DebugSymbols() []string {
out = append(out, fmt.Sprintf("[global] %s -> R%d", k, r))
}
for k, v := range st.params {
out = append(out, fmt.Sprintf("[param] %s -> %s", k, v))
}
for _, c := range st.constants.All() {
out = append(out, fmt.Sprintf("[constant] %s", c.String()))
}
return out
}

0
pkg/compiler/internal/core/symbol_table_test.go → pkg/compiler/internal/core/symbols_test.go
View File

9
pkg/compiler/internal/expr.go
View File

@@ -18,10 +18,10 @@ const (
)
type ExprCompiler struct {
ctx *FuncContext
ctx *CompilerContext
}
func NewExprCompiler(ctx *FuncContext) *ExprCompiler {
func NewExprCompiler(ctx *CompilerContext) *ExprCompiler {
return &ExprCompiler{ctx: ctx}
}
@@ -385,8 +385,11 @@ func (ec *ExprCompiler) CompileFunctionCallExpression(ctx fql.IFunctionCallExpre
}
func (ec *ExprCompiler) CompileFunctionCall(ctx fql.IFunctionCallContext, protected bool) vm.Operand {
return ec.CompileFunctionCallWith(ctx, protected, ec.CompileArgumentList(ctx.ArgumentList()))
}
func (ec *ExprCompiler) CompileFunctionCallWith(ctx fql.IFunctionCallContext, protected bool, seq core.RegisterSequence) vm.Operand {
name := ec.functionName(ctx)
seq := ec.CompileArgumentList(ctx.ArgumentList())
switch name {
case runtimeLength:

6
pkg/compiler/internal/helpers.go
View File

@@ -13,12 +13,16 @@ import (
"github.com/pkg/errors"
)
func loadConstant(ctx *FuncContext, value runtime.Value) vm.Operand {
func loadConstant(ctx *CompilerContext, value runtime.Value) vm.Operand {
reg := ctx.Registers.Allocate(core.Temp)
ctx.Emitter.EmitLoadConst(reg, ctx.Symbols.AddConstant(value))
return reg
}
func loadConstantTo(ctx *CompilerContext, constant runtime.Value, reg vm.Operand) {
ctx.Emitter.EmitAB(vm.OpLoadConst, reg, ctx.Symbols.AddConstant(constant))
}
func sortDirection(dir antlr.TerminalNode) runtime.SortDirection {
if dir == nil {
return runtime.SortDirectionAsc

4
pkg/compiler/internal/literal.go
View File

@@ -13,10 +13,10 @@ import (
)
type LiteralCompiler struct {
ctx *FuncContext
ctx *CompilerContext
}
func NewLiteralCompiler(ctx *FuncContext) *LiteralCompiler {
func NewLiteralCompiler(ctx *CompilerContext) *LiteralCompiler {
return &LiteralCompiler{
ctx: ctx,
}

22
pkg/compiler/internal/loop.go
View File

@@ -10,10 +10,10 @@ import (
)
type LoopCompiler struct {
ctx *FuncContext
ctx *CompilerContext
}
func NewLoopCompiler(ctx *FuncContext) *LoopCompiler {
func NewLoopCompiler(ctx *CompilerContext) *LoopCompiler {
return &LoopCompiler{ctx: ctx}
}
@@ -49,7 +49,7 @@ func (lc *LoopCompiler) Compile(ctx fql.IForExpressionContext) vm.Operand {
} else {
}
lc.emitLoopBegin(loop)
lc.EmitLoopBegin(loop)
// body
if body := ctx.AllForExpressionBody(); body != nil && len(body) > 0 {
@@ -76,7 +76,7 @@ func (lc *LoopCompiler) Compile(ctx fql.IForExpressionContext) vm.Operand {
}
}
res := lc.emitLoopEnd(loop)
res := lc.EmitLoopEnd(loop)
lc.ctx.Symbols.ExitScope()
lc.ctx.Loops.Pop()
@@ -252,15 +252,15 @@ func (lc *LoopCompiler) CompileSortClause(ctx fql.ISortClauseContext) {
lc.ctx.Emitter.EmitAB(vm.OpMove, loop.Src, loop.Result)
// Create a new loop
lc.emitLoopBegin(loop)
lc.EmitLoopBegin(loop)
}
func (lc *LoopCompiler) CompileCollectClause(ctx fql.ICollectClauseContext) {
lc.ctx.CollectCompiler.Compile(ctx)
}
// emitIterValue emits an instruction to get the value from the iterator
func (lc *LoopCompiler) emitLoopBegin(loop *core.Loop) {
// EmitLoopBegin emits an instruction to get the value from the iterator
func (lc *LoopCompiler) EmitLoopBegin(loop *core.Loop) {
if loop.Allocate {
lc.ctx.Emitter.EmitAb(vm.OpDataSet, loop.Result, loop.Distinct)
loop.ResultPos = lc.ctx.Emitter.Size() - 1
@@ -286,8 +286,8 @@ func (lc *LoopCompiler) emitLoopBegin(loop *core.Loop) {
}
}
// emitPatchLoop replaces the source of the loop with a modified dataset
func (lc *LoopCompiler) emitPatchLoop(loop *core.Loop) {
// PatchLoop replaces the source of the loop with a modified dataset
func (lc *LoopCompiler) PatchLoop(loop *core.Loop) {
// Replace source with sorted array
lc.ctx.Emitter.EmitAB(vm.OpMove, loop.Src, loop.Result)
@@ -295,10 +295,10 @@ func (lc *LoopCompiler) emitPatchLoop(loop *core.Loop) {
lc.ctx.Symbols.EnterScope()
// Create new for loop
lc.emitLoopBegin(loop)
lc.EmitLoopBegin(loop)
}
func (lc *LoopCompiler) emitLoopEnd(loop *core.Loop) vm.Operand {
func (lc *LoopCompiler) EmitLoopEnd(loop *core.Loop) vm.Operand {
lc.ctx.Emitter.EmitJump(loop.Jump - loop.JumpOffset)
// TODO: Do not allocate for pass-through Loops

405
pkg/compiler/internal/loop_collect.go
View File

@@ -1,56 +1,375 @@
package internal
import (
"github.com/MontFerret/ferret/pkg/compiler/internal/core"
"github.com/MontFerret/ferret/pkg/parser/fql"
"github.com/MontFerret/ferret/pkg/runtime"
"github.com/MontFerret/ferret/pkg/vm"
"github.com/antlr4-go/antlr/v4"
)
type CollectCompiler struct {
Ctx *FuncContext
ctx *CompilerContext
}
func NewCollectCompiler(ctx *FuncContext) *CollectCompiler {
return &CollectCompiler{Ctx: ctx}
func NewCollectCompiler(ctx *CompilerContext) *CollectCompiler {
return &CollectCompiler{ctx: ctx}
}
func (cc *CollectCompiler) Compile(ctx fql.ICollectClauseContext) {
//loop := cc.Ctx.Loops.Current()
//if loop == nil {
// panic("COLLECT clause must appear inside a loop")
//}
//
//// Grouping by key
//if group := ctx.CollectGrouping(); group != nil {
// // Example: COLLECT key = expr
// keyName := group.Variable().GetText()
// keyExpr := group.Expression()
// keyReg := cc.Ctx.ExprCompiler.Compile(keyExpr)
//
// loop.Result = cc.Ctx.Registers.Allocate(Result)
//
// cc.Ctx.Emitter.EmitABC(vm.OpCollect, loop.Result, keyReg, keyReg) // src1=key, src2=key (single-group)
// cc.Ctx.Symbols.DeclareLocal(keyName)
//}
//
//// Aggregation
//if agg := ctx.CollectAggregator(); agg != nil {
// for _, part := range agg.AllCollectGroupVariable() {
// name := part.Variable().GetText()
// expr := part.Expression()
//
// src := cc.Ctx.ExprCompiler.Compile(expr)
// dst := cc.Ctx.Registers.Allocate(Result)
//
// cc.Ctx.Emitter.EmitABC(vm.OpCollect, dst, src, src)
// cc.Ctx.Symbols.DeclareLocal(name)
// }
//}
//
//// Optional counter
//if counter := ctx.CollectCounter(); counter != nil {
// name := counter.Variable().GetText()
// dst := cc.Ctx.Registers.Allocate(Result)
//
// cc.Ctx.Emitter.EmitAB(vm.OpCount, dst, loop.Value)
// cc.Ctx.Symbols.DeclareLocal(name)
//}
// TODO: Undefine original loop variables
loop := cc.ctx.Loops.Current()
// We collect the aggregation keys
// And wrap each loop element by a KeyValuePair
// Where a key is either a single value or a list of values
// These KeyValuePairs are then added to the dataset
var kvKeyReg, kvValReg vm.Operand
var groupSelectors []fql.ICollectSelectorContext
var isGrouping bool
grouping := ctx.CollectGrouping()
counter := ctx.CollectCounter()
aggregator := ctx.CollectAggregator()
isCollecting := grouping != nil || counter != nil
if isCollecting {
if grouping != nil {
isGrouping = true
groupSelectors = grouping.AllCollectSelector()
kvKeyReg = cc.compileCollectGroupKeySelectors(groupSelectors)
}
kvValReg = cc.ctx.Registers.Allocate(core.Temp)
loop.EmitValue(kvKeyReg, cc.ctx.Emitter)
var projectionVariableName string
collectorType := core.CollectorTypeKey
// If we have a collect group variable, we need to project it
if groupVar := ctx.CollectGroupVariable(); groupVar != nil {
// Projection can be either a default projection (identifier) or a custom projection (selector expression)
if identifier := groupVar.Identifier(); identifier != nil {
projectionVariableName = cc.compileDefaultGroupProjection(loop, kvValReg, identifier, groupVar.CollectGroupVariableKeeper())
} else if selector := groupVar.CollectSelector(); selector != nil {
projectionVariableName = cc.compileCustomGroupProjection(loop, kvValReg, selector)
}
collectorType = core.CollectorTypeKeyGroup
} else if counter != nil {
projectionVariableName = counter.Identifier().GetText()
if isGrouping {
collectorType = core.CollectorTypeKeyCounter
} else {
collectorType = core.CollectorTypeCounter
}
}
// If we use aggregators, we need to collect group items by key
if aggregator != nil && collectorType != core.CollectorTypeKeyGroup {
// We need to patch the loop result to be a collector
collectorType = core.CollectorTypeKeyGroup
}
// We replace DataSet initialization with Collector initialization
cc.ctx.Emitter.PatchSwapAx(loop.ResultPos, vm.OpDataSetCollector, loop.Result, int(collectorType))
cc.ctx.Emitter.EmitABC(vm.OpPushKV, loop.Result, kvKeyReg, kvValReg)
loop.EmitFinalization(cc.ctx.Emitter)
// Replace the source with the collector
cc.ctx.LoopCompiler.PatchLoop(loop)
// If the projection is used, we allocate a new register for the variable and put the iterator's value into it
if projectionVariableName != "" {
// Now we need to expand group variables from the dataset
loop.EmitKey(kvValReg, cc.ctx.Emitter)
loop.EmitValue(cc.ctx.Symbols.DeclareLocal(projectionVariableName), cc.ctx.Emitter)
} else {
loop.EmitKey(kvKeyReg, cc.ctx.Emitter)
loop.EmitValue(kvValReg, cc.ctx.Emitter)
}
}
// Aggregation loop
if aggregator != nil {
cc.compileAggregator(aggregator, loop, isCollecting)
}
// TODO: Reuse the Registers
cc.ctx.Registers.Free(loop.Value)
cc.ctx.Registers.Free(loop.Key)
loop.Value = vm.NoopOperand
loop.Key = vm.NoopOperand
if isCollecting && isGrouping {
// Now we are defining new variables for the group selectors
cc.compileCollectGroupKeySelectorVariables(groupSelectors, kvKeyReg, kvValReg, aggregator != nil)
}
}
func (cc *CollectCompiler) compileAggregator(c fql.ICollectAggregatorContext, parentLoop *core.Loop, isCollected bool) {
var accums []vm.Operand
var loop *core.Loop
selectors := c.AllCollectAggregateSelector()
// If data is collected, we need to allocate a temporary accumulators to store aggregation results
if isCollected {
// First of all, we allocate registers for accumulators
accums = make([]vm.Operand, len(selectors))
// We need to allocate a register for each accumulator
for i := 0; i < len(selectors); i++ {
reg := cc.ctx.Registers.Allocate(core.Temp)
accums[i] = reg
// TODO: Select persistent List type, we do not know how many items we will have
cc.ctx.Emitter.EmitA(vm.OpList, reg)
}
loop = cc.ctx.Loops.NewLoop(core.TemporalLoop, core.ForLoop, false)
// Now we iterate over the grouped items
parentLoop.EmitValue(loop.Iterator, cc.ctx.Emitter)
// We just re-use the same register
cc.ctx.Emitter.EmitAB(vm.OpIter, loop.Iterator, loop.Iterator)
// jumpPlaceholder is a placeholder for the exit aggrIterJump position
loop.Jump = cc.ctx.Emitter.EmitJumpc(vm.OpIterNext, core.JumpPlaceholder, loop.Iterator)
loop.ValueName = parentLoop.ValueName
} else {
loop = parentLoop
// Otherwise, we create a custom collector for aggregators
cc.ctx.Emitter.PatchSwapAx(loop.ResultPos, vm.OpDataSetCollector, loop.Result, int(core.CollectorTypeKeyGroup))
}
// Store upper scope for aggregators
//mainScope := cc.ctx.Symbols.Scope()
// Nested scope for aggregators
cc.ctx.Symbols.EnterScope()
aggrIterVal := cc.ctx.Symbols.DeclareLocal(loop.ValueName)
cc.ctx.Emitter.EmitAB(vm.OpIterValue, aggrIterVal, loop.Iterator)
// Now we add value selectors to the accumulators
for i := 0; i < len(selectors); i++ {
selector := selectors[i]
fcx := selector.FunctionCallExpression()
args := cc.ctx.ExprCompiler.CompileArgumentList(fcx.FunctionCall().ArgumentList())
if len(args) == 0 {
// TODO: Better error handling
panic("No arguments provided for the function call in the aggregate selector")
}
if len(args) > 1 {
// TODO: Better error handling
panic("Too many arguments")
}
resultReg := args[0]
if isCollected {
cc.ctx.Emitter.EmitAB(vm.OpPush, accums[i], resultReg)
} else {
aggrKeyName := selector.Identifier().GetText()
aggrKeyReg := loadConstant(cc.ctx, runtime.String(aggrKeyName))
cc.ctx.Emitter.EmitABC(vm.OpPushKV, loop.Result, aggrKeyReg, resultReg)
cc.ctx.Registers.Free(aggrKeyReg)
}
cc.ctx.Registers.Free(resultReg)
}
// Now we can iterate over the grouped items
loop.EmitFinalization(cc.ctx.Emitter)
// Now we can iterate over the selectors and execute the aggregation functions by passing the accumulators
// And define variables for each accumulator result
if isCollected {
for i, selector := range selectors {
fcx := selector.FunctionCallExpression()
// We won't make any checks here, as we already did it before
selectorVarName := selector.Identifier().GetText()
// We execute the function call with the accumulator as an argument
accum := accums[i]
result := cc.ctx.ExprCompiler.CompileFunctionCallWith(fcx.FunctionCall(), fcx.ErrorOperator() != nil, core.RegisterSequence{accum})
// We define the variable for the selector result in the upper scope
// Since this temporary scope is only for aggregators and will be closed after the aggregation
varReg := cc.ctx.Symbols.DeclareLocal(selectorVarName)
cc.ctx.Emitter.EmitAB(vm.OpMove, varReg, result)
cc.ctx.Registers.Free(result)
}
cc.ctx.Loops.Pop()
// Now close the aggregators scope
cc.ctx.Symbols.ExitScope()
} else {
// Now close the aggregators scope
cc.ctx.Symbols.ExitScope()
parentLoop.ValueName = ""
parentLoop.KeyName = ""
// Since we we in the middle of the loop, we need to patch the loop result
// Now we just create a range with 1 item to push the aggregated values to the dataset
// Replace source with sorted array
zero := loadConstant(cc.ctx, runtime.Int(0))
one := loadConstant(cc.ctx, runtime.Int(1))
aggregator := cc.ctx.Registers.Allocate(core.Temp)
cc.ctx.Emitter.EmitAB(vm.OpMove, aggregator, loop.Result)
cc.ctx.Symbols.ExitScope()
cc.ctx.Symbols.EnterScope()
// Create new for loop
cc.ctx.Emitter.EmitABC(vm.OpRange, loop.Src, zero, one)
cc.ctx.Emitter.EmitAb(vm.OpDataSet, loop.Result, loop.Distinct)
// In case of non-collected aggregators, we just iterate over the grouped items
// Retrieve the grouped values by key, execute aggregation funcs and assign variable names to the results
for _, selector := range selectors {
fcx := selector.FunctionCallExpression()
// We won't make any checks here, as we already did it before
selectorVarName := selector.Identifier().GetText()
// We execute the function call with the accumulator as an argument
key := loadConstant(cc.ctx, runtime.String(selectorVarName))
value := cc.ctx.Registers.Allocate(core.Temp)
cc.ctx.Emitter.EmitABC(vm.OpLoadKey, value, aggregator, key)
result := cc.ctx.ExprCompiler.CompileFunctionCallWith(fcx.FunctionCall(), fcx.ErrorOperator() != nil, core.RegisterSequence{value})
// We define the variable for the selector result in the upper scope
// Since this temporary scope is only for aggregators and will be closed after the aggregation
varReg := cc.ctx.Symbols.DeclareLocal(selectorVarName)
cc.ctx.Emitter.EmitAB(vm.OpMove, varReg, result)
cc.ctx.Registers.Free(result)
cc.ctx.Registers.Free(value)
cc.ctx.Registers.Free(key)
}
cc.ctx.Registers.Free(aggregator)
}
// Free the registers for accumulators
for _, reg := range accums {
cc.ctx.Registers.Free(reg)
}
// Free the register for the iterator value
cc.ctx.Registers.Free(aggrIterVal)
}
func (cc *CollectCompiler) compileCollectGroupKeySelectors(selectors []fql.ICollectSelectorContext) vm.Operand {
if len(selectors) == 0 {
return vm.NoopOperand
}
var kvKeyReg vm.Operand
if len(selectors) > 1 {
// We create a sequence of Registers for the clauses
// To pack them into an array
selectorRegs := cc.ctx.Registers.AllocateSequence(len(selectors))
for i, selector := range selectors {
reg := cc.ctx.ExprCompiler.Compile(selector.Expression())
cc.ctx.Emitter.EmitAB(vm.OpMove, selectorRegs[i], reg)
// Free the register after moving its value to the sequence register
cc.ctx.Registers.Free(reg)
}
kvKeyReg = cc.ctx.Registers.Allocate(core.Temp)
cc.ctx.Emitter.EmitAs(vm.OpList, kvKeyReg, selectorRegs)
cc.ctx.Registers.FreeSequence(selectorRegs)
} else {
kvKeyReg = cc.ctx.ExprCompiler.Compile(selectors[0].Expression())
}
return kvKeyReg
}
func (cc *CollectCompiler) compileCollectGroupKeySelectorVariables(selectors []fql.ICollectSelectorContext, kvKeyReg, kvValReg vm.Operand, isAggregation bool) {
if len(selectors) > 1 {
variables := make([]vm.Operand, len(selectors))
for i, selector := range selectors {
name := selector.Identifier().GetText()
if variables[i] == vm.NoopOperand {
variables[i] = cc.ctx.Symbols.DeclareLocal(name)
}
reg := kvValReg
if isAggregation {
reg = kvKeyReg
}
cc.ctx.Emitter.EmitABC(vm.OpLoadIndex, variables[i], reg, loadConstant(cc.ctx, runtime.Int(i)))
}
// Free the register after moving its value to the variable
for _, reg := range variables {
cc.ctx.Registers.Free(reg)
}
} else {
// Get the variable name
name := selectors[0].Identifier().GetText()
// Define a variable for each selector
varReg := cc.ctx.Symbols.DeclareLocal(name)
reg := kvValReg
if isAggregation {
reg = kvKeyReg
}
// If we have a single selector, we can just move the value
cc.ctx.Emitter.EmitAB(vm.OpMove, varReg, reg)
}
}
func (cc *CollectCompiler) compileDefaultGroupProjection(loop *core.Loop, kvValReg vm.Operand, identifier antlr.TerminalNode, keeper fql.ICollectGroupVariableKeeperContext) string {
if keeper == nil {
seq := cc.ctx.Registers.AllocateSequence(2) // Key and Value for Map
// TODO: Review this. It's quite a questionable ArrangoDB feature of wrapping group items by a nested object
// We will keep it for now for backward compatibility.
loadConstantTo(cc.ctx, runtime.String(loop.ValueName), seq[0]) // Map key
cc.ctx.Emitter.EmitAB(vm.OpMove, seq[1], kvValReg) // Map value
cc.ctx.Emitter.EmitAs(vm.OpMap, kvValReg, seq)
cc.ctx.Registers.FreeSequence(seq)
} else {
variables := keeper.AllIdentifier()
seq := cc.ctx.Registers.AllocateSequence(len(variables) * 2)
for i, j := 0, 0; i < len(variables); i, j = i+1, j+2 {
varName := variables[i].GetText()
loadConstantTo(cc.ctx, runtime.String(varName), seq[j])
variable, _, found := cc.ctx.Symbols.Resolve(varName)
if !found {
panic("variable not found: " + varName)
}
cc.ctx.Emitter.EmitAB(vm.OpMove, seq[j+1], variable)
}
cc.ctx.Emitter.EmitAs(vm.OpMap, kvValReg, seq)
cc.ctx.Registers.FreeSequence(seq)
}
return identifier.GetText()
}
func (cc *CollectCompiler) compileCustomGroupProjection(_ *core.Loop, kvValReg vm.Operand, selector fql.ICollectSelectorContext) string {
selectorReg := cc.ctx.ExprCompiler.Compile(selector.Expression())
cc.ctx.Emitter.EmitMove(kvValReg, selectorReg)
cc.ctx.Registers.Free(selectorReg)
return selector.Identifier().GetText()
}

4
pkg/compiler/internal/stmt.go
View File

@@ -7,10 +7,10 @@ import (
)
type StmtCompiler struct {
ctx *FuncContext
ctx *CompilerContext
}
func NewStmtCompiler(ctx *FuncContext) *StmtCompiler {
func NewStmtCompiler(ctx *CompilerContext) *StmtCompiler {
return &StmtCompiler{
ctx: ctx,
}

4
pkg/compiler/internal/wait.go
View File

@@ -8,10 +8,10 @@ import (
)
type WaitCompiler struct {
ctx *FuncContext
ctx *CompilerContext
}
func NewWaitCompiler(ctx *FuncContext) *WaitCompiler {
func NewWaitCompiler(ctx *CompilerContext) *WaitCompiler {
return &WaitCompiler{
ctx: ctx,
}

488
pkg/compiler/visitor.go
View File

@@ -8,7 +8,7 @@ import (
type Visitor struct {
*fql.BaseFqlParserVisitor
Ctx *internal.FuncContext
Ctx *internal.CompilerContext
Err error
Src string
}
@@ -16,7 +16,7 @@ type Visitor struct {
func NewVisitor(src string) *Visitor {
v := new(Visitor)
v.BaseFqlParserVisitor = new(fql.BaseFqlParserVisitor)
v.Ctx = internal.NewFuncContext()
v.Ctx = internal.NewCompilerContext()
v.Src = src
@@ -36,487 +36,3 @@ func (v *Visitor) VisitProgram(ctx *fql.ProgramContext) interface{} {
func (v *Visitor) VisitHead(_ *fql.HeadContext) interface{} {
return nil
}
//func (v *Visitor) VisitCollectClause(Ctx *fql.CollectClauseContext) interface{} {
// // TODO: Undefine original loop variables
// loop := v.Loops.Current()
//
// // We collect the aggregation keys
// // And wrap each loop element by a KeyValuePair
// // Where a key is either a single value or a list of values
// // These KeyValuePairs are then added to the dataset
// var kvKeyReg, kvValReg vm.Operand
// var groupSelectors []fql.ICollectSelectorContext
// var isGrouping bool
// grouping := Ctx.CollectGrouping()
// counter := Ctx.CollectCounter()
// aggregator := Ctx.CollectAggregator()
//
// isCollecting := grouping != nil || counter != nil
//
// if isCollecting {
// if grouping != nil {
// isGrouping = true
// groupSelectors = grouping.AllCollectSelector()
// kvKeyReg = v.emitCollectGroupKeySelectors(groupSelectors)
// }
//
// kvValReg = v.Registers.Allocate(Temp)
// v.emitIterValue(loop, kvValReg)
//
// var projectionVariableName string
// collectorType := CollectorTypeKey
//
// // If we have a collect group variable, we need to project it
// if groupVar := Ctx.CollectGroupVariable(); groupVar != nil {
// // Projection can be either a default projection (identifier) or a custom projection (selector expression)
// if identifier := groupVar.Identifier(); identifier != nil {
// projectionVariableName = v.emitCollectDefaultGroupProjection(loop, kvValReg, identifier, groupVar.CollectGroupVariableKeeper())
// } else if selector := groupVar.CollectSelector(); selector != nil {
// projectionVariableName = v.emitCollectCustomGroupProjection(loop, kvValReg, selector)
// }
//
// collectorType = CollectorTypeKeyGroup
// } else if counter != nil {
// projectionVariableName = v.emitCollectCountProjection(loop, kvValReg, counter)
//
// if isGrouping {
// collectorType = CollectorTypeKeyCounter
// } else {
// collectorType = CollectorTypeCounter
// }
// }
//
// // If we use aggregators, we need to collect group items by key
// if aggregator != nil && collectorType != CollectorTypeKeyGroup {
// // We need to patch the loop result to be a collector
// collectorType = CollectorTypeKeyGroup
// }
//
// // We replace DataSet initialization with Collector initialization
// v.Emitter.PatchSwapAx(loop.ResultPos, vm.OpDataSetCollector, loop.Result, int(collectorType))
// v.Emitter.EmitABC(vm.OpPushKV, loop.Result, kvKeyReg, kvValReg)
// v.emitIterJumpOrClose(loop)
//
// // Replace the source with the collector
// v.emitPatchLoop(loop)
//
// // If the projection is used, we allocate a new register for the variable and put the iterator's value into it
// if projectionVariableName != "" {
// // Now we need to expand group variables from the dataset
// v.emitIterKey(loop, kvValReg)
// v.emitIterValue(loop, v.Symbols.DeclareLocal(projectionVariableName))
// } else {
// v.emitIterKey(loop, kvKeyReg)
// v.emitIterValue(loop, kvValReg)
// }
// }
//
// // Aggregation loop
// if aggregator != nil {
// v.emitCollectAggregator(aggregator, loop, isCollecting)
// }
//
// // TODO: Reuse the Registers
// v.Registers.Free(loop.Value)
// v.Registers.Free(loop.Key)
// loop.Value = vm.NoopOperand
// loop.Key = vm.NoopOperand
//
// if isCollecting && isGrouping {
// // Now we are defining new variables for the group selectors
// v.emitCollectGroupKeySelectorVariables(groupSelectors, kvKeyReg, kvValReg, aggregator != nil)
// }
//
// return nil
//}
//
//func (v *Visitor) emitCollectAggregator(c fql.ICollectAggregatorContext, parentLoop *Loop, isCollected bool) {
// var accums []vm.Operand
// var loop *Loop
// selectors := c.AllCollectAggregateSelector()
//
// // If data is collected, we need to allocate a temporary accumulators to store aggregation results
// if isCollected {
// // First of all, we allocate registers for accumulators
// accums = make([]vm.Operand, len(selectors))
//
// // We need to allocate a register for each accumulator
// for i := 0; i < len(selectors); i++ {
// reg := v.Registers.Allocate(Temp)
// accums[i] = reg
// // TODO: Select persistent List type, we do not know how many items we will have
// v.Emitter.EmitA(vm.OpList, reg)
// }
//
// loop = v.Loops.NewLoop(TemporalLoop, ForLoop, false)
//
// // Now we iterate over the grouped items
// v.emitIterValue(parentLoop, loop.Iterator)
// // We just re-use the same register
// v.Emitter.EmitAB(vm.OpIter, loop.Iterator, loop.Iterator)
// // jumpPlaceholder is a placeholder for the exit aggrIterJump position
// loop.Jump = v.Emitter.EmitJumpc(vm.OpIterNext, jumpPlaceholder, loop.Iterator)
// loop.ValueName = parentLoop.ValueName
// } else {
// loop = parentLoop
// // Otherwise, we create a custom collector for aggregators
// v.Emitter.PatchSwapAx(loop.ResultPos, vm.OpDataSetCollector, loop.Result, int(CollectorTypeKeyGroup))
// }
//
// // Store upper scope for aggregators
// //mainScope := v.Symbols.Scope()
// // Nested scope for aggregators
// v.Symbols.EnterScope()
//
// aggrIterVal := v.Symbols.DeclareLocal(loop.ValueName)
// v.Emitter.EmitAB(vm.OpIterValue, aggrIterVal, loop.Iterator)
//
// // Now we add value selectors to the accumulators
// for i := 0; i < len(selectors); i++ {
// selector := selectors[i]
// fcx := selector.FunctionCallExpression()
// args := fcx.FunctionCall().ArgumentList().AllExpression()
//
// if len(args) == 0 {
// // TODO: Better error handling
// panic("No arguments provided for the function call in the aggregate selector")
// }
//
// if len(args) > 1 {
// // TODO: Better error handling
// panic("Too many arguments")
// }
//
// resultReg := args[0].Accept(v).(vm.Operand)
//
// if isCollected {
// v.Emitter.EmitAB(vm.OpPush, accums[i], resultReg)
// } else {
// aggrKeyName := selector.Identifier().GetText()
// aggrKeyReg := v.loadConstant(runtime.String(aggrKeyName))
// v.Emitter.EmitABC(vm.OpPushKV, loop.Result, aggrKeyReg, resultReg)
// v.Registers.Free(aggrKeyReg)
// }
//
// v.Registers.Free(resultReg)
// }
//
// // Now we can iterate over the grouped items
// v.emitIterJumpOrClose(loop)
//
// // Now we can iterate over the selectors and execute the aggregation functions by passing the accumulators
// // And define variables for each accumulator result
// if isCollected {
// for i, selector := range selectors {
// fcx := selector.FunctionCallExpression()
// // We won't make any checks here, as we already did it before
// selectorVarName := selector.Identifier().GetText()
//
// // We execute the function call with the accumulator as an argument
// accum := accums[i]
// result := v.emitFunctionCall(fcx.FunctionCall(), fcx.ErrorOperator() != nil, RegisterSequence{accum})
//
// // We define the variable for the selector result in the upper scope
// // Since this temporary scope is only for aggregators and will be closed after the aggregation
// varReg := v.Symbols.DeclareLocal(selectorVarName)
// v.Emitter.EmitAB(vm.OpMove, varReg, result)
// v.Registers.Free(result)
// }
//
// v.Loops.Pop()
// // Now close the aggregators scope
// v.Symbols.ExitScope()
// } else {
// // Now close the aggregators scope
// v.Symbols.ExitScope()
//
// parentLoop.ValueName = ""
// parentLoop.KeyName = ""
//
// // Since we we in the middle of the loop, we need to patch the loop result
// // Now we just create a range with 1 item to push the aggregated values to the dataset
// // Replace source with sorted array
// zero := v.loadConstant(runtime.Int(0))
// one := v.loadConstant(runtime.Int(1))
// aggregator := v.Registers.Allocate(Temp)
// v.Emitter.EmitAB(vm.OpMove, aggregator, loop.Result)
// v.Symbols.ExitScope()
//
// v.Symbols.EnterScope()
//
// // Create new for loop
// v.Emitter.EmitABC(vm.OpRange, loop.Src, zero, one)
// v.Emitter.EmitAb(vm.OpDataSet, loop.Result, loop.Distinct)
//
// // In case of non-collected aggregators, we just iterate over the grouped items
// // Retrieve the grouped values by key, execute aggregation funcs and assign variable names to the results
// for _, selector := range selectors {
// fcx := selector.FunctionCallExpression()
// // We won't make any checks here, as we already did it before
// selectorVarName := selector.Identifier().GetText()
//
// // We execute the function call with the accumulator as an argument
// key := v.loadConstant(runtime.String(selectorVarName))
// value := v.Registers.Allocate(Temp)
// v.Emitter.EmitABC(vm.OpLoadKey, value, aggregator, key)
//
// result := v.emitFunctionCall(fcx.FunctionCall(), fcx.ErrorOperator() != nil, RegisterSequence{value})
//
// // We define the variable for the selector result in the upper scope
// // Since this temporary scope is only for aggregators and will be closed after the aggregation
// varReg := v.Symbols.DeclareLocal(selectorVarName)
// v.Emitter.EmitAB(vm.OpMove, varReg, result)
// v.Registers.Free(result)
// v.Registers.Free(value)
// v.Registers.Free(key)
// }
//
// v.Registers.Free(aggregator)
// }
//
// // Free the registers for accumulators
// for _, reg := range accums {
// v.Registers.Free(reg)
// }
//
// // Free the register for the iterator value
// v.Registers.Free(aggrIterVal)
//}
//
//func (v *Visitor) emitCollectGroupKeySelectors(selectors []fql.ICollectSelectorContext) vm.Operand {
// if len(selectors) == 0 {
// return vm.NoopOperand
// }
//
// var kvKeyReg vm.Operand
//
// if len(selectors) > 1 {
// // We create a sequence of Registers for the clauses
// // To pack them into an array
// selectorRegs := v.Registers.AllocateSequence(len(selectors))
//
// for i, selector := range selectors {
// reg := selector.Accept(v).(vm.Operand)
// v.Emitter.EmitAB(vm.OpMove, selectorRegs[i], reg)
// // Free the register after moving its value to the sequence register
// v.Registers.Free(reg)
// }
//
// kvKeyReg = v.Registers.Allocate(Temp)
// v.Emitter.EmitAs(vm.OpList, kvKeyReg, selectorRegs)
// v.Registers.FreeSequence(selectorRegs)
// } else {
// kvKeyReg = selectors[0].Accept(v).(vm.Operand)
// }
//
// return kvKeyReg
//}
//
//func (v *Visitor) emitCollectGroupKeySelectorVariables(selectors []fql.ICollectSelectorContext, kvKeyReg, kvValReg vm.Operand, isAggregation bool) {
// if len(selectors) > 1 {
// variables := make([]vm.Operand, len(selectors))
//
// for i, selector := range selectors {
// name := selector.Identifier().GetText()
//
// if variables[i] == vm.NoopOperand {
// variables[i] = v.Symbols.DeclareLocal(name)
// }
//
// reg := kvValReg
//
// if isAggregation {
// reg = kvKeyReg
// }
//
// v.Emitter.EmitABC(vm.OpLoadIndex, variables[i], reg, v.loadConstant(runtime.Int(i)))
// }
//
// // Free the register after moving its value to the variable
// for _, reg := range variables {
// v.Registers.Free(reg)
// }
// } else {
// // Get the variable name
// name := selectors[0].Identifier().GetText()
// // Define a variable for each selector
// varReg := v.Symbols.DeclareLocal(name)
//
// reg := kvValReg
//
// if isAggregation {
// reg = kvKeyReg
// }
//
// // If we have a single selector, we can just move the value
// v.Emitter.EmitAB(vm.OpMove, varReg, reg)
// }
//}
//
//func (v *Visitor) emitCollectDefaultGroupProjection(loop *Loop, kvValReg vm.Operand, identifier antlr.TerminalNode, keeper fql.ICollectGroupVariableKeeperContext) string {
// if keeper == nil {
// seq := v.Registers.AllocateSequence(2) // Key and Value for Map
//
// // TODO: Review this. It's quite a questionable ArrangoDB feature of wrapping group items by a nested object
// // We will keep it for now for backward compatibility.
// v.loadConstantTo(runtime.String(loop.ValueName), seq[0]) // Map key
// v.Emitter.EmitAB(vm.OpMove, seq[1], kvValReg) // Map value
// v.Emitter.EmitAs(vm.OpMap, kvValReg, seq)
//
// v.Registers.FreeSequence(seq)
// } else {
// variables := keeper.AllIdentifier()
// seq := v.Registers.AllocateSequence(len(variables) * 2)
//
// for i, j := 0, 0; i < len(variables); i, j = i+1, j+2 {
// varName := variables[i].GetText()
// v.loadConstantTo(runtime.String(varName), seq[j])
//
// variable, _, found := v.Symbols.Resolve(varName)
//
// if !found {
// panic("variable not found: " + varName)
// }
//
// v.Emitter.EmitAB(vm.OpMove, seq[j+1], variable)
// }
//
// v.Emitter.EmitAs(vm.OpMap, kvValReg, seq)
// v.Registers.FreeSequence(seq)
// }
//
// return identifier.GetText()
//}
//
//func (v *Visitor) emitCollectCustomGroupProjection(_ *Loop, kvValReg vm.Operand, selector fql.ICollectSelectorContext) string {
// selectorReg := selector.Expression().Accept(v).(vm.Operand)
// v.Emitter.EmitAB(vm.OpMove, kvValReg, selectorReg)
// v.Registers.Free(selectorReg)
//
// return selector.Identifier().GetText()
//}
//
//func (v *Visitor) emitCollectCountProjection(_ *Loop, _ vm.Operand, selector fql.ICollectCounterContext) string {
// return selector.Identifier().GetText()
//}
//
//func (v *Visitor) VisitCollectSelector(Ctx *fql.CollectSelectorContext) interface{} {
// if c := Ctx.Expression(); c != nil {
// return c.Accept(v)
// }
//
// panic(runtime.Error(ErrUnexpectedToken, Ctx.GetText()))
//}
//
//func (v *Visitor) VisitForExpressionStatement(Ctx *fql.ForExpressionStatementContext) interface{} {
// if c := Ctx.VariableDeclaration(); c != nil {
// return c.Accept(v)
// }
//
// if c := Ctx.FunctionCallExpression(); c != nil {
// return c.Accept(v)
// }
//
// panic(runtime.Error(ErrUnexpectedToken, Ctx.GetText()))
//}
//
//func (v *Visitor) VisitExpression(Ctx *fql.ExpressionContext) interface{} {
// return v.Ctx.ExprCompiler.Compile(Ctx)
//}
//
//// emitIterValue emits an instruction to get the value from the iterator
//func (v *Visitor) emitLoopBegin(loop *Loop) {
// if loop.Allocate {
// v.Emitter.EmitAb(vm.OpDataSet, loop.Result, loop.Distinct)
// loop.ResultPos = v.Emitter.Size() - 1
// }
//
// loop.Iterator = v.Registers.Allocate(State)
//
// if loop.Kind == ForLoop {
// v.Emitter.EmitAB(vm.OpIter, loop.Iterator, loop.Src)
// // jumpPlaceholder is a placeholder for the exit jump position
// loop.Jump = v.Emitter.EmitJumpc(vm.OpIterNext, jumpPlaceholder, loop.Iterator)
//
// if loop.Value != vm.NoopOperand {
// v.Emitter.EmitAB(vm.OpIterValue, loop.Value, loop.Iterator)
// }
//
// if loop.Key != vm.NoopOperand {
// v.Emitter.EmitAB(vm.OpIterKey, loop.Key, loop.Iterator)
// }
// } else {
// //counterReg := v.Registers.Allocate(Storage)
// // TODO: Set JumpOffset here
// }
//}
//
//// emitIterValue emits an instruction to get the value from the iterator
//func (v *Visitor) emitIterValue(loop *Loop, reg vm.Operand) {
// v.Emitter.EmitAB(vm.OpIterValue, reg, loop.Iterator)
//}
//
//// emitIterKey emits an instruction to get the key from the iterator
//func (v *Visitor) emitIterKey(loop *Loop, reg vm.Operand) {
// v.Emitter.EmitAB(vm.OpIterKey, reg, loop.Iterator)
//}
//
//// emitIterJumpOrClose emits an instruction to jump to the end of the loop or close the iterator
//func (v *Visitor) emitIterJumpOrClose(loop *Loop) {
// v.Emitter.EmitJump(loop.Jump - loop.JumpOffset)
// v.Emitter.EmitA(vm.OpClose, loop.Iterator)
//
// if loop.Kind == ForLoop {
// v.Emitter.PatchJump(loop.Jump)
// } else {
// v.Emitter.PatchJumpAB(loop.Jump)
// }
//}
//
//// emitPatchLoop replaces the source of the loop with a modified dataset
//func (v *Visitor) emitPatchLoop(loop *Loop) {
// // Replace source with sorted array
// v.Emitter.EmitAB(vm.OpMove, loop.Src, loop.Result)
//
// v.Symbols.ExitScope()
// v.Symbols.EnterScope()
//
// // Create new for loop
// v.emitLoopBegin(loop)
//}
//
//func (v *Visitor) emitLoopEnd(loop *Loop) vm.Operand {
// v.Emitter.EmitJump(loop.Jump - loop.JumpOffset)
//
// // TODO: Do not allocate for pass-through Loops
// dst := v.Registers.Allocate(Temp)
//
// if loop.Allocate {
// // TODO: Reuse the dsReg register
// v.Emitter.EmitA(vm.OpClose, loop.Iterator)
// v.Emitter.EmitAB(vm.OpMove, dst, loop.Result)
//
// if loop.Kind == ForLoop {
// v.Emitter.PatchJump(loop.Jump)
// } else {
// v.Emitter.PatchJumpAB(loop.Jump)
// }
// } else {
// if loop.Kind == ForLoop {
// v.Emitter.PatchJumpNext(loop.Jump)
// } else {
// v.Emitter.PatchJumpNextAB(loop.Jump)
// }
// }
//
// return dst
//}
//
//func (v *Visitor) loadConstant(constant runtime.Value) vm.Operand {
// return loadConstant(v.Ctx, constant)
//}
//
//func (v *Visitor) loadConstantTo(constant runtime.Value, reg vm.Operand) {
// v.Emitter.EmitAB(vm.OpLoadConst, reg, v.Symbols.AddConstant(constant))
//}