Refactor compiler internals; rename files and update context usage for improved clarity and structure

Makefile

@@ -44,8 +44,8 @@ doc:
 
 # http://golang.org/cmd/go/#hdr-Run_gofmt_on_package_sources
 fmt:
-	go fmt ${DIR_PKG}/... && \
-	goimports -w -local github.com/MontFerret ./pkg ./test/e2e
+	go fmt ${DIR_PKG}/... ${DIR_INTEG}/... ${DIR_E2E}/... && \
+	goimports -w -local github.com/MontFerret ${DIR_PKG} ${DIR_INTEG} ${DIR_E2E}
 
 # https://github.com/mgechev/revive
 # go get github.com/mgechev/revive

pkg/compiler/compiler.go

@@ -2,10 +2,9 @@ package compiler
 
 import (
 	"errors"
-	"github.com/MontFerret/ferret/pkg/compiler/internal/core"
 	goruntime "runtime"
 
-	"github.com/MontFerret/ferret/pkg/compiler/internal"
+	"github.com/MontFerret/ferret/pkg/compiler/internal/core"
 
 	"github.com/MontFerret/ferret/pkg/stdlib"
 	"github.com/MontFerret/ferret/pkg/vm"
@@ -65,7 +64,7 @@ func (c *Compiler) Compile(query string) (program *vm.Program, err error) {
 	p := parser.New(query)
 	p.AddErrorListener(newErrorListener())
 
-	l := internal.NewVisitor(query)
+	l := NewVisitor(query)
 
 	p.Visit(l)
 
@@ -74,11 +73,11 @@ func (c *Compiler) Compile(query string) (program *vm.Program, err error) {
 	}
 
 	program = &vm.Program{}
-	program.Bytecode = l.Emitter.Bytecode()
-	program.Constants = l.Symbols.Constants()
-	program.CatchTable = l.CatchTable.All()
-	program.Registers = l.Registers.Size()
-	program.Params = l.Symbols.Params()
+	program.Bytecode = l.Ctx.Emitter.Bytecode()
+	program.Constants = l.Ctx.Symbols.Constants()
+	program.CatchTable = l.Ctx.CatchTable.All()
+	program.Registers = l.Ctx.Registers.Size()
+	program.Params = l.Ctx.Symbols.Params()
 
 	return program, err
 }

pkg/compiler/internal/core/constant_pool.go

@@ -1,9 +1,10 @@
 package core
 
 import (
+	"strconv"
+
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"
-	"strconv"
 )
 
 // ConstantPool stores and deduplicates constants

pkg/compiler/internal/core/emitter_helpers.go

@@ -58,8 +58,8 @@ func (e *Emitter) EmitLoadGlobal(dst, constant vm.Operand) {
 	e.EmitAB(vm.OpLoadGlobal, dst, constant)
 }
 
-func (e *Emitter) EmitLoadParam(constant vm.Operand) {
-	e.EmitA(vm.OpLoadParam, constant)
+func (e *Emitter) EmitLoadParam(dst, constant vm.Operand) {
+	e.EmitAB(vm.OpLoadParam, dst, constant)
 }
 
 func (e *Emitter) EmitBoolean(dst vm.Operand, value bool) {

pkg/compiler/internal/core/symbol_table.go

@@ -2,6 +2,7 @@ package core
 
 import (
 	"fmt"
+
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"
 )

pkg/compiler/internal/expr.go

@@ -1,12 +1,13 @@
 package internal
 
 import (
+	"regexp"
+	"strings"
+
 	"github.com/MontFerret/ferret/pkg/compiler/internal/core"
 	"github.com/MontFerret/ferret/pkg/parser/fql"
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"
-	"regexp"
-	"strings"
 )
 
 // Runtime functions
@@ -370,8 +371,8 @@ func (ec *ExprCompiler) CompileVariable(ctx fql.IVariableContext) vm.Operand {
 
 func (ec *ExprCompiler) CompileParam(ctx fql.IParamContext) vm.Operand {
 	name := ctx.Identifier().GetText()
-	reg := ec.ctx.Symbols.BindParam(name)
-	ec.ctx.Emitter.EmitLoadParam(reg)
+	reg := ec.ctx.Registers.Allocate(core.Temp)
+	ec.ctx.Emitter.EmitLoadParam(reg, ec.ctx.Symbols.BindParam(name))
 
 	return reg
 }

pkg/compiler/internal/helpers.go

@@ -1,10 +1,12 @@
 package internal
 
 import (
+	"strings"
+
+	"github.com/antlr4-go/antlr/v4"
+
 	"github.com/MontFerret/ferret/pkg/compiler/internal/core"
 	"github.com/MontFerret/ferret/pkg/vm"
-	"github.com/antlr4-go/antlr/v4"
-	"strings"
 
 	"github.com/MontFerret/ferret/pkg/runtime"
 

pkg/compiler/internal/literal.go

@@ -1,13 +1,15 @@
 package internal
 
 import (
+	"strconv"
+	"strings"
+
+	"github.com/antlr4-go/antlr/v4"
+
 	"github.com/MontFerret/ferret/pkg/compiler/internal/core"
 	"github.com/MontFerret/ferret/pkg/parser/fql"
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"
-	"github.com/antlr4-go/antlr/v4"
-	"strconv"
-	"strings"
 )
 
 type LiteralCompiler struct {

pkg/compiler/internal/loop.go

@@ -1,11 +1,12 @@
 package internal
 
 import (
+	"github.com/antlr4-go/antlr/v4"
+
 	"github.com/MontFerret/ferret/pkg/compiler/internal/core"
 	"github.com/MontFerret/ferret/pkg/parser/fql"
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"
-	"github.com/antlr4-go/antlr/v4"
 )
 
 type LoopCompiler struct {

pkg/compiler/visitor.go

@@ -1,34 +1,24 @@
-package internal
+package compiler
 
 import (
-	"github.com/MontFerret/ferret/pkg/compiler/internal/core"
+	"github.com/MontFerret/ferret/pkg/compiler/internal"
 	"github.com/MontFerret/ferret/pkg/parser/fql"
 )
 
 type Visitor struct {
 	*fql.BaseFqlParserVisitor
-	ctx *FuncContext
 
+	Ctx *internal.FuncContext
 	Err error
 	Src string
-	Emitter    *core.Emitter
-	Registers  *core.RegisterAllocator
-	Symbols    *core.SymbolTable
-	Loops      *core.LoopTable
-	CatchTable *core.CatchStack
 }
 
 func NewVisitor(src string) *Visitor {
 	v := new(Visitor)
 	v.BaseFqlParserVisitor = new(fql.BaseFqlParserVisitor)
-	v.ctx = NewFuncContext()
+	v.Ctx = internal.NewFuncContext()
 
 	v.Src = src
-	v.Registers = v.ctx.Registers
-	v.Symbols = v.ctx.Symbols
-	v.Loops = v.ctx.Loops
-	v.Emitter = v.ctx.Emitter
-	v.CatchTable = v.ctx.CatchTable
 
 	return v
 }
@@ -38,7 +28,7 @@ func (v *Visitor) VisitProgram(ctx *fql.ProgramContext) interface{} {
 		v.VisitHead(head.(*fql.HeadContext))
 	}
 
-	v.ctx.StmtCompiler.Compile(ctx.Body())
+	v.Ctx.StmtCompiler.Compile(ctx.Body())
 
 	return nil
 }
@@ -47,7 +37,7 @@ func (v *Visitor) VisitHead(_ *fql.HeadContext) interface{} {
 	return nil
 }
 
-//func (v *Visitor) VisitCollectClause(ctx *fql.CollectClauseContext) interface{} {
+//func (v *Visitor) VisitCollectClause(Ctx *fql.CollectClauseContext) interface{} {
 //	// TODO: Undefine original loop variables
 //	loop := v.Loops.Current()
 //
@@ -58,9 +48,9 @@ func (v *Visitor) VisitHead(_ *fql.HeadContext) interface{} {
 //	var kvKeyReg, kvValReg vm.Operand
 //	var groupSelectors []fql.ICollectSelectorContext
 //	var isGrouping bool
-//	grouping := ctx.CollectGrouping()
-//	counter := ctx.CollectCounter()
-//	aggregator := ctx.CollectAggregator()
+//	grouping := Ctx.CollectGrouping()
+//	counter := Ctx.CollectCounter()
+//	aggregator := Ctx.CollectAggregator()
 //
 //	isCollecting := grouping != nil || counter != nil
 //
@@ -78,7 +68,7 @@ func (v *Visitor) VisitHead(_ *fql.HeadContext) interface{} {
 //		collectorType := CollectorTypeKey
 //
 //		// If we have a collect group variable, we need to project it
-//		if groupVar := ctx.CollectGroupVariable(); groupVar != nil {
+//		if groupVar := Ctx.CollectGroupVariable(); groupVar != nil {
 //			// Projection can be either a default projection (identifier) or a custom projection (selector expression)
 //			if identifier := groupVar.Identifier(); identifier != nil {
 //				projectionVariableName = v.emitCollectDefaultGroupProjection(loop, kvValReg, identifier, groupVar.CollectGroupVariableKeeper())
@@ -411,28 +401,28 @@ func (v *Visitor) VisitHead(_ *fql.HeadContext) interface{} {
 //	return selector.Identifier().GetText()
 //}
 //
-//func (v *Visitor) VisitCollectSelector(ctx *fql.CollectSelectorContext) interface{} {
-//	if c := ctx.Expression(); c != nil {
+//func (v *Visitor) VisitCollectSelector(Ctx *fql.CollectSelectorContext) interface{} {
+//	if c := Ctx.Expression(); c != nil {
 //		return c.Accept(v)
 //	}
 //
-//	panic(runtime.Error(ErrUnexpectedToken, ctx.GetText()))
+//	panic(runtime.Error(ErrUnexpectedToken, Ctx.GetText()))
 //}
 //
-//func (v *Visitor) VisitForExpressionStatement(ctx *fql.ForExpressionStatementContext) interface{} {
-//	if c := ctx.VariableDeclaration(); c != nil {
+//func (v *Visitor) VisitForExpressionStatement(Ctx *fql.ForExpressionStatementContext) interface{} {
+//	if c := Ctx.VariableDeclaration(); c != nil {
 //		return c.Accept(v)
 //	}
 //
-//	if c := ctx.FunctionCallExpression(); c != nil {
+//	if c := Ctx.FunctionCallExpression(); c != nil {
 //		return c.Accept(v)
 //	}
 //
-//	panic(runtime.Error(ErrUnexpectedToken, ctx.GetText()))
+//	panic(runtime.Error(ErrUnexpectedToken, Ctx.GetText()))
 //}
 //
-//func (v *Visitor) VisitExpression(ctx *fql.ExpressionContext) interface{} {
-//	return v.ctx.ExprCompiler.Compile(ctx)
+//func (v *Visitor) VisitExpression(Ctx *fql.ExpressionContext) interface{} {
+//	return v.Ctx.ExprCompiler.Compile(Ctx)
 //}
 //
 //// emitIterValue emits an instruction to get the value from the iterator
@@ -524,7 +514,7 @@ func (v *Visitor) VisitHead(_ *fql.HeadContext) interface{} {
 //}
 //
 //func (v *Visitor) loadConstant(constant runtime.Value) vm.Operand {
-//	return loadConstant(v.ctx, constant)
+//	return loadConstant(v.Ctx, constant)
 //}
 //
 //func (v *Visitor) loadConstantTo(constant runtime.Value, reg vm.Operand) {

pkg/parser/fql/fql_lexer.go

@@ -4,9 +4,10 @@ package fql
 
 import (
 	"fmt"
-	"github.com/antlr4-go/antlr/v4"
 	"sync"
 	"unicode"
+
+	"github.com/antlr4-go/antlr/v4"
 )
 
 // Suppress unused import error

test/integration/benchmarks/compiler_bench_test.go

@@ -1,7 +1,6 @@
 package benchmarks_test
 
 import (
-	"github.com/MontFerret/ferret/test/integration/setup"
 	"testing"
 )
 

test/integration/bytecode/compiler_test.go

@@ -2,7 +2,6 @@ package bytecode_test
 
 import (
 	"fmt"
-	"github.com/MontFerret/ferret/test/integration/setup"
 	"testing"
 
 	"github.com/MontFerret/ferret/pkg/runtime"

test/integration/vm/vm_for_collect_agg_test.go

@@ -1,8 +1,9 @@
 package vm_test
 
 import (
-	. "github.com/MontFerret/ferret/test/integration/base"
 	"testing"
+
+	. "github.com/MontFerret/ferret/test/integration/base"
 )
 
 func TestCollectAggregate(t *testing.T) {

test/integration/vm/vm_for_collect_test.go

@@ -1,8 +1,9 @@
 package vm_test
 
 import (
-	. "github.com/MontFerret/ferret/test/integration/base"
 	"testing"
+
+	. "github.com/MontFerret/ferret/test/integration/base"
 )
 
 // COLLECT vs. RETURN DISTINCT

test/integration/vm/vm_for_filter_test.go

@@ -2,10 +2,11 @@ package vm_test
 
 import (
 	"context"
+	"testing"
+
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"
 	. "github.com/MontFerret/ferret/test/integration/base"
-	"testing"
 )
 
 func TestForFilter(t *testing.T) {

test/integration/vm/vm_for_limit_test.go

@@ -2,10 +2,11 @@ package vm_test
 
 import (
 	"context"
+	"testing"
+
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"
 	. "github.com/MontFerret/ferret/test/integration/base"
-	"testing"
 )
 
 func TestForLimit(t *testing.T) {

test/integration/vm/vm_for_sort_test.go

@@ -2,10 +2,11 @@ package vm_test
 
 import (
 	"context"
+	"testing"
+
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"
 	. "github.com/MontFerret/ferret/test/integration/base"
-	"testing"
 )
 
 func TestForSort(t *testing.T) {

test/integration/vm/vm_for_ternary_test.go

@@ -1,8 +1,9 @@
 package vm_test
 
 import (
-	. "github.com/MontFerret/ferret/test/integration/base"
 	"testing"
+
+	. "github.com/MontFerret/ferret/test/integration/base"
 )
 
 func TestForTernaryExpression(t *testing.T) {

test/integration/vm/vm_for_test.go

@@ -2,9 +2,10 @@ package vm_test
 
 import (
 	"context"
-	. "github.com/MontFerret/ferret/test/integration/base"
 	"testing"
 
+	. "github.com/MontFerret/ferret/test/integration/base"
+
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"
 )

test/integration/vm/vm_func_test.go

@@ -1,8 +1,9 @@
 package vm_test
 
 import (
-	. "github.com/MontFerret/ferret/test/integration/base"
 	"testing"
+
+	. "github.com/MontFerret/ferret/test/integration/base"
 )
 
 func TestFunctionCall(t *testing.T) {

test/integration/vm/vm_member_test.go

@@ -3,15 +3,16 @@ package vm_test
 import (
 	"context"
 	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+
 	"github.com/MontFerret/ferret/pkg/compiler"
 	"github.com/MontFerret/ferret/pkg/parser"
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"
 	. "github.com/MontFerret/ferret/test/integration/base"
-	"regexp"
-	"strconv"
-	"strings"
-	"testing"
 
 	. "github.com/smartystreets/goconvey/convey"
 )

test/integration/vm/vm_op_array_test.go

@@ -1,8 +1,9 @@
 package vm_test
 
 import (
-	. "github.com/MontFerret/ferret/test/integration/base"
 	"testing"
+
+	. "github.com/MontFerret/ferret/test/integration/base"
 )
 
 func TestArrayAllOperator(t *testing.T) {

test/integration/vm/vm_op_equality_test.go

@@ -1,8 +1,9 @@
 package vm_test
 
 import (
-	. "github.com/MontFerret/ferret/test/integration/base"
 	"testing"
+
+	. "github.com/MontFerret/ferret/test/integration/base"
 )
 
 func TestEqualityOperators(t *testing.T) {

test/integration/vm/vm_op_in_test.go

@@ -1,8 +1,9 @@
 package vm_test
 
 import (
-	. "github.com/MontFerret/ferret/test/integration/base"
 	"testing"
+
+	. "github.com/MontFerret/ferret/test/integration/base"
 )
 
 func TestInOperator(t *testing.T) {

test/integration/vm/vm_op_like_test.go

@@ -1,8 +1,9 @@
 package vm_test
 
 import (
-	. "github.com/MontFerret/ferret/test/integration/base"
 	"testing"
+
+	. "github.com/MontFerret/ferret/test/integration/base"
 )
 
 func TestLikeOperator(t *testing.T) {

test/integration/vm/vm_op_logical_test.go

@@ -3,10 +3,11 @@ package vm_test
 import (
 	"context"
 	"fmt"
+	"testing"
+
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"
 	. "github.com/MontFerret/ferret/test/integration/base"
-	"testing"
 )
 
 func TestLogicalOperators(t *testing.T) {

test/integration/vm/vm_op_math_test.go

@@ -1,8 +1,9 @@
 package vm_test
 
 import (
-	. "github.com/MontFerret/ferret/test/integration/base"
 	"testing"
+
+	. "github.com/MontFerret/ferret/test/integration/base"
 )
 
 func TestMathOperators(t *testing.T) {

test/integration/vm/vm_op_regex_test.go

@@ -3,9 +3,10 @@ package vm_test
 import (
 	"context"
 	"fmt"
-	. "github.com/MontFerret/ferret/test/integration/base"
 	"testing"
 
+	. "github.com/MontFerret/ferret/test/integration/base"
+
 	"github.com/MontFerret/ferret/pkg/compiler"
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"

test/integration/vm/vm_op_ternary_test.go

@@ -2,9 +2,11 @@ package vm_test
 
 import (
 	"fmt"
+
+	. "github.com/smartystreets/goconvey/convey"
+
 	"github.com/MontFerret/ferret/pkg/compiler"
 	. "github.com/MontFerret/ferret/test/integration/base"
-	. "github.com/smartystreets/goconvey/convey"
 
 	"testing"
 )

test/integration/vm/vm_op_unary_test.go

@@ -1,12 +1,14 @@
 package vm_test
 
 import (
+	"testing"
+
 	"github.com/MontFerret/ferret/pkg/compiler"
 	"github.com/MontFerret/ferret/pkg/vm"
 	. "github.com/MontFerret/ferret/test/integration/base"
-	"testing"
 
 	gocontext "context"
+
 	. "github.com/smartystreets/goconvey/convey"
 )
 

test/integration/vm/vm_param_test.go

@@ -1,10 +1,11 @@
 package vm_test
 
 import (
+	"testing"
+
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"
 	. "github.com/MontFerret/ferret/test/integration/base"
-	"testing"
 )
 
 func TestParam(t *testing.T) {

test/integration/vm/vm_range_test.go

@@ -1,17 +1,18 @@
 package vm_test
 
 import (
+	"testing"
+
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"
 	. "github.com/MontFerret/ferret/test/integration/base"
-	"testing"
 )
 
 func TestRange(t *testing.T) {
 	RunUseCases(t, []UseCase{
-		SkipCaseArray("RETURN 1..10", []any{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, "Should return a range from 1 to 10"),
-		SkipCaseArray("RETURN 10..1", []any{10, 9, 8, 7, 6, 5, 4, 3, 2, 1}, "Should return a range from 10 to 1"),
-		SkipCaseArray(
+		CaseArray("RETURN 1..10", []any{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, "Should return a range from 1 to 10"),
+		CaseArray("RETURN 10..1", []any{10, 9, 8, 7, 6, 5, 4, 3, 2, 1}, "Should return a range from 10 to 1"),
+		CaseArray(
 			`
 		LET start = 1
 		LET end = 10

test/integration/vm/vm_string_test.go

@@ -2,8 +2,9 @@ package vm_test
 
 import (
 	"fmt"
-	. "github.com/MontFerret/ferret/test/integration/base"
 	"testing"
+
+	. "github.com/MontFerret/ferret/test/integration/base"
 )
 
 func TestString(t *testing.T) {

test/integration/vm/vm_variable_test.go

@@ -2,13 +2,15 @@ package vm_test
 
 import (
 	"fmt"
+	"testing"
+
 	"github.com/MontFerret/ferret/pkg/compiler"
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"
 	. "github.com/MontFerret/ferret/test/integration/base"
-	"testing"
 
 	gocontext "context"
+
 	. "github.com/smartystreets/goconvey/convey"
 )
 

test/integration/vm/vm_waitfor_event_test.go

@@ -1,8 +1,9 @@
 package vm_test
 
 import (
-	. "github.com/MontFerret/ferret/test/integration/base"
 	"testing"
+
+	. "github.com/MontFerret/ferret/test/integration/base"
 )
 
 func TestWaitforEvent(t *testing.T) {

test/integration/vm/vm_while_do_test.go

@@ -2,9 +2,10 @@ package vm_test
 
 import (
 	"context"
-	. "github.com/MontFerret/ferret/test/integration/base"
 	"testing"
 
+	. "github.com/MontFerret/ferret/test/integration/base"
+
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"
 )

test/integration/vm/vm_while_ternary_test.go

@@ -2,9 +2,10 @@ package vm_test
 
 import (
 	"context"
-	. "github.com/MontFerret/ferret/test/integration/base"
 	"testing"
 
+	. "github.com/MontFerret/ferret/test/integration/base"
+
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"
 )

test/integration/vm/vm_while_test.go

@@ -2,9 +2,10 @@ package vm_test
 
 import (
 	"context"
-	. "github.com/MontFerret/ferret/test/integration/base"
 	"testing"
 
+	. "github.com/MontFerret/ferret/test/integration/base"
+
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/MontFerret/ferret/pkg/vm"
 )