go/ferret
go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2024-12-14 11:23:02 +02:00
Code Issues Releases Activity

Feature/#5 collect keyword alt (#141)

Browse Source 
Implemented COLLECT key word
This commit is contained in:
Tim Voronov 2018-10-24 21:30:05 -04:00 committed by GitHub
parent b02c554214
commit 549b4abd3b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
67 changed files with 4913 additions and 2811 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

309
pkg/compiler/compiler_collect_aggregate_test.go Normal file
View File

@ -0,0 +1,309 @@
package compiler_test
import (
"context"
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/runtime"
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func TestAggregate(t *testing.T) {
Convey("Should aggregate values without grouping", t, func() {
c := compiler.New()
prog, err := c.Compile(`
LET users = [
{
active: true,
married: true,
age: 31,
gender: "m"
},
{
active: true,
married: false,
age: 25,
gender: "f"
},
{
active: true,
married: false,
age: 36,
gender: "m"
},
{
active: false,
married: true,
age: 69,
gender: "m"
},
{
active: true,
married: true,
age: 45,
gender: "f"
}
]
FOR u IN users
COLLECT AGGREGATE minAge = MIN(u.age), maxAge = MAX(u.age)
RETURN {
minAge,
maxAge
}
`)
So(err, ShouldBeNil)
So(prog, ShouldHaveSameTypeAs, &runtime.Program{})
out, err := prog.Run(context.Background())
So(err, ShouldBeNil)
So(string(out), ShouldEqual, `[{"maxAge":69,"minAge":25}]`)
})
Convey("Should aggregate values without grouping with multiple arguments", t, func() {
c := compiler.New()
prog, err := c.Compile(`
LET users = [
{
active: true,
married: true,
age: 31,
gender: "m"
},
{
active: true,
married: false,
age: 25,
gender: "f"
},
{
active: true,
married: false,
age: 36,
gender: "m"
},
{
active: false,
married: true,
age: 69,
gender: "m"
},
{
active: true,
married: true,
age: 45,
gender: "f"
}
]
FOR u IN users
COLLECT AGGREGATE ages = UNION(u.age, u.age)
RETURN ages
`)
So(err, ShouldBeNil)
So(prog, ShouldHaveSameTypeAs, &runtime.Program{})
out, err := prog.Run(context.Background())
So(err, ShouldBeNil)
So(string(out), ShouldEqual, `[[31,25,36,69,45,31,25,36,69,45]]`)
})
Convey("Should aggregate values with grouping", t, func() {
c := compiler.New()
prog, err := c.Compile(`
LET users = [
{
active: true,
married: true,
age: 31,
gender: "m"
},
{
active: true,
married: false,
age: 25,
gender: "f"
},
{
active: true,
married: false,
age: 36,
gender: "m"
},
{
active: false,
married: true,
age: 69,
gender: "m"
},
{
active: true,
married: true,
age: 45,
gender: "f"
}
]
FOR u IN users
COLLECT ageGroup = FLOOR(u.age / 5) * 5
AGGREGATE minAge = MIN(u.age), maxAge = MAX(u.age)
RETURN {
ageGroup,
minAge,
maxAge
}
`)
So(err, ShouldBeNil)
So(prog, ShouldHaveSameTypeAs, &runtime.Program{})
out, err := prog.Run(context.Background())
So(err, ShouldBeNil)
So(string(out), ShouldEqual, `[{"ageGroup":25,"maxAge":25,"minAge":25},{"ageGroup":30,"maxAge":31,"minAge":31},{"ageGroup":35,"maxAge":36,"minAge":36},{"ageGroup":45,"maxAge":45,"minAge":45},{"ageGroup":65,"maxAge":69,"minAge":69}]`)
})
}
func BenchmarkAggregate(b *testing.B) {
p := compiler.New().MustCompile(`
LET users = [
{
active: true,
married: true,
age: 31,
gender: "m"
},
{
active: true,
married: false,
age: 25,
gender: "f"
},
{
active: true,
married: false,
age: 36,
gender: "m"
},
{
active: false,
married: true,
age: 69,
gender: "m"
},
{
active: true,
married: true,
age: 45,
gender: "f"
}
]
FOR u IN users
COLLECT AGGREGATE minAge = MIN(u.age), maxAge = MAX(u.age)
RETURN {
minAge,
maxAge
}
`)
for n := 0; n < b.N; n++ {
p.Run(context.Background())
}
}
func BenchmarkAggregate2(b *testing.B) {
p := compiler.New().MustCompile(`
LET users = [
{
active: true,
married: true,
age: 31,
gender: "m"
},
{
active: true,
married: false,
age: 25,
gender: "f"
},
{
active: true,
married: false,
age: 36,
gender: "m"
},
{
active: false,
married: true,
age: 69,
gender: "m"
},
{
active: true,
married: true,
age: 45,
gender: "f"
}
]
FOR u IN users
COLLECT AGGREGATE ages = UNION(u.age, u.age)
RETURN ages
`)
for n := 0; n < b.N; n++ {
p.Run(context.Background())
}
}
func BenchmarkAggregate3(b *testing.B) {
p := compiler.New().MustCompile(`
LET users = [
{
active: true,
married: true,
age: 31,
gender: "m"
},
{
active: true,
married: false,
age: 25,
gender: "f"
},
{
active: true,
married: false,
age: 36,
gender: "m"
},
{
active: false,
married: true,
age: 69,
gender: "m"
},
{
active: true,
married: true,
age: 45,
gender: "f"
}
]
FOR u IN users
COLLECT ageGroup = FLOOR(u.age / 5) * 5
AGGREGATE minAge = MIN(u.age), maxAge = MAX(u.age)
RETURN {
ageGroup,
minAge,
maxAge
}
`)
for n := 0; n < b.N; n++ {
p.Run(context.Background())
}
}

105
pkg/compiler/compiler_collect_count_test.go Normal file
View File

@ -0,0 +1,105 @@
package compiler_test
import (
"context"
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/runtime"
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func TestCollectCount(t *testing.T) {
Convey("Should count grouped values", t, func() {
c := compiler.New()
prog, err := c.Compile(`
LET users = [
{
active: true,
age: 31,
gender: "m",
married: true
},
{
active: true,
age: 25,
gender: "f",
married: false
},
{
active: true,
age: 36,
gender: "m",
married: false
},
{
active: false,
age: 69,
gender: "m",
married: true
},
{
active: true,
age: 45,
gender: "f",
married: true
}
]
FOR i IN users
COLLECT WITH COUNT INTO c
RETURN c
`)
So(err, ShouldBeNil)
So(prog, ShouldHaveSameTypeAs, &runtime.Program{})
out, err := prog.Run(context.Background())
So(err, ShouldBeNil)
So(string(out), ShouldEqual, `[5]`)
})
}
func BenchmarkCollectCount(b *testing.B) {
p := compiler.New().MustCompile(`
LET users = [
{
active: true,
age: 31,
gender: "m",
married: true
},
{
active: true,
age: 25,
gender: "f",
married: false
},
{
active: true,
age: 36,
gender: "m",
married: false
},
{
active: false,
age: 69,
gender: "m",
married: true
},
{
active: true,
age: 45,
gender: "f",
married: true
}
]
FOR i IN users
COLLECT WITH COUNT INTO c
RETURN c
`)
for n := 0; n < b.N; n++ {
p.Run(context.Background())
}
}

265
pkg/compiler/compiler_collect_into_test.go Normal file
View File

@ -0,0 +1,265 @@
package compiler_test
import (
"context"
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/runtime"
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func TestCollectInto(t *testing.T) {
Convey("Should create default projection", t, func() {
c := compiler.New()
prog, err := c.Compile(`
LET users = [
{
active: true,
age: 31,
gender: "m",
married: true
},
{
active: true,
age: 25,
gender: "f",
married: false
},
{
active: true,
age: 36,
gender: "m",
married: false
},
{
active: false,
age: 69,
gender: "m",
married: true
},
{
active: true,
age: 45,
gender: "f",
married: true
}
]
FOR i IN users
COLLECT gender = i.gender INTO genders
RETURN {
gender,
values: genders
}
`)
So(err, ShouldBeNil)
So(prog, ShouldHaveSameTypeAs, &runtime.Program{})
out, err := prog.Run(context.Background())
So(err, ShouldBeNil)
So(string(out), ShouldEqual, `[{"gender":"f","values":[{"i":{"active":true,"age":25,"gender":"f","married":false}},{"i":{"active":true,"age":45,"gender":"f","married":true}}]},{"gender":"m","values":[{"i":{"active":true,"age":31,"gender":"m","married":true}},{"i":{"active":true,"age":36,"gender":"m","married":false}},{"i":{"active":false,"age":69,"gender":"m","married":true}}]}]`)
})
Convey("Should create custom projection", t, func() {
c := compiler.New()
prog, err := c.Compile(`
LET users = [
{
active: true,
age: 31,
gender: "m",
married: true
},
{
active: true,
age: 25,
gender: "f",
married: false
},
{
active: true,
age: 36,
gender: "m",
married: false
},
{
active: false,
age: 69,
gender: "m",
married: true
},
{
active: true,
age: 45,
gender: "f",
married: true
}
]
FOR i IN users
COLLECT gender = i.gender INTO genders = { active: i.active }
RETURN {
gender,
values: genders
}
`)
So(err, ShouldBeNil)
So(prog, ShouldHaveSameTypeAs, &runtime.Program{})
out, err := prog.Run(context.Background())
So(err, ShouldBeNil)
So(string(out), ShouldEqual, `[{"gender":"f","values":[{"active":true},{"active":true}]},{"gender":"m","values":[{"active":true},{"active":true},{"active":false}]}]`)
})
Convey("Should create custom projection grouped by miltiple keys", t, func() {
c := compiler.New()
prog, err := c.Compile(`
LET users = [
{
active: true,
age: 31,
gender: "m",
married: true
},
{
active: true,
age: 25,
gender: "f",
married: false
},
{
active: true,
age: 36,
gender: "m",
married: false
},
{
active: false,
age: 69,
gender: "m",
married: true
},
{
active: true,
age: 45,
gender: "f",
married: true
}
]
FOR i IN users
COLLECT gender = i.gender, age = i.age INTO genders = { active: i.active }
RETURN {
age,
gender,
values: genders
}
`)
So(err, ShouldBeNil)
So(prog, ShouldHaveSameTypeAs, &runtime.Program{})
out, err := prog.Run(context.Background())
So(err, ShouldBeNil)
So(string(out), ShouldEqual, `[{"age":25,"gender":"f","values":[{"active":true}]},{"age":45,"gender":"f","values":[{"active":true}]},{"age":31,"gender":"m","values":[{"active":true}]},{"age":36,"gender":"m","values":[{"active":true}]},{"age":69,"gender":"m","values":[{"active":false}]}]`)
})
}
func BenchmarkCollectInto(b *testing.B) {
p := compiler.New().MustCompile(`
LET users = [
{
active: true,
age: 31,
gender: "m",
married: true
},
{
active: true,
age: 25,
gender: "f",
married: false
},
{
active: true,
age: 36,
gender: "m",
married: false
},
{
active: false,
age: 69,
gender: "m",
married: true
},
{
active: true,
age: 45,
gender: "f",
married: true
}
]
FOR i IN users
COLLECT gender = i.gender INTO genders
RETURN {
gender,
values: genders
}
`)
for n := 0; n < b.N; n++ {
p.Run(context.Background())
}
}
func BenchmarkCollectInto2(b *testing.B) {
p := compiler.New().MustCompile(`
LET users = [
{
active: true,
age: 31,
gender: "m",
married: true
},
{
active: true,
age: 25,
gender: "f",
married: false
},
{
active: true,
age: 36,
gender: "m",
married: false
},
{
active: false,
age: 69,
gender: "m",
married: true
},
{
active: true,
age: 45,
gender: "f",
married: true
}
]
FOR i IN users
COLLECT gender = i.gender INTO genders = { active: i.active }
RETURN {
gender,
values: genders
}
`)
for n := 0; n < b.N; n++ {
p.Run(context.Background())
}
}

246
pkg/compiler/compiler_collect_test.go Normal file
View File

@ -0,0 +1,246 @@
package compiler_test
import (
"context"
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/runtime"
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func TestCollect(t *testing.T) {
Convey("Should not have access to initial variables", t, func() {
c := compiler.New()
_, err := c.Compile(`
LET users = [
{
active: true,
married: true,
age: 31,
gender: "m"
},
{
active: true,
married: false,
age: 25,
gender: "f"
},
{
active: true,
married: false,
age: 36,
gender: "m"
},
{
active: false,
married: true,
age: 69,
gender: "m"
},
{
active: true,
married: true,
age: 45,
gender: "f"
}
]
FOR i IN users
COLLECT gender = i.gender
RETURN {
user: i,
gender: gender
}
`)
So(err, ShouldNotBeNil)
})
Convey("Should group result by a single key", t, func() {
c := compiler.New()
prog, err := c.Compile(`
LET users = [
{
active: true,
married: true,
age: 31,
gender: "m"
},
{
active: true,
married: false,
age: 25,
gender: "f"
},
{
active: true,
married: false,
age: 36,
gender: "m"
},
{
active: false,
married: true,
age: 69,
gender: "m"
},
{
active: true,
married: true,
age: 45,
gender: "f"
}
]
FOR i IN users
COLLECT gender = i.gender
RETURN gender
`)
So(err, ShouldBeNil)
So(prog, ShouldHaveSameTypeAs, &runtime.Program{})
out, err := prog.Run(context.Background())
So(err, ShouldBeNil)
So(string(out), ShouldEqual, `["f","m"]`)
})
Convey("Should group result by multiple keys", t, func() {
c := compiler.New()
prog, err := c.Compile(`
LET users = [
{
active: true,
married: true,
age: 31,
gender: "m"
},
{
active: true,
married: false,
age: 25,
gender: "f"
},
{
active: true,
married: false,
age: 36,
gender: "m"
},
{
active: false,
married: true,
age: 69,
gender: "m"
},
{
active: true,
married: true,
age: 45,
gender: "f"
}
]
FOR i IN users
COLLECT gender = i.gender, age = i.age
RETURN {age, gender}
`)
So(err, ShouldBeNil)
So(prog, ShouldHaveSameTypeAs, &runtime.Program{})
out, err := prog.Run(context.Background())
So(err, ShouldBeNil)
So(string(out), ShouldEqual, `[{"age":25,"gender":"f"},{"age":45,"gender":"f"},{"age":31,"gender":"m"},{"age":36,"gender":"m"},{"age":69,"gender":"m"}]`)
})
}
func BenchmarkCollect(b *testing.B) {
p := compiler.New().MustCompile(`
LET users = [
{
active: true,
married: true,
age: 31,
gender: "m"
},
{
active: true,
married: false,
age: 25,
gender: "f"
},
{
active: true,
married: false,
age: 36,
gender: "m"
},
{
active: false,
married: true,
age: 69,
gender: "m"
},
{
active: true,
married: true,
age: 45,
gender: "f"
}
]
FOR i IN users
COLLECT gender = i.gender
RETURN gender
`)
for n := 0; n < b.N; n++ {
p.Run(context.Background())
}
}
func BenchmarkCollect2(b *testing.B) {
p := compiler.New().MustCompile(`
LET users = [
{
active: true,
married: true,
age: 31,
gender: "m"
},
{
active: true,
married: false,
age: 25,
gender: "f"
},
{
active: true,
married: false,
age: 36,
gender: "m"
},
{
active: false,
married: true,
age: 69,
gender: "m"
},
{
active: true,
married: true,
age: 45,
gender: "f"
}
]
FOR i IN users
COLLECT gender = i.gender, age = i.age
RETURN {age, gender}
`)
for n := 0; n < b.N; n++ {
p.Run(context.Background())
}
}

199
pkg/compiler/compiler_collect_with_count_test.go Normal file
View File

@ -0,0 +1,199 @@
package compiler_test
import (
"context"
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/runtime"
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func TestCollectWithCount(t *testing.T) {
Convey("Should count grouped values", t, func() {
c := compiler.New()
prog, err := c.Compile(`
LET users = [
{
active: true,
age: 31,
gender: "m",
married: true
},
{
active: true,
age: 25,
gender: "f",
married: false
},
{
active: true,
age: 36,
gender: "m",
married: false
},
{
active: false,
age: 69,
gender: "m",
married: true
},
{
active: true,
age: 45,
gender: "f",
married: true
}
]
FOR i IN users
COLLECT gender = i.gender WITH COUNT INTO genders
RETURN {gender, genders}
`)
So(err, ShouldBeNil)
So(prog, ShouldHaveSameTypeAs, &runtime.Program{})
out, err := prog.Run(context.Background())
So(err, ShouldBeNil)
So(string(out), ShouldEqual, `[{"gender":"f","genders":2},{"gender":"m","genders":3}]`)
})
Convey("Should count grouped values with multiple keys", t, func() {
c := compiler.New()
prog, err := c.Compile(`
LET users = [
{
active: true,
age: 31,
gender: "m",
married: true
},
{
active: true,
age: 25,
gender: "f",
married: false
},
{
active: true,
age: 36,
gender: "m",
married: false
},
{
active: false,
age: 69,
gender: "m",
married: true
},
{
active: true,
age: 45,
gender: "f",
married: true
}
]
FOR i IN users
COLLECT gender = i.gender, married = i.married WITH COUNT INTO genders
RETURN {gender, married, genders}
`)
So(err, ShouldBeNil)
So(prog, ShouldHaveSameTypeAs, &runtime.Program{})
out, err := prog.Run(context.Background())
So(err, ShouldBeNil)
So(string(out), ShouldEqual, `[{"gender":"f","genders":1,"married":false},{"gender":"f","genders":1,"married":true},{"gender":"m","genders":1,"married":false},{"gender":"m","genders":2,"married":true}]`)
})
}
func BenchmarkWithCount(b *testing.B) {
p := compiler.New().MustCompile(`
LET users = [
{
active: true,
age: 31,
gender: "m",
married: true
},
{
active: true,
age: 25,
gender: "f",
married: false
},
{
active: true,
age: 36,
gender: "m",
married: false
},
{
active: false,
age: 69,
gender: "m",
married: true
},
{
active: true,
age: 45,
gender: "f",
married: true
}
]
FOR i IN users
COLLECT gender = i.gender WITH COUNT INTO genders
RETURN {gender, genders}
`)
for n := 0; n < b.N; n++ {
p.Run(context.Background())
}
}
func BenchmarkWithCount2(b *testing.B) {
p := compiler.New().MustCompile(`
LET users = [
{
active: true,
age: 31,
gender: "m",
married: true
},
{
active: true,
age: 25,
gender: "f",
married: false
},
{
active: true,
age: 36,
gender: "m",
married: false
},
{
active: false,
age: 69,
gender: "m",
married: true
},
{
active: true,
age: 45,
gender: "f",
married: true
}
]
FOR i IN users
COLLECT gender = i.gender, married = i.married WITH COUNT INTO genders
RETURN {gender, married, genders}
`)
for n := 0; n < b.N; n++ {
p.Run(context.Background())
}
}

7
pkg/compiler/compiler_param_test.go
View File

@ -47,13 +47,14 @@ func TestParam(t *testing.T) {
})
Convey("Should be possible to use in range", t, func() {
out := compiler.New().
prog := compiler.New().
MustCompile(`
FOR i IN @start..@end
SORT i
RETURN i
`).
MustRun(
`)
out := prog.MustRun(
context.Background(),
runtime.WithParam("start", 1),
runtime.WithParam("end", 4),

12
pkg/compiler/scope.go
View File

@ -58,6 +58,18 @@ func (s *scope) SetVariable(name string) error {
return nil
}
func (s *scope) RemoveVariable(name string) error {
_, exists := s.vars[name]
if !exists {
return errors.Wrap(ErrVariableNotFound, name)
}
delete(s.vars, name)
return nil
}
func (s *scope) Fork() *scope {
return newScope(s)
}

355
pkg/compiler/visitor.go
View File

@ -16,11 +16,15 @@ import (
"strings"
)
type visitor struct {
type (
forOption func(f *expressions.ForExpression) error
visitor struct {
*fql.BaseFqlParserVisitor
src string
funcs map[string]core.Function
}
}
)
func newVisitor(src string, funcs map[string]core.Function) *visitor {
return &visitor{
@ -155,6 +159,7 @@ func (v *visitor) doVisitForExpression(ctx *fql.ForExpressionContext, scope *sco
var valVarName string
var keyVarName string
parsedClauses := make([]forOption, 0, 10)
valVar := ctx.ForExpressionValueVariable()
valVarName = valVar.GetText()
forInScope := scope.Fork()
@ -167,12 +172,95 @@ func (v *visitor) doVisitForExpression(ctx *fql.ForExpressionContext, scope *sco
forInScope.SetVariable(keyVarName)
}
src, err := v.doVisitForExpressionSource(ctx.ForExpressionSource().(*fql.ForExpressionSourceContext), forInScope)
srcCtx := ctx.ForExpressionSource().(*fql.ForExpressionSourceContext)
srcExp, err := v.doVisitForExpressionSource(srcCtx, forInScope)
if err != nil {
return nil, err
}
src, err := expressions.NewDataSource(
v.getSourceMap(srcCtx),
valVarName,
keyVarName,
srcExp,
)
// Clauses.
// We put clauses parsing before parsing the query body because COLLECT clause overrides scope variables
for _, clause := range ctx.AllForExpressionClause() {
clause := clause.(*fql.ForExpressionClauseContext)
limitCtx := clause.LimitClause()
if limitCtx != nil {
limit, offset, err := v.createLimit(limitCtx.(*fql.LimitClauseContext))
if err != nil {
return nil, err
}
parsedClauses = append(parsedClauses, func(f *expressions.ForExpression) error {
return f.AddLimit(v.getSourceMap(limitCtx), limit, offset)
})
continue
}
filterCtx := clause.FilterClause()
if filterCtx != nil {
filterExp, err := v.createFilter(filterCtx.(*fql.FilterClauseContext), forInScope)
if err != nil {
return nil, err
}
parsedClauses = append(parsedClauses, func(f *expressions.ForExpression) error {
return f.AddFilter(v.getSourceMap(filterCtx), filterExp)
})
continue
}
sortCtx := clause.SortClause()
if sortCtx != nil {
sortCtx := sortCtx.(*fql.SortClauseContext)
sortExps, err := v.createSort(sortCtx, forInScope)
if err != nil {
return nil, err
}
parsedClauses = append(parsedClauses, func(f *expressions.ForExpression) error {
return f.AddSort(v.getSourceMap(sortCtx), sortExps...)
})
}
collectCtx := clause.CollectClause()
if collectCtx != nil {
collectCtx := collectCtx.(*fql.CollectClauseContext)
params, err := v.createCollect(collectCtx, forInScope, valVarName)
if err != nil {
return nil, err
}
forInScope.RemoveVariable(valVarName)
if keyVarName != "" {
forInScope.RemoveVariable(keyVarName)
}
parsedClauses = append(parsedClauses, func(f *expressions.ForExpression) error {
return f.AddCollect(v.getSourceMap(collectCtx), params)
})
}
}
body := ctx.AllForExpressionBody()
predicate := expressions.NewBlockExpression(len(body) + 1)
@ -192,7 +280,6 @@ func (v *visitor) doVisitForExpression(ctx *fql.ForExpressionContext, scope *sco
var spread bool
var distinct bool
var distinctSrc core.SourceMap
forRetCtx := ctx.ForExpressionReturn().(*fql.ForExpressionReturnContext)
returnCtx := forRetCtx.ReturnExpression()
@ -208,8 +295,6 @@ func (v *visitor) doVisitForExpression(ctx *fql.ForExpressionContext, scope *sco
if distinctCtx != nil {
distinct = true
token := distinctCtx.GetSymbol()
distinctSrc = core.NewSourceMap(token.GetText(), token.GetLine(), token.GetColumn())
}
predicate.Add(returnExp)
@ -228,10 +313,9 @@ func (v *visitor) doVisitForExpression(ctx *fql.ForExpressionContext, scope *sco
forExp, err := expressions.NewForExpression(
v.getSourceMap(ctx),
valVarName,
keyVarName,
src,
predicate,
distinct,
spread,
)
@ -239,53 +323,11 @@ func (v *visitor) doVisitForExpression(ctx *fql.ForExpressionContext, scope *sco
return nil, err
}
if distinct {
forExp.AddDistinct(distinctSrc)
}
for _, clause := range ctx.AllForExpressionClause() {
clause := clause.(*fql.ForExpressionClauseContext)
limitCtx := clause.LimitClause()
if limitCtx != nil {
limit, offset, err := v.createLimit(limitCtx.(*fql.LimitClauseContext))
if err != nil {
// add all available clauses
for _, clause := range parsedClauses {
if err := clause(forExp); err != nil {
return nil, err
}
forExp.AddLimit(v.getSourceMap(limitCtx), limit, offset)
continue
}
filterCtx := clause.FilterClause()
if filterCtx != nil {
filterExp, err := v.createFilter(filterCtx.(*fql.FilterClauseContext), forInScope)
if err != nil {
return nil, err
}
forExp.AddFilter(v.getSourceMap(filterCtx), filterExp)
continue
}
sortCtx := clause.SortClause()
if sortCtx != nil {
sortCtx := sortCtx.(*fql.SortClauseContext)
sortExps, err := v.createSort(sortCtx, forInScope)
if err != nil {
return nil, err
}
forExp.AddSort(v.getSourceMap(sortCtx), sortExps...)
}
}
return forExp, nil
@ -392,7 +434,190 @@ func (v *visitor) createSort(ctx *fql.SortClauseContext, scope *scope) ([]*claus
return res, nil
}
func (v *visitor) doVisitForExpressionSource(ctx *fql.ForExpressionSourceContext, scope *scope) (collections.IterableExpression, error) {
func (v *visitor) createCollect(ctx *fql.CollectClauseContext, scope *scope, valVarName string) (*clauses.Collect, error) {
var err error
var selectors []*clauses.CollectSelector
var projection *clauses.CollectProjection
var count *clauses.CollectCount
var aggregate *clauses.CollectAggregate
groupingCtx := ctx.CollectGrouping()
if groupingCtx != nil {
groupingCtx := groupingCtx.(*fql.CollectGroupingContext)
collectSelectors := groupingCtx.AllCollectSelector()
// group selectors
if collectSelectors != nil && len(collectSelectors) > 0 {
selectors = make([]*clauses.CollectSelector, 0, len(collectSelectors))
for _, cs := range collectSelectors {
selector, err := v.createCollectSelector(cs.(*fql.CollectSelectorContext), scope)
if err != nil {
return nil, err
}
selectors = append(selectors, selector)
if err := scope.SetVariable(selector.Variable()); err != nil {
return nil, err
}
}
}
projectionCtx := ctx.CollectGroupVariable()
if projectionCtx != nil {
projectionCtx := projectionCtx.(*fql.CollectGroupVariableContext)
projectionSelectorCtx := projectionCtx.CollectSelector()
var projectionSelector *clauses.CollectSelector
// if projection expression is defined like WITH group = { foo: i.bar }
if projectionSelectorCtx != nil {
selector, err := v.createCollectSelector(projectionSelectorCtx.(*fql.CollectSelectorContext), scope)
if err != nil {
return nil, err
}
projectionSelector = selector
} else {
// otherwise, use default expression WITH group = { i }
projectionIdentifier := projectionCtx.Identifier(0)
if projectionIdentifier != nil {
varExp, err := expressions.NewVariableExpression(v.getSourceMap(projectionCtx), valVarName)
if err != nil {
return nil, err
}
strLitExp := literals.NewStringLiteral(valVarName)
propExp, err := literals.NewObjectPropertyAssignment(
strLitExp,
varExp,
)
if err != nil {
return nil, err
}
projectionSelectorExp := literals.NewObjectLiteralWith(propExp)
if err != nil {
return nil, err
}
selector, err := clauses.NewCollectSelector(projectionIdentifier.GetText(), projectionSelectorExp)
if err != nil {
return nil, err
}
projectionSelector = selector
}
}
if projectionSelector != nil {
if err := scope.SetVariable(projectionSelector.Variable()); err != nil {
return nil, err
}
projection, err = clauses.NewCollectProjection(projectionSelector)
if err != nil {
return nil, err
}
}
}
}
countCtx := ctx.CollectCounter()
if countCtx != nil {
countCtx := countCtx.(*fql.CollectCounterContext)
variable := countCtx.Identifier().GetText()
if err := scope.SetVariable(variable); err != nil {
return nil, err
}
count, err = clauses.NewCollectCount(variable)
if err != nil {
return nil, err
}
}
aggrCtx := ctx.CollectAggregator()
if aggrCtx != nil {
aggrCtx := aggrCtx.(*fql.CollectAggregatorContext)
selectorCtxs := aggrCtx.AllCollectAggregateSelector()
selectors := make([]*clauses.CollectAggregateSelector, 0, len(selectorCtxs))
for _, sc := range selectorCtxs {
selector, err := v.createCollectAggregateSelector(sc.(*fql.CollectAggregateSelectorContext), scope)
if err != nil {
return nil, err
}
selectors = append(selectors, selector)
}
aggregate, err = clauses.NewCollectAggregate(selectors)
if err != nil {
return nil, err
}
}
return clauses.NewCollect(selectors, projection, count, aggregate)
}
func (v *visitor) createCollectSelector(ctx *fql.CollectSelectorContext, scope *scope) (*clauses.CollectSelector, error) {
variable := ctx.Identifier().GetText()
exp, err := v.doVisitExpression(ctx.Expression().(*fql.ExpressionContext), scope)
if err != nil {
return nil, err
}
return clauses.NewCollectSelector(variable, exp)
}
func (v *visitor) createCollectAggregateSelector(ctx *fql.CollectAggregateSelectorContext, scope *scope) (*clauses.CollectAggregateSelector, error) {
variable := ctx.Identifier().GetText()
fnCtx := ctx.FunctionCallExpression()
if fnCtx != nil {
exp, err := v.doVisitFunctionCallExpression(fnCtx.(*fql.FunctionCallExpressionContext), scope)
if err != nil {
return nil, err
}
fnExp, ok := exp.(*expressions.FunctionCallExpression)
if !ok {
return nil, core.Error(core.ErrInvalidType, "expected function expression")
}
if err := scope.SetVariable(variable); err != nil {
return nil, err
}
return clauses.NewCollectAggregateSelector(variable, fnExp.Arguments(), fnExp.Function())
}
return nil, core.Error(core.ErrNotFound, "function expression")
}
func (v *visitor) doVisitForExpressionSource(ctx *fql.ForExpressionSourceContext, scope *scope) (core.Expression, error) {
arr := ctx.ArrayLiteral()
if arr != nil {
@ -454,7 +679,7 @@ func (v *visitor) doVisitForExpressionBody(ctx *fql.ForExpressionBodyContext, sc
return nil, v.unexpectedToken(ctx)
}
func (v *visitor) doVisitMemberExpression(ctx *fql.MemberExpressionContext, scope *scope) (collections.IterableExpression, error) {
func (v *visitor) doVisitMemberExpression(ctx *fql.MemberExpressionContext, scope *scope) (core.Expression, error) {
varName := ctx.Identifier().GetText()
_, err := scope.GetVariable(varName)
@ -516,7 +741,7 @@ func (v *visitor) doVisitMemberExpression(ctx *fql.MemberExpressionContext, scop
return member, nil
}
func (v *visitor) doVisitObjectLiteral(ctx *fql.ObjectLiteralContext, scope *scope) (collections.IterableExpression, error) {
func (v *visitor) doVisitObjectLiteral(ctx *fql.ObjectLiteralContext, scope *scope) (core.Expression, error) {
assignments := ctx.AllPropertyAssignment()
props := make([]*literals.ObjectPropertyAssignment, 0, len(assignments))
@ -553,7 +778,13 @@ func (v *visitor) doVisitObjectLiteral(ctx *fql.ObjectLiteralContext, scope *sco
return nil, err
}
props = append(props, literals.NewObjectPropertyAssignment(name, value))
pa, err := literals.NewObjectPropertyAssignment(name, value)
if err != nil {
return nil, err
}
props = append(props, pa)
}
return literals.NewObjectLiteralWith(props...), nil
@ -598,7 +829,7 @@ func (v *visitor) doVisitShorthandPropertyNameContext(ctx *fql.ShorthandProperty
return literals.NewStringLiteral(ctx.Variable().GetText()), nil
}
func (v *visitor) doVisitArrayLiteral(ctx *fql.ArrayLiteralContext, scope *scope) (collections.IterableExpression, error) {
func (v *visitor) doVisitArrayLiteral(ctx *fql.ArrayLiteralContext, scope *scope) (core.Expression, error) {
listCtx := ctx.ArrayElementList()
if listCtx == nil {
@ -657,7 +888,7 @@ func (v *visitor) doVisitNoneLiteral(_ *fql.NoneLiteralContext) (core.Expression
return literals.None, nil
}
func (v *visitor) doVisitVariable(ctx *fql.VariableContext, scope *scope) (collections.IterableExpression, error) {
func (v *visitor) doVisitVariable(ctx *fql.VariableContext, scope *scope) (core.Expression, error) {
name := ctx.Identifier().GetText()
// check whether the variable is defined
@ -711,7 +942,7 @@ func (v *visitor) doVisitVariableDeclaration(ctx *fql.VariableDeclarationContext
)
}
func (v *visitor) doVisitRangeOperator(ctx *fql.RangeOperatorContext, scope *scope) (collections.IterableExpression, error) {
func (v *visitor) doVisitRangeOperator(ctx *fql.RangeOperatorContext, scope *scope) (core.Expression, error) {
exp, err := v.doVisitChildren(ctx, scope)
if err != nil {
@ -732,7 +963,7 @@ func (v *visitor) doVisitRangeOperator(ctx *fql.RangeOperatorContext, scope *sco
)
}
func (v *visitor) doVisitFunctionCallExpression(context *fql.FunctionCallExpressionContext, scope *scope) (collections.IterableExpression, error) {
func (v *visitor) doVisitFunctionCallExpression(context *fql.FunctionCallExpressionContext, scope *scope) (core.Expression, error) {
args := make([]core.Expression, 0, 5)
argsCtx := context.Arguments()
@ -765,7 +996,7 @@ func (v *visitor) doVisitFunctionCallExpression(context *fql.FunctionCallExpress
)
}
func (v *visitor) doVisitParamContext(context *fql.ParamContext, _ *scope) (collections.IterableExpression, error) {
func (v *visitor) doVisitParamContext(context *fql.ParamContext, _ *scope) (core.Expression, error) {
name := context.Identifier().GetText()
return expressions.NewParameterExpression(

52
pkg/parser/antlr/FqlParser.g4
View File

@ -75,41 +75,37 @@ sortClauseExpression
;
collectClause
: Collect collectVariable Assign expression
| Collect collectVariable Assign expression Into collectGroupVariable
| Collect collectVariable Assign expression Into collectGroupVariable Keep collectKeepVariable
| Collect collectVariable Assign expression With Count collectCountVariable
| Collect collectVariable Assign expression Aggregate collectAggregateVariable Assign collectAggregateExpression
| Collect Aggregate collectAggregateVariable Assign collectAggregateExpression
| Collect With Count Into collectCountVariable
: Collect collectCounter
| Collect collectAggregator
| Collect collectGrouping collectAggregator
| Collect collectGrouping collectGroupVariable
| Collect collectGrouping collectCounter
| Collect collectGrouping
;
collectVariable
: Identifier
collectSelector
: Identifier Assign expression
;
collectGrouping
: collectSelector (Comma collectSelector)*
;
collectAggregator
: Aggregate collectAggregateSelector (Comma collectAggregateSelector)*
;
collectAggregateSelector
: Identifier Assign functionCallExpression
;
collectGroupVariable
: Identifier
: Into collectSelector
| Into Identifier (Keep Identifier)?
;
collectKeepVariable
: Identifier
;
collectCountVariable
: Identifier
;
collectAggregateVariable
: Identifier
;
collectAggregateExpression
: expression
;
collectOption
:
collectCounter
: With Count Into Identifier
;
forExpressionBody

13
pkg/parser/fql/FqlParser.interp
View File

File diff suppressed because one or more lines are too long

4
pkg/parser/fql/fql_lexer.go
View File

@ -276,7 +276,7 @@ var lexerSymbolicNames = []string{
"And", "Or", "Range", "Assign", "QuestionMark", "RegexNotMatch", "RegexMatch",
"For", "Return", "Distinct", "Filter", "Sort", "Limit", "Let", "Collect",
"SortDirection", "None", "Null", "BooleanLiteral", "Into", "Keep", "With",
"Count", "All", "Any", "Aggregate", "Like", "Not", "In", "Param", "Identifier",
"count", "All", "Any", "aggregate", "Like", "Not", "In", "Param", "Identifier",
"StringLiteral", "TemplateStringLiteral", "IntegerLiteral", "FloatLiteral",
}
@ -288,7 +288,7 @@ var lexerRuleNames = []string{
"And", "Or", "Range", "Assign", "QuestionMark", "RegexNotMatch", "RegexMatch",
"For", "Return", "Distinct", "Filter", "Sort", "Limit", "Let", "Collect",
"SortDirection", "None", "Null", "BooleanLiteral", "Into", "Keep", "With",
"Count", "All", "Any", "Aggregate", "Like", "Not", "In", "Param", "Identifier",
"count", "All", "Any", "aggregate", "Like", "Not", "In", "Param", "Identifier",
"StringLiteral", "TemplateStringLiteral", "IntegerLiteral", "FloatLiteral",
"HexDigit", "DecimalIntegerLiteral", "ExponentPart", "Letter", "Symbols",
"Digit", "DQSring", "SQString",

2699
pkg/parser/fql/fql_parser.go
View File

File diff suppressed because it is too large Load Diff

60
pkg/parser/fql/fqlparser_base_listener.go
View File

@ -112,11 +112,29 @@ func (s *BaseFqlParserListener) EnterCollectClause(ctx *CollectClauseContext) {}
// ExitCollectClause is called when production collectClause is exited.
func (s *BaseFqlParserListener) ExitCollectClause(ctx *CollectClauseContext) {}
// EnterCollectVariable is called when production collectVariable is entered.
func (s *BaseFqlParserListener) EnterCollectVariable(ctx *CollectVariableContext) {}
// EnterCollectSelector is called when production collectSelector is entered.
func (s *BaseFqlParserListener) EnterCollectSelector(ctx *CollectSelectorContext) {}
// ExitCollectVariable is called when production collectVariable is exited.
func (s *BaseFqlParserListener) ExitCollectVariable(ctx *CollectVariableContext) {}
// ExitCollectSelector is called when production collectSelector is exited.
func (s *BaseFqlParserListener) ExitCollectSelector(ctx *CollectSelectorContext) {}
// EnterCollectGrouping is called when production collectGrouping is entered.
func (s *BaseFqlParserListener) EnterCollectGrouping(ctx *CollectGroupingContext) {}
// ExitCollectGrouping is called when production collectGrouping is exited.
func (s *BaseFqlParserListener) ExitCollectGrouping(ctx *CollectGroupingContext) {}
// EnterCollectAggregator is called when production collectAggregator is entered.
func (s *BaseFqlParserListener) EnterCollectAggregator(ctx *CollectAggregatorContext) {}
// ExitCollectAggregator is called when production collectAggregator is exited.
func (s *BaseFqlParserListener) ExitCollectAggregator(ctx *CollectAggregatorContext) {}
// EnterCollectAggregateSelector is called when production collectAggregateSelector is entered.
func (s *BaseFqlParserListener) EnterCollectAggregateSelector(ctx *CollectAggregateSelectorContext) {}
// ExitCollectAggregateSelector is called when production collectAggregateSelector is exited.
func (s *BaseFqlParserListener) ExitCollectAggregateSelector(ctx *CollectAggregateSelectorContext) {}
// EnterCollectGroupVariable is called when production collectGroupVariable is entered.
func (s *BaseFqlParserListener) EnterCollectGroupVariable(ctx *CollectGroupVariableContext) {}
@ -124,37 +142,11 @@ func (s *BaseFqlParserListener) EnterCollectGroupVariable(ctx *CollectGroupVaria
// ExitCollectGroupVariable is called when production collectGroupVariable is exited.
func (s *BaseFqlParserListener) ExitCollectGroupVariable(ctx *CollectGroupVariableContext) {}
// EnterCollectKeepVariable is called when production collectKeepVariable is entered.
func (s *BaseFqlParserListener) EnterCollectKeepVariable(ctx *CollectKeepVariableContext) {}
// EnterCollectCounter is called when production collectCounter is entered.
func (s *BaseFqlParserListener) EnterCollectCounter(ctx *CollectCounterContext) {}
// ExitCollectKeepVariable is called when production collectKeepVariable is exited.
func (s *BaseFqlParserListener) ExitCollectKeepVariable(ctx *CollectKeepVariableContext) {}
// EnterCollectCountVariable is called when production collectCountVariable is entered.
func (s *BaseFqlParserListener) EnterCollectCountVariable(ctx *CollectCountVariableContext) {}
// ExitCollectCountVariable is called when production collectCountVariable is exited.
func (s *BaseFqlParserListener) ExitCollectCountVariable(ctx *CollectCountVariableContext) {}
// EnterCollectAggregateVariable is called when production collectAggregateVariable is entered.
func (s *BaseFqlParserListener) EnterCollectAggregateVariable(ctx *CollectAggregateVariableContext) {}
// ExitCollectAggregateVariable is called when production collectAggregateVariable is exited.
func (s *BaseFqlParserListener) ExitCollectAggregateVariable(ctx *CollectAggregateVariableContext) {}
// EnterCollectAggregateExpression is called when production collectAggregateExpression is entered.
func (s *BaseFqlParserListener) EnterCollectAggregateExpression(ctx *CollectAggregateExpressionContext) {
}
// ExitCollectAggregateExpression is called when production collectAggregateExpression is exited.
func (s *BaseFqlParserListener) ExitCollectAggregateExpression(ctx *CollectAggregateExpressionContext) {
}
// EnterCollectOption is called when production collectOption is entered.
func (s *BaseFqlParserListener) EnterCollectOption(ctx *CollectOptionContext) {}
// ExitCollectOption is called when production collectOption is exited.
func (s *BaseFqlParserListener) ExitCollectOption(ctx *CollectOptionContext) {}
// ExitCollectCounter is called when production collectCounter is exited.
func (s *BaseFqlParserListener) ExitCollectCounter(ctx *CollectCounterContext) {}
// EnterForExpressionBody is called when production forExpressionBody is entered.
func (s *BaseFqlParserListener) EnterForExpressionBody(ctx *ForExpressionBodyContext) {}

32
pkg/parser/fql/fqlparser_base_visitor.go
View File

@ -67,7 +67,19 @@ func (v *BaseFqlParserVisitor) VisitCollectClause(ctx *CollectClauseContext) int
return v.VisitChildren(ctx)
}
func (v *BaseFqlParserVisitor) VisitCollectVariable(ctx *CollectVariableContext) interface{} {
func (v *BaseFqlParserVisitor) VisitCollectSelector(ctx *CollectSelectorContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BaseFqlParserVisitor) VisitCollectGrouping(ctx *CollectGroupingContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BaseFqlParserVisitor) VisitCollectAggregator(ctx *CollectAggregatorContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BaseFqlParserVisitor) VisitCollectAggregateSelector(ctx *CollectAggregateSelectorContext) interface{} {
return v.VisitChildren(ctx)
}
@ -75,23 +87,7 @@ func (v *BaseFqlParserVisitor) VisitCollectGroupVariable(ctx *CollectGroupVariab
return v.VisitChildren(ctx)
}
func (v *BaseFqlParserVisitor) VisitCollectKeepVariable(ctx *CollectKeepVariableContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BaseFqlParserVisitor) VisitCollectCountVariable(ctx *CollectCountVariableContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BaseFqlParserVisitor) VisitCollectAggregateVariable(ctx *CollectAggregateVariableContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BaseFqlParserVisitor) VisitCollectAggregateExpression(ctx *CollectAggregateExpressionContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BaseFqlParserVisitor) VisitCollectOption(ctx *CollectOptionContext) interface{} {
func (v *BaseFqlParserVisitor) VisitCollectCounter(ctx *CollectCounterContext) interface{} {
return v.VisitChildren(ctx)
}

58
pkg/parser/fql/fqlparser_listener.go
View File

@ -52,26 +52,23 @@ type FqlParserListener interface {
// EnterCollectClause is called when entering the collectClause production.
EnterCollectClause(c *CollectClauseContext)
// EnterCollectVariable is called when entering the collectVariable production.
EnterCollectVariable(c *CollectVariableContext)
// EnterCollectSelector is called when entering the collectSelector production.
EnterCollectSelector(c *CollectSelectorContext)
// EnterCollectGrouping is called when entering the collectGrouping production.
EnterCollectGrouping(c *CollectGroupingContext)
// EnterCollectAggregator is called when entering the collectAggregator production.
EnterCollectAggregator(c *CollectAggregatorContext)
// EnterCollectAggregateSelector is called when entering the collectAggregateSelector production.
EnterCollectAggregateSelector(c *CollectAggregateSelectorContext)
// EnterCollectGroupVariable is called when entering the collectGroupVariable production.
EnterCollectGroupVariable(c *CollectGroupVariableContext)
// EnterCollectKeepVariable is called when entering the collectKeepVariable production.
EnterCollectKeepVariable(c *CollectKeepVariableContext)
// EnterCollectCountVariable is called when entering the collectCountVariable production.
EnterCollectCountVariable(c *CollectCountVariableContext)
// EnterCollectAggregateVariable is called when entering the collectAggregateVariable production.
EnterCollectAggregateVariable(c *CollectAggregateVariableContext)
// EnterCollectAggregateExpression is called when entering the collectAggregateExpression production.
EnterCollectAggregateExpression(c *CollectAggregateExpressionContext)
// EnterCollectOption is called when entering the collectOption production.
EnterCollectOption(c *CollectOptionContext)
// EnterCollectCounter is called when entering the collectCounter production.
EnterCollectCounter(c *CollectCounterContext)
// EnterForExpressionBody is called when entering the forExpressionBody production.
EnterForExpressionBody(c *ForExpressionBodyContext)
@ -208,26 +205,23 @@ type FqlParserListener interface {
// ExitCollectClause is called when exiting the collectClause production.
ExitCollectClause(c *CollectClauseContext)
// ExitCollectVariable is called when exiting the collectVariable production.
ExitCollectVariable(c *CollectVariableContext)
// ExitCollectSelector is called when exiting the collectSelector production.
ExitCollectSelector(c *CollectSelectorContext)
// ExitCollectGrouping is called when exiting the collectGrouping production.
ExitCollectGrouping(c *CollectGroupingContext)
// ExitCollectAggregator is called when exiting the collectAggregator production.
ExitCollectAggregator(c *CollectAggregatorContext)
// ExitCollectAggregateSelector is called when exiting the collectAggregateSelector production.
ExitCollectAggregateSelector(c *CollectAggregateSelectorContext)
// ExitCollectGroupVariable is called when exiting the collectGroupVariable production.
ExitCollectGroupVariable(c *CollectGroupVariableContext)
// ExitCollectKeepVariable is called when exiting the collectKeepVariable production.
ExitCollectKeepVariable(c *CollectKeepVariableContext)
// ExitCollectCountVariable is called when exiting the collectCountVariable production.
ExitCollectCountVariable(c *CollectCountVariableContext)
// ExitCollectAggregateVariable is called when exiting the collectAggregateVariable production.
ExitCollectAggregateVariable(c *CollectAggregateVariableContext)
// ExitCollectAggregateExpression is called when exiting the collectAggregateExpression production.
ExitCollectAggregateExpression(c *CollectAggregateExpressionContext)
// ExitCollectOption is called when exiting the collectOption production.
ExitCollectOption(c *CollectOptionContext)
// ExitCollectCounter is called when exiting the collectCounter production.
ExitCollectCounter(c *CollectCounterContext)
// ExitForExpressionBody is called when exiting the forExpressionBody production.
ExitForExpressionBody(c *ForExpressionBodyContext)

29
pkg/parser/fql/fqlparser_visitor.go
View File

@ -52,26 +52,23 @@ type FqlParserVisitor interface {
// Visit a parse tree produced by FqlParser#collectClause.
VisitCollectClause(ctx *CollectClauseContext) interface{}
// Visit a parse tree produced by FqlParser#collectVariable.
VisitCollectVariable(ctx *CollectVariableContext) interface{}
// Visit a parse tree produced by FqlParser#collectSelector.
VisitCollectSelector(ctx *CollectSelectorContext) interface{}
// Visit a parse tree produced by FqlParser#collectGrouping.
VisitCollectGrouping(ctx *CollectGroupingContext) interface{}
// Visit a parse tree produced by FqlParser#collectAggregator.
VisitCollectAggregator(ctx *CollectAggregatorContext) interface{}
// Visit a parse tree produced by FqlParser#collectAggregateSelector.
VisitCollectAggregateSelector(ctx *CollectAggregateSelectorContext) interface{}
// Visit a parse tree produced by FqlParser#collectGroupVariable.
VisitCollectGroupVariable(ctx *CollectGroupVariableContext) interface{}
// Visit a parse tree produced by FqlParser#collectKeepVariable.
VisitCollectKeepVariable(ctx *CollectKeepVariableContext) interface{}
// Visit a parse tree produced by FqlParser#collectCountVariable.
VisitCollectCountVariable(ctx *CollectCountVariableContext) interface{}
// Visit a parse tree produced by FqlParser#collectAggregateVariable.
VisitCollectAggregateVariable(ctx *CollectAggregateVariableContext) interface{}
// Visit a parse tree produced by FqlParser#collectAggregateExpression.
VisitCollectAggregateExpression(ctx *CollectAggregateExpressionContext) interface{}
// Visit a parse tree produced by FqlParser#collectOption.
VisitCollectOption(ctx *CollectOptionContext) interface{}
// Visit a parse tree produced by FqlParser#collectCounter.
VisitCollectCounter(ctx *CollectCounterContext) interface{}
// Visit a parse tree produced by FqlParser#forExpressionBody.
VisitForExpressionBody(ctx *ForExpressionBodyContext) interface{}

24
pkg/runtime/collections/collection.go
View File

@ -1,7 +1,25 @@
package collections
import "github.com/MontFerret/ferret/pkg/runtime/values"
import (
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type Collection interface {
type (
Collection interface {
Length() values.Int
}
}
IndexedCollection interface {
Collection
Get(idx values.Int) core.Value
Set(idx values.Int, value core.Value) error
}
KeyedCollection interface {
Collection
Keys() []string
Get(key values.String) (core.Value, values.Boolean)
Set(key values.String, value core.Value)
}
)

108
pkg/runtime/collections/data_set.go Normal file
View File

@ -0,0 +1,108 @@
package collections
import (
"encoding/binary"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"hash/fnv"
"sort"
)
type DataSet map[string]core.Value
func NewDataSet() DataSet {
return make(DataSet)
}
func (ds DataSet) Apply(scope *core.Scope, variables Variables) error {
if err := ValidateDataSet(ds, variables); err != nil {
return err
}
for _, variable := range variables {
if variable != "" {
value, found := ds[variable]
if !found {
return core.Errorf(core.ErrNotFound, "variable not found in a given data set: %s", variable)
}
scope.SetVariable(variable, value)
}
}
return nil
}
func (ds DataSet) Set(key string, value core.Value) {
ds[key] = value
}
func (ds DataSet) Get(key string) core.Value {
val, found := ds[key]
if found {
return val
}
return values.None
}
func (ds DataSet) Hash() uint64 {
h := fnv.New64a()
keys := make([]string, 0, len(ds))
for key := range ds {
keys = append(keys, key)
}
// order does not really matter
// but it will give us a consistent hash sum
sort.Strings(keys)
endIndex := len(keys) - 1
h.Write([]byte("{"))
for idx, key := range keys {
h.Write([]byte(key))
h.Write([]byte(":"))
el := ds[key]
bytes := make([]byte, 8)
binary.LittleEndian.PutUint64(bytes, el.Hash())
h.Write(bytes)
if idx != endIndex {
h.Write([]byte(","))
}
}
h.Write([]byte("}"))
return h.Sum64()
}
func (ds DataSet) Compare(other DataSet) int {
if len(ds) > len(ds) {
return 1
}
if len(ds) < len(ds) {
return -1
}
var res = 0
for key, otherVal := range other {
res = -1
if val, exists := ds[key]; exists {
res = val.Compare(otherVal)
}
}
return res
}

9
pkg/runtime/collections/errors.go
View File

@ -6,4 +6,13 @@ import (
var (
ErrExhausted = core.Error(core.ErrInvalidOperation, "iterator has been exhausted")
ErrResultSetMismatch = core.Error(core.ErrInvalidArgument, "count of values in result set is less that count of variables")
)
func ValidateDataSet(set DataSet, variables Variables) error {
if len(variables) > len(set) {
return ErrResultSetMismatch
}
return nil
}

28
pkg/runtime/collections/filter.go
View File

@ -2,17 +2,16 @@ package collections
import (
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/pkg/errors"
)
type (
FilterPredicate func(val core.Value, key core.Value) (bool, error)
FilterPredicate func(set DataSet) (bool, error)
FilterIterator struct {
src Iterator
predicate FilterPredicate
value core.Value
key core.Value
dataSet DataSet
ready bool
}
)
@ -35,34 +34,33 @@ func (iterator *FilterIterator) HasNext() bool {
iterator.ready = true
}
return iterator.value != nil && iterator.value.Type() != core.NoneType
return iterator.dataSet != nil
}
func (iterator *FilterIterator) Next() (core.Value, core.Value, error) {
func (iterator *FilterIterator) Next() (DataSet, error) {
if iterator.HasNext() == true {
val := iterator.value
key := iterator.key
ds := iterator.dataSet
iterator.filter()
return val, key, nil
return ds, nil
}
return values.None, values.None, ErrExhausted
return nil, ErrExhausted
}
func (iterator *FilterIterator) filter() {
var doNext bool
for iterator.src.HasNext() {
val, key, err := iterator.src.Next()
set, err := iterator.src.Next()
if err != nil {
doNext = false
break
}
take, err := iterator.predicate(val, key)
take, err := iterator.predicate(set)
if err != nil {
doNext = false
@ -71,14 +69,12 @@ func (iterator *FilterIterator) filter() {
if take == true {
doNext = true
iterator.value = val
iterator.key = key
iterator.dataSet = set
break
}
}
if doNext == false {
iterator.value = nil
iterator.key = nil
iterator.dataSet = nil
}
}

53
pkg/runtime/collections/filter_test.go
View File

@ -20,15 +20,15 @@ func TestFilter(t *testing.T) {
values.NewInt(5),
}
predicate := func(val core.Value, _ core.Value) (bool, error) {
i := float64(val.Unwrap().(int))
predicate := func(ds collections.DataSet) (bool, error) {
i := float64(ds.Get(collections.DefaultValueVar).Unwrap().(int))
calc := float64(i / 2)
return calc == math.Floor(calc), nil
}
iter, err := collections.NewFilterIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
predicate,
)
@ -37,7 +37,7 @@ func TestFilter(t *testing.T) {
res := make([]core.Value, 0, len(arr))
for iter.HasNext() {
item, _, err := iter.Next()
item, _, err := next(iter)
So(err, ShouldBeNil)
@ -47,7 +47,7 @@ func TestFilter(t *testing.T) {
So(res, ShouldHaveLength, 2)
})
Convey("Should filter out non-even keys", t, func() {
Convey("Should filter out non-even groupKeys", t, func() {
arr := []core.Value{
values.NewInt(1),
values.NewInt(2),
@ -56,8 +56,8 @@ func TestFilter(t *testing.T) {
values.NewInt(5),
}
predicate := func(_ core.Value, key core.Value) (bool, error) {
i := float64(key.Unwrap().(int))
predicate := func(ds collections.DataSet) (bool, error) {
i := float64(ds.Get(collections.DefaultKeyVar).Unwrap().(int))
if i == 0 {
return false, nil
@ -69,7 +69,7 @@ func TestFilter(t *testing.T) {
}
iter, err := collections.NewFilterIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
predicate,
)
@ -78,7 +78,7 @@ func TestFilter(t *testing.T) {
res := make([]core.Value, 0, len(arr))
for iter.HasNext() {
item, _, err := iter.Next()
item, _, err := next(iter)
So(err, ShouldBeNil)
@ -97,12 +97,12 @@ func TestFilter(t *testing.T) {
values.NewInt(5),
}
predicate := func(val core.Value, _ core.Value) (bool, error) {
predicate := func(_ collections.DataSet) (bool, error) {
return false, nil
}
iter, err := collections.NewFilterIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
predicate,
)
@ -111,7 +111,7 @@ func TestFilter(t *testing.T) {
res := make([]core.Value, 0, len(arr))
for iter.HasNext() {
item, _, err := iter.Next()
item, _, err := next(iter)
So(err, ShouldBeNil)
@ -130,12 +130,12 @@ func TestFilter(t *testing.T) {
values.NewInt(5),
}
predicate := func(val core.Value, _ core.Value) (bool, error) {
predicate := func(_ collections.DataSet) (bool, error) {
return true, nil
}
iter, err := collections.NewFilterIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
predicate,
)
@ -144,7 +144,7 @@ func TestFilter(t *testing.T) {
res := make([]core.Value, 0, len(arr))
for iter.HasNext() {
item, _, err := iter.Next()
item, _, err := next(iter)
So(err, ShouldBeNil)
@ -163,12 +163,12 @@ func TestFilter(t *testing.T) {
values.NewInt(5),
}
predicate := func(val core.Value, _ core.Value) (bool, error) {
predicate := func(_ collections.DataSet) (bool, error) {
return true, nil
}
iter, err := collections.NewFilterIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
predicate,
)
@ -177,15 +177,16 @@ func TestFilter(t *testing.T) {
res := make([]core.Value, 0, len(arr))
for iter.HasNext() {
item, _, err := iter.Next()
item, _, err := next(iter)
So(err, ShouldBeNil)
res = append(res, item)
}
_, _, err = iter.Next()
item, _, err := next(iter)
So(item, ShouldBeNil)
So(err, ShouldBeError)
})
@ -199,17 +200,17 @@ func TestFilter(t *testing.T) {
}
// i < 5
predicate1 := func(val core.Value, _ core.Value) (bool, error) {
return val.Compare(values.NewInt(5)) == -1, nil
predicate1 := func(ds collections.DataSet) (bool, error) {
return ds.Get(collections.DefaultValueVar).Compare(values.NewInt(5)) == -1, nil
}
// i > 2
predicate2 := func(val core.Value, _ core.Value) (bool, error) {
return val.Compare(values.NewInt(2)) == 1, nil
predicate2 := func(ds collections.DataSet) (bool, error) {
return ds.Get(collections.DefaultValueVar).Compare(values.NewInt(2)) == 1, nil
}
it, _ := collections.NewFilterIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
predicate1,
)
@ -220,10 +221,12 @@ func TestFilter(t *testing.T) {
So(err, ShouldBeNil)
res, err := collections.ToSlice(iter)
sets, err := collections.ToSlice(iter)
So(err, ShouldBeNil)
res := toArrayOfValues(sets)
js, _ := json.Marshal(res)
So(string(js), ShouldEqual, `[3,4]`)

85
pkg/runtime/collections/group.go
View File

@ -1,85 +0,0 @@
package collections
import (
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type (
GroupKey func(value core.Value) (core.Value, error)
GroupIterator struct {
src Iterator
keys []GroupKey
ready bool
values *MapIterator
}
)
func NewGroupIterator(
src Iterator,
keys ...GroupKey,
) (*GroupIterator, error) {
if core.IsNil(src) {
return nil, core.Error(core.ErrMissedArgument, "source")
}
if len(keys) == 0 {
return nil, core.Error(core.ErrMissedArgument, "key(s)")
}
return &GroupIterator{src, keys, false, nil}, nil
}
func (iterator *GroupIterator) HasNext() bool {
if !iterator.ready {
iterator.ready = true
groups, err := iterator.group()
if err != nil {
iterator.values = NewMapIterator(map[string]core.Value{})
return false
}
iterator.values = groups
}
return iterator.values.HasNext()
}
func (iterator *GroupIterator) Next() (core.Value, core.Value, error) {
return iterator.values.Next()
}
func (iterator *GroupIterator) group() (*MapIterator, error) {
groups := make(map[string]core.Value)
for iterator.src.HasNext() {
for _, keyFn := range iterator.keys {
val, _, err := iterator.src.Next()
if err != nil {
return nil, err
}
keyVal, err := keyFn(val)
if err != nil {
return nil, err
}
key := keyVal.String()
group, exists := groups[key]
if !exists {
group = values.NewArray(10)
groups[key] = group
}
group.(*values.Array).Push(val)
}
}
return NewMapIterator(groups), nil
}

53
pkg/runtime/collections/group_test.go
View File

@ -1,53 +0,0 @@
package collections_test
import (
"encoding/json"
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func TestGroup(t *testing.T) {
makeObj := func(active bool, age int, city, gender string) *values.Object {
obj := values.NewObject()
obj.Set("active", values.NewBoolean(active))
obj.Set("age", values.NewInt(age))
obj.Set("city", values.NewString(city))
obj.Set("gender", values.NewString(gender))
return obj
}
Convey("Should group by a single key", t, func() {
arr := []core.Value{
makeObj(true, 31, "D.C.", "m"),
makeObj(true, 29, "L.A.", "f"),
makeObj(true, 36, "D.C.", "m"),
makeObj(true, 34, "N.Y.C.", "f"),
makeObj(true, 28, "L.A.", "f"),
makeObj(true, 41, "Boston", "m"),
}
iter, err := collections.NewGroupIterator(
collections.NewSliceIterator(arr),
func(value core.Value) (core.Value, error) {
val, _ := value.(*values.Object).Get("gender")
return val, nil
},
)
So(err, ShouldBeNil)
res, err := collections.ToMap(iter)
So(err, ShouldBeNil)
j, _ := json.Marshal(res)
So(string(j), ShouldEqual, `{"f":[{"active":true,"age":29,"city":"L.A.","gender":"f"},{"active":true,"age":34,"city":"N.Y.C.","gender":"f"},{"active":true,"age":28,"city":"L.A.","gender":"f"}],"m":[{"active":true,"age":31,"city":"D.C.","gender":"m"},{"active":true,"age":36,"city":"D.C.","gender":"m"},{"active":true,"age":41,"city":"Boston","gender":"m"}]}`)
})
}

25
pkg/runtime/collections/hash-table.go
View File

@ -1,25 +0,0 @@
package collections
import "github.com/MontFerret/ferret/pkg/runtime/core"
func ToHashTable(iterator Iterator) (map[uint64]core.Value, error) {
result := make(map[uint64]core.Value)
for iterator.HasNext() {
val, _, err := iterator.Next()
if err != nil {
return nil, err
}
h := val.Hash()
_, exists := result[h]
if !exists {
result[h] = val
}
}
return result, nil
}

40
pkg/runtime/collections/html.go Normal file
View File

@ -0,0 +1,40 @@
package collections
import (
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type HTMLNodeIterator struct {
valVar string
keyVar string
values values.HTMLNode
pos int
}
func NewHTMLNodeIterator(
valVar,
keyVar string,
input values.HTMLNode,
) Iterator {
return &HTMLNodeIterator{valVar, keyVar, input, 0}
}
func (iterator *HTMLNodeIterator) HasNext() bool {
return iterator.values.Length() > values.NewInt(iterator.pos)
}
func (iterator *HTMLNodeIterator) Next() (DataSet, error) {
if iterator.values.Length() > values.NewInt(iterator.pos) {
idx := values.NewInt(iterator.pos)
val := iterator.values.GetChildNode(idx)
iterator.pos++
return DataSet{
iterator.valVar: val,
iterator.keyVar: idx,
}, nil
}
return nil, ErrExhausted
}

50
pkg/runtime/collections/indexed.go Normal file
View File

@ -0,0 +1,50 @@
package collections
import (
"github.com/MontFerret/ferret/pkg/runtime/values"
)
const (
DefaultValueVar = "value"
DefaultKeyVar = "key"
)
type IndexedIterator struct {
valVar string
keyVar string
values IndexedCollection
pos int
}
func NewIndexedIterator(
valVar,
keyVar string,
input IndexedCollection,
) Iterator {
return &IndexedIterator{valVar, keyVar, input, 0}
}
func NewDefaultIndexedIterator(
input IndexedCollection,
) Iterator {
return &IndexedIterator{DefaultValueVar, DefaultKeyVar, input, 0}
}
func (iterator *IndexedIterator) HasNext() bool {
return int(iterator.values.Length()) > iterator.pos
}
func (iterator *IndexedIterator) Next() (DataSet, error) {
if int(iterator.values.Length()) > iterator.pos {
idx := values.NewInt(iterator.pos)
val := iterator.values.Get(idx)
iterator.pos++
return DataSet{
iterator.valVar: val,
iterator.keyVar: idx,
}, nil
}
return nil, ErrExhausted
}

129
pkg/runtime/collections/indexed_test.go Normal file
View File

@ -0,0 +1,129 @@
package collections_test
import (
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func next(iterator collections.Iterator) (core.Value, core.Value, error) {
ds, err := iterator.Next()
if err != nil {
return nil, nil, err
}
val := ds[collections.DefaultValueVar]
key := ds[collections.DefaultKeyVar]
return val, key, nil
}
func arrayIterator(arr *values.Array) collections.Iterator {
return collections.NewDefaultIndexedIterator(arr)
}
func TestArrayIterator(t *testing.T) {
Convey("Should iterate over an array", t, func() {
arr := values.NewArrayWith(
values.NewInt(1),
values.NewInt(2),
values.NewInt(3),
values.NewInt(4),
values.NewInt(5),
)
iter := arrayIterator(arr)
res := make([]core.Value, 0, arr.Length())
pos := 0
for iter.HasNext() {
item, key, err := next(iter)
So(err, ShouldBeNil)
So(key.Unwrap(), ShouldEqual, pos)
res = append(res, item)
pos += 1
}
So(res, ShouldHaveLength, arr.Length())
})
Convey("Should iterate over an array in the same order", t, func() {
arr := values.NewArrayWith(
values.NewInt(1),
values.NewInt(2),
values.NewInt(3),
values.NewInt(4),
values.NewInt(5),
)
iter := arrayIterator(arr)
res := make([]core.Value, 0, arr.Length())
for iter.HasNext() {
item, _, err := next(iter)
So(err, ShouldBeNil)
res = append(res, item)
}
arr.ForEach(func(expected core.Value, idx int) bool {
actual := res[idx]
So(actual, ShouldEqual, expected)
return true
})
})
Convey("Should return an error when exhausted", t, func() {
arr := values.NewArrayWith(
values.NewInt(1),
values.NewInt(2),
values.NewInt(3),
values.NewInt(4),
values.NewInt(5),
)
iter := arrayIterator(arr)
res := make([]core.Value, 0, arr.Length())
for iter.HasNext() {
item, _, err := next(iter)
So(err, ShouldBeNil)
res = append(res, item)
}
item, _, err := next(iter)
So(item, ShouldBeNil)
So(err, ShouldBeError)
})
Convey("Should NOT iterate over an empty array", t, func() {
arr := values.NewArray(10)
iter := arrayIterator(arr)
var iterated bool
for iter.HasNext() {
iterated = true
}
So(iterated, ShouldBeFalse)
})
}

216
pkg/runtime/collections/iterator.go
View File

@ -3,234 +3,34 @@ package collections
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type (
Variables []string
Iterator interface {
HasNext() bool
Next() (value core.Value, key core.Value, err error)
Next() (DataSet, error)
}
Iterable interface {
Iterate() Iterator
}
IterableExpression interface {
core.Expression
Variables() Variables
Iterate(ctx context.Context, scope *core.Scope) (Iterator, error)
}
SliceIterator struct {
values []core.Value
pos int
}
MapIterator struct {
values map[string]core.Value
keys []string
pos int
}
ArrayIterator struct {
values *values.Array
pos int
}
ObjectIterator struct {
values *values.Object
keys []string
pos int
}
HTMLNodeIterator struct {
values values.HTMLNode
pos int
}
)
func ToIterator(value core.Value) (Iterator, error) {
switch value.Type() {
case core.ArrayType:
return NewArrayIterator(value.(*values.Array)), nil
case core.ObjectType:
return NewObjectIterator(value.(*values.Object)), nil
case core.HTMLElementType, core.HTMLDocumentType:
return NewHTMLNodeIterator(value.(values.HTMLNode)), nil
default:
return nil, core.TypeError(
value.Type(),
core.ArrayType,
core.ObjectType,
core.HTMLDocumentType,
core.HTMLElementType,
)
}
}
func ToSlice(iterator Iterator) ([]core.Value, error) {
res := make([]core.Value, 0, 10)
func ToSlice(iterator Iterator) ([]DataSet, error) {
res := make([]DataSet, 0, 10)
for iterator.HasNext() {
item, _, err := iterator.Next()
ds, err := iterator.Next()
if err != nil {
return nil, err
}
res = append(res, item)
res = append(res, ds)
}
return res, nil
}
func ToMap(iterator Iterator) (map[string]core.Value, error) {
res := make(map[string]core.Value)
for iterator.HasNext() {
item, key, err := iterator.Next()
if err != nil {
return nil, err
}
res[key.String()] = item
}
return res, nil
}
func ToArray(iterator Iterator) (*values.Array, error) {
res := values.NewArray(10)
for iterator.HasNext() {
item, _, err := iterator.Next()
if err != nil {
return nil, err
}
res.Push(item)
}
return res, nil
}
func NewSliceIterator(input []core.Value) *SliceIterator {
return &SliceIterator{input, 0}
}
func (iterator *SliceIterator) HasNext() bool {
return len(iterator.values) > iterator.pos
}
func (iterator *SliceIterator) Next() (core.Value, core.Value, error) {
if len(iterator.values) > iterator.pos {
idx := iterator.pos
val := iterator.values[idx]
iterator.pos++
return val, values.NewInt(idx), nil
}
return values.None, values.None, ErrExhausted
}
func NewMapIterator(input map[string]core.Value) *MapIterator {
return &MapIterator{input, nil, 0}
}
func (iterator *MapIterator) HasNext() bool {
// lazy initialization
if iterator.keys == nil {
keys := make([]string, len(iterator.values))
i := 0
for k := range iterator.values {
keys[i] = k
i++
}
iterator.keys = keys
}
return len(iterator.keys) > iterator.pos
}
func (iterator *MapIterator) Next() (core.Value, core.Value, error) {
if len(iterator.keys) > iterator.pos {
key := iterator.keys[iterator.pos]
val := iterator.values[key]
iterator.pos++
return val, values.NewString(key), nil
}
return values.None, values.None, ErrExhausted
}
func NewArrayIterator(input *values.Array) *ArrayIterator {
return &ArrayIterator{input, 0}
}
func (iterator *ArrayIterator) HasNext() bool {
return int(iterator.values.Length()) > iterator.pos
}
func (iterator *ArrayIterator) Next() (core.Value, core.Value, error) {
if int(iterator.values.Length()) > iterator.pos {
idx := iterator.pos
val := iterator.values.Get(values.NewInt(idx))
iterator.pos++
return val, values.NewInt(idx), nil
}
return values.None, values.None, ErrExhausted
}
func NewObjectIterator(input *values.Object) *ObjectIterator {
return &ObjectIterator{input, nil, 0}
}
func (iterator *ObjectIterator) HasNext() bool {
// lazy initialization
if iterator.keys == nil {
iterator.keys = iterator.values.Keys()
}
return len(iterator.keys) > iterator.pos
}
func (iterator *ObjectIterator) Next() (core.Value, core.Value, error) {
if len(iterator.keys) > iterator.pos {
key := iterator.keys[iterator.pos]
val, _ := iterator.values.Get(values.NewString(key))
iterator.pos++
return val, values.NewString(key), nil
}
return values.None, values.None, ErrExhausted
}
func NewHTMLNodeIterator(input values.HTMLNode) *HTMLNodeIterator {
return &HTMLNodeIterator{input, 0}
}
func (iterator *HTMLNodeIterator) HasNext() bool {
return iterator.values.Length() > values.NewInt(iterator.pos)
}
func (iterator *HTMLNodeIterator) Next() (core.Value, core.Value, error) {
if iterator.values.Length() > values.NewInt(iterator.pos) {
idx := iterator.pos
val := iterator.values.GetChildNode(values.NewInt(idx))
iterator.pos++
return val, values.NewInt(idx), nil
}
return values.None, values.None, ErrExhausted
}

356
pkg/runtime/collections/iterator_test.go
View File

@ -1,356 +0,0 @@
package collections_test
import (
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func TestSliceIterator(t *testing.T) {
Convey("Should iterate over a slice", t, func() {
arr := []core.Value{
values.NewInt(1),
values.NewInt(2),
values.NewInt(3),
values.NewInt(4),
values.NewInt(5),
}
iter := collections.NewSliceIterator(arr)
res := make([]core.Value, 0, len(arr))
pos := 0
for iter.HasNext() {
item, key, err := iter.Next()
So(err, ShouldBeNil)
So(key.Unwrap(), ShouldEqual, pos)
res = append(res, item)
pos += 1
}
So(res, ShouldHaveLength, len(arr))
})
Convey("Should iterate over a slice in the same order", t, func() {
arr := []core.Value{
values.NewInt(1),
values.NewInt(2),
values.NewInt(3),
values.NewInt(4),
values.NewInt(5),
}
iter := collections.NewSliceIterator(arr)
res := make([]core.Value, 0, len(arr))
for iter.HasNext() {
item, _, err := iter.Next()
So(err, ShouldBeNil)
res = append(res, item)
}
for idx := range arr {
expected := arr[idx]
actual := res[idx]
So(actual, ShouldEqual, expected)
}
})
Convey("Should return an error when exhausted", t, func() {
arr := []core.Value{
values.NewInt(1),
values.NewInt(2),
values.NewInt(3),
values.NewInt(4),
values.NewInt(5),
}
iter := collections.NewSliceIterator(arr)
res := make([]core.Value, 0, len(arr))
for iter.HasNext() {
item, _, err := iter.Next()
So(err, ShouldBeNil)
res = append(res, item)
}
item, _, err := iter.Next()
So(item, ShouldEqual, values.None)
So(err, ShouldBeError)
})
Convey("Should NOT iterate over an empty slice", t, func() {
arr := []core.Value{}
iter := collections.NewSliceIterator(arr)
var iterated bool
for iter.HasNext() {
iterated = true
}
So(iterated, ShouldBeFalse)
})
}
func TestMapIterator(t *testing.T) {
Convey("Should iterate over a map", t, func() {
m := map[string]core.Value{
"one": values.NewInt(1),
"two": values.NewInt(2),
"three": values.NewInt(3),
"four": values.NewInt(4),
"five": values.NewInt(5),
}
iter := collections.NewMapIterator(m)
res := make([]core.Value, 0, len(m))
for iter.HasNext() {
item, key, err := iter.Next()
So(err, ShouldBeNil)
expected, exists := m[key.String()]
So(exists, ShouldBeTrue)
So(expected, ShouldEqual, item)
res = append(res, item)
}
So(res, ShouldHaveLength, len(m))
})
Convey("Should return an error when exhausted", t, func() {
m := map[string]core.Value{
"one": values.NewInt(1),
"two": values.NewInt(2),
"three": values.NewInt(3),
"four": values.NewInt(4),
"five": values.NewInt(5),
}
iter := collections.NewMapIterator(m)
res := make([]core.Value, 0, len(m))
for iter.HasNext() {
item, _, err := iter.Next()
So(err, ShouldBeNil)
res = append(res, item)
}
item, _, err := iter.Next()
So(item, ShouldEqual, values.None)
So(err, ShouldBeError)
})
Convey("Should NOT iterate over a empty map", t, func() {
m := make(map[string]core.Value)
iter := collections.NewMapIterator(m)
var iterated bool
for iter.HasNext() {
iterated = true
}
So(iterated, ShouldBeFalse)
})
}
func TestArrayIterator(t *testing.T) {
Convey("Should iterate over an array", t, func() {
arr := values.NewArrayWith(
values.NewInt(1),
values.NewInt(2),
values.NewInt(3),
values.NewInt(4),
values.NewInt(5),
)
iter := collections.NewArrayIterator(arr)
res := make([]core.Value, 0, arr.Length())
pos := 0
for iter.HasNext() {
item, key, err := iter.Next()
So(err, ShouldBeNil)
So(key.Unwrap(), ShouldEqual, pos)
res = append(res, item)
pos += 1
}
So(res, ShouldHaveLength, arr.Length())
})
Convey("Should iterate over an array in the same order", t, func() {
arr := values.NewArrayWith(
values.NewInt(1),
values.NewInt(2),
values.NewInt(3),
values.NewInt(4),
values.NewInt(5),
)
iter := collections.NewArrayIterator(arr)
res := make([]core.Value, 0, arr.Length())
for iter.HasNext() {
item, _, err := iter.Next()
So(err, ShouldBeNil)
res = append(res, item)
}
arr.ForEach(func(expected core.Value, idx int) bool {
actual := res[idx]
So(actual, ShouldEqual, expected)
return true
})
})
Convey("Should return an error when exhausted", t, func() {
arr := values.NewArrayWith(
values.NewInt(1),
values.NewInt(2),
values.NewInt(3),
values.NewInt(4),
values.NewInt(5),
)
iter := collections.NewArrayIterator(arr)
res := make([]core.Value, 0, arr.Length())
for iter.HasNext() {
item, _, err := iter.Next()
So(err, ShouldBeNil)
res = append(res, item)
}
item, _, err := iter.Next()
So(item, ShouldEqual, values.None)
So(err, ShouldBeError)
})
Convey("Should NOT iterate over an empty array", t, func() {
arr := values.NewArray(10)
iter := collections.NewArrayIterator(arr)
var iterated bool
for iter.HasNext() {
iterated = true
}
So(iterated, ShouldBeFalse)
})
}
func TestObjectIterator(t *testing.T) {
Convey("Should iterate over a map", t, func() {
m := values.NewObjectWith(
values.NewObjectProperty("one", values.NewInt(1)),
values.NewObjectProperty("two", values.NewInt(2)),
values.NewObjectProperty("three", values.NewInt(3)),
values.NewObjectProperty("four", values.NewInt(4)),
values.NewObjectProperty("five", values.NewInt(5)),
)
iter := collections.NewObjectIterator(m)
res := make([]core.Value, 0, m.Length())
for iter.HasNext() {
item, key, err := iter.Next()
So(err, ShouldBeNil)
expected, exists := m.Get(values.NewString(key.String()))
So(bool(exists), ShouldBeTrue)
So(expected, ShouldEqual, item)
res = append(res, item)
}
So(res, ShouldHaveLength, m.Length())
})
Convey("Should return an error when exhausted", t, func() {
m := values.NewObjectWith(
values.NewObjectProperty("one", values.NewInt(1)),
values.NewObjectProperty("two", values.NewInt(2)),
values.NewObjectProperty("three", values.NewInt(3)),
values.NewObjectProperty("four", values.NewInt(4)),
values.NewObjectProperty("five", values.NewInt(5)),
)
iter := collections.NewObjectIterator(m)
res := make([]core.Value, 0, m.Length())
for iter.HasNext() {
item, _, err := iter.Next()
So(err, ShouldBeNil)
res = append(res, item)
}
item, _, err := iter.Next()
So(item, ShouldEqual, values.None)
So(err, ShouldBeError)
})
Convey("Should NOT iterate over a empty map", t, func() {
m := values.NewObject()
iter := collections.NewObjectIterator(m)
var iterated bool
for iter.HasNext() {
iterated = true
}
So(iterated, ShouldBeFalse)
})
}

49
pkg/runtime/collections/keyed.go Normal file
View File

@ -0,0 +1,49 @@
package collections
import (
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type KeyedIterator struct {
valVar string
keyVar string
values KeyedCollection
keys []string
pos int
}
func NewKeyedIterator(
valVar,
keyVar string,
input KeyedCollection,
) Iterator {
return &KeyedIterator{valVar, keyVar, input, nil, 0}
}
func NewDefaultKeyedIterator(input KeyedCollection) Iterator {
return NewKeyedIterator(DefaultValueVar, DefaultKeyVar, input)
}
func (iterator *KeyedIterator) HasNext() bool {
// lazy initialization
if iterator.keys == nil {
iterator.keys = iterator.values.Keys()
}
return len(iterator.keys) > iterator.pos
}
func (iterator *KeyedIterator) Next() (DataSet, error) {
if len(iterator.keys) > iterator.pos {
key := values.NewString(iterator.keys[iterator.pos])
val, _ := iterator.values.Get(key)
iterator.pos++
return DataSet{
iterator.valVar: val,
iterator.keyVar: key,
}, nil
}
return nil, ErrExhausted
}

85
pkg/runtime/collections/keyed_test.go Normal file
View File

@ -0,0 +1,85 @@
package collections_test
import (
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func objectIterator(obj *values.Object) collections.Iterator {
return collections.NewDefaultKeyedIterator(obj)
}
func TestObjectIterator(t *testing.T) {
Convey("Should iterate over a map", t, func() {
m := values.NewObjectWith(
values.NewObjectProperty("one", values.NewInt(1)),
values.NewObjectProperty("two", values.NewInt(2)),
values.NewObjectProperty("three", values.NewInt(3)),
values.NewObjectProperty("four", values.NewInt(4)),
values.NewObjectProperty("five", values.NewInt(5)),
)
iter := objectIterator(m)
res := make([]core.Value, 0, m.Length())
for iter.HasNext() {
item, key, err := next(iter)
So(err, ShouldBeNil)
expected, exists := m.Get(values.NewString(key.String()))
So(bool(exists), ShouldBeTrue)
So(expected, ShouldEqual, item)
res = append(res, item)
}
So(res, ShouldHaveLength, m.Length())
})
Convey("Should return an error when exhausted", t, func() {
m := values.NewObjectWith(
values.NewObjectProperty("one", values.NewInt(1)),
values.NewObjectProperty("two", values.NewInt(2)),
values.NewObjectProperty("three", values.NewInt(3)),
values.NewObjectProperty("four", values.NewInt(4)),
values.NewObjectProperty("five", values.NewInt(5)),
)
iter := objectIterator(m)
res := make([]core.Value, 0, m.Length())
for iter.HasNext() {
item, _, err := next(iter)
So(err, ShouldBeNil)
res = append(res, item)
}
item, _, err := next(iter)
So(item, ShouldBeNil)
So(err, ShouldBeError)
})
Convey("Should NOT iterate over a empty map", t, func() {
m := values.NewObject()
iter := objectIterator(m)
var iterated bool
for iter.HasNext() {
iterated = true
}
So(iterated, ShouldBeFalse)
})
}

4
pkg/runtime/collections/limit.go
View File

@ -30,14 +30,14 @@ func (i *LimitIterator) HasNext() bool {
return i.counter() < i.count
}
func (i *LimitIterator) Next() (core.Value, core.Value, error) {
func (i *LimitIterator) Next() (DataSet, error) {
if i.counter() <= i.count {
i.currCount++
return i.src.Next()
}
return nil, nil, ErrExhausted
return nil, ErrExhausted
}
func (i *LimitIterator) counter() int {

20
pkg/runtime/collections/limit_test.go
View File

@ -19,7 +19,7 @@ func TestLimit(t *testing.T) {
}
src, err := collections.NewLimitIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
1,
0,
)
@ -29,7 +29,7 @@ func TestLimit(t *testing.T) {
res := make([]core.Value, 0, len(arr))
for src.HasNext() {
item, _, err := src.Next()
item, _, err := next(src)
So(err, ShouldBeNil)
@ -49,7 +49,7 @@ func TestLimit(t *testing.T) {
}
src, err := collections.NewLimitIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
2,
0,
)
@ -59,7 +59,7 @@ func TestLimit(t *testing.T) {
res := make([]core.Value, 0, len(arr))
for src.HasNext() {
item, _, err := src.Next()
item, _, err := next(src)
So(err, ShouldBeNil)
@ -80,7 +80,7 @@ func TestLimit(t *testing.T) {
offset := 2
src, err := collections.NewLimitIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
2,
offset,
)
@ -90,7 +90,7 @@ func TestLimit(t *testing.T) {
res := make([]core.Value, 0, len(arr))
for src.HasNext() {
item, _, err := src.Next()
item, _, err := next(src)
So(err, ShouldBeNil)
@ -118,7 +118,7 @@ func TestLimit(t *testing.T) {
offset := 3
src, err := collections.NewLimitIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
2,
offset,
)
@ -128,7 +128,7 @@ func TestLimit(t *testing.T) {
res := make([]core.Value, 0, len(arr))
for src.HasNext() {
item, _, err := src.Next()
item, _, err := next(src)
So(err, ShouldBeNil)
@ -156,7 +156,7 @@ func TestLimit(t *testing.T) {
offset := 4
src, err := collections.NewLimitIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
2,
offset,
)
@ -166,7 +166,7 @@ func TestLimit(t *testing.T) {
res := make([]core.Value, 0, len(arr))
for src.HasNext() {
item, _, err := src.Next()
item, _, err := next(src)
So(err, ShouldBeNil)

60
pkg/runtime/collections/map.go Normal file
View File

@ -0,0 +1,60 @@
package collections
import (
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type MapIterator struct {
valVar string
keyVar string
values map[string]core.Value
keys []string
pos int
}
func NewMapIterator(
valVar,
keyVar string,
input map[string]core.Value,
) Iterator {
return &MapIterator{valVar, keyVar, input, nil, 0}
}
func NewDefaultMapIterator(
input map[string]core.Value,
) Iterator {
return &MapIterator{DefaultValueVar, DefaultKeyVar, input, nil, 0}
}
func (iterator *MapIterator) HasNext() bool {
// lazy initialization
if iterator.keys == nil {
keys := make([]string, len(iterator.values))
i := 0
for k := range iterator.values {
keys[i] = k
i++
}
iterator.keys = keys
}
return len(iterator.keys) > iterator.pos
}
func (iterator *MapIterator) Next() (DataSet, error) {
if len(iterator.keys) > iterator.pos {
key := iterator.keys[iterator.pos]
val := iterator.values[key]
iterator.pos++
return DataSet{
iterator.valVar: val,
iterator.keyVar: values.NewString(key),
}, nil
}
return nil, ErrExhausted
}

85
pkg/runtime/collections/map_test.go Normal file
View File

@ -0,0 +1,85 @@
package collections_test
import (
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func mapIterator(m map[string]core.Value) collections.Iterator {
return collections.NewDefaultMapIterator(m)
}
func TestMapIterator(t *testing.T) {
Convey("Should iterate over a map", t, func() {
m := map[string]core.Value{
"one": values.NewInt(1),
"two": values.NewInt(2),
"three": values.NewInt(3),
"four": values.NewInt(4),
"five": values.NewInt(5),
}
iter := mapIterator(m)
res := make([]core.Value, 0, len(m))
for iter.HasNext() {
item, key, err := next(iter)
So(err, ShouldBeNil)
expected, exists := m[key.String()]
So(exists, ShouldBeTrue)
So(expected, ShouldEqual, item)
res = append(res, item)
}
So(res, ShouldHaveLength, len(m))
})
Convey("Should return an error when exhausted", t, func() {
m := map[string]core.Value{
"one": values.NewInt(1),
"two": values.NewInt(2),
"three": values.NewInt(3),
"four": values.NewInt(4),
"five": values.NewInt(5),
}
iter := mapIterator(m)
res := make([]core.Value, 0, len(m))
for iter.HasNext() {
item, _, err := next(iter)
So(err, ShouldBeNil)
res = append(res, item)
}
item, _, err := next(iter)
So(item, ShouldBeNil)
So(err, ShouldBeError)
})
Convey("Should NOT iterate over a empty map", t, func() {
m := make(map[string]core.Value)
iter := mapIterator(m)
var iterated bool
for iter.HasNext() {
iterated = true
}
So(iterated, ShouldBeFalse)
})
}

13
pkg/runtime/collections/noop.go Normal file
View File

@ -0,0 +1,13 @@
package collections
type noopIterator struct{}
var NoopIterator = &noopIterator{}
func (iterator *noopIterator) HasNext() bool {
return false
}
func (iterator *noopIterator) Next() (DataSet, error) {
return nil, ErrExhausted
}

21
pkg/runtime/collections/reducer.go
View File

@ -1,21 +0,0 @@
package collections
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
)
type (
Reducer interface {
Reduce(collection core.Value, value core.Value) (core.Value, error)
}
Reducible interface {
Reduce() Reducer
}
ReducibleExpression interface {
core.Expression
Reduce(ctx context.Context, scope *core.Scope) (Reducer, error)
}
)

44
pkg/runtime/collections/slice.go Normal file
View File

@ -0,0 +1,44 @@
package collections
import (
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type SliceIterator struct {
valVar string
keyVar string
values []core.Value
pos int
}
func NewSliceIterator(
valVar,
keyVar string,
input []core.Value,
) Iterator {
return &SliceIterator{valVar, keyVar, input, 0}
}
func NewDefaultSliceIterator(input []core.Value) Iterator {
return NewSliceIterator(DefaultValueVar, DefaultKeyVar, input)
}
func (iterator *SliceIterator) HasNext() bool {
return len(iterator.values) > iterator.pos
}
func (iterator *SliceIterator) Next() (DataSet, error) {
if len(iterator.values) > iterator.pos {
idx := iterator.pos
val := iterator.values[idx]
iterator.pos++
return DataSet{
iterator.valVar: val,
iterator.keyVar: values.NewInt(idx),
}, nil
}
return nil, ErrExhausted
}

114
pkg/runtime/collections/slice_test.go Normal file
View File

@ -0,0 +1,114 @@
package collections_test
import (
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func sliceIterator(value []core.Value) collections.Iterator {
return collections.NewDefaultSliceIterator(value)
}
func TestSliceIterator(t *testing.T) {
Convey("Should iterate over a slice", t, func() {
arr := []core.Value{
values.NewInt(1),
values.NewInt(2),
values.NewInt(3),
values.NewInt(4),
values.NewInt(5),
}
iter := sliceIterator(arr)
res := make([]core.Value, 0, len(arr))
pos := 0
for iter.HasNext() {
item, key, err := next(iter)
So(err, ShouldBeNil)
So(key.Unwrap(), ShouldEqual, pos)
res = append(res, item)
pos += 1
}
So(res, ShouldHaveLength, len(arr))
})
Convey("Should iterate over a slice in the same order", t, func() {
arr := []core.Value{
values.NewInt(1),
values.NewInt(2),
values.NewInt(3),
values.NewInt(4),
values.NewInt(5),
}
iter := sliceIterator(arr)
res := make([]core.Value, 0, len(arr))
for iter.HasNext() {
item, _, err := next(iter)
So(err, ShouldBeNil)
res = append(res, item)
}
for idx := range arr {
expected := arr[idx]
actual := res[idx]
So(actual, ShouldEqual, expected)
}
})
Convey("Should return an error when exhausted", t, func() {
arr := []core.Value{
values.NewInt(1),
values.NewInt(2),
values.NewInt(3),
values.NewInt(4),
values.NewInt(5),
}
iter := sliceIterator(arr)
res := make([]core.Value, 0, len(arr))
for iter.HasNext() {
item, _, err := next(iter)
So(err, ShouldBeNil)
res = append(res, item)
}
item, _, err := next(iter)
So(item, ShouldBeNil)
So(err, ShouldBeError)
})
Convey("Should NOT iterate over an empty slice", t, func() {
arr := []core.Value{}
iter := sliceIterator(arr)
var iterated bool
for iter.HasNext() {
iterated = true
}
So(iterated, ShouldBeFalse)
})
}

48
pkg/runtime/collections/sort.go
View File

@ -10,7 +10,7 @@ import (
type (
SortDirection int
Comparator func(first core.Value, second core.Value) (int, error)
Comparator func(first DataSet, second DataSet) (int, error)
Sorter struct {
fn Comparator
@ -21,7 +21,9 @@ type (
src Iterator
sorters []*Sorter
ready bool
values *SliceIterator
values []DataSet
err error
pos int
}
)
@ -71,33 +73,53 @@ func NewSortIterator(
return nil, errors.Wrap(core.ErrMissedArgument, "comparator")
}
return &SortIterator{src, comparators, false, nil}, nil
return &SortIterator{
src,
comparators,
false,
nil, nil,
0,
}, nil
}
func (iterator *SortIterator) HasNext() bool {
// we need to initialize the iterator
if iterator.ready == false {
iterator.ready = true
values, err := iterator.sort()
sorted, err := iterator.sort()
if err != nil {
// set to true because we do not want to initialize next time anymore
iterator.values = NewSliceIterator(make([]core.Value, 0, 0))
// dataSet to true because we do not want to initialize next time anymore
iterator.values = nil
iterator.err = err
return false
// if there is an error, we need to show it during Next()
return true
}
iterator.values = values
iterator.values = sorted
}
return iterator.values.HasNext()
return iterator.values != nil && len(iterator.values) > iterator.pos
}
func (iterator *SortIterator) Next() (core.Value, core.Value, error) {
return iterator.values.Next()
func (iterator *SortIterator) Next() (DataSet, error) {
if iterator.err != nil {
return nil, iterator.err
}
if len(iterator.values) > iterator.pos {
idx := iterator.pos
val := iterator.values[idx]
iterator.pos++
return val, nil
}
return nil, ErrExhausted
}
func (iterator *SortIterator) sort() (*SliceIterator, error) {
func (iterator *SortIterator) sort() ([]DataSet, error) {
res, err := ToSlice(iterator.src)
if err != nil {
@ -147,5 +169,5 @@ func (iterator *SortIterator) sort() (*SliceIterator, error) {
return nil, failure
}
return NewSliceIterator(res), nil
return res, nil
}

102
pkg/runtime/collections/sort_test.go
View File

@ -9,6 +9,20 @@ import (
"testing"
)
func toValues(sets []collections.DataSet) []core.Value {
res := make([]core.Value, 0, len(sets))
for _, ds := range sets {
res = append(res, ds.Get(collections.DefaultValueVar))
}
return res
}
func toArrayOfValues(sets []collections.DataSet) *values.Array {
return values.NewArrayWith(toValues(sets)...)
}
func TestSort(t *testing.T) {
Convey("Should sort asc", t, func() {
arr := []core.Value{
@ -20,14 +34,14 @@ func TestSort(t *testing.T) {
}
s, _ := collections.NewSorter(
func(first core.Value, second core.Value) (int, error) {
return first.Compare(second), nil
func(first collections.DataSet, second collections.DataSet) (int, error) {
return first.Get(collections.DefaultValueVar).Compare(second.Get(collections.DefaultValueVar)), nil
},
collections.SortDirectionAsc,
)
src, err := collections.NewSortIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
s,
)
@ -40,7 +54,7 @@ func TestSort(t *testing.T) {
numbers := []int{1, 2, 3, 4, 5}
for idx, num := range numbers {
So(res[idx].Unwrap(), ShouldEqual, num)
So(res[idx].Get(collections.DefaultValueVar).Unwrap(), ShouldEqual, num)
}
})
@ -54,14 +68,14 @@ func TestSort(t *testing.T) {
}
s, _ := collections.NewSorter(
func(first core.Value, second core.Value) (int, error) {
return first.Compare(second), nil
func(first collections.DataSet, second collections.DataSet) (int, error) {
return first.Get(collections.DefaultValueVar).Compare(second.Get(collections.DefaultValueVar)), nil
},
collections.SortDirectionDesc,
)
src, err := collections.NewSortIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
s,
)
@ -74,7 +88,7 @@ func TestSort(t *testing.T) {
numbers := []int{5, 4, 3, 2, 1}
for idx, num := range numbers {
So(res[idx].Unwrap(), ShouldEqual, num)
So(res[idx].Get(collections.DefaultValueVar).Unwrap(), ShouldEqual, num)
}
})
@ -100,9 +114,9 @@ func TestSort(t *testing.T) {
}
s1, _ := collections.NewSorter(
func(first core.Value, second core.Value) (int, error) {
o1, _ := first.(*values.Object).Get("one")
o2, _ := second.(*values.Object).Get("one")
func(first collections.DataSet, second collections.DataSet) (int, error) {
o1, _ := first.Get(collections.DefaultValueVar).(*values.Object).Get("one")
o2, _ := second.Get(collections.DefaultValueVar).(*values.Object).Get("one")
return o1.Compare(o2), nil
},
@ -110,9 +124,9 @@ func TestSort(t *testing.T) {
)
s2, _ := collections.NewSorter(
func(first core.Value, second core.Value) (int, error) {
o1, _ := first.(*values.Object).Get("two")
o2, _ := second.(*values.Object).Get("two")
func(first collections.DataSet, second collections.DataSet) (int, error) {
o1, _ := first.Get(collections.DefaultValueVar).(*values.Object).Get("two")
o2, _ := second.Get(collections.DefaultValueVar).(*values.Object).Get("two")
return o1.Compare(o2), nil
},
@ -120,17 +134,19 @@ func TestSort(t *testing.T) {
)
src, err := collections.NewSortIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
s1,
s2,
)
So(err, ShouldBeNil)
res, err := collections.ToSlice(src)
sets, err := collections.ToSlice(src)
So(err, ShouldBeNil)
res := toValues(sets)
j, _ := json.Marshal(res)
So(string(j), ShouldEqual, `[{"one":1,"two":1},{"one":1,"two":2},{"one":2,"two":1},{"one":2,"two":2},{"one":3,"two":1},{"one":3,"two":2},{"one":4,"two":1},{"one":4,"two":2}]`)
@ -158,9 +174,9 @@ func TestSort(t *testing.T) {
}
s1, _ := collections.NewSorter(
func(first core.Value, second core.Value) (int, error) {
o1, _ := first.(*values.Object).Get("one")
o2, _ := second.(*values.Object).Get("one")
func(first collections.DataSet, second collections.DataSet) (int, error) {
o1, _ := first.Get(collections.DefaultValueVar).(*values.Object).Get("one")
o2, _ := second.Get(collections.DefaultValueVar).(*values.Object).Get("one")
return o1.Compare(o2), nil
},
@ -168,9 +184,9 @@ func TestSort(t *testing.T) {
)
s2, _ := collections.NewSorter(
func(first core.Value, second core.Value) (int, error) {
o1, _ := first.(*values.Object).Get("two")
o2, _ := second.(*values.Object).Get("two")
func(first collections.DataSet, second collections.DataSet) (int, error) {
o1, _ := first.Get(collections.DefaultValueVar).(*values.Object).Get("two")
o2, _ := second.Get(collections.DefaultValueVar).(*values.Object).Get("two")
return o1.Compare(o2), nil
},
@ -178,17 +194,19 @@ func TestSort(t *testing.T) {
)
src, err := collections.NewSortIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
s1,
s2,
)
So(err, ShouldBeNil)
res, err := collections.ToSlice(src)
sets, err := collections.ToSlice(src)
So(err, ShouldBeNil)
res := toValues(sets)
j, _ := json.Marshal(res)
So(string(j), ShouldEqual, `[{"one":4,"two":2},{"one":4,"two":1},{"one":3,"two":2},{"one":3,"two":1},{"one":2,"two":2},{"one":2,"two":1},{"one":1,"two":2},{"one":1,"two":1}]`)
@ -216,9 +234,9 @@ func TestSort(t *testing.T) {
}
s1, _ := collections.NewSorter(
func(first core.Value, second core.Value) (int, error) {
o1, _ := first.(*values.Object).Get("one")
o2, _ := second.(*values.Object).Get("one")
func(first collections.DataSet, second collections.DataSet) (int, error) {
o1, _ := first.Get(collections.DefaultValueVar).(*values.Object).Get("one")
o2, _ := second.Get(collections.DefaultValueVar).(*values.Object).Get("one")
return o1.Compare(o2), nil
},
@ -226,9 +244,9 @@ func TestSort(t *testing.T) {
)
s2, _ := collections.NewSorter(
func(first core.Value, second core.Value) (int, error) {
o1, _ := first.(*values.Object).Get("two")
o2, _ := second.(*values.Object).Get("two")
func(first collections.DataSet, second collections.DataSet) (int, error) {
o1, _ := first.Get(collections.DefaultValueVar).(*values.Object).Get("two")
o2, _ := second.Get(collections.DefaultValueVar).(*values.Object).Get("two")
return o1.Compare(o2), nil
},
@ -236,17 +254,19 @@ func TestSort(t *testing.T) {
)
src, err := collections.NewSortIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
s1,
s2,
)
So(err, ShouldBeNil)
res, err := collections.ToSlice(src)
sets, err := collections.ToSlice(src)
So(err, ShouldBeNil)
res := toValues(sets)
j, _ := json.Marshal(res)
So(string(j), ShouldEqual, `[{"one":1,"two":2},{"one":1,"two":1},{"one":2,"two":2},{"one":2,"two":1},{"one":3,"two":2},{"one":3,"two":1},{"one":4,"two":2},{"one":4,"two":1}]`)
@ -274,9 +294,9 @@ func TestSort(t *testing.T) {
}
s1, _ := collections.NewSorter(
func(first core.Value, second core.Value) (int, error) {
o1, _ := first.(*values.Object).Get("one")
o2, _ := second.(*values.Object).Get("one")
func(first collections.DataSet, second collections.DataSet) (int, error) {
o1, _ := first.Get(collections.DefaultValueVar).(*values.Object).Get("one")
o2, _ := second.Get(collections.DefaultValueVar).(*values.Object).Get("one")
return o1.Compare(o2), nil
},
@ -284,9 +304,9 @@ func TestSort(t *testing.T) {
)
s2, _ := collections.NewSorter(
func(first core.Value, second core.Value) (int, error) {
o1, _ := first.(*values.Object).Get("two")
o2, _ := second.(*values.Object).Get("two")
func(first collections.DataSet, second collections.DataSet) (int, error) {
o1, _ := first.Get(collections.DefaultValueVar).(*values.Object).Get("two")
o2, _ := second.Get(collections.DefaultValueVar).(*values.Object).Get("two")
return o1.Compare(o2), nil
},
@ -294,17 +314,19 @@ func TestSort(t *testing.T) {
)
src, err := collections.NewSortIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
s1,
s2,
)
So(err, ShouldBeNil)
res, err := collections.ToSlice(src)
sets, err := collections.ToSlice(src)
So(err, ShouldBeNil)
res := toValues(sets)
j, _ := json.Marshal(res)
So(string(j), ShouldEqual, `[{"one":4,"two":1},{"one":4,"two":2},{"one":3,"two":1},{"one":3,"two":2},{"one":2,"two":1},{"one":2,"two":2},{"one":1,"two":1},{"one":1,"two":2}]`)

23
pkg/runtime/collections/unique.go
View File

@ -8,13 +8,13 @@ type (
UniqueIterator struct {
src Iterator
hashes map[uint64]bool
value core.Value
key core.Value
hashKey string
dataSet DataSet
err error
}
)
func NewUniqueIterator(src Iterator) (*UniqueIterator, error) {
func NewUniqueIterator(src Iterator, hashKey string) (*UniqueIterator, error) {
if src == nil {
return nil, core.Error(core.ErrMissedArgument, "source")
}
@ -22,6 +22,7 @@ func NewUniqueIterator(src Iterator) (*UniqueIterator, error) {
return &UniqueIterator{
src: src,
hashes: make(map[uint64]bool),
hashKey: hashKey,
}, nil
}
@ -36,26 +37,25 @@ func (iterator *UniqueIterator) HasNext() bool {
return false
}
if !core.IsNil(iterator.value) {
if iterator.dataSet != nil {
return true
}
return false
}
func (iterator *UniqueIterator) Next() (core.Value, core.Value, error) {
return iterator.value, iterator.key, iterator.err
func (iterator *UniqueIterator) Next() (DataSet, error) {
return iterator.dataSet, iterator.err
}
func (iterator *UniqueIterator) doNext() {
// reset state
iterator.err = nil
iterator.value = nil
iterator.key = nil
iterator.dataSet = nil
// iterate over source until we find a non-unique item
for iterator.src.HasNext() {
val, key, err := iterator.src.Next()
ds, err := iterator.src.Next()
if err != nil {
iterator.err = err
@ -63,7 +63,7 @@ func (iterator *UniqueIterator) doNext() {
return
}
h := val.Hash()
h := ds.Get(iterator.hashKey).Hash()
_, exists := iterator.hashes[h]
@ -72,8 +72,7 @@ func (iterator *UniqueIterator) doNext() {
}
iterator.hashes[h] = true
iterator.key = key
iterator.value = val
iterator.dataSet = ds
return
}

21
pkg/runtime/collections/unique_test.go
View File

@ -24,15 +24,18 @@ func TestUniqueIterator(t *testing.T) {
}
iter, err := collections.NewUniqueIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
collections.DefaultValueVar,
)
So(err, ShouldBeNil)
res, err := collections.ToArray(iter)
sets, err := collections.ToSlice(iter)
So(err, ShouldBeNil)
res := toArrayOfValues(sets)
So(res.String(), ShouldEqual, `[1,2,3,4,5,6]`)
})
@ -47,15 +50,18 @@ func TestUniqueIterator(t *testing.T) {
}
iter, err := collections.NewUniqueIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
collections.DefaultValueVar,
)
So(err, ShouldBeNil)
res, err := collections.ToArray(iter)
sets, err := collections.ToSlice(iter)
So(err, ShouldBeNil)
res := toArrayOfValues(sets)
So(res.String(), ShouldEqual, `[1]`)
})
@ -75,15 +81,18 @@ func TestUniqueIterator(t *testing.T) {
}
iter, err := collections.NewUniqueIterator(
collections.NewSliceIterator(arr),
sliceIterator(arr),
collections.DefaultValueVar,
)
So(err, ShouldBeNil)
res, err := collections.ToArray(iter)
sets, err := collections.ToSlice(iter)
So(err, ShouldBeNil)
res := toArrayOfValues(sets)
So(res.String(), ShouldEqual, `["a","b","c","d","e","f"]`)
})
}

4
pkg/runtime/core/errors.go
View File

@ -50,6 +50,10 @@ func Error(err error, msg string) error {
return errors.Errorf("%s: %s", err.Error(), msg)
}
func Errorf(err error, format string, args ...interface{}) error {
return errors.Errorf("%s: %s", err.Error(), fmt.Sprintf(format, args...))
}
func Errors(err ...error) error {
message := ""

33
pkg/runtime/expressions/clauses/clause.go
View File

@ -1,33 +0,0 @@
package clauses
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type baseClause struct {
src core.SourceMap
dataSource collections.IterableExpression
}
func (clause *baseClause) Iterate(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
src, err := clause.dataSource.Iterate(ctx, scope)
if err != nil {
return nil, err
}
return src, nil
}
func (clause *baseClause) Exec(ctx context.Context, scope *core.Scope) (core.Value, error) {
iterator, err := clause.Iterate(ctx, scope)
if err != nil {
return values.None, err
}
return collections.ToArray(iterator)
}

170
pkg/runtime/expressions/clauses/collect.go Normal file
View File

@ -0,0 +1,170 @@
package clauses
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
)
type (
Collect struct {
group *CollectGroup
count *CollectCount
aggregate *CollectAggregate
}
CollectGroup struct {
selectors []*CollectSelector
projection *CollectProjection
count *CollectCount
aggregate *CollectAggregate
}
CollectCount struct {
variable string
}
CollectProjection struct {
selector *CollectSelector
}
CollectAggregate struct {
selectors []*CollectAggregateSelector
}
CollectClause struct {
src core.SourceMap
dataSource collections.Iterable
params *Collect
}
)
func NewCollect(
selectors []*CollectSelector,
projection *CollectProjection,
count *CollectCount,
aggregate *CollectAggregate,
) (*Collect, error) {
collect := new(Collect)
// grouping
if selectors != nil {
collect.group = new(CollectGroup)
collect.group.selectors = selectors
if projection == nil && count == nil && aggregate == nil {
return collect, nil
}
if projection != nil && count == nil && aggregate == nil {
collect.group.projection = projection
} else if projection == nil && count != nil && aggregate == nil {
collect.group.count = count
} else if projection == nil && count == nil && aggregate != nil {
collect.group.aggregate = aggregate
} else {
return nil, core.Error(core.ErrInvalidOperation, "projection, count and aggregate cannot be used together")
}
return collect, nil
}
if count == nil && aggregate != nil {
collect.aggregate = aggregate
} else if count != nil && aggregate == nil {
collect.count = count
} else {
return nil, core.Error(core.ErrInvalidOperation, "count and aggregate cannot be used together")
}
return collect, nil
}
func NewCollectCount(variable string) (*CollectCount, error) {
if variable == "" {
return nil, core.Error(core.ErrMissedArgument, "count variable")
}
return &CollectCount{variable}, nil
}
func NewCollectProjection(selector *CollectSelector) (*CollectProjection, error) {
if selector == nil {
return nil, core.Error(core.ErrMissedArgument, "projection selector")
}
return &CollectProjection{selector}, nil
}
func NewCollectAggregate(selectors []*CollectAggregateSelector) (*CollectAggregate, error) {
if selectors == nil {
return nil, core.Error(core.ErrMissedArgument, "aggregate selectors")
}
return &CollectAggregate{selectors}, nil
}
func NewCollectClause(
src core.SourceMap,
dataSource collections.Iterable,
params *Collect,
) (collections.Iterable, error) {
if dataSource == nil {
return nil, core.Error(core.ErrMissedArgument, "dataSource source")
}
return &CollectClause{src, dataSource, params}, nil
}
func (clause *CollectClause) Variables() collections.Variables {
vars := make(collections.Variables, 0, 10)
if clause.params.group != nil {
grouping := clause.params.group
for _, selector := range grouping.selectors {
vars = append(vars, selector.variable)
}
if grouping.projection != nil {
vars = append(vars, clause.params.group.projection.selector.variable)
}
if grouping.count != nil {
vars = append(vars, clause.params.group.count.variable)
}
if grouping.aggregate != nil {
for _, selector := range grouping.aggregate.selectors {
vars = append(vars, selector.variable)
}
}
} else if clause.params.count != nil {
vars = append(vars, clause.params.count.variable)
} else if clause.params.aggregate != nil {
for _, selector := range clause.params.aggregate.selectors {
vars = append(vars, selector.variable)
}
}
return vars
}
func (clause *CollectClause) Iterate(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
srcIterator, err := clause.dataSource.Iterate(ctx, scope)
if err != nil {
return nil, core.SourceError(clause.src, err)
}
srcVariables := clause.dataSource.Variables()
return NewCollectIterator(
clause.src,
clause.params,
srcIterator,
srcVariables,
ctx,
scope,
)
}

412
pkg/runtime/expressions/clauses/collect_iterator.go Normal file
View File

@ -0,0 +1,412 @@
package clauses
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type CollectIterator struct {
ready bool
values []collections.DataSet
pos int
src core.SourceMap
params *Collect
dataSource collections.Iterator
variables collections.Variables
ctx context.Context
scope *core.Scope
}
//revive:disable context-as-argument
func NewCollectIterator(
src core.SourceMap,
params *Collect,
dataSource collections.Iterator,
variables collections.Variables,
ctx context.Context,
scope *core.Scope,
) (*CollectIterator, error) {
if params.group != nil {
if params.group.selectors != nil {
var err error
sorters := make([]*collections.Sorter, len(params.group.selectors))
for i, selector := range params.group.selectors {
sorter, err := newGroupSorter(ctx, scope, variables, selector)
if err != nil {
return nil, err
}
sorters[i] = sorter
}
dataSource, err = collections.NewSortIterator(dataSource, sorters...)
if err != nil {
return nil, err
}
}
if params.group.count != nil && params.group.projection != nil {
return nil, core.Error(core.ErrInvalidArgumentNumber, "counter and projection cannot be used together")
}
}
return &CollectIterator{
false,
nil,
0,
src,
params,
dataSource,
variables,
ctx,
scope,
}, nil
}
func newGroupSorter(ctx context.Context, scope *core.Scope, variables collections.Variables, selector *CollectSelector) (*collections.Sorter, error) {
return collections.NewSorter(func(first collections.DataSet, second collections.DataSet) (int, error) {
scope1 := scope.Fork()
first.Apply(scope1, variables)
f, err := selector.expression.Exec(ctx, scope1)
if err != nil {
return -1, err
}
scope2 := scope.Fork()
second.Apply(scope2, variables)
s, err := selector.expression.Exec(ctx, scope2)
if err != nil {
return -1, err
}
return f.Compare(s), nil
}, collections.SortDirectionAsc)
}
func (iterator *CollectIterator) HasNext() bool {
if !iterator.ready {
iterator.ready = true
groups, err := iterator.init()
if err != nil {
iterator.values = nil
return false
}
iterator.values = groups
}
return len(iterator.values) > iterator.pos
}
func (iterator *CollectIterator) Next() (collections.DataSet, error) {
if len(iterator.values) > iterator.pos {
val := iterator.values[iterator.pos]
iterator.pos++
return val, nil
}
return nil, collections.ErrExhausted
}
func (iterator *CollectIterator) init() ([]collections.DataSet, error) {
if iterator.params.group != nil {
return iterator.group()
}
if iterator.params.count != nil {
return iterator.count()
}
if iterator.params.aggregate != nil {
return iterator.aggregate()
}
return nil, core.ErrInvalidOperation
}
func (iterator *CollectIterator) group() ([]collections.DataSet, error) {
// TODO: honestly, this code is ugly. it needs to be refactored in more chained way with much less if statements
// slice of groups
collected := make([]collections.DataSet, 0, 10)
// hash table of unique values
// key is a DataSet hash
// value is its index in result slice (collected)
hashTable := make(map[uint64]int)
ctx := iterator.ctx
groupSelectors := iterator.params.group.selectors
proj := iterator.params.group.projection
count := iterator.params.group.count
aggr := iterator.params.group.aggregate
// iterating over underlying data source
for iterator.dataSource.HasNext() {
set, err := iterator.dataSource.Next()
if err != nil {
return nil, err
}
if len(set) == 0 {
continue
}
// creating a new scope for all further operations
childScope := iterator.scope.Fork()
// populate the new scope with results from an underlying source and its exposed variables
if err := set.Apply(childScope, iterator.variables); err != nil {
return nil, err
}
// this data set represents a data of a given iteration with values retrieved by selectors
ds := collections.NewDataSet()
// iterate over each selector for a current data set
for _, selector := range groupSelectors {
// execute a selector and get a value
// e.g. COLLECT age = u.age
value, err := selector.expression.Exec(ctx, childScope)
if err != nil {
return nil, err
}
ds.Set(selector.variable, value)
}
// it important to get hash value before projection and counting
// otherwise hash value will be inaccurate
h := ds.Hash()
_, exists := hashTable[h]
if !exists {
collected = append(collected, ds)
hashTable[h] = len(collected) - 1
if proj != nil {
// create a new variable for keeping projection
ds.Set(proj.selector.variable, values.NewArray(10))
} else if count != nil {
// create a new variable for keeping counter
ds.Set(count.variable, values.ZeroInt)
} else if aggr != nil {
// create a new variable for keeping aggregated values
for _, selector := range aggr.selectors {
arr := values.NewArray(len(selector.aggregators))
for range selector.aggregators {
arr.Push(values.None)
}
ds.Set(selector.variable, arr)
}
}
}
if proj != nil {
idx := hashTable[h]
ds := collected[idx]
groupValue := ds.Get(proj.selector.variable)
arr, ok := groupValue.(*values.Array)
if !ok {
return nil, core.TypeError(groupValue.Type(), core.IntType)
}
value, err := proj.selector.expression.Exec(ctx, childScope)
if err != nil {
return nil, err
}
arr.Push(value)
} else if count != nil {
idx := hashTable[h]
ds := collected[idx]
groupValue := ds.Get(count.variable)
counter, ok := groupValue.(values.Int)
if !ok {
return nil, core.TypeError(groupValue.Type(), core.IntType)
}
groupValue = counter + 1
// set a new value
ds.Set(count.variable, groupValue)
} else if aggr != nil {
idx := hashTable[h]
ds := collected[idx]
// iterate over each selector for a current data set
for _, selector := range aggr.selectors {
vv := ds.Get(selector.variable).(*values.Array)
// execute a selector and get a value
// e.g. AGGREGATE age = CONCAT(u.age, u.dob)
// u.age and u.dob get executed
for idx, exp := range selector.aggregators {
arg, err := exp.Exec(ctx, childScope)
if err != nil {
return nil, err
}
var args *values.Array
idx := values.NewInt(idx)
if vv.Get(idx) == values.None {
args = values.NewArray(10)
vv.Set(idx, args)
} else {
args = vv.Get(idx).(*values.Array)
}
args.Push(arg)
}
}
}
}
if aggr != nil {
for _, ds := range collected {
for _, selector := range aggr.selectors {
arr := ds[selector.variable].(*values.Array)
matrix := make([]core.Value, arr.Length())
arr.ForEach(func(value core.Value, idx int) bool {
matrix[idx] = value
return true
})
reduced, err := selector.reducer(ctx, matrix...)
if err != nil {
return nil, err
}
// replace value with calculated one
ds.Set(selector.variable, reduced)
}
}
}
return collected, nil
}
func (iterator *CollectIterator) count() ([]collections.DataSet, error) {
var counter int
// iterating over underlying data source
for iterator.dataSource.HasNext() {
_, err := iterator.dataSource.Next()
if err != nil {
return nil, err
}
counter++
}
return []collections.DataSet{
{
iterator.params.count.variable: values.NewInt(counter),
},
}, nil
}
func (iterator *CollectIterator) aggregate() ([]collections.DataSet, error) {
ds := collections.NewDataSet()
// matrix of aggregated expressions
// string key of the map is a selector variable
// value of the map is a matrix of arguments
// e.g. x = CONCAT(arg1, arg2, argN...)
// x is a string key where a nested array is an array of all values of argN expressions
aggregated := make(map[string][]core.Value)
ctx := iterator.ctx
selectors := iterator.params.aggregate.selectors
// iterating over underlying data source
for iterator.dataSource.HasNext() {
set, err := iterator.dataSource.Next()
if err != nil {
return nil, err
}
if len(set) == 0 {
continue
}
// creating a new scope for all further operations
childScope := iterator.scope.Fork()
// populate the new scope with results from an underlying source and its exposed variables
if err := set.Apply(childScope, iterator.variables); err != nil {
return nil, err
}
// iterate over each selector for a current data set
for _, selector := range selectors {
vv, exists := aggregated[selector.variable]
if !exists {
vv = make([]core.Value, len(selector.aggregators))
aggregated[selector.variable] = vv
}
// execute a selector and get a value
// e.g. AGGREGATE age = CONCAT(u.age, u.dob)
// u.age and u.dob get executed
for idx, exp := range selector.aggregators {
arg, err := exp.Exec(ctx, childScope)
if err != nil {
return nil, err
}
var args *values.Array
if vv[idx] == nil {
args = values.NewArray(10)
vv[idx] = args
} else {
args = vv[idx].(*values.Array)
}
args.Push(arg)
}
}
}
for _, selector := range selectors {
matrix := aggregated[selector.variable]
reduced, err := selector.reducer(ctx, matrix...)
if err != nil {
return nil, err
}
ds.Set(selector.variable, reduced)
}
return []collections.DataSet{ds}, nil
}

64
pkg/runtime/expressions/clauses/collect_selector.go Normal file
View File

@ -0,0 +1,64 @@
package clauses
import "github.com/MontFerret/ferret/pkg/runtime/core"
type (
CollectSelector struct {
variable string
expression core.Expression
}
CollectAggregateSelector struct {
variable string
aggregators []core.Expression
reducer core.Function
}
)
func NewCollectSelector(variable string, exp core.Expression) (*CollectSelector, error) {
if variable == "" {
return nil, core.Error(core.ErrMissedArgument, "selector variable")
}
if exp == nil {
return nil, core.Error(core.ErrMissedArgument, "selector reducer")
}
return &CollectSelector{variable, exp}, nil
}
func (selector *CollectSelector) Variable() string {
return selector.variable
}
func (selector *CollectSelector) Expression() core.Expression {
return selector.expression
}
func NewCollectAggregateSelector(variable string, aggr []core.Expression, reducer core.Function) (*CollectAggregateSelector, error) {
if variable == "" {
return nil, core.Error(core.ErrMissedArgument, "selector variable")
}
if reducer == nil {
return nil, core.Error(core.ErrMissedArgument, "selector reducer")
}
if aggr == nil {
return nil, core.Error(core.ErrMissedArgument, "selector aggregators")
}
return &CollectAggregateSelector{variable, aggr, reducer}, nil
}
func (selector *CollectAggregateSelector) Variable() string {
return selector.variable
}
func (selector *CollectAggregateSelector) Expression() core.Function {
return selector.reducer
}
func (selector *CollectAggregateSelector) Aggregators() []core.Expression {
return selector.aggregators
}

28
pkg/runtime/expressions/clauses/distinct.go
View File

@ -1,28 +0,0 @@
package clauses
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
)
type DistinctClause struct {
*baseClause
}
func NewDistinctClause(
src core.SourceMap,
dataSource collections.IterableExpression,
) *DistinctClause {
return &DistinctClause{&baseClause{src, dataSource}}
}
func (clause *DistinctClause) Iterate(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
src, err := clause.dataSource.Iterate(ctx, scope)
if err != nil {
return nil, err
}
return collections.NewUniqueIterator(src)
}

41
pkg/runtime/expressions/clauses/filter.go
View File

@ -8,25 +8,32 @@ import (
)
type FilterClause struct {
*baseClause
valVar string
keyVar string
src core.SourceMap
dataSource collections.Iterable
predicate core.Expression
}
func NewFilterClause(
src core.SourceMap,
dataSource collections.IterableExpression,
valVar string,
keyVar string,
dataSource collections.Iterable,
predicate core.Expression,
) *FilterClause {
return &FilterClause{
&baseClause{src, dataSource},
valVar,
keyVar,
predicate,
) (collections.Iterable, error) {
if dataSource == nil {
return nil, core.Error(core.ErrMissedArgument, "dataSource source")
}
if predicate == nil {
return nil, core.Error(core.ErrMissedArgument, "predicate")
}
return &FilterClause{
src, dataSource,
predicate,
}, nil
}
func (clause *FilterClause) Variables() collections.Variables {
return clause.dataSource.Variables()
}
func (clause *FilterClause) Iterate(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
@ -36,13 +43,15 @@ func (clause *FilterClause) Iterate(ctx context.Context, scope *core.Scope) (col
return nil, err
}
return collections.NewFilterIterator(src, func(val core.Value, key core.Value) (bool, error) {
variables := clause.dataSource.Variables()
return collections.NewFilterIterator(src, func(set collections.DataSet) (bool, error) {
innerScope := scope.Fork()
innerScope.SetVariable(clause.valVar, val)
err := set.Apply(innerScope, variables)
if clause.keyVar != "" {
innerScope.SetVariable(clause.keyVar, key)
if err != nil {
return false, core.SourceError(clause.src, err)
}
ret, err := clause.predicate.Exec(ctx, innerScope)

31
pkg/runtime/expressions/clauses/limit.go
View File

@ -7,26 +7,45 @@ import (
)
type LimitClause struct {
*baseClause
src core.SourceMap
dataSource collections.Iterable
count int
offset int
}
func NewLimitClause(
src core.SourceMap,
dataSource collections.IterableExpression,
dataSource collections.Iterable,
count int,
offset int,
) *LimitClause {
return &LimitClause{&baseClause{src, dataSource}, count, offset}
) (collections.Iterable, error) {
if dataSource == nil {
return nil, core.Error(core.ErrMissedArgument, "dataSource source")
}
return &LimitClause{src, dataSource, count, offset}, nil
}
func (clause *LimitClause) Variables() collections.Variables {
return clause.dataSource.Variables()
}
func (clause *LimitClause) Iterate(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
src, err := clause.dataSource.Iterate(ctx, scope)
if err != nil {
return nil, err
return nil, core.SourceError(clause.src, err)
}
return collections.NewLimitIterator(src, clause.count, clause.offset)
iterator, err := collections.NewLimitIterator(
src,
clause.count,
clause.offset,
)
if err != nil {
return nil, core.SourceError(clause.src, err)
}
return iterator, nil
}

34
pkg/runtime/expressions/clauses/sort.go
View File

@ -12,15 +12,15 @@ type (
direction collections.SortDirection
}
SortClause struct {
*baseClause
variableName string
src core.SourceMap
dataSource collections.Iterable
sorters []*SorterExpression
}
)
func NewSorterExpression(expression core.Expression, direction collections.SortDirection) (*SorterExpression, error) {
if expression == nil {
return nil, core.Error(core.ErrMissedArgument, "expression")
return nil, core.Error(core.ErrMissedArgument, "reducer")
}
if !collections.IsValidSortDirection(direction) {
@ -32,11 +32,22 @@ func NewSorterExpression(expression core.Expression, direction collections.SortD
func NewSortClause(
src core.SourceMap,
dataSource collections.IterableExpression,
variableName string,
dataSource collections.Iterable,
sorters ...*SorterExpression,
) *SortClause {
return &SortClause{&baseClause{src, dataSource}, variableName, sorters}
) (collections.Iterable, error) {
if dataSource == nil {
return nil, core.Error(core.ErrMissedArgument, "dataSource source")
}
if len(sorters) == 0 {
return nil, core.Error(core.ErrMissedArgument, "sorters")
}
return &SortClause{src, dataSource, sorters}, nil
}
func (clause *SortClause) Variables() collections.Variables {
return clause.dataSource.Variables()
}
func (clause *SortClause) Iterate(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
@ -47,12 +58,13 @@ func (clause *SortClause) Iterate(ctx context.Context, scope *core.Scope) (colle
}
sorters := make([]*collections.Sorter, len(clause.sorters))
variables := clause.dataSource.Variables()
// converting sorter expression into collections.Sorter
// converting sorter reducer into collections.Sorter
for idx, srt := range clause.sorters {
sorter, err := collections.NewSorter(func(first core.Value, second core.Value) (int, error) {
sorter, err := collections.NewSorter(func(first collections.DataSet, second collections.DataSet) (int, error) {
scope1 := scope.Fork()
scope1.SetVariable(clause.variableName, first)
first.Apply(scope1, variables)
f, err := srt.expression.Exec(ctx, scope1)
@ -61,7 +73,7 @@ func (clause *SortClause) Iterate(ctx context.Context, scope *core.Scope) (colle
}
scope2 := scope.Fork()
scope2.SetVariable(clause.variableName, second)
second.Apply(scope2, variables)
s, err := srt.expression.Exec(ctx, scope2)

71
pkg/runtime/expressions/data_source.go Normal file
View File

@ -0,0 +1,71 @@
package expressions
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type DataSource struct {
src core.SourceMap
variables collections.Variables
exp core.Expression
}
func NewDataSource(
src core.SourceMap,
valVariable,
keyVariable string,
exp core.Expression,
) (collections.Iterable, error) {
if exp == nil {
return nil, core.Error(core.ErrMissedArgument, "expression")
}
return &DataSource{
src,
collections.Variables{valVariable, keyVariable},
exp,
}, nil
}
func (ds *DataSource) Variables() collections.Variables {
return ds.variables
}
func (ds *DataSource) Iterate(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
data, err := ds.exp.Exec(ctx, scope)
if err != nil {
return nil, core.SourceError(ds.src, err)
}
valVar := ds.variables[0]
keyVar := ds.variables[1]
switch data.Type() {
case core.ArrayType:
return collections.NewIndexedIterator(valVar, keyVar, data.(collections.IndexedCollection)), nil
case core.ObjectType:
return collections.NewKeyedIterator(valVar, keyVar, data.(collections.KeyedCollection)), nil
case core.HTMLElementType, core.HTMLDocumentType:
return collections.NewHTMLNodeIterator(valVar, keyVar, data.(values.HTMLNode)), nil
default:
// fallback to user defined types
switch data.(type) {
case collections.KeyedCollection:
return collections.NewIndexedIterator(valVar, keyVar, data.(collections.IndexedCollection)), nil
case collections.IndexedCollection:
return collections.NewKeyedIterator(valVar, keyVar, data.(collections.KeyedCollection)), nil
default:
return nil, core.TypeError(
data.Type(),
core.ArrayType,
core.ObjectType,
core.HTMLDocumentType,
core.HTMLElementType,
)
}
}
}

98
pkg/runtime/expressions/for.go
View File

@ -11,25 +11,19 @@ import (
type ForExpression struct {
src core.SourceMap
valVar string
keyVar string
dataSource collections.IterableExpression
dataSource collections.Iterable
predicate core.Expression
distinct bool
spread bool
}
func NewForExpression(
src core.SourceMap,
valVar string,
keyVar string,
dataSource collections.IterableExpression,
dataSource collections.Iterable,
predicate core.Expression,
distinct,
spread bool,
) (*ForExpression, error) {
if valVar == "" {
return nil, errors.Wrap(core.ErrInvalidArgument, "valVar is empty")
}
if core.IsNil(dataSource) {
return nil, errors.Wrap(core.ErrMissedArgument, "missed source expression")
}
@ -40,27 +34,59 @@ func NewForExpression(
return &ForExpression{
src,
valVar, keyVar,
dataSource,
predicate,
distinct,
spread,
}, nil
}
func (e *ForExpression) AddLimit(src core.SourceMap, size, count int) {
e.dataSource = clauses.NewLimitClause(src, e.dataSource, size, count)
func (e *ForExpression) AddLimit(src core.SourceMap, size, count int) error {
limit, err := clauses.NewLimitClause(src, e.dataSource, size, count)
if err != nil {
return err
}
e.dataSource = limit
return nil
}
func (e *ForExpression) AddFilter(src core.SourceMap, exp core.Expression) {
e.dataSource = clauses.NewFilterClause(src, e.dataSource, e.valVar, e.keyVar, exp)
func (e *ForExpression) AddFilter(src core.SourceMap, exp core.Expression) error {
filter, err := clauses.NewFilterClause(src, e.dataSource, exp)
if err != nil {
return err
}
e.dataSource = filter
return nil
}
func (e *ForExpression) AddSort(src core.SourceMap, sorters ...*clauses.SorterExpression) {
e.dataSource = clauses.NewSortClause(src, e.dataSource, e.valVar, sorters...)
func (e *ForExpression) AddSort(src core.SourceMap, sorters ...*clauses.SorterExpression) error {
sort, err := clauses.NewSortClause(src, e.dataSource, sorters...)
if err != nil {
return err
}
e.dataSource = sort
return nil
}
func (e *ForExpression) AddDistinct(src core.SourceMap) {
e.dataSource = clauses.NewDistinctClause(src, e.dataSource)
func (e *ForExpression) AddCollect(src core.SourceMap, params *clauses.Collect) error {
collect, err := clauses.NewCollectClause(src, e.dataSource, params)
if err != nil {
return err
}
e.dataSource = collect
return nil
}
func (e *ForExpression) Exec(ctx context.Context, scope *core.Scope) (core.Value, error) {
@ -70,20 +96,27 @@ func (e *ForExpression) Exec(ctx context.Context, scope *core.Scope) (core.Value
return values.None, err
}
// Hash map for a check for uniqueness
var hashTable map[uint64]bool
if e.distinct {
hashTable = make(map[uint64]bool)
}
res := values.NewArray(10)
variables := e.dataSource.Variables()
for iterator.HasNext() {
val, key, err := iterator.Next()
ds, err := iterator.Next()
if err != nil {
return values.None, core.SourceError(e.src, err)
}
innerScope := scope.Fork()
innerScope.SetVariable(e.valVar, val)
if e.keyVar != "" {
innerScope.SetVariable(e.keyVar, key)
if err := ds.Apply(innerScope, variables); err != nil {
return values.None, err
}
out, err := e.predicate.Exec(ctx, innerScope)
@ -92,6 +125,24 @@ func (e *ForExpression) Exec(ctx context.Context, scope *core.Scope) (core.Value
return values.None, err
}
var add bool
// The result shouldn't be distinct
// Just add the output
if !e.distinct {
add = true
} else {
// We need to check whether the value already exists in the result set
hash := out.Hash()
_, exists := hashTable[hash]
if !exists {
hashTable[hash] = true
add = true
}
}
if add {
if !e.spread {
res.Push(out)
} else {
@ -108,6 +159,7 @@ func (e *ForExpression) Exec(ctx context.Context, scope *core.Scope) (core.Value
})
}
}
}
return res, nil
}

19
pkg/runtime/expressions/func_call.go
View File

@ -2,7 +2,6 @@ package expressions
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -25,20 +24,12 @@ func NewFunctionCallExpression(
return &FunctionCallExpression{src, fun, args}, nil
}
func (e *FunctionCallExpression) Iterate(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
value, err := e.Exec(ctx, scope)
func (e *FunctionCallExpression) Arguments() []core.Expression {
return e.args
}
if err != nil {
return nil, core.SourceError(e.src, err)
}
iter, err := collections.ToIterator(value)
if err != nil {
return nil, core.SourceError(e.src, err)
}
return iter, nil
func (e *FunctionCallExpression) Function() core.Function {
return e.fun
}
func (e *FunctionCallExpression) Exec(ctx context.Context, scope *core.Scope) (core.Value, error) {

23
pkg/runtime/expressions/literals/array.go
View File

@ -2,7 +2,6 @@ package literals
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -23,34 +22,14 @@ func (l *ArrayLiteral) Push(expression core.Expression) {
l.elements = append(l.elements, expression)
}
func (l *ArrayLiteral) Iterate(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
arr, err := l.doExec(ctx, scope)
if err != nil {
return nil, err
}
return collections.NewArrayIterator(arr), nil
}
func (l *ArrayLiteral) Exec(ctx context.Context, scope *core.Scope) (core.Value, error) {
arr, err := l.doExec(ctx, scope)
if err != nil {
return values.None, err
}
return arr, nil
}
func (l *ArrayLiteral) doExec(ctx context.Context, scope *core.Scope) (*values.Array, error) {
arr := values.NewArray(len(l.elements))
for _, el := range l.elements {
val, err := el.Exec(ctx, scope)
if err != nil {
return nil, err
return values.None, err
}
arr.Push(val)

41
pkg/runtime/expressions/literals/object.go
View File

@ -2,7 +2,6 @@ package literals
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -18,56 +17,40 @@ type (
}
)
func NewObjectPropertyAssignment(name, value core.Expression) *ObjectPropertyAssignment {
return &ObjectPropertyAssignment{name, value}
}
func NewObjectPropertyAssignment(name, value core.Expression) (*ObjectPropertyAssignment, error) {
if name == nil {
return nil, core.Error(core.ErrMissedArgument, "property name expression")
}
func NewObjectLiteral() *ObjectLiteral {
return &ObjectLiteral{make([]*ObjectPropertyAssignment, 0, 10)}
if value == nil {
return nil, core.Error(core.ErrMissedArgument, "property value expression")
}
return &ObjectPropertyAssignment{name, value}, nil
}
func NewObjectLiteralWith(props ...*ObjectPropertyAssignment) *ObjectLiteral {
return &ObjectLiteral{props}
}
func (l *ObjectLiteral) Iterate(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
obj, err := l.doExec(ctx, scope)
if err != nil {
return nil, err
}
return collections.NewObjectIterator(obj), nil
}
func (l *ObjectLiteral) Exec(ctx context.Context, scope *core.Scope) (core.Value, error) {
arr, err := l.doExec(ctx, scope)
if err != nil {
return values.None, err
}
return arr, nil
}
func (l *ObjectLiteral) doExec(ctx context.Context, scope *core.Scope) (*values.Object, error) {
obj := values.NewObject()
for _, el := range l.properties {
name, err := el.name.Exec(ctx, scope)
if err != nil {
return nil, err
return values.None, err
}
val, err := el.value.Exec(ctx, scope)
if err != nil {
return nil, err
return values.None, err
}
if name.Type() != core.StringType {
return nil, core.TypeError(name.Type(), core.StringType)
return values.None, core.TypeError(name.Type(), core.StringType)
}
obj.Set(name.(values.String), val)

17
pkg/runtime/expressions/member.go
View File

@ -2,7 +2,6 @@ package expressions
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/pkg/errors"
@ -26,22 +25,6 @@ func NewMemberExpression(src core.SourceMap, variableName string, path []core.Ex
return &MemberExpression{src, variableName, path}, nil
}
func (e *MemberExpression) Iterate(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
value, err := e.Exec(ctx, scope)
if err != nil {
return nil, core.SourceError(e.src, err)
}
iter, err := collections.ToIterator(value)
if err != nil {
return nil, core.SourceError(e.src, err)
}
return iter, nil
}
func (e *MemberExpression) Exec(ctx context.Context, scope *core.Scope) (core.Value, error) {
val, err := scope.GetVariable(e.variableName)

11
pkg/runtime/expressions/operators/range.go
View File

@ -2,7 +2,6 @@ package operators
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -27,16 +26,6 @@ func NewRangeOperator(
return &RangeOperator{&baseOperator{src, left, right}}, nil
}
func (operator *RangeOperator) Iterate(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
arr, err := operator.Exec(ctx, scope)
if err != nil {
return nil, err
}
return collections.NewArrayIterator(arr.(*values.Array)), nil
}
func (operator *RangeOperator) Exec(ctx context.Context, scope *core.Scope) (core.Value, error) {
left, err := operator.left.Exec(ctx, scope)

17
pkg/runtime/expressions/param.go
View File

@ -2,7 +2,6 @@ package expressions
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -20,22 +19,6 @@ func NewParameterExpression(src core.SourceMap, name string) (*ParameterExpressi
return &ParameterExpression{src, name}, nil
}
func (e *ParameterExpression) Iterate(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
value, err := e.Exec(ctx, scope)
if err != nil {
return nil, core.SourceError(e.src, err)
}
iter, err := collections.ToIterator(value)
if err != nil {
return nil, core.SourceError(e.src, err)
}
return iter, nil
}
func (e *ParameterExpression) Exec(ctx context.Context, _ *core.Scope) (core.Value, error) {
param, err := core.ParamFrom(ctx, e.name)

17
pkg/runtime/expressions/variable.go
View File

@ -2,7 +2,6 @@ package expressions
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/pkg/errors"
@ -42,22 +41,6 @@ func NewVariableDeclarationExpression(src core.SourceMap, name string, init core
return &VariableDeclarationExpression{v, init}, nil
}
func (e *VariableExpression) Iterate(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
value, err := e.Exec(ctx, scope)
if err != nil {
return nil, core.SourceError(e.src, err)
}
iter, err := collections.ToIterator(value)
if err != nil {
return nil, core.SourceError(e.src, err)
}
return iter, nil
}
func (e *VariableExpression) Exec(_ context.Context, scope *core.Scope) (core.Value, error) {
return scope.GetVariable(e.name)
}

62
pkg/runtime/values/helpers.go
View File

@ -279,6 +279,17 @@ func Unmarshal(value json.RawMessage) (core.Value, error) {
return Parse(o), nil
}
func IsCloneable(value core.Value) Boolean {
switch value.Type() {
case core.ArrayType:
return NewBoolean(true)
case core.ObjectType:
return NewBoolean(true)
default:
return NewBoolean(false)
}
}
func ToBoolean(input core.Value) core.Value {
switch input.Type() {
case core.BooleanType:
@ -296,13 +307,54 @@ func ToBoolean(input core.Value) core.Value {
}
}
func IsCloneable(value core.Value) Boolean {
switch value.Type() {
func ToArray(input core.Value) core.Value {
switch input.Type() {
case core.BooleanType,
core.IntType,
core.FloatType,
core.StringType,
core.DateTimeType:
return NewArrayWith(input)
case core.HTMLElementType,
core.HTMLDocumentType:
val := input.(HTMLNode)
attrs := val.GetAttributes()
obj, ok := attrs.(*Object)
if !ok {
return NewArray(0)
}
arr := NewArray(int(obj.Length()))
obj.ForEach(func(value core.Value, key string) bool {
arr.Push(value)
return true
})
return obj
case core.ArrayType:
return NewBoolean(true)
return input.Copy()
case core.ObjectType:
return NewBoolean(true)
obj, ok := input.(*Object)
if !ok {
return NewArray(0)
}
arr := NewArray(int(obj.Length()))
obj.ForEach(func(value core.Value, key string) bool {
arr.Push(value)
return true
})
return obj
default:
return NewBoolean(false)
return NewArray(0)
}
}

22
pkg/stdlib/arrays/lib.go
View File

@ -1,6 +1,10 @@
package arrays
import "github.com/MontFerret/ferret/pkg/runtime/core"
import (
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
func NewLib() map[string]core.Function {
return map[string]core.Function{
@ -29,3 +33,19 @@ func NewLib() map[string]core.Function {
"UNSHIFT": Unshift,
}
}
func toArray(iterator collections.Iterator) (core.Value, error) {
arr := values.NewArray(10)
for iterator.HasNext() {
ds, err := iterator.Next()
if err != nil {
return values.None, err
}
arr.Push(ds.Get(collections.DefaultValueVar))
}
return arr, nil
}

8
pkg/stdlib/arrays/sorted.go
View File

@ -31,8 +31,8 @@ func Sorted(_ context.Context, args ...core.Value) (core.Value, error) {
return values.NewArray(0), nil
}
sorter, err := collections.NewSorter(func(first core.Value, second core.Value) (int, error) {
return first.Compare(second), nil
sorter, err := collections.NewSorter(func(first collections.DataSet, second collections.DataSet) (int, error) {
return first.Get(collections.DefaultValueVar).Compare(second.Get(collections.DefaultValueVar)), nil
}, collections.SortDirectionAsc)
if err != nil {
@ -40,7 +40,7 @@ func Sorted(_ context.Context, args ...core.Value) (core.Value, error) {
}
iterator, err := collections.NewSortIterator(
collections.NewArrayIterator(arr),
collections.NewDefaultIndexedIterator(arr),
sorter,
)
@ -48,5 +48,5 @@ func Sorted(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
return collections.ToArray(iterator)
return toArray(iterator)
}

11
pkg/stdlib/arrays/sorted_unique.go
View File

@ -32,15 +32,18 @@ func SortedUnique(_ context.Context, args ...core.Value) (core.Value, error) {
return values.NewArray(0), nil
}
sorter, err := collections.NewSorter(func(first core.Value, second core.Value) (int, error) {
return first.Compare(second), nil
sorter, err := collections.NewSorter(func(first collections.DataSet, second collections.DataSet) (int, error) {
return first.Get(collections.DefaultValueVar).Compare(second.Get(collections.DefaultValueVar)), nil
}, collections.SortDirectionAsc)
if err != nil {
return values.None, err
}
uniqIterator, err := collections.NewUniqueIterator(collections.NewArrayIterator(arr))
uniqIterator, err := collections.NewUniqueIterator(
collections.NewDefaultIndexedIterator(arr),
collections.DefaultValueVar,
)
if err != nil {
return values.None, err
@ -55,5 +58,5 @@ func SortedUnique(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
return collections.ToArray(iterator)
return toArray(iterator)
}

5
pkg/stdlib/arrays/unique.go
View File

@ -31,12 +31,13 @@ func Unique(_ context.Context, args ...core.Value) (core.Value, error) {
}
iterator, err := collections.NewUniqueIterator(
collections.NewArrayIterator(arr),
collections.NewDefaultIndexedIterator(arr),
collections.DefaultValueVar,
)
if err != nil {
return values.None, err
}
return collections.ToArray(iterator)
return toArray(iterator)
}

31
pkg/stdlib/types/to_array.go
View File

@ -2,13 +2,11 @@ package types
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
// ToArray takes an input value of any type and convert it into an array value.
// toArray takes an input value of any type and convert it into an array value.
// @param (Value) - Input value of arbitrary type.
// @returns (Array)
// None is converted to an empty array
@ -24,30 +22,5 @@ func ToArray(_ context.Context, args ...core.Value) (core.Value, error) {
arg := args[0]
switch arg.Type() {
case core.BooleanType,
core.IntType,
core.FloatType,
core.StringType,
core.DateTimeType:
return values.NewArrayWith(arg), nil
case core.HTMLElementType,
core.HTMLDocumentType:
val := arg.(values.HTMLNode)
attrs := val.GetAttributes()
obj, ok := attrs.(*values.Object)
if !ok {
return values.NewArray(0), nil
}
return collections.ToArray(collections.NewObjectIterator(obj))
case core.ArrayType:
return arg, nil
case core.ObjectType:
return collections.ToArray(collections.NewObjectIterator(arg.(*values.Object)))
default:
return values.NewArray(0), nil
}
return values.ToArray(arg), nil
}