Merge branch 'master' into feature/issue_#286

.golangci.yml

@@ -11,6 +11,7 @@ run:
   skip-dirs:
     - pkg/parser/fql
     - pkg/parser/antlr
+    - examples
 
 linters:
   disable:

go.mod

@@ -5,8 +5,8 @@ go 1.13
 require (
 	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/PuerkitoBio/goquery v1.5.0
-	github.com/antchfx/htmlquery v1.0.0
+	github.com/antchfx/htmlquery v1.1.0
-	github.com/antchfx/xpath v1.0.0
+	github.com/antchfx/xpath v1.1.0
 	github.com/antlr/antlr4 v0.0.0-20191005235324-c81b4e69b6c3
 	github.com/chzyer/logex v1.1.10 // indirect
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
@@ -17,7 +17,7 @@ require (
 	github.com/google/go-cmp v0.3.1 // indirect
 	github.com/gorilla/css v1.0.0
 	github.com/gorilla/websocket v1.4.1 // indirect
-	github.com/labstack/echo/v4 v4.1.10
+	github.com/labstack/echo/v4 v4.1.11
 	github.com/mafredri/cdp v0.24.2
 	github.com/natefinch/lumberjack v2.0.0+incompatible
 	github.com/pkg/errors v0.8.1

go.sum

@@ -7,12 +7,12 @@ github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP
 github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
 github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
 github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
-github.com/antchfx/htmlquery v1.0.0 h1:O5IXz8fZF3B3MW+B33MZWbTHBlYmcfw0BAxgErHuaMA=
+github.com/antchfx/htmlquery v1.1.0 h1:KMS88sLl5KP9GUVU2MQIDcQXNQ0M5MGlkC9WlYgAQqY=
-github.com/antchfx/htmlquery v1.0.0/go.mod h1:MS9yksVSQXls00iXkiMqXr0J+umL/AmxXKuP28SUJM8=
+github.com/antchfx/htmlquery v1.1.0/go.mod h1:MS9yksVSQXls00iXkiMqXr0J+umL/AmxXKuP28SUJM8=
-github.com/antchfx/xpath v1.0.0 h1:Q5gFgh2O40VTSwMOVbFE7nFNRBu3tS21Tn0KAWeEjtk=
+github.com/antchfx/xpath v1.1.0 h1:mJTvYpiHvxNQRD4Lbfin/FodHVCHh2a5KrOFr4ZxMOI=
-github.com/antchfx/xpath v1.0.0/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
+github.com/antchfx/xpath v1.1.0/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
-github.com/antlr/antlr4 v0.0.0-20191005235324-c81b4e69b6c3 h1:bmbhqj60LNfApeR0sPmGCUXDm20s/gJ8EyYK9+hMe3k=
+github.com/antlr/antlr4 v0.0.0-20190819145818-b43a4c3a8015 h1:StuiJFxQUsxSCzcby6NFZRdEhPkXD5vxN7TZ4MD6T84=
-github.com/antlr/antlr4 v0.0.0-20191005235324-c81b4e69b6c3/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y=
+github.com/antlr/antlr4 v0.0.0-20190819145818-b43a4c3a8015/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y=
 github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
@@ -40,8 +40,8 @@ github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvK
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/labstack/echo/v4 v4.1.10 h1:/yhIpO50CBInUbE/nHJtGIyhBv0dJe2cDAYxc3V3uMo=
+github.com/labstack/echo/v4 v4.1.11 h1:z0BZoArY4FqdpUEl+wlHp4hnr/oSR6MTmQmv8OHSoww=
-github.com/labstack/echo/v4 v4.1.10/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
+github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
 github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/mafredri/cdp v0.24.2 h1:Rzhj/EQw9opbiwUpNML7P+4Hvf0/nSYPaDbiCEpILOM=

pkg/compiler/compiler_param_test.go

@@ -111,6 +111,38 @@ func TestParam(t *testing.T) {
 		)
 
 		So(string(out), ShouldEqual, `"baz"`)
+	})
 
+	Convey("Should return an error if param values are not passed", t, func() {
+		prog := compiler.New().
+			MustCompile(`
+			LET doc = { foo: { bar: "baz" } }
+
+			RETURN doc.@attr.@subattr
+		`)
+
+		_, err := prog.Run(
+			context.Background(),
+		)
+
+		So(err, ShouldNotBeNil)
+		So(err.Error(), ShouldContainSubstring, runtime.ErrMissedParam.Error())
+	})
+
+	Convey("Should be possible to use in member expression as segments", t, func() {
+		prog := compiler.New().
+			MustCompile(`
+			LET doc = { foo: { bar: "baz" } }
+
+			RETURN doc.@attr.@subattr
+		`)
+
+		_, err := prog.Run(
+			context.Background(),
+			runtime.WithParam("attr", "foo"),
+		)
+
+		So(err, ShouldNotBeNil)
+		So(err.Error(), ShouldContainSubstring, "subattr")
 	})
 }

pkg/compiler/scope.go

@@ -2,47 +2,56 @@ package compiler
 
 import (
 	"github.com/MontFerret/ferret/pkg/runtime/core"
-	"github.com/MontFerret/ferret/pkg/runtime/values/types"
 )
 
 type (
+	globalScope struct {
+		params map[string]struct{}
+	}
+
 	scope struct {
+		global *globalScope
 		parent *scope
-		vars   map[string]core.Type
+		vars   map[string]struct{}
 	}
 )
 
-func newRootScope() *scope {
+func newGlobalScope() *globalScope {
+	return &globalScope{
+		params: map[string]struct{}{},
+	}
+}
+
+func newRootScope(global *globalScope) *scope {
 	return &scope{
-		vars: make(map[string]core.Type),
+		global: global,
+		vars:   make(map[string]struct{}),
 	}
 }
 
 func newScope(parent *scope) *scope {
-	s := newRootScope()
+	s := newRootScope(parent.global)
 	s.parent = parent
 
 	return s
 }
 
-func (s *scope) GetVariable(name string) (core.Type, error) {
+func (s *scope) AddParam(name string) {
-	local, exists := s.vars[name]
+	s.global.params[name] = struct{}{}
+}
+
+func (s *scope) HasVariable(name string) bool {
+	_, exists := s.vars[name]
 
 	if exists {
-		return local, nil
+		return true
 	}
 
 	if s.parent != nil {
-		parents, err := s.parent.GetVariable(name)
+		return s.parent.HasVariable(name)
-
-		if err != nil {
-			return types.None, err
 	}
 
-		return parents, nil
+	return false
-	}
-
-	return types.None, core.Error(ErrVariableNotFound, name)
 }
 
 func (s *scope) SetVariable(name string) error {
@@ -53,7 +62,7 @@ func (s *scope) SetVariable(name string) error {
 	}
 
 	// TODO: add type detection
-	s.vars[name] = types.None
+	s.vars[name] = struct{}{}
 
 	return nil
 }
@@ -71,7 +80,7 @@ func (s *scope) RemoveVariable(name string) error {
 }
 
 func (s *scope) ClearVariables() {
-	s.vars = make(map[string]core.Type)
+	s.vars = make(map[string]struct{})
 }
 
 func (s *scope) Fork() *scope {

pkg/compiler/visitor.go

@@ -38,14 +38,15 @@ func newVisitor(src string, funcs *core.Functions) *visitor {
 
 func (v *visitor) VisitProgram(ctx *fql.ProgramContext) interface{} {
 	return newResultFrom(func() (interface{}, error) {
-		rootScope := newRootScope()
+		gs := newGlobalScope()
-		block, err := v.doVisitBody(ctx.Body().(*fql.BodyContext), rootScope)
+		rs := newRootScope(gs)
+		block, err := v.doVisitBody(ctx.Body().(*fql.BodyContext), rs)
 
 		if err != nil {
 			return nil, err
 		}
 
-		return runtime.NewProgram(v.src, block)
+		return runtime.NewProgram(v.src, block, gs.params)
 	})
 }
 
@@ -801,10 +802,8 @@ func (v *visitor) doVisitMember(ctx *fql.MemberContext, scope *scope) (core.Expr
 	if identifier != nil {
 		varName := ctx.Identifier().GetText()
 
-		_, err := scope.GetVariable(varName)
+		if !scope.HasVariable(varName) {
-
+			return nil, core.Error(ErrVariableNotFound, varName)
-		if err != nil {
-			return nil, err
 		}
 
 		exp, err := expressions.NewVariableExpression(v.getSourceMap(ctx), varName)
@@ -927,10 +926,8 @@ func (v *visitor) doVisitComputedPropertyNameContext(ctx *fql.ComputedPropertyNa
 func (v *visitor) doVisitShorthandPropertyNameContext(ctx *fql.ShorthandPropertyNameContext, scope *scope) (core.Expression, error) {
 	name := ctx.Variable().GetText()
 
-	_, err := scope.GetVariable(name)
+	if !scope.HasVariable(name) {
-
+		return nil, core.Error(ErrVariableNotFound, name)
-	if err != nil {
-		return nil, err
 	}
 
 	return literals.NewStringLiteral(ctx.Variable().GetText()), nil
@@ -1005,10 +1002,8 @@ func (v *visitor) doVisitVariable(ctx *fql.VariableContext, scope *scope) (core.
 	name := ctx.Identifier().GetText()
 
 	// check whether the variable is defined
-	_, err := scope.GetVariable(name)
+	if !scope.HasVariable(name) {
-
+		return nil, core.Error(ErrVariableNotFound, name)
-	if err != nil {
-		return nil, err
 	}
 
 	return expressions.NewVariableExpression(v.getSourceMap(ctx), name)
@@ -1117,9 +1112,11 @@ func (v *visitor) doVisitFunctionCallExpression(context *fql.FunctionCallExpress
 	)
 }
 
-func (v *visitor) doVisitParamContext(context *fql.ParamContext, _ *scope) (core.Expression, error) {
+func (v *visitor) doVisitParamContext(context *fql.ParamContext, s *scope) (core.Expression, error) {
 	name := context.Identifier().GetText()
 
+	s.AddParam(name)
+
 	return expressions.NewParameterExpression(
 		v.getSourceMap(context),
 		name,

pkg/drivers/http/driver.go

@@ -152,7 +152,7 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
 
 	defer resp.Body.Close()
 
-	if resp.StatusCode != http.StatusOK {
+	if !drv.responseCodeAllowed(resp) {
 		return nil, errors.New(resp.Status)
 	}
 
@@ -168,7 +168,6 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
 		return nil, err
 	}
 
-	// HTTPResponse is temporarily unavailable when the status code != OK
 	r := drivers.HTTPResponse{
 		StatusCode: resp.StatusCode,
 		Status:     resp.Status,
@@ -178,6 +177,11 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
 	return NewHTMLPage(doc, params.URL, &r, cookies)
 }
 
+func (drv *Driver) responseCodeAllowed(resp *http.Response) bool {
+	_, exists := drv.options.AllowedHTTPCodes[resp.StatusCode]
+	return exists
+}
+
 func (drv *Driver) Parse(_ context.Context, str values.String) (drivers.HTMLPage, error) {
 	buf := bytes.NewBuffer([]byte(str))
 

pkg/drivers/http/options.go

@@ -1,6 +1,8 @@
 package http
 
 import (
+	stdhttp "net/http"
+
 	"github.com/MontFerret/ferret/pkg/drivers"
 	"github.com/sethgrid/pester"
 )
@@ -17,6 +19,7 @@ type (
 		UserAgent        string
 		Headers          drivers.HTTPHeaders
 		Cookies          drivers.HTTPCookies
+		AllowedHTTPCodes map[int]struct{}
 	}
 )
 
@@ -26,6 +29,7 @@ func newOptions(setters []Option) *Options {
 	opts.Backoff = pester.ExponentialBackoff
 	opts.Concurrency = 3
 	opts.MaxRetries = 5
+	opts.AllowedHTTPCodes = map[int]struct{}{stdhttp.StatusOK: struct{}{}}
 
 	for _, setter := range setters {
 		setter(opts)
@@ -125,3 +129,17 @@ func WithCookies(cookies []drivers.HTTPCookie) Option {
 		}
 	}
 }
+
+func WithAllowedHTTPCode(httpCode int) Option {
+	return func(opts *Options) {
+		opts.AllowedHTTPCodes[httpCode] = struct{}{}
+	}
+}
+
+func WithAllowedHTTPCodes(httpCodes []int) Option {
+	return func(opts *Options) {
+		for _, code := range httpCodes {
+			opts.AllowedHTTPCodes[code] = struct{}{}
+		}
+	}
+}

pkg/runtime/errors.go

@@ -0,0 +1,9 @@
+package runtime
+
+import (
+	"github.com/pkg/errors"
+)
+
+var (
+	ErrMissedParam = errors.New("missed value for parameter(s)")
+)

pkg/runtime/program.go

@@ -3,6 +3,7 @@ package runtime
 import (
 	"context"
 	"runtime"
+	"strings"
 
 	"github.com/MontFerret/ferret/pkg/runtime/core"
 	"github.com/MontFerret/ferret/pkg/runtime/logging"
@@ -13,9 +14,10 @@ import (
 type Program struct {
 	src    string
 	body   core.Expression
+	params map[string]struct{}
 }
 
-func NewProgram(src string, body core.Expression) (*Program, error) {
+func NewProgram(src string, body core.Expression, params map[string]struct{}) (*Program, error) {
 	if src == "" {
 		return nil, core.Error(core.ErrMissedArgument, "source")
 	}
@@ -24,16 +26,33 @@ func NewProgram(src string, body core.Expression) (*Program, error) {
 		return nil, core.Error(core.ErrMissedArgument, "body")
 	}
 
-	return &Program{src, body}, nil
+	return &Program{src, body, params}, nil
 }
 
 func (p *Program) Source() string {
 	return p.src
 }
 
-func (p *Program) Run(ctx context.Context, setters ...Option) (result []byte, err error) {
+func (p *Program) Params() []string {
-	ctx = NewOptions(setters).WithContext(ctx)
+	res := make([]string, 0, len(p.params))
 
+	for name := range p.params {
+		res = append(res, name)
+	}
+
+	return res
+}
+
+func (p *Program) Run(ctx context.Context, setters ...Option) (result []byte, err error) {
+	opts := NewOptions(setters)
+
+	err = p.validateParams(opts)
+
+	if err != nil {
+		return nil, err
+	}
+
+	ctx = opts.WithContext(ctx)
 	logger := logging.FromContext(ctx)
 
 	defer func() {
@@ -92,3 +111,31 @@ func (p *Program) MustRun(ctx context.Context, setters ...Option) []byte {
 
 	return out
 }
+
+func (p *Program) validateParams(opts *Options) error {
+	if len(p.params) == 0 {
+		return nil
+	}
+
+	// There might be no errors.
+	// Thus, we allocate this slice lazily, on a first error.
+	var missedParams []string
+
+	for n := range p.params {
+		_, exists := opts.params[n]
+
+		if !exists {
+			if missedParams == nil {
+				missedParams = make([]string, 0, len(p.params))
+			}
+
+			missedParams = append(missedParams, "@"+n)
+		}
+	}
+
+	if len(missedParams) > 0 {
+		return core.Error(ErrMissedParam, strings.Join(missedParams, ", "))
+	}
+
+	return nil
+}