go/ferret
go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2024-12-12 11:15:14 +02:00
Code Issues Releases Activity

Merge branch 'master' into feature/issue_#286

Browse Source
This commit is contained in:
Tim Voronov 2019-10-17 16:58:41 -04:00 committed by GitHub
commit c3cdb48078
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 179 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.golangci.yml
View File

@ -11,6 +11,7 @@ run:
skip-dirs:
- pkg/parser/fql
- pkg/parser/antlr
- examples
linters:
disable:

6
go.mod
View File

@ -5,8 +5,8 @@ go 1.13
require (
github.com/BurntSushi/toml v0.3.1 // indirect
github.com/PuerkitoBio/goquery v1.5.0
github.com/antchfx/htmlquery v1.0.0
github.com/antchfx/xpath v1.0.0
github.com/antchfx/htmlquery v1.1.0
github.com/antchfx/xpath v1.1.0
github.com/antlr/antlr4 v0.0.0-20191005235324-c81b4e69b6c3
github.com/chzyer/logex v1.1.10 // indirect
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
@ -17,7 +17,7 @@ require (
github.com/google/go-cmp v0.3.1 // indirect
github.com/gorilla/css v1.0.0
github.com/gorilla/websocket v1.4.1 // indirect
github.com/labstack/echo/v4 v4.1.10
github.com/labstack/echo/v4 v4.1.11
github.com/mafredri/cdp v0.24.2
github.com/natefinch/lumberjack v2.0.0+incompatible
github.com/pkg/errors v0.8.1

16
go.sum
View File

@ -7,12 +7,12 @@ github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP
github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/antchfx/htmlquery v1.0.0 h1:O5IXz8fZF3B3MW+B33MZWbTHBlYmcfw0BAxgErHuaMA=
github.com/antchfx/htmlquery v1.0.0/go.mod h1:MS9yksVSQXls00iXkiMqXr0J+umL/AmxXKuP28SUJM8=
github.com/antchfx/xpath v1.0.0 h1:Q5gFgh2O40VTSwMOVbFE7nFNRBu3tS21Tn0KAWeEjtk=
github.com/antchfx/xpath v1.0.0/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
github.com/antlr/antlr4 v0.0.0-20191005235324-c81b4e69b6c3 h1:bmbhqj60LNfApeR0sPmGCUXDm20s/gJ8EyYK9+hMe3k=
github.com/antlr/antlr4 v0.0.0-20191005235324-c81b4e69b6c3/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y=
github.com/antchfx/htmlquery v1.1.0 h1:KMS88sLl5KP9GUVU2MQIDcQXNQ0M5MGlkC9WlYgAQqY=
github.com/antchfx/htmlquery v1.1.0/go.mod h1:MS9yksVSQXls00iXkiMqXr0J+umL/AmxXKuP28SUJM8=
github.com/antchfx/xpath v1.1.0 h1:mJTvYpiHvxNQRD4Lbfin/FodHVCHh2a5KrOFr4ZxMOI=
github.com/antchfx/xpath v1.1.0/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
github.com/antlr/antlr4 v0.0.0-20190819145818-b43a4c3a8015 h1:StuiJFxQUsxSCzcby6NFZRdEhPkXD5vxN7TZ4MD6T84=
github.com/antlr/antlr4 v0.0.0-20190819145818-b43a4c3a8015/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y=
github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
@ -40,8 +40,8 @@ github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvK
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/labstack/echo/v4 v4.1.10 h1:/yhIpO50CBInUbE/nHJtGIyhBv0dJe2cDAYxc3V3uMo=
github.com/labstack/echo/v4 v4.1.10/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
github.com/labstack/echo/v4 v4.1.11 h1:z0BZoArY4FqdpUEl+wlHp4hnr/oSR6MTmQmv8OHSoww=
github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/mafredri/cdp v0.24.2 h1:Rzhj/EQw9opbiwUpNML7P+4Hvf0/nSYPaDbiCEpILOM=

32
pkg/compiler/compiler_param_test.go
View File

@ -111,6 +111,38 @@ func TestParam(t *testing.T) {
)
So(string(out), ShouldEqual, `"baz"`)
})
Convey("Should return an error if param values are not passed", t, func() {
prog := compiler.New().
MustCompile(`
LET doc = { foo: { bar: "baz" } }
RETURN doc.@attr.@subattr
`)
_, err := prog.Run(
context.Background(),
)
So(err, ShouldNotBeNil)
So(err.Error(), ShouldContainSubstring, runtime.ErrMissedParam.Error())
})
Convey("Should be possible to use in member expression as segments", t, func() {
prog := compiler.New().
MustCompile(`
LET doc = { foo: { bar: "baz" } }
RETURN doc.@attr.@subattr
`)
_, err := prog.Run(
context.Background(),
runtime.WithParam("attr", "foo"),
)
So(err, ShouldNotBeNil)
So(err.Error(), ShouldContainSubstring, "subattr")
})
}

45
pkg/compiler/scope.go
View File

@ -2,47 +2,56 @@ package compiler
import (
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
type (
globalScope struct {
params map[string]struct{}
}
scope struct {
global *globalScope
parent *scope
vars map[string]core.Type
vars map[string]struct{}
}
)
func newRootScope() *scope {
func newGlobalScope() *globalScope {
return &globalScope{
params: map[string]struct{}{},
}
}
func newRootScope(global *globalScope) *scope {
return &scope{
vars: make(map[string]core.Type),
global: global,
vars: make(map[string]struct{}),
}
}
func newScope(parent *scope) *scope {
s := newRootScope()
s := newRootScope(parent.global)
s.parent = parent
return s
}
func (s *scope) GetVariable(name string) (core.Type, error) {
local, exists := s.vars[name]
func (s *scope) AddParam(name string) {
s.global.params[name] = struct{}{}
}
func (s *scope) HasVariable(name string) bool {
_, exists := s.vars[name]
if exists {
return local, nil
return true
}
if s.parent != nil {
parents, err := s.parent.GetVariable(name)
if err != nil {
return types.None, err
return s.parent.HasVariable(name)
}
return parents, nil
}
return types.None, core.Error(ErrVariableNotFound, name)
return false
}
func (s *scope) SetVariable(name string) error {
@ -53,7 +62,7 @@ func (s *scope) SetVariable(name string) error {
}
// TODO: add type detection
s.vars[name] = types.None
s.vars[name] = struct{}{}
return nil
}
@ -71,7 +80,7 @@ func (s *scope) RemoveVariable(name string) error {
}
func (s *scope) ClearVariables() {
s.vars = make(map[string]core.Type)
s.vars = make(map[string]struct{})
}
func (s *scope) Fork() *scope {

29
pkg/compiler/visitor.go
View File

@ -38,14 +38,15 @@ func newVisitor(src string, funcs *core.Functions) *visitor {
func (v *visitor) VisitProgram(ctx *fql.ProgramContext) interface{} {
return newResultFrom(func() (interface{}, error) {
rootScope := newRootScope()
block, err := v.doVisitBody(ctx.Body().(*fql.BodyContext), rootScope)
gs := newGlobalScope()
rs := newRootScope(gs)
block, err := v.doVisitBody(ctx.Body().(*fql.BodyContext), rs)
if err != nil {
return nil, err
}
return runtime.NewProgram(v.src, block)
return runtime.NewProgram(v.src, block, gs.params)
})
}
@ -801,10 +802,8 @@ func (v *visitor) doVisitMember(ctx *fql.MemberContext, scope *scope) (core.Expr
if identifier != nil {
varName := ctx.Identifier().GetText()
_, err := scope.GetVariable(varName)
if err != nil {
return nil, err
if !scope.HasVariable(varName) {
return nil, core.Error(ErrVariableNotFound, varName)
}
exp, err := expressions.NewVariableExpression(v.getSourceMap(ctx), varName)
@ -927,10 +926,8 @@ func (v *visitor) doVisitComputedPropertyNameContext(ctx *fql.ComputedPropertyNa
func (v *visitor) doVisitShorthandPropertyNameContext(ctx *fql.ShorthandPropertyNameContext, scope *scope) (core.Expression, error) {
name := ctx.Variable().GetText()
_, err := scope.GetVariable(name)
if err != nil {
return nil, err
if !scope.HasVariable(name) {
return nil, core.Error(ErrVariableNotFound, name)
}
return literals.NewStringLiteral(ctx.Variable().GetText()), nil
@ -1005,10 +1002,8 @@ func (v *visitor) doVisitVariable(ctx *fql.VariableContext, scope *scope) (core.
name := ctx.Identifier().GetText()
// check whether the variable is defined
_, err := scope.GetVariable(name)
if err != nil {
return nil, err
if !scope.HasVariable(name) {
return nil, core.Error(ErrVariableNotFound, name)
}
return expressions.NewVariableExpression(v.getSourceMap(ctx), name)
@ -1117,9 +1112,11 @@ func (v *visitor) doVisitFunctionCallExpression(context *fql.FunctionCallExpress
)
}
func (v *visitor) doVisitParamContext(context *fql.ParamContext, _ *scope) (core.Expression, error) {
func (v *visitor) doVisitParamContext(context *fql.ParamContext, s *scope) (core.Expression, error) {
name := context.Identifier().GetText()
s.AddParam(name)
return expressions.NewParameterExpression(
v.getSourceMap(context),
name,

8
pkg/drivers/http/driver.go
View File

@ -152,7 +152,7 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
if !drv.responseCodeAllowed(resp) {
return nil, errors.New(resp.Status)
}
@ -168,7 +168,6 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
return nil, err
}
// HTTPResponse is temporarily unavailable when the status code != OK
r := drivers.HTTPResponse{
StatusCode: resp.StatusCode,
Status: resp.Status,
@ -178,6 +177,11 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
return NewHTMLPage(doc, params.URL, &r, cookies)
}
func (drv *Driver) responseCodeAllowed(resp *http.Response) bool {
_, exists := drv.options.AllowedHTTPCodes[resp.StatusCode]
return exists
}
func (drv *Driver) Parse(_ context.Context, str values.String) (drivers.HTMLPage, error) {
buf := bytes.NewBuffer([]byte(str))

18
pkg/drivers/http/options.go
View File

@ -1,6 +1,8 @@
package http
import (
stdhttp "net/http"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/sethgrid/pester"
)
@ -17,6 +19,7 @@ type (
UserAgent string
Headers drivers.HTTPHeaders
Cookies drivers.HTTPCookies
AllowedHTTPCodes map[int]struct{}
}
)
@ -26,6 +29,7 @@ func newOptions(setters []Option) *Options {
opts.Backoff = pester.ExponentialBackoff
opts.Concurrency = 3
opts.MaxRetries = 5
opts.AllowedHTTPCodes = map[int]struct{}{stdhttp.StatusOK: struct{}{}}
for _, setter := range setters {
setter(opts)
@ -125,3 +129,17 @@ func WithCookies(cookies []drivers.HTTPCookie) Option {
}
}
}
func WithAllowedHTTPCode(httpCode int) Option {
return func(opts *Options) {
opts.AllowedHTTPCodes[httpCode] = struct{}{}
}
}
func WithAllowedHTTPCodes(httpCodes []int) Option {
return func(opts *Options) {
for _, code := range httpCodes {
opts.AllowedHTTPCodes[code] = struct{}{}
}
}
}

9
pkg/runtime/errors.go Normal file
View File

@ -0,0 +1,9 @@
package runtime
import (
"github.com/pkg/errors"
)
var (
ErrMissedParam = errors.New("missed value for parameter(s)")
)

55
pkg/runtime/program.go
View File

@ -3,6 +3,7 @@ package runtime
import (
"context"
"runtime"
"strings"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/logging"
@ -13,9 +14,10 @@ import (
type Program struct {
src string
body core.Expression
params map[string]struct{}
}
func NewProgram(src string, body core.Expression) (*Program, error) {
func NewProgram(src string, body core.Expression, params map[string]struct{}) (*Program, error) {
if src == "" {
return nil, core.Error(core.ErrMissedArgument, "source")
}
@ -24,16 +26,33 @@ func NewProgram(src string, body core.Expression) (*Program, error) {
return nil, core.Error(core.ErrMissedArgument, "body")
}
return &Program{src, body}, nil
return &Program{src, body, params}, nil
}
func (p *Program) Source() string {
return p.src
}
func (p *Program) Run(ctx context.Context, setters ...Option) (result []byte, err error) {
ctx = NewOptions(setters).WithContext(ctx)
func (p *Program) Params() []string {
res := make([]string, 0, len(p.params))
for name := range p.params {
res = append(res, name)
}
return res
}
func (p *Program) Run(ctx context.Context, setters ...Option) (result []byte, err error) {
opts := NewOptions(setters)
err = p.validateParams(opts)
if err != nil {
return nil, err
}
ctx = opts.WithContext(ctx)
logger := logging.FromContext(ctx)
defer func() {
@ -92,3 +111,31 @@ func (p *Program) MustRun(ctx context.Context, setters ...Option) []byte {
return out
}
func (p *Program) validateParams(opts *Options) error {
if len(p.params) == 0 {
return nil
}
// There might be no errors.
// Thus, we allocate this slice lazily, on a first error.
var missedParams []string
for n := range p.params {
_, exists := opts.params[n]
if !exists {
if missedParams == nil {
missedParams = make([]string, 0, len(p.params))
}
missedParams = append(missedParams, "@"+n)
}
}
if len(missedParams) > 0 {
return core.Error(ErrMissedParam, strings.Join(missedParams, ", "))
}
return nil
}