go/ferret
go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2024-12-12 11:15:14 +02:00
Code Issues Releases Activity

Merge branch 'master' into feature/issue_#286

Browse Source
This commit is contained in:
Tim Voronov 2019-10-17 16:58:41 -04:00 committed by GitHub
commit c3cdb48078
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 179 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.golangci.yml
View File

@ -11,6 +11,7 @@ run:
skip-dirs:
- pkg/parser/fql
- pkg/parser/antlr
- examples
linters:
disable:
@ -30,4 +31,4 @@ issues:
# no need to include all autogenerated files, we confidently recognize
# autogenerated files. If it's not please let us know.
skip-files:
- "*_test.go"
- "*_test.go"

6
go.mod
View File

@ -5,8 +5,8 @@ go 1.13
require (
github.com/BurntSushi/toml v0.3.1 // indirect
github.com/PuerkitoBio/goquery v1.5.0
github.com/antchfx/htmlquery v1.0.0
github.com/antchfx/xpath v1.0.0
github.com/antchfx/htmlquery v1.1.0
github.com/antchfx/xpath v1.1.0
github.com/antlr/antlr4 v0.0.0-20191005235324-c81b4e69b6c3
github.com/chzyer/logex v1.1.10 // indirect
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
@ -17,7 +17,7 @@ require (
github.com/google/go-cmp v0.3.1 // indirect
github.com/gorilla/css v1.0.0
github.com/gorilla/websocket v1.4.1 // indirect
github.com/labstack/echo/v4 v4.1.10
github.com/labstack/echo/v4 v4.1.11
github.com/mafredri/cdp v0.24.2
github.com/natefinch/lumberjack v2.0.0+incompatible
github.com/pkg/errors v0.8.1

16
go.sum
View File

@ -7,12 +7,12 @@ github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP
github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/antchfx/htmlquery v1.0.0 h1:O5IXz8fZF3B3MW+B33MZWbTHBlYmcfw0BAxgErHuaMA=
github.com/antchfx/htmlquery v1.0.0/go.mod h1:MS9yksVSQXls00iXkiMqXr0J+umL/AmxXKuP28SUJM8=
github.com/antchfx/xpath v1.0.0 h1:Q5gFgh2O40VTSwMOVbFE7nFNRBu3tS21Tn0KAWeEjtk=
github.com/antchfx/xpath v1.0.0/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
github.com/antlr/antlr4 v0.0.0-20191005235324-c81b4e69b6c3 h1:bmbhqj60LNfApeR0sPmGCUXDm20s/gJ8EyYK9+hMe3k=
github.com/antlr/antlr4 v0.0.0-20191005235324-c81b4e69b6c3/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y=
github.com/antchfx/htmlquery v1.1.0 h1:KMS88sLl5KP9GUVU2MQIDcQXNQ0M5MGlkC9WlYgAQqY=
github.com/antchfx/htmlquery v1.1.0/go.mod h1:MS9yksVSQXls00iXkiMqXr0J+umL/AmxXKuP28SUJM8=
github.com/antchfx/xpath v1.1.0 h1:mJTvYpiHvxNQRD4Lbfin/FodHVCHh2a5KrOFr4ZxMOI=
github.com/antchfx/xpath v1.1.0/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
github.com/antlr/antlr4 v0.0.0-20190819145818-b43a4c3a8015 h1:StuiJFxQUsxSCzcby6NFZRdEhPkXD5vxN7TZ4MD6T84=
github.com/antlr/antlr4 v0.0.0-20190819145818-b43a4c3a8015/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y=
github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
@ -40,8 +40,8 @@ github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvK
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/labstack/echo/v4 v4.1.10 h1:/yhIpO50CBInUbE/nHJtGIyhBv0dJe2cDAYxc3V3uMo=
github.com/labstack/echo/v4 v4.1.10/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
github.com/labstack/echo/v4 v4.1.11 h1:z0BZoArY4FqdpUEl+wlHp4hnr/oSR6MTmQmv8OHSoww=
github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/mafredri/cdp v0.24.2 h1:Rzhj/EQw9opbiwUpNML7P+4Hvf0/nSYPaDbiCEpILOM=

32
pkg/compiler/compiler_param_test.go
View File

@ -111,6 +111,38 @@ func TestParam(t *testing.T) {
)
So(string(out), ShouldEqual, `"baz"`)
})
Convey("Should return an error if param values are not passed", t, func() {
prog := compiler.New().
MustCompile(`
LET doc = { foo: { bar: "baz" } }
RETURN doc.@attr.@subattr
`)
_, err := prog.Run(
context.Background(),
)
So(err, ShouldNotBeNil)
So(err.Error(), ShouldContainSubstring, runtime.ErrMissedParam.Error())
})
Convey("Should be possible to use in member expression as segments", t, func() {
prog := compiler.New().
MustCompile(`
LET doc = { foo: { bar: "baz" } }
RETURN doc.@attr.@subattr
`)
_, err := prog.Run(
context.Background(),
runtime.WithParam("attr", "foo"),
)
So(err, ShouldNotBeNil)
So(err.Error(), ShouldContainSubstring, "subattr")
})
}

45
pkg/compiler/scope.go
View File

@ -2,47 +2,56 @@ package compiler
import (
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
type (
globalScope struct {
params map[string]struct{}
}
scope struct {
global *globalScope
parent *scope
vars map[string]core.Type
vars map[string]struct{}
}
)
func newRootScope() *scope {
func newGlobalScope() *globalScope {
return &globalScope{
params: map[string]struct{}{},
}
}
func newRootScope(global *globalScope) *scope {
return &scope{
vars: make(map[string]core.Type),
global: global,
vars: make(map[string]struct{}),
}
}
func newScope(parent *scope) *scope {
s := newRootScope()
s := newRootScope(parent.global)
s.parent = parent
return s
}
func (s *scope) GetVariable(name string) (core.Type, error) {
local, exists := s.vars[name]
func (s *scope) AddParam(name string) {
s.global.params[name] = struct{}{}
}
func (s *scope) HasVariable(name string) bool {
_, exists := s.vars[name]
if exists {
return local, nil
return true
}
if s.parent != nil {
parents, err := s.parent.GetVariable(name)
if err != nil {
return types.None, err
}
return parents, nil
return s.parent.HasVariable(name)
}
return types.None, core.Error(ErrVariableNotFound, name)
return false
}
func (s *scope) SetVariable(name string) error {
@ -53,7 +62,7 @@ func (s *scope) SetVariable(name string) error {
}
// TODO: add type detection
s.vars[name] = types.None
s.vars[name] = struct{}{}
return nil
}
@ -71,7 +80,7 @@ func (s *scope) RemoveVariable(name string) error {
}
func (s *scope) ClearVariables() {
s.vars = make(map[string]core.Type)
s.vars = make(map[string]struct{})
}
func (s *scope) Fork() *scope {

29
pkg/compiler/visitor.go
View File

@ -38,14 +38,15 @@ func newVisitor(src string, funcs *core.Functions) *visitor {
func (v *visitor) VisitProgram(ctx *fql.ProgramContext) interface{} {
return newResultFrom(func() (interface{}, error) {
rootScope := newRootScope()
block, err := v.doVisitBody(ctx.Body().(*fql.BodyContext), rootScope)
gs := newGlobalScope()
rs := newRootScope(gs)
block, err := v.doVisitBody(ctx.Body().(*fql.BodyContext), rs)
if err != nil {
return nil, err
}
return runtime.NewProgram(v.src, block)
return runtime.NewProgram(v.src, block, gs.params)
})
}
@ -801,10 +802,8 @@ func (v *visitor) doVisitMember(ctx *fql.MemberContext, scope *scope) (core.Expr
if identifier != nil {
varName := ctx.Identifier().GetText()
_, err := scope.GetVariable(varName)
if err != nil {
return nil, err
if !scope.HasVariable(varName) {
return nil, core.Error(ErrVariableNotFound, varName)
}
exp, err := expressions.NewVariableExpression(v.getSourceMap(ctx), varName)
@ -927,10 +926,8 @@ func (v *visitor) doVisitComputedPropertyNameContext(ctx *fql.ComputedPropertyNa
func (v *visitor) doVisitShorthandPropertyNameContext(ctx *fql.ShorthandPropertyNameContext, scope *scope) (core.Expression, error) {
name := ctx.Variable().GetText()
_, err := scope.GetVariable(name)
if err != nil {
return nil, err
if !scope.HasVariable(name) {
return nil, core.Error(ErrVariableNotFound, name)
}
return literals.NewStringLiteral(ctx.Variable().GetText()), nil
@ -1005,10 +1002,8 @@ func (v *visitor) doVisitVariable(ctx *fql.VariableContext, scope *scope) (core.
name := ctx.Identifier().GetText()
// check whether the variable is defined
_, err := scope.GetVariable(name)
if err != nil {
return nil, err
if !scope.HasVariable(name) {
return nil, core.Error(ErrVariableNotFound, name)
}
return expressions.NewVariableExpression(v.getSourceMap(ctx), name)
@ -1117,9 +1112,11 @@ func (v *visitor) doVisitFunctionCallExpression(context *fql.FunctionCallExpress
)
}
func (v *visitor) doVisitParamContext(context *fql.ParamContext, _ *scope) (core.Expression, error) {
func (v *visitor) doVisitParamContext(context *fql.ParamContext, s *scope) (core.Expression, error) {
name := context.Identifier().GetText()
s.AddParam(name)
return expressions.NewParameterExpression(
v.getSourceMap(context),
name,

8
pkg/drivers/http/driver.go
View File

@ -152,7 +152,7 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
if !drv.responseCodeAllowed(resp) {
return nil, errors.New(resp.Status)
}
@ -168,7 +168,6 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
return nil, err
}
// HTTPResponse is temporarily unavailable when the status code != OK
r := drivers.HTTPResponse{
StatusCode: resp.StatusCode,
Status: resp.Status,
@ -178,6 +177,11 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
return NewHTMLPage(doc, params.URL, &r, cookies)
}
func (drv *Driver) responseCodeAllowed(resp *http.Response) bool {
_, exists := drv.options.AllowedHTTPCodes[resp.StatusCode]
return exists
}
func (drv *Driver) Parse(_ context.Context, str values.String) (drivers.HTMLPage, error) {
buf := bytes.NewBuffer([]byte(str))

34
pkg/drivers/http/options.go
View File

@ -1,6 +1,8 @@
package http
import (
stdhttp "net/http"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/sethgrid/pester"
)
@ -9,14 +11,15 @@ type (
Option func(opts *Options)
Options struct {
Name string
Backoff pester.BackoffStrategy
MaxRetries int
Concurrency int
Proxy string
UserAgent string
Headers drivers.HTTPHeaders
Cookies drivers.HTTPCookies
Name string
Backoff pester.BackoffStrategy
MaxRetries int
Concurrency int
Proxy string
UserAgent string
Headers drivers.HTTPHeaders
Cookies drivers.HTTPCookies
AllowedHTTPCodes map[int]struct{}
}
)
@ -26,6 +29,7 @@ func newOptions(setters []Option) *Options {
opts.Backoff = pester.ExponentialBackoff
opts.Concurrency = 3
opts.MaxRetries = 5
opts.AllowedHTTPCodes = map[int]struct{}{stdhttp.StatusOK: struct{}{}}
for _, setter := range setters {
setter(opts)
@ -125,3 +129,17 @@ func WithCookies(cookies []drivers.HTTPCookie) Option {
}
}
}
func WithAllowedHTTPCode(httpCode int) Option {
return func(opts *Options) {
opts.AllowedHTTPCodes[httpCode] = struct{}{}
}
}
func WithAllowedHTTPCodes(httpCodes []int) Option {
return func(opts *Options) {
for _, code := range httpCodes {
opts.AllowedHTTPCodes[code] = struct{}{}
}
}
}

9
pkg/runtime/errors.go Normal file
View File

@ -0,0 +1,9 @@
package runtime
import (
"github.com/pkg/errors"
)
var (
ErrMissedParam = errors.New("missed value for parameter(s)")
)

59
pkg/runtime/program.go
View File

@ -3,6 +3,7 @@ package runtime
import (
"context"
"runtime"
"strings"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/logging"
@ -11,11 +12,12 @@ import (
)
type Program struct {
src string
body core.Expression
src string
body core.Expression
params map[string]struct{}
}
func NewProgram(src string, body core.Expression) (*Program, error) {
func NewProgram(src string, body core.Expression, params map[string]struct{}) (*Program, error) {
if src == "" {
return nil, core.Error(core.ErrMissedArgument, "source")
}
@ -24,16 +26,33 @@ func NewProgram(src string, body core.Expression) (*Program, error) {
return nil, core.Error(core.ErrMissedArgument, "body")
}
return &Program{src, body}, nil
return &Program{src, body, params}, nil
}
func (p *Program) Source() string {
return p.src
}
func (p *Program) Run(ctx context.Context, setters ...Option) (result []byte, err error) {
ctx = NewOptions(setters).WithContext(ctx)
func (p *Program) Params() []string {
res := make([]string, 0, len(p.params))
for name := range p.params {
res = append(res, name)
}
return res
}
func (p *Program) Run(ctx context.Context, setters ...Option) (result []byte, err error) {
opts := NewOptions(setters)
err = p.validateParams(opts)
if err != nil {
return nil, err
}
ctx = opts.WithContext(ctx)
logger := logging.FromContext(ctx)
defer func() {
@ -92,3 +111,31 @@ func (p *Program) MustRun(ctx context.Context, setters ...Option) []byte {
return out
}
func (p *Program) validateParams(opts *Options) error {
if len(p.params) == 0 {
return nil
}
// There might be no errors.
// Thus, we allocate this slice lazily, on a first error.
var missedParams []string
for n := range p.params {
_, exists := opts.params[n]
if !exists {
if missedParams == nil {
missedParams = make([]string, 0, len(p.params))
}
missedParams = append(missedParams, "@"+n)
}
}
if len(missedParams) > 0 {
return core.Error(ErrMissedParam, strings.Join(missedParams, ", "))
}
return nil
}