From 85b3430319b5e8c22e83c2bef6662b48b5639e0c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2019 06:02:03 +0000 Subject: [PATCH 01/23] Bump github.com/labstack/echo/v4 from 4.1.10 to 4.1.11 Bumps [github.com/labstack/echo/v4](https://github.com/labstack/echo) from 4.1.10 to 4.1.11. - [Release notes](https://github.com/labstack/echo/releases) - [Commits](https://github.com/labstack/echo/compare/v4.1.10...v4.1.11) Signed-off-by: dependabot-preview[bot] --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 040ad07b..fe347bf6 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/gobwas/glob v0.2.3 github.com/gorilla/css v1.0.0 github.com/gorilla/websocket v1.4.1 // indirect - github.com/labstack/echo/v4 v4.1.10 + github.com/labstack/echo/v4 v4.1.11 github.com/mafredri/cdp v0.24.2 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/pkg/errors v0.8.1 diff --git a/go.sum b/go.sum index 3bf476a0..d781479b 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/labstack/echo/v4 v4.1.10 h1:/yhIpO50CBInUbE/nHJtGIyhBv0dJe2cDAYxc3V3uMo= github.com/labstack/echo/v4 v4.1.10/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= +github.com/labstack/echo/v4 v4.1.11 h1:z0BZoArY4FqdpUEl+wlHp4hnr/oSR6MTmQmv8OHSoww= +github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/mafredri/cdp v0.24.2 h1:Rzhj/EQw9opbiwUpNML7P+4Hvf0/nSYPaDbiCEpILOM= From 666511e7b307e44fcb98b67e0f59e2a0fb9c73dc Mon Sep 17 00:00:00 2001 From: Tim Voronov Date: Mon, 7 Oct 2019 14:25:07 -0400 Subject: [PATCH 02/23] Update go.sum --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index d781479b..1411b2d7 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,6 @@ github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvK github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/labstack/echo/v4 v4.1.10 h1:/yhIpO50CBInUbE/nHJtGIyhBv0dJe2cDAYxc3V3uMo= -github.com/labstack/echo/v4 v4.1.10/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/echo/v4 v4.1.11 h1:z0BZoArY4FqdpUEl+wlHp4hnr/oSR6MTmQmv8OHSoww= github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= From 9e7998f9c8f57f32828d86e1dd298529d9b16487 Mon Sep 17 00:00:00 2001 From: Tim Voronov Date: Fri, 11 Oct 2019 17:31:23 -0400 Subject: [PATCH 03/23] Added params check before execution --- Makefile | 2 +- pkg/compiler/compiler_param_test.go | 31 ++++++++++++++++++++ pkg/compiler/scope.go | 45 +++++++++++++++++------------ pkg/compiler/visitor.go | 29 +++++++++---------- pkg/runtime/errors.go | 10 +++++++ pkg/runtime/program.go | 37 ++++++++++++++++++++---- 6 files changed, 113 insertions(+), 41 deletions(-) create mode 100644 pkg/runtime/errors.go diff --git a/Makefile b/Makefile index dc295a0b..91c0d138 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ compile: ./main.go test: - go test -race ${DIR_PKG}/... + go test -v -race ${DIR_PKG}/... cover: go test -race -coverprofile=coverage.txt -covermode=atomic ${DIR_PKG}/... && \ diff --git a/pkg/compiler/compiler_param_test.go b/pkg/compiler/compiler_param_test.go index a44d17a5..7948871a 100644 --- a/pkg/compiler/compiler_param_test.go +++ b/pkg/compiler/compiler_param_test.go @@ -111,6 +111,37 @@ func TestParam(t *testing.T) { ) So(string(out), ShouldEqual, `"baz"`) + }) + Convey("Should return an error if param values are not passed", t, func() { + prog := compiler.New(). + MustCompile(` + LET doc = { foo: { bar: "baz" } } + + RETURN doc.@attr.@subattr + `) + + _, err := prog.Run( + context.Background(), + ) + + So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, runtime.ErrMissedParams.Error()) + }) + Convey("Should be possible to use in member expression as segments", t, func() { + prog := compiler.New(). + MustCompile(` + LET doc = { foo: { bar: "baz" } } + + RETURN doc.@attr.@subattr + `) + + _, err := prog.Run( + context.Background(), + runtime.WithParam("attr", "foo"), + ) + + So(err, ShouldNotBeNil) + So(err.Error(), ShouldContainSubstring, "subattr") }) } diff --git a/pkg/compiler/scope.go b/pkg/compiler/scope.go index 9acac22e..d7aea34e 100644 --- a/pkg/compiler/scope.go +++ b/pkg/compiler/scope.go @@ -2,47 +2,56 @@ package compiler import ( "github.com/MontFerret/ferret/pkg/runtime/core" - "github.com/MontFerret/ferret/pkg/runtime/values/types" ) type ( + globalScope struct { + params map[string]struct{} + } + scope struct { + global *globalScope parent *scope - vars map[string]core.Type + vars map[string]struct{} } ) -func newRootScope() *scope { +func newGlobalScope() *globalScope { + return &globalScope{ + params: map[string]struct{}{}, + } +} + +func newRootScope(global *globalScope) *scope { return &scope{ - vars: make(map[string]core.Type), + global: global, + vars: make(map[string]struct{}), } } func newScope(parent *scope) *scope { - s := newRootScope() + s := newRootScope(parent.global) s.parent = parent return s } -func (s *scope) GetVariable(name string) (core.Type, error) { - local, exists := s.vars[name] +func (s *scope) AddParam(name string) { + s.global.params[name] = struct{}{} +} + +func (s *scope) HasVariable(name string) bool { + _, exists := s.vars[name] if exists { - return local, nil + return true } if s.parent != nil { - parents, err := s.parent.GetVariable(name) - - if err != nil { - return types.None, err - } - - return parents, nil + return s.parent.HasVariable(name) } - return types.None, core.Error(ErrVariableNotFound, name) + return false } func (s *scope) SetVariable(name string) error { @@ -53,7 +62,7 @@ func (s *scope) SetVariable(name string) error { } // TODO: add type detection - s.vars[name] = types.None + s.vars[name] = struct{}{} return nil } @@ -71,7 +80,7 @@ func (s *scope) RemoveVariable(name string) error { } func (s *scope) ClearVariables() { - s.vars = make(map[string]core.Type) + s.vars = make(map[string]struct{}) } func (s *scope) Fork() *scope { diff --git a/pkg/compiler/visitor.go b/pkg/compiler/visitor.go index dedd6cad..49979537 100644 --- a/pkg/compiler/visitor.go +++ b/pkg/compiler/visitor.go @@ -38,14 +38,15 @@ func newVisitor(src string, funcs map[string]core.Function) *visitor { func (v *visitor) VisitProgram(ctx *fql.ProgramContext) interface{} { return newResultFrom(func() (interface{}, error) { - rootScope := newRootScope() - block, err := v.doVisitBody(ctx.Body().(*fql.BodyContext), rootScope) + gs := newGlobalScope() + rs := newRootScope(gs) + block, err := v.doVisitBody(ctx.Body().(*fql.BodyContext), rs) if err != nil { return nil, err } - return runtime.NewProgram(v.src, block) + return runtime.NewProgram(v.src, block, gs.params) }) } @@ -801,10 +802,8 @@ func (v *visitor) doVisitMember(ctx *fql.MemberContext, scope *scope) (core.Expr if identifier != nil { varName := ctx.Identifier().GetText() - _, err := scope.GetVariable(varName) - - if err != nil { - return nil, err + if !scope.HasVariable(varName) { + return nil, core.Error(ErrVariableNotFound, varName) } exp, err := expressions.NewVariableExpression(v.getSourceMap(ctx), varName) @@ -927,10 +926,8 @@ func (v *visitor) doVisitComputedPropertyNameContext(ctx *fql.ComputedPropertyNa func (v *visitor) doVisitShorthandPropertyNameContext(ctx *fql.ShorthandPropertyNameContext, scope *scope) (core.Expression, error) { name := ctx.Variable().GetText() - _, err := scope.GetVariable(name) - - if err != nil { - return nil, err + if !scope.HasVariable(name) { + return nil, core.Error(ErrVariableNotFound, name) } return literals.NewStringLiteral(ctx.Variable().GetText()), nil @@ -1005,10 +1002,8 @@ func (v *visitor) doVisitVariable(ctx *fql.VariableContext, scope *scope) (core. name := ctx.Identifier().GetText() // check whether the variable is defined - _, err := scope.GetVariable(name) - - if err != nil { - return nil, err + if !scope.HasVariable(name) { + return nil, core.Error(ErrVariableNotFound, name) } return expressions.NewVariableExpression(v.getSourceMap(ctx), name) @@ -1117,9 +1112,11 @@ func (v *visitor) doVisitFunctionCallExpression(context *fql.FunctionCallExpress ) } -func (v *visitor) doVisitParamContext(context *fql.ParamContext, _ *scope) (core.Expression, error) { +func (v *visitor) doVisitParamContext(context *fql.ParamContext, s *scope) (core.Expression, error) { name := context.Identifier().GetText() + s.AddParam(name) + return expressions.NewParameterExpression( v.getSourceMap(context), name, diff --git a/pkg/runtime/errors.go b/pkg/runtime/errors.go new file mode 100644 index 00000000..98172dad --- /dev/null +++ b/pkg/runtime/errors.go @@ -0,0 +1,10 @@ +package runtime + +import ( + "github.com/pkg/errors" +) + +var ( + ErrMissedParams = errors.New("missed parameter values") + ErrMissedParam = errors.New("missed value for parameter") +) diff --git a/pkg/runtime/program.go b/pkg/runtime/program.go index d51a46c0..e756dc25 100644 --- a/pkg/runtime/program.go +++ b/pkg/runtime/program.go @@ -11,11 +11,12 @@ import ( ) type Program struct { - src string - body core.Expression + src string + body core.Expression + params map[string]struct{} } -func NewProgram(src string, body core.Expression) (*Program, error) { +func NewProgram(src string, body core.Expression, params map[string]struct{}) (*Program, error) { if src == "" { return nil, core.Error(core.ErrMissedArgument, "source") } @@ -24,16 +25,40 @@ func NewProgram(src string, body core.Expression) (*Program, error) { return nil, core.Error(core.ErrMissedArgument, "body") } - return &Program{src, body}, nil + return &Program{src, body, params}, nil } func (p *Program) Source() string { return p.src } -func (p *Program) Run(ctx context.Context, setters ...Option) (result []byte, err error) { - ctx = NewOptions(setters).WithContext(ctx) +func (p *Program) Params() []string { + res := make([]string, 0, len(p.params)) + for name := range p.params { + res = append(res, name) + } + + return res +} + +func (p *Program) Run(ctx context.Context, setters ...Option) (result []byte, err error) { + opts := NewOptions(setters) + + // Check params + if len(p.params) > 0 && len(opts.params) == 0 { + return nil, ErrMissedParams + } + + for n := range p.params { + _, exists := opts.params[n] + + if !exists { + return nil, errors.Wrap(ErrMissedParam, n) + } + } + + ctx = opts.WithContext(ctx) logger := logging.FromContext(ctx) defer func() { From c65dd6b3c8ded390dcfe7f789cb7b9ace522c3f0 Mon Sep 17 00:00:00 2001 From: Tim Voronov Date: Fri, 11 Oct 2019 17:32:32 -0400 Subject: [PATCH 04/23] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 91c0d138..dc295a0b 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ compile: ./main.go test: - go test -v -race ${DIR_PKG}/... + go test -race ${DIR_PKG}/... cover: go test -race -coverprofile=coverage.txt -covermode=atomic ${DIR_PKG}/... && \ From c2e7315008241fa5e905a4d9a8a401d2236c2327 Mon Sep 17 00:00:00 2001 From: Tim Voronov Date: Fri, 11 Oct 2019 21:30:42 -0400 Subject: [PATCH 05/23] Improved error message --- pkg/compiler/compiler_param_test.go | 3 ++- pkg/runtime/errors.go | 3 +-- pkg/runtime/program.go | 40 +++++++++++++++++++++-------- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/pkg/compiler/compiler_param_test.go b/pkg/compiler/compiler_param_test.go index 7948871a..5e202d08 100644 --- a/pkg/compiler/compiler_param_test.go +++ b/pkg/compiler/compiler_param_test.go @@ -126,8 +126,9 @@ func TestParam(t *testing.T) { ) So(err, ShouldNotBeNil) - So(err.Error(), ShouldEqual, runtime.ErrMissedParams.Error()) + So(err.Error(), ShouldContainSubstring, runtime.ErrMissedParam.Error()) }) + Convey("Should be possible to use in member expression as segments", t, func() { prog := compiler.New(). MustCompile(` diff --git a/pkg/runtime/errors.go b/pkg/runtime/errors.go index 98172dad..3d336cb5 100644 --- a/pkg/runtime/errors.go +++ b/pkg/runtime/errors.go @@ -5,6 +5,5 @@ import ( ) var ( - ErrMissedParams = errors.New("missed parameter values") - ErrMissedParam = errors.New("missed value for parameter") + ErrMissedParam = errors.New("missed value for parameter(s)") ) diff --git a/pkg/runtime/program.go b/pkg/runtime/program.go index e756dc25..e61dd255 100644 --- a/pkg/runtime/program.go +++ b/pkg/runtime/program.go @@ -3,6 +3,7 @@ package runtime import ( "context" "runtime" + "strings" "github.com/MontFerret/ferret/pkg/runtime/core" "github.com/MontFerret/ferret/pkg/runtime/logging" @@ -45,17 +46,10 @@ func (p *Program) Params() []string { func (p *Program) Run(ctx context.Context, setters ...Option) (result []byte, err error) { opts := NewOptions(setters) - // Check params - if len(p.params) > 0 && len(opts.params) == 0 { - return nil, ErrMissedParams - } + err = p.validateParams(opts) - for n := range p.params { - _, exists := opts.params[n] - - if !exists { - return nil, errors.Wrap(ErrMissedParam, n) - } + if err != nil { + return nil, err } ctx = opts.WithContext(ctx) @@ -117,3 +111,29 @@ func (p *Program) MustRun(ctx context.Context, setters ...Option) []byte { return out } + +func (p *Program) validateParams(opts *Options) error { + if len(p.params) == 0 { + return nil + } + + var missedParams []string + + for n := range p.params { + _, exists := opts.params[n] + + if !exists { + if missedParams == nil { + missedParams = make([]string, 0, len(p.params)) + } + + missedParams = append(missedParams, "@"+n) + } + } + + if len(missedParams) > 0 { + return core.Error(ErrMissedParam, strings.Join(missedParams, ", ")) + } + + return nil +} From 0e06ee2589e74324245a8a7fee9a59f4585fa2c3 Mon Sep 17 00:00:00 2001 From: Gabriel Marinkovic Date: Sun, 13 Oct 2019 19:23:15 +0200 Subject: [PATCH 06/23] added http response status 500 as valid --- pkg/drivers/http/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/drivers/http/driver.go b/pkg/drivers/http/driver.go index 3fd26f72..5ab279b0 100644 --- a/pkg/drivers/http/driver.go +++ b/pkg/drivers/http/driver.go @@ -152,7 +152,7 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusInternalServerError { return nil, errors.New(resp.Status) } From dce2badb58b59534467cb14470099fc0f7828be7 Mon Sep 17 00:00:00 2001 From: Gabriel Marinkovic Date: Sun, 13 Oct 2019 19:41:53 +0200 Subject: [PATCH 07/23] added WithAllowedHTTPCode(s) option to http driver --- pkg/drivers/http/driver.go | 11 ++++++++++- pkg/drivers/http/options.go | 38 +++++++++++++++++++++++++++++-------- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/pkg/drivers/http/driver.go b/pkg/drivers/http/driver.go index 5ab279b0..196975b1 100644 --- a/pkg/drivers/http/driver.go +++ b/pkg/drivers/http/driver.go @@ -152,7 +152,16 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM defer resp.Body.Close() - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusInternalServerError { + + responseAllowed := false + for _, code := range drv.options.AllowedHTTPCodes { + if resp.StatusCode == code { + responseAllowed = true + break + } + } + + if !responseAllowed { return nil, errors.New(resp.Status) } diff --git a/pkg/drivers/http/options.go b/pkg/drivers/http/options.go index 03769106..c3937ba3 100644 --- a/pkg/drivers/http/options.go +++ b/pkg/drivers/http/options.go @@ -1,6 +1,8 @@ package http import ( + stdhttp "net/http" + "github.com/MontFerret/ferret/pkg/drivers" "github.com/sethgrid/pester" ) @@ -9,14 +11,15 @@ type ( Option func(opts *Options) Options struct { - Name string - Backoff pester.BackoffStrategy - MaxRetries int - Concurrency int - Proxy string - UserAgent string - Headers drivers.HTTPHeaders - Cookies drivers.HTTPCookies + Name string + Backoff pester.BackoffStrategy + MaxRetries int + Concurrency int + Proxy string + UserAgent string + Headers drivers.HTTPHeaders + Cookies drivers.HTTPCookies + AllowedHTTPCodes []int } ) @@ -26,6 +29,7 @@ func newOptions(setters []Option) *Options { opts.Backoff = pester.ExponentialBackoff opts.Concurrency = 3 opts.MaxRetries = 5 + opts.AllowedHTTPCodes = []int{stdhttp.StatusOK} for _, setter := range setters { setter(opts) @@ -125,3 +129,21 @@ func WithCookies(cookies []drivers.HTTPCookie) Option { } } } + +func WithAllowedHTTPCode(httpCode int) Option { + return func(opts *Options) { + if opts.AllowedHTTPCodes == nil { + opts.AllowedHTTPCodes = make([]int, 0, 1) + } + opts.AllowedHTTPCodes = append(opts.AllowedHTTPCodes, httpCode) + } +} + +func WithAllowedHTTPCodes(httpCodes []int) Option { + return func(opts *Options) { + if opts.AllowedHTTPCodes == nil { + opts.AllowedHTTPCodes = httpCodes + } + opts.AllowedHTTPCodes = append(opts.AllowedHTTPCodes, httpCodes...) + } +} From 475e17383a369974d9c0f02589f1142252e92e62 Mon Sep 17 00:00:00 2001 From: Gabriel Marinkovic Date: Sun, 13 Oct 2019 19:46:13 +0200 Subject: [PATCH 08/23] fixed formatting --- pkg/drivers/http/driver.go | 1 - pkg/drivers/http/options.go | 18 +++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/pkg/drivers/http/driver.go b/pkg/drivers/http/driver.go index 196975b1..4f82d750 100644 --- a/pkg/drivers/http/driver.go +++ b/pkg/drivers/http/driver.go @@ -152,7 +152,6 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM defer resp.Body.Close() - responseAllowed := false for _, code := range drv.options.AllowedHTTPCodes { if resp.StatusCode == code { diff --git a/pkg/drivers/http/options.go b/pkg/drivers/http/options.go index c3937ba3..c1b257a2 100644 --- a/pkg/drivers/http/options.go +++ b/pkg/drivers/http/options.go @@ -11,15 +11,15 @@ type ( Option func(opts *Options) Options struct { - Name string - Backoff pester.BackoffStrategy - MaxRetries int - Concurrency int - Proxy string - UserAgent string - Headers drivers.HTTPHeaders - Cookies drivers.HTTPCookies - AllowedHTTPCodes []int + Name string + Backoff pester.BackoffStrategy + MaxRetries int + Concurrency int + Proxy string + UserAgent string + Headers drivers.HTTPHeaders + Cookies drivers.HTTPCookies + AllowedHTTPCodes []int } ) From bb37852ba4f797d79dccc55c14c38ef8b2a5fcaa Mon Sep 17 00:00:00 2001 From: Gabriel Marinkovic Date: Mon, 14 Oct 2019 17:40:06 +0200 Subject: [PATCH 09/23] removed unneeded nil check --- pkg/drivers/http/options.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/drivers/http/options.go b/pkg/drivers/http/options.go index c1b257a2..d5461ece 100644 --- a/pkg/drivers/http/options.go +++ b/pkg/drivers/http/options.go @@ -132,18 +132,12 @@ func WithCookies(cookies []drivers.HTTPCookie) Option { func WithAllowedHTTPCode(httpCode int) Option { return func(opts *Options) { - if opts.AllowedHTTPCodes == nil { - opts.AllowedHTTPCodes = make([]int, 0, 1) - } opts.AllowedHTTPCodes = append(opts.AllowedHTTPCodes, httpCode) } } func WithAllowedHTTPCodes(httpCodes []int) Option { return func(opts *Options) { - if opts.AllowedHTTPCodes == nil { - opts.AllowedHTTPCodes = httpCodes - } opts.AllowedHTTPCodes = append(opts.AllowedHTTPCodes, httpCodes...) } } From 04e7f1ecbc299abc322b630c3865eac8aba1b63a Mon Sep 17 00:00:00 2001 From: Gabriel Marinkovic Date: Mon, 14 Oct 2019 17:40:33 +0200 Subject: [PATCH 10/23] moved http response code validation into a separate function --- pkg/drivers/http/driver.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pkg/drivers/http/driver.go b/pkg/drivers/http/driver.go index 4f82d750..edb5fa7c 100644 --- a/pkg/drivers/http/driver.go +++ b/pkg/drivers/http/driver.go @@ -152,15 +152,7 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM defer resp.Body.Close() - responseAllowed := false - for _, code := range drv.options.AllowedHTTPCodes { - if resp.StatusCode == code { - responseAllowed = true - break - } - } - - if !responseAllowed { + if !drv.responseCodeAllowed(resp) { return nil, errors.New(resp.Status) } @@ -186,6 +178,15 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM return NewHTMLPage(doc, params.URL, &r, cookies) } +func (drv *Driver) responseCodeAllowed(resp *stdhttp.Response) bool { + for _, code := range drv.options.AllowedHTTPCodes { + if resp.StatusCode == code { + return true + } + } + return false +} + func (drv *Driver) Parse(_ context.Context, str values.String) (drivers.HTMLPage, error) { buf := bytes.NewBuffer([]byte(str)) From b9a55a7582d1e42e1e7ffecadc0e1ac3af103a62 Mon Sep 17 00:00:00 2001 From: gabriel-m Date: Mon, 14 Oct 2019 17:46:30 +0200 Subject: [PATCH 11/23] typo --- pkg/drivers/http/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/drivers/http/driver.go b/pkg/drivers/http/driver.go index edb5fa7c..30cea7bf 100644 --- a/pkg/drivers/http/driver.go +++ b/pkg/drivers/http/driver.go @@ -178,7 +178,7 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM return NewHTMLPage(doc, params.URL, &r, cookies) } -func (drv *Driver) responseCodeAllowed(resp *stdhttp.Response) bool { +func (drv *Driver) responseCodeAllowed(resp *http.Response) bool { for _, code := range drv.options.AllowedHTTPCodes { if resp.StatusCode == code { return true From e5b85976aeecee29d9940e65e16098f0158de05b Mon Sep 17 00:00:00 2001 From: gabriel-m Date: Mon, 14 Oct 2019 23:30:28 +0200 Subject: [PATCH 12/23] removed obsolete comment --- pkg/drivers/http/driver.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/drivers/http/driver.go b/pkg/drivers/http/driver.go index 30cea7bf..a514c14e 100644 --- a/pkg/drivers/http/driver.go +++ b/pkg/drivers/http/driver.go @@ -168,7 +168,6 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM return nil, err } - // HTTPResponse is temporarily unavailable when the status code != OK r := drivers.HTTPResponse{ StatusCode: resp.StatusCode, Status: resp.Status, From fbaafec40a54675a608743c8fe0247a0be9e365a Mon Sep 17 00:00:00 2001 From: gabriel-m Date: Mon, 14 Oct 2019 23:56:08 +0200 Subject: [PATCH 13/23] skip issue reporting for examples --- .golangci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 14a4ad11..abca27ef 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -30,4 +30,6 @@ issues: # no need to include all autogenerated files, we confidently recognize # autogenerated files. If it's not please let us know. skip-files: - - "*_test.go" \ No newline at end of file + - "*_test.go" + skip-dirs: + - examples From f4c63c0c6c663c4c66f255603901a975eda12d9d Mon Sep 17 00:00:00 2001 From: gabriel-m Date: Tue, 15 Oct 2019 08:11:15 +0200 Subject: [PATCH 14/23] Update .golangci.yml --- .golangci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index abca27ef..af609fb7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -11,6 +11,7 @@ run: skip-dirs: - pkg/parser/fql - pkg/parser/antlr + - examples linters: disable: @@ -31,5 +32,3 @@ issues: # autogenerated files. If it's not please let us know. skip-files: - "*_test.go" - skip-dirs: - - examples From 95b40fe14a6ae08cb4f9a65c9134ebeeb5d9825d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2019 13:27:58 +0000 Subject: [PATCH 15/23] Bump github.com/antchfx/htmlquery from 1.0.0 to 1.1.0 Bumps [github.com/antchfx/htmlquery](https://github.com/antchfx/htmlquery) from 1.0.0 to 1.1.0. - [Release notes](https://github.com/antchfx/htmlquery/releases) - [Commits](https://github.com/antchfx/htmlquery/compare/v1.0.0...1.1.0) Signed-off-by: dependabot-preview[bot] --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index fe347bf6..7edad35d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/PuerkitoBio/goquery v1.5.0 - github.com/antchfx/htmlquery v1.0.0 + github.com/antchfx/htmlquery v1.1.0 github.com/antchfx/xpath v1.0.0 github.com/antlr/antlr4 v0.0.0-20190819145818-b43a4c3a8015 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e diff --git a/go.sum b/go.sum index 1411b2d7..a679f94f 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRy github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/antchfx/htmlquery v1.0.0 h1:O5IXz8fZF3B3MW+B33MZWbTHBlYmcfw0BAxgErHuaMA= github.com/antchfx/htmlquery v1.0.0/go.mod h1:MS9yksVSQXls00iXkiMqXr0J+umL/AmxXKuP28SUJM8= +github.com/antchfx/htmlquery v1.1.0 h1:KMS88sLl5KP9GUVU2MQIDcQXNQ0M5MGlkC9WlYgAQqY= +github.com/antchfx/htmlquery v1.1.0/go.mod h1:MS9yksVSQXls00iXkiMqXr0J+umL/AmxXKuP28SUJM8= github.com/antchfx/xpath v1.0.0 h1:Q5gFgh2O40VTSwMOVbFE7nFNRBu3tS21Tn0KAWeEjtk= github.com/antchfx/xpath v1.0.0/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/antlr/antlr4 v0.0.0-20190819145818-b43a4c3a8015 h1:StuiJFxQUsxSCzcby6NFZRdEhPkXD5vxN7TZ4MD6T84= From 428c64bfae14d0ec8b05f4c7da635b738732727f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2019 13:28:17 +0000 Subject: [PATCH 16/23] Bump github.com/antchfx/xpath from 1.0.0 to 1.1.0 Bumps [github.com/antchfx/xpath](https://github.com/antchfx/xpath) from 1.0.0 to 1.1.0. - [Release notes](https://github.com/antchfx/xpath/releases) - [Commits](https://github.com/antchfx/xpath/compare/v1.0.0...1.1.0) Signed-off-by: dependabot-preview[bot] --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index fe347bf6..9288776f 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.13 require ( github.com/PuerkitoBio/goquery v1.5.0 github.com/antchfx/htmlquery v1.0.0 - github.com/antchfx/xpath v1.0.0 + github.com/antchfx/xpath v1.1.0 github.com/antlr/antlr4 v0.0.0-20190819145818-b43a4c3a8015 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e github.com/corpix/uarand v0.1.1 diff --git a/go.sum b/go.sum index 1411b2d7..cf077d67 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/antchfx/htmlquery v1.0.0 h1:O5IXz8fZF3B3MW+B33MZWbTHBlYmcfw0BAxgErHua github.com/antchfx/htmlquery v1.0.0/go.mod h1:MS9yksVSQXls00iXkiMqXr0J+umL/AmxXKuP28SUJM8= github.com/antchfx/xpath v1.0.0 h1:Q5gFgh2O40VTSwMOVbFE7nFNRBu3tS21Tn0KAWeEjtk= github.com/antchfx/xpath v1.0.0/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/antchfx/xpath v1.1.0 h1:mJTvYpiHvxNQRD4Lbfin/FodHVCHh2a5KrOFr4ZxMOI= +github.com/antchfx/xpath v1.1.0/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/antlr/antlr4 v0.0.0-20190819145818-b43a4c3a8015 h1:StuiJFxQUsxSCzcby6NFZRdEhPkXD5vxN7TZ4MD6T84= github.com/antlr/antlr4 v0.0.0-20190819145818-b43a4c3a8015/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= From 2104791e36e2af4b685f1cd4946df88c642fdaab Mon Sep 17 00:00:00 2001 From: Tim Voronov Date: Tue, 15 Oct 2019 13:00:44 -0400 Subject: [PATCH 17/23] Update go.sum --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index cf077d67..e72c91c5 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,6 @@ github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRy github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/antchfx/htmlquery v1.0.0 h1:O5IXz8fZF3B3MW+B33MZWbTHBlYmcfw0BAxgErHuaMA= github.com/antchfx/htmlquery v1.0.0/go.mod h1:MS9yksVSQXls00iXkiMqXr0J+umL/AmxXKuP28SUJM8= -github.com/antchfx/xpath v1.0.0 h1:Q5gFgh2O40VTSwMOVbFE7nFNRBu3tS21Tn0KAWeEjtk= -github.com/antchfx/xpath v1.0.0/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/antchfx/xpath v1.1.0 h1:mJTvYpiHvxNQRD4Lbfin/FodHVCHh2a5KrOFr4ZxMOI= github.com/antchfx/xpath v1.1.0/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/antlr/antlr4 v0.0.0-20190819145818-b43a4c3a8015 h1:StuiJFxQUsxSCzcby6NFZRdEhPkXD5vxN7TZ4MD6T84= From ff4459076288796d99afff2c12d932e54092afda Mon Sep 17 00:00:00 2001 From: Tim Voronov Date: Tue, 15 Oct 2019 13:03:19 -0400 Subject: [PATCH 18/23] Update go.sum --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index a679f94f..d89a9228 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,6 @@ github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= -github.com/antchfx/htmlquery v1.0.0 h1:O5IXz8fZF3B3MW+B33MZWbTHBlYmcfw0BAxgErHuaMA= -github.com/antchfx/htmlquery v1.0.0/go.mod h1:MS9yksVSQXls00iXkiMqXr0J+umL/AmxXKuP28SUJM8= github.com/antchfx/htmlquery v1.1.0 h1:KMS88sLl5KP9GUVU2MQIDcQXNQ0M5MGlkC9WlYgAQqY= github.com/antchfx/htmlquery v1.1.0/go.mod h1:MS9yksVSQXls00iXkiMqXr0J+umL/AmxXKuP28SUJM8= github.com/antchfx/xpath v1.0.0 h1:Q5gFgh2O40VTSwMOVbFE7nFNRBu3tS21Tn0KAWeEjtk= From d0a3218b9ac77769cbd2cab255c773700b52bee9 Mon Sep 17 00:00:00 2001 From: Gabriel Marinkovic Date: Wed, 16 Oct 2019 00:07:01 +0200 Subject: [PATCH 19/23] changed AllowedHTTPCodes from []int to map[int]struct{} --- pkg/drivers/http/driver.go | 11 +---------- pkg/drivers/http/options.go | 10 ++++++---- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/pkg/drivers/http/driver.go b/pkg/drivers/http/driver.go index a514c14e..5880d580 100644 --- a/pkg/drivers/http/driver.go +++ b/pkg/drivers/http/driver.go @@ -152,7 +152,7 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM defer resp.Body.Close() - if !drv.responseCodeAllowed(resp) { + if _, exists := drv.options.AllowedHTTPCodes[resp.StatusCode]; !exists { return nil, errors.New(resp.Status) } @@ -177,15 +177,6 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM return NewHTMLPage(doc, params.URL, &r, cookies) } -func (drv *Driver) responseCodeAllowed(resp *http.Response) bool { - for _, code := range drv.options.AllowedHTTPCodes { - if resp.StatusCode == code { - return true - } - } - return false -} - func (drv *Driver) Parse(_ context.Context, str values.String) (drivers.HTMLPage, error) { buf := bytes.NewBuffer([]byte(str)) diff --git a/pkg/drivers/http/options.go b/pkg/drivers/http/options.go index d5461ece..4615ee1a 100644 --- a/pkg/drivers/http/options.go +++ b/pkg/drivers/http/options.go @@ -19,7 +19,7 @@ type ( UserAgent string Headers drivers.HTTPHeaders Cookies drivers.HTTPCookies - AllowedHTTPCodes []int + AllowedHTTPCodes map[int]struct{} } ) @@ -29,7 +29,7 @@ func newOptions(setters []Option) *Options { opts.Backoff = pester.ExponentialBackoff opts.Concurrency = 3 opts.MaxRetries = 5 - opts.AllowedHTTPCodes = []int{stdhttp.StatusOK} + opts.AllowedHTTPCodes = map[int]struct{}{stdhttp.StatusOK: struct{}{}} for _, setter := range setters { setter(opts) @@ -132,12 +132,14 @@ func WithCookies(cookies []drivers.HTTPCookie) Option { func WithAllowedHTTPCode(httpCode int) Option { return func(opts *Options) { - opts.AllowedHTTPCodes = append(opts.AllowedHTTPCodes, httpCode) + opts.AllowedHTTPCodes[httpCode] = struct{}{} } } func WithAllowedHTTPCodes(httpCodes []int) Option { return func(opts *Options) { - opts.AllowedHTTPCodes = append(opts.AllowedHTTPCodes, httpCodes...) + for _, code := range httpCodes { + opts.AllowedHTTPCodes[code] = struct{}{} + } } } From cdc6573d0f82673a4cce8571b272ee43e80f3b8d Mon Sep 17 00:00:00 2001 From: gabriel-m Date: Wed, 16 Oct 2019 00:54:42 +0200 Subject: [PATCH 20/23] added back allowedHTTPCodes function --- pkg/drivers/http/driver.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/drivers/http/driver.go b/pkg/drivers/http/driver.go index 5880d580..0c483f96 100644 --- a/pkg/drivers/http/driver.go +++ b/pkg/drivers/http/driver.go @@ -152,7 +152,7 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM defer resp.Body.Close() - if _, exists := drv.options.AllowedHTTPCodes[resp.StatusCode]; !exists { + if !drv.responseCodeAllowed(resp) { return nil, errors.New(resp.Status) } @@ -177,6 +177,11 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM return NewHTMLPage(doc, params.URL, &r, cookies) } +func (drv *Driver) responseCodeAllowed(resp *http.Response) bool { + _, exists := drv.options.AllowedHTTPCodes[resp.StatusCode] + return exists +} + func (drv *Driver) Parse(_ context.Context, str values.String) (drivers.HTMLPage, error) { buf := bytes.NewBuffer([]byte(str)) From dab50a1c535d4b983d7543b9009f760543959b17 Mon Sep 17 00:00:00 2001 From: gabriel-m Date: Wed, 16 Oct 2019 01:00:28 +0200 Subject: [PATCH 21/23] formatting --- pkg/drivers/http/driver.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/drivers/http/driver.go b/pkg/drivers/http/driver.go index 0c483f96..60cfd2f1 100644 --- a/pkg/drivers/http/driver.go +++ b/pkg/drivers/http/driver.go @@ -178,8 +178,8 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM } func (drv *Driver) responseCodeAllowed(resp *http.Response) bool { - _, exists := drv.options.AllowedHTTPCodes[resp.StatusCode] - return exists + _, exists := drv.options.AllowedHTTPCodes[resp.StatusCode] + return exists } func (drv *Driver) Parse(_ context.Context, str values.String) (drivers.HTMLPage, error) { From 40f043b4143892813195d32a9e4542947ca42237 Mon Sep 17 00:00:00 2001 From: gabriel-m Date: Wed, 16 Oct 2019 01:00:28 +0200 Subject: [PATCH 22/23] formatting --- pkg/drivers/http/driver.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/drivers/http/driver.go b/pkg/drivers/http/driver.go index 0c483f96..9e0e576e 100644 --- a/pkg/drivers/http/driver.go +++ b/pkg/drivers/http/driver.go @@ -178,8 +178,8 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM } func (drv *Driver) responseCodeAllowed(resp *http.Response) bool { - _, exists := drv.options.AllowedHTTPCodes[resp.StatusCode] - return exists + _, exists := drv.options.AllowedHTTPCodes[resp.StatusCode] + return exists } func (drv *Driver) Parse(_ context.Context, str values.String) (drivers.HTMLPage, error) { From 95ed4ba18c1a7887e5437c296a1fc529aa66cf47 Mon Sep 17 00:00:00 2001 From: Tim Voronov Date: Wed, 16 Oct 2019 12:27:43 -0400 Subject: [PATCH 23/23] Added a comment --- pkg/runtime/program.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/runtime/program.go b/pkg/runtime/program.go index e61dd255..dd4391bd 100644 --- a/pkg/runtime/program.go +++ b/pkg/runtime/program.go @@ -117,6 +117,8 @@ func (p *Program) validateParams(opts *Options) error { return nil } + // There might be no errors. + // Thus, we allocate this slice lazily, on a first error. var missedParams []string for n := range p.params {