From 46009da1e7ba9518563fa4d1fb74328f74bf2db8 Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 9 Apr 2021 16:58:42 +0300 Subject: [PATCH] Fixes and changes 2 (#614) * feat(core): added SourceErrorDetail * fix(driver): fix sets user headers and cookies * fix(driver): check users params on nil * fix formating * fix(tests): fix tests and change SourceErrorDetail --- pkg/drivers/http/driver.go | 85 ++++++++++++++++++-------- pkg/runtime/core/errors.go | 14 ++++- pkg/runtime/expressions/body_test.go | 2 +- pkg/runtime/expressions/param_test.go | 2 +- pkg/runtime/expressions/return_test.go | 5 +- 5 files changed, 78 insertions(+), 30 deletions(-) diff --git a/pkg/drivers/http/driver.go b/pkg/drivers/http/driver.go index 5e8886a4..e9fd343d 100644 --- a/pkg/drivers/http/driver.go +++ b/pkg/drivers/http/driver.go @@ -3,6 +3,8 @@ package http import ( "bytes" "context" + "github.com/MontFerret/ferret/pkg/runtime/logging" + "github.com/MontFerret/ferret/pkg/runtime/values" "github.com/gobwas/glob" "io" "net/http" @@ -16,7 +18,6 @@ import ( "github.com/MontFerret/ferret/pkg/drivers" "github.com/MontFerret/ferret/pkg/drivers/common" - "github.com/MontFerret/ferret/pkg/runtime/logging" ) const DriverName = "http" @@ -87,40 +88,18 @@ func (drv *Driver) Name() string { func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTMLPage, error) { req, err := http.NewRequest(http.MethodGet, params.URL, nil) - if err != nil { return nil, err } - logger := logging.FromContext(ctx) - - req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8") - req.Header.Set("Accept-Language", "en-US,en;q=0.9,ru;q=0.8") - req.Header.Set("Cache-Control", "no-cache") - req.Header.Set("Pragma", "no-cache") - params = drivers.SetDefaultParams(drv.options.Options, params) - req = req.WithContext(ctx) - - ua := common.GetUserAgent(params.UserAgent) - - logger. - Debug(). - Timestamp(). - Str("user-agent", ua). - Msg("using User-Agent") - - if ua != "" { - req.Header.Set("User-Agent", ua) - } + drv.makeRequest(ctx, req, params) resp, err := drv.client.Do(req) - if err != nil { return nil, errors.Wrapf(err, "failed to retrieve a document %s", params.URL) } - defer resp.Body.Close() var queryFilters []drivers.StatusCodeFilter @@ -147,7 +126,6 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM } cookies, err := toDriverCookies(resp.Cookies()) - if err != nil { return nil, err } @@ -228,3 +206,60 @@ func (drv *Driver) convertToUTF8(reader io.Reader, srcCharset string) (data io.R return } + +func (drv *Driver) makeRequest(ctx context.Context, req *http.Request, params drivers.Params) { + logger := logging.FromContext(ctx) + + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8") + req.Header.Set("Accept-Language", "en-US,en;q=0.9,ru;q=0.8") + req.Header.Set("Cache-Control", "no-cache") + req.Header.Set("Pragma", "no-cache") + + if params.Headers != nil { + params.Headers.ForEach(func(value []string, key string) bool { + v := params.Headers.Get(key) + + req.Header.Add(key, v) + + logger. + Debug(). + Timestamp(). + Str("header", key). + Msg("set header") + + return true + }) + } + + if params.Cookies != nil { + params.Cookies.ForEach(func(value drivers.HTTPCookie, key values.String) bool { + v, exist := params.Cookies.Get(key) + if !exist { + return false + } + + req.AddCookie(fromDriverCookie(v)) + + logger. + Debug(). + Timestamp(). + Str("cookie", key.String()). + Msg("set cookie") + + return true + }) + } + + ua := common.GetUserAgent(params.UserAgent) + logger. + Debug(). + Timestamp(). + Str("user-agent", ua). + Msg("using User-Agent") + + if ua != "" { + req.Header.Set("User-Agent", ua) + } + + req = req.WithContext(ctx) +} diff --git a/pkg/runtime/core/errors.go b/pkg/runtime/core/errors.go index 2c732e5d..93b34593 100644 --- a/pkg/runtime/core/errors.go +++ b/pkg/runtime/core/errors.go @@ -7,6 +7,15 @@ import ( "github.com/pkg/errors" ) +type SourceErrorDetail struct { + BaseError error + ComputeError error +} + +func (e *SourceErrorDetail) Error() string { + return e.ComputeError.Error() +} + var ( ErrMissedArgument = errors.New("missed argument") ErrInvalidArgument = errors.New("invalid argument") @@ -26,7 +35,10 @@ var ( const typeErrorTemplate = "expected %s, but got %s" func SourceError(src SourceMap, err error) error { - return errors.Errorf("%s: %s", err.Error(), src.String()) + return &SourceErrorDetail{ + BaseError: err, + ComputeError: errors.Errorf("%s: %s", err.Error(), src.String()), + } } func TypeError(actual Type, expected ...Type) error { diff --git a/pkg/runtime/expressions/body_test.go b/pkg/runtime/expressions/body_test.go index fd4a2840..a070a90a 100644 --- a/pkg/runtime/expressions/body_test.go +++ b/pkg/runtime/expressions/body_test.go @@ -104,7 +104,7 @@ func TestBody(t *testing.T) { value, err := s.Exec(context.Background(), scope) So(err, ShouldNotBeNil) - So(err, ShouldHaveSameTypeAs, core.ErrNotFound) + So(err.(*core.SourceErrorDetail).BaseError, ShouldHaveSameTypeAs, core.ErrNotFound) So(value, ShouldHaveSameTypeAs, values.None) }) diff --git a/pkg/runtime/expressions/param_test.go b/pkg/runtime/expressions/param_test.go index 79f5cd00..05057fac 100644 --- a/pkg/runtime/expressions/param_test.go +++ b/pkg/runtime/expressions/param_test.go @@ -61,7 +61,7 @@ func TestParameterExpressionExec(t *testing.T) { value, err := notExistExp.Exec(ctx, &core.Scope{}) So(err, ShouldNotBeNil) - So(err, ShouldHaveSameTypeAs, core.ErrNotFound) + So(err.(*core.SourceErrorDetail).BaseError, ShouldHaveSameTypeAs, core.ErrNotFound) So(value.Type().Equals(types.None), ShouldBeTrue) }) } diff --git a/pkg/runtime/expressions/return_test.go b/pkg/runtime/expressions/return_test.go index 00cf5d0e..93705dc4 100644 --- a/pkg/runtime/expressions/return_test.go +++ b/pkg/runtime/expressions/return_test.go @@ -2,9 +2,10 @@ package expressions_test import ( "context" + "testing" + "github.com/MontFerret/ferret/pkg/runtime/expressions/literals" "github.com/MontFerret/ferret/pkg/runtime/values" - "testing" "github.com/MontFerret/ferret/pkg/runtime/core" "github.com/MontFerret/ferret/pkg/runtime/expressions" @@ -64,7 +65,7 @@ func TestReturnExpression(t *testing.T) { value, err := exp.Exec(context.Background(), scope) So(err, ShouldNotBeNil) - So(err, ShouldHaveSameTypeAs, core.ErrNotFound) + So(err.(*core.SourceErrorDetail).BaseError, ShouldHaveSameTypeAs, core.ErrNotFound) So(value, ShouldHaveSameTypeAs, values.None) })