go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2025-07-17 01:32:22 +02:00
Code Issues Releases Activity

Feature/#360 default driver params (#372)

Browse Source 
* Added default headers and cookies

* wip

* Added tests

* Added default headers and cookies to HTTP driver

* Removed unused struct prop
This commit is contained in:
Tim Voronov
2019-09-05 11:49:21 -04:00
committed by GitHub
parent 4b1633269a
commit 9b762d32ee
21 changed files with 635 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
e2e/runner/runner.go
View File

@ -60,12 +60,108 @@ func (r *Runner) Run(ctx context.Context) error {
cdp.NewDriver(cdp.WithAddress(r.settings.CDPAddress)), cdp.NewDriver(cdp.WithAddress(r.settings.CDPAddress)),
) )
ctx = drivers.WithContext(
ctx,
cdp.NewDriver(cdp.WithAddress(r.settings.CDPAddress),
cdp.WithCustomName("cdp_headers"),
cdp.WithHeader("Single_header", []string{"single_header_value"}),
cdp.WithHeaders(drivers.HTTPHeaders{
"Multi_set_header": []string{"multi_set_header_value"},
"Multi_set_header2": []string{"multi_set_header2_value"},
}),
),
)
ctx = drivers.WithContext(
ctx,
cdp.NewDriver(cdp.WithAddress(r.settings.CDPAddress),
cdp.WithCustomName("cdp_cookies"),
cdp.WithCookie(drivers.HTTPCookie{
Name: "single_cookie",
Value: "single_cookie_value",
Path: "/",
MaxAge: 0,
Secure: false,
HTTPOnly: false,
SameSite: 0,
}),
cdp.WithCookies([]drivers.HTTPCookie{
{
Name: "multi_set_cookie",
Value: "multi_set_cookie_value",
Path: "/",
MaxAge: 0,
Secure: false,
HTTPOnly: false,
SameSite: 0,
},
{
Name: "multi_set_cookie2",
Value: "multi_set_cookie2_value",
Path: "/",
MaxAge: 0,
Secure: false,
HTTPOnly: false,
SameSite: 0,
},
}),
),
)
ctx = drivers.WithContext( ctx = drivers.WithContext(
ctx, ctx,
http.NewDriver(), http.NewDriver(),
drivers.AsDefault(), drivers.AsDefault(),
) )
ctx = drivers.WithContext(
ctx,
http.NewDriver(
http.WithCustomName("http_headers"),
http.WithHeader("Single_header", []string{"single_header_value"}),
http.WithHeaders(drivers.HTTPHeaders{
"Multi_set_header": []string{"multi_set_header_value"},
"Multi_set_header2": []string{"multi_set_header2_value"},
}),
),
)
ctx = drivers.WithContext(
ctx,
http.NewDriver(
http.WithCustomName("http_cookies"),
http.WithCookie(drivers.HTTPCookie{
Name: "single_cookie",
Value: "single_cookie_value",
Path: "/",
MaxAge: 0,
Secure: false,
HTTPOnly: false,
SameSite: 0,
}),
http.WithCookies([]drivers.HTTPCookie{
{
Name: "multi_set_cookie",
Value: "multi_set_cookie_value",
Path: "/",
MaxAge: 0,
Secure: false,
HTTPOnly: false,
SameSite: 0,
},
{
Name: "multi_set_cookie2",
Value: "multi_set_cookie2_value",
Path: "/",
MaxAge: 0,
Secure: false,
HTTPOnly: false,
SameSite: 0,
},
}),
),
)
results, err := r.runQueries(ctx, r.settings.Dir) results, err := r.runQueries(ctx, r.settings.Dir)
if err != nil { if err != nil {

15
e2e/server/server.go
View File

@ -54,6 +54,18 @@ func New(settings Settings) *Server {
headers = string(b) headers = string(b)
} }
var cookies string
if len(ctx.Request().Cookies()) > 0 {
b, err := json.Marshal(ctx.Request().Cookies())
if err != nil {
return err
}
cookies = string(b)
}
ts := time.Now().Format("2006-01-02 15:04:05") ts := time.Now().Format("2006-01-02 15:04:05")
return ctx.HTML(http.StatusOK, fmt.Sprintf(` return ctx.HTML(http.StatusOK, fmt.Sprintf(`
@ -65,9 +77,10 @@ func New(settings Settings) *Server {
<body> <body>
<span id="timestamp">%s</span> <span id="timestamp">%s</span>
<span id="headers">%s</span> <span id="headers">%s</span>
<span id="cookies">%s</span>
</body> </body>
</html> </html>
`, ts, headers)) `, ts, headers, cookies))
}) })
api.GET("/ping", func(ctx echo.Context) error { api.GET("/ping", func(ctx echo.Context) error {
return ctx.JSON(http.StatusOK, echo.Map{ return ctx.JSON(http.StatusOK, echo.Map{

21
e2e/tests/dynamic/doc/cookies/default.fql Normal file
View File

@ -0,0 +1,21 @@
LET url = @static + "/api/ts"
LET page = DOCUMENT(url, {
driver: "cdp_cookies"
})
LET el = ELEMENT(page, "#cookies")
LET actual = (
FOR c IN JSON_PARSE(el.innerText)
SORT c.Name
RETURN c
)
LET expected = {
"Single_cookie": "single_cookie_value",
"Multi_set_cookie": "multi_set_cookie_value",
}
RETURN EXPECT(expected, {
"Single_cookie": actual[2].Value,
"Multi_set_cookie": actual[0].Value,
})

31
e2e/tests/dynamic/doc/cookies/override_default.fql Normal file
View File

@ -0,0 +1,31 @@
LET url = @static + "/api/ts"
LET page = DOCUMENT(url, {
driver: "cdp_cookies",
cookies: [
{
name: "Single_cookie",
value: "Foo"
},
{
name: "Multi_set_cookie",
value: "Bar"
}
]
})
LET el = ELEMENT(page, "#cookies")
LET actual = (
FOR c IN JSON_PARSE(el.innerText)
SORT c.Name
RETURN c
)
LET expected = {
"Single_cookie": "Foo",
"Multi_set_cookie": "Bar",
}
RETURN EXPECT(expected, {
"Single_cookie": actual[1].Value,
"Multi_set_cookie": actual[0].Value,
})

17
e2e/tests/dynamic/doc/headers/default.fql Normal file
View File

@ -0,0 +1,17 @@
LET url = @static + "/api/ts"
LET page = DOCUMENT(url, {
driver: "cdp_headers"
})
LET el = ELEMENT(page, "#headers")
LET actual = JSON_PARSE(el.innerText)
LET expected = {
"Single_header": ["single_header_value"],
"Multi_set_header":["multi_set_header_value"],
}
RETURN EXPECT(expected, {
"Single_header": actual["Single_header"],
"Multi_set_header": actual["Multi_set_header"],
})

20
e2e/tests/dynamic/doc/headers/override_default.fql Normal file
View File

@ -0,0 +1,20 @@
LET url = @static + "/api/ts"
LET page = DOCUMENT(url, {
driver: "cdp_headers",
headers: {
"single_header": "foo"
}
})
LET el = ELEMENT(page, "#headers")
LET actual = JSON_PARSE(el.innerText)
LET expected = {
"Single_header": ["foo"],
"Multi_set_header":["multi_set_header_value"],
}
RETURN EXPECT(expected, {
"Single_header": actual["Single_header"],
"Multi_set_header": actual["Multi_set_header"],
})

21
e2e/tests/static/doc/cookies/default.fql Normal file
View File

@ -0,0 +1,21 @@
LET url = @static + "/api/ts"
LET page = DOCUMENT(url, {
driver: "http_cookies"
})
LET el = ELEMENT(page, "#cookies")
LET actual = (
FOR c IN JSON_PARSE(el.innerText)
SORT c.Name
RETURN c
)
LET expected = {
"Single_cookie": "single_cookie_value",
"Multi_set_cookie": "multi_set_cookie_value",
}
RETURN EXPECT(expected, {
"Single_cookie": actual[2].Value,
"Multi_set_cookie": actual[0].Value,
})

10
e2e/tests/static/doc/cookies/get.fql Normal file
View File

@ -0,0 +1,10 @@
LET url = @static + "/api/ts"
LET doc = DOCUMENT(url, {
driver: "http"
})
LET cookiesPath = LENGTH(doc.cookies) > 0 ? "ok" : "false"
LET cookie = COOKIE_GET(doc, "x-ferret")
LET expected = "ok e2e"
RETURN EXPECT(expected, cookiesPath + " " + cookie.value)

31
e2e/tests/static/doc/cookies/override_default.fql Normal file
View File

@ -0,0 +1,31 @@
LET url = @static + "/api/ts"
LET page = DOCUMENT(url, {
driver: "http_cookies",
cookies: [
{
name: "Single_cookie",
value: "Foo"
},
{
name: "Multi_set_cookie",
value: "Bar"
}
]
})
LET el = ELEMENT(page, "#cookies")
LET actual = (
FOR c IN JSON_PARSE(el.innerText)
SORT c.Name
RETURN c
)
LET expected = {
"Single_cookie": "Foo",
"Multi_set_cookie": "Bar",
}
RETURN EXPECT(expected, {
"Single_cookie": actual[1].Value,
"Multi_set_cookie": actual[0].Value,
})

17
e2e/tests/static/doc/headers/default.fql Normal file
View File

@ -0,0 +1,17 @@
LET url = @static + "/api/ts"
LET page = DOCUMENT(url, {
driver: "http_headers"
})
LET el = ELEMENT(page, "#headers")
LET actual = JSON_PARSE(el.innerText)
LET expected = {
"Single_header": ["single_header_value"],
"Multi_set_header":["multi_set_header_value"],
}
RETURN EXPECT(expected, {
"Single_header": actual["Single_header"],
"Multi_set_header": actual["Multi_set_header"],
})

20
e2e/tests/static/doc/headers/override_default.fql Normal file
View File

@ -0,0 +1,20 @@
LET url = @static + "/api/ts"
LET page = DOCUMENT(url, {
driver: "http_headers",
headers: {
"single_header": "foo"
}
})
LET el = ELEMENT(page, "#headers")
LET actual = JSON_PARSE(el.innerText)
LET expected = {
"Single_header": ["foo"],
"Multi_set_header":["multi_set_header_value"],
}
RETURN EXPECT(expected, {
"Single_header": actual["Single_header"],
"Multi_set_header": actual["Multi_set_header"],
})

28
pkg/drivers/cdp/driver.go
View File

@ -107,6 +107,34 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
params.Viewport = defaultViewport params.Viewport = defaultViewport
} }
if drv.options.Headers != nil && params.Headers == nil {
params.Headers = make(drivers.HTTPHeaders)
}
// set default headers
for k, v := range drv.options.Headers {
_, exists := params.Headers[k]
// do not override user's set values
if !exists {
params.Headers[k] = v
}
}
if drv.options.Cookies != nil && params.Cookies == nil {
params.Cookies = make(drivers.HTTPCookies)
}
// set default cookies
for k, v := range drv.options.Cookies {
_, exists := params.Cookies[k]
// do not override user's set values
if !exists {
params.Cookies[k] = v
}
}
return LoadHTMLPage(ctx, conn, params) return LoadHTMLPage(ctx, conn, params)
} }

48
pkg/drivers/cdp/options.go
View File

@ -1,5 +1,7 @@
package cdp package cdp
import "github.com/MontFerret/ferret/pkg/drivers"
type ( type (
Options struct { Options struct {
Name string Name string
@ -7,6 +9,8 @@ type (
UserAgent string UserAgent string
Address string Address string
KeepCookies bool KeepCookies bool
Headers drivers.HTTPHeaders
Cookies drivers.HTTPCookies
} }
Option func(opts *Options) Option func(opts *Options)
@ -57,3 +61,47 @@ func WithCustomName(name string) Option {
opts.Name = name opts.Name = name
} }
} }
func WithHeader(name string, value []string) Option {
return func(opts *Options) {
if opts.Headers == nil {
opts.Headers = make(drivers.HTTPHeaders)
}
opts.Headers[name] = value
}
}
func WithHeaders(headers drivers.HTTPHeaders) Option {
return func(opts *Options) {
if opts.Headers == nil {
opts.Headers = make(drivers.HTTPHeaders)
}
for k, v := range headers {
opts.Headers[k] = v
}
}
}
func WithCookie(cookie drivers.HTTPCookie) Option {
return func(opts *Options) {
if opts.Cookies == nil {
opts.Cookies = make(drivers.HTTPCookies)
}
opts.Cookies[cookie.Name] = cookie
}
}
func WithCookies(cookies []drivers.HTTPCookie) Option {
return func(opts *Options) {
if opts.Cookies == nil {
opts.Cookies = make(drivers.HTTPCookies)
}
for _, c := range cookies {
opts.Cookies[c.Name] = c
}
}
}

4
pkg/drivers/cdp/page.go
View File

@ -143,7 +143,7 @@ func LoadHTMLPage(
return nil, err return nil, err
} }
if params.Cookies != nil { if len(params.Cookies) > 0 {
cookies := make([]network.CookieParam, 0, len(params.Cookies)) cookies := make([]network.CookieParam, 0, len(params.Cookies))
for _, c := range params.Cookies { for _, c := range params.Cookies {
@ -166,7 +166,7 @@ func LoadHTMLPage(
} }
} }
if params.Headers != nil { if len(params.Headers) > 0 {
j, err := json.Marshal(params.Headers) j, err := json.Marshal(params.Headers)
if err != nil { if err != nil {

2
pkg/drivers/cookie.go
View File

@ -30,6 +30,8 @@ type (
HTTPOnly bool HTTPOnly bool
SameSite SameSite SameSite SameSite
} }
HTTPCookies map[string]HTTPCookie
) )
const ( const (

57
pkg/drivers/http/driver.go
View File

@ -26,7 +26,7 @@ func NewDriver(opts ...Option) *Driver {
drv := new(Driver) drv := new(Driver)
drv.options = newOptions(opts) drv.options = newOptions(opts)
if drv.options.proxy == "" { if drv.options.Proxy == "" {
drv.client = pester.New() drv.client = pester.New()
} else { } else {
client, err := newClientWithProxy(drv.options) client, err := newClientWithProxy(drv.options)
@ -38,15 +38,15 @@ func NewDriver(opts ...Option) *Driver {
} }
} }
drv.client.Concurrency = drv.options.concurrency drv.client.Concurrency = drv.options.Concurrency
drv.client.MaxRetries = drv.options.maxRetries drv.client.MaxRetries = drv.options.MaxRetries
drv.client.Backoff = drv.options.backoff drv.client.Backoff = drv.options.Backoff
return drv return drv
} }
func newClientWithProxy(options *Options) (*http.Client, error) { func newClientWithProxy(options *Options) (*http.Client, error) {
proxyURL, err := url.Parse(options.proxy) proxyURL, err := url.Parse(options.Proxy)
if err != nil { if err != nil {
return nil, err return nil, err
@ -59,7 +59,7 @@ func newClientWithProxy(options *Options) (*http.Client, error) {
} }
func (drv *Driver) Name() string { func (drv *Driver) Name() string {
return DriverName return drv.options.Name
} }
func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTMLPage, error) { func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTMLPage, error) {
@ -76,7 +76,20 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
req.Header.Set("Cache-Control", "no-cache") req.Header.Set("Cache-Control", "no-cache")
req.Header.Set("Pragma", "no-cache") req.Header.Set("Pragma", "no-cache")
if params.Headers != nil { if drv.options.Headers != nil && params.Headers == nil {
params.Headers = make(drivers.HTTPHeaders)
}
// Set default headers
for k, v := range drv.options.Headers {
_, exists := params.Headers[k]
// do not override user's set values
if !exists {
params.Headers[k] = v
}
}
for k := range params.Headers { for k := range params.Headers {
req.Header.Add(k, params.Headers.Get(k)) req.Header.Add(k, params.Headers.Get(k))
@ -86,14 +99,23 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
Str("header", k). Str("header", k).
Msg("set header") Msg("set header")
} }
if drv.options.Cookies != nil && params.Cookies == nil {
params.Cookies = make(drivers.HTTPCookies)
}
// set default cookies
for k, v := range drv.options.Cookies {
_, exists := params.Cookies[k]
// do not override user's set values
if !exists {
params.Cookies[k] = v
}
} }
if params.Cookies != nil {
for _, c := range params.Cookies { for _, c := range params.Cookies {
req.AddCookie(&http.Cookie{ req.AddCookie(fromDriverCookie(c))
Name: c.Name,
Value: c.Value,
})
logger. logger.
Debug(). Debug().
@ -101,7 +123,6 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
Str("cookie", c.Name). Str("cookie", c.Name).
Msg("set cookie") Msg("set cookie")
} }
}
req = req.WithContext(ctx) req = req.WithContext(ctx)
@ -110,7 +131,7 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
if params.UserAgent != "" { if params.UserAgent != "" {
ua = common.GetUserAgent(params.UserAgent) ua = common.GetUserAgent(params.UserAgent)
} else { } else {
ua = common.GetUserAgent(drv.options.userAgent) ua = common.GetUserAgent(drv.options.UserAgent)
} }
logger. logger.
@ -141,7 +162,13 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
return nil, errors.Wrapf(err, "failed to parse a document %s", params.URL) return nil, errors.Wrapf(err, "failed to parse a document %s", params.URL)
} }
return NewHTMLPage(doc, params.URL, params.Cookies) cookies, err := toDriverCookies(resp.Cookies())
if err != nil {
return nil, err
}
return NewHTMLPage(doc, params.URL, cookies)
} }
func (drv *Driver) Parse(_ context.Context, str values.String) (drivers.HTMLPage, error) { func (drv *Driver) Parse(_ context.Context, str values.String) (drivers.HTMLPage, error) {

78
pkg/drivers/http/helpers.go
View File

@ -2,14 +2,16 @@ package http
import ( import (
"bytes" "bytes"
"golang.org/x/net/html" HTTP "net/http"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"github.com/antchfx/htmlquery" "github.com/antchfx/htmlquery"
"github.com/antchfx/xpath" "github.com/antchfx/xpath"
"golang.org/x/net/html"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
) )
func parseXPathNode(nav *htmlquery.NodeNavigator) (core.Value, error) { func parseXPathNode(nav *htmlquery.NodeNavigator) (core.Value, error) {
@ -45,3 +47,71 @@ func outerHTML(s *goquery.Selection) (string, error) {
return buf.String(), nil return buf.String(), nil
} }
func toDriverCookies(cookies []*HTTP.Cookie) (drivers.HTTPCookies, error) {
res := make(drivers.HTTPCookies)
for _, c := range cookies {
dc, err := toDriverCookie(c)
if err != nil {
return nil, err
}
res[dc.Name] = dc
}
return res, nil
}
func toDriverCookie(cookie *HTTP.Cookie) (drivers.HTTPCookie, error) {
res := drivers.HTTPCookie{}
if cookie == nil {
return res, core.Error(core.ErrMissedArgument, "cookie")
}
res.Name = cookie.Name
res.Value = cookie.Value
res.Path = cookie.Path
res.Domain = cookie.Domain
res.Expires = cookie.Expires
res.MaxAge = cookie.MaxAge
res.Secure = cookie.Secure
res.HTTPOnly = cookie.HttpOnly
switch cookie.SameSite {
case HTTP.SameSiteLaxMode:
res.SameSite = drivers.SameSiteLaxMode
case HTTP.SameSiteStrictMode:
res.SameSite = drivers.SameSiteStrictMode
default:
res.SameSite = drivers.SameSiteDefaultMode
}
return res, nil
}
func fromDriverCookie(cookie drivers.HTTPCookie) *HTTP.Cookie {
res := &HTTP.Cookie{}
res.Name = cookie.Name
res.Value = cookie.Value
res.Path = cookie.Path
res.Domain = cookie.Domain
res.Expires = cookie.Expires
res.MaxAge = cookie.MaxAge
res.Secure = cookie.Secure
res.HttpOnly = cookie.HTTPOnly
switch cookie.SameSite {
case drivers.SameSiteLaxMode:
res.SameSite = HTTP.SameSiteLaxMode
case drivers.SameSiteStrictMode:
res.SameSite = HTTP.SameSiteStrictMode
default:
res.SameSite = HTTP.SameSiteDefaultMode
}
return res
}

85
pkg/drivers/http/options.go
View File

@ -1,6 +1,7 @@
package http package http
import ( import (
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/sethgrid/pester" "github.com/sethgrid/pester"
) )
@ -8,19 +9,23 @@ type (
Option func(opts *Options) Option func(opts *Options)
Options struct { Options struct {
backoff pester.BackoffStrategy Name string
maxRetries int Backoff pester.BackoffStrategy
concurrency int MaxRetries int
proxy string Concurrency int
userAgent string Proxy string
UserAgent string
Headers drivers.HTTPHeaders
Cookies drivers.HTTPCookies
} }
) )
func newOptions(setters []Option) *Options { func newOptions(setters []Option) *Options {
opts := new(Options) opts := new(Options)
opts.backoff = pester.ExponentialBackoff opts.Name = DriverName
opts.concurrency = 3 opts.Backoff = pester.ExponentialBackoff
opts.maxRetries = 5 opts.Concurrency = 3
opts.MaxRetries = 5
for _, setter := range setters { for _, setter := range setters {
setter(opts) setter(opts)
@ -31,42 +36,92 @@ func newOptions(setters []Option) *Options {
func WithDefaultBackoff() Option { func WithDefaultBackoff() Option {
return func(opts *Options) { return func(opts *Options) {
opts.backoff = pester.DefaultBackoff opts.Backoff = pester.DefaultBackoff
} }
} }
func WithLinearBackoff() Option { func WithLinearBackoff() Option {
return func(opts *Options) { return func(opts *Options) {
opts.backoff = pester.LinearBackoff opts.Backoff = pester.LinearBackoff
} }
} }
func WithExponentialBackoff() Option { func WithExponentialBackoff() Option {
return func(opts *Options) { return func(opts *Options) {
opts.backoff = pester.ExponentialBackoff opts.Backoff = pester.ExponentialBackoff
} }
} }
func WithMaxRetries(value int) Option { func WithMaxRetries(value int) Option {
return func(opts *Options) { return func(opts *Options) {
opts.maxRetries = value opts.MaxRetries = value
} }
} }
func WithConcurrency(value int) Option { func WithConcurrency(value int) Option {
return func(opts *Options) { return func(opts *Options) {
opts.concurrency = value opts.Concurrency = value
} }
} }
func WithProxy(address string) Option { func WithProxy(address string) Option {
return func(opts *Options) { return func(opts *Options) {
opts.proxy = address opts.Proxy = address
} }
} }
func WithUserAgent(value string) Option { func WithUserAgent(value string) Option {
return func(opts *Options) { return func(opts *Options) {
opts.userAgent = value opts.UserAgent = value
}
}
func WithCustomName(name string) Option {
return func(opts *Options) {
opts.Name = name
}
}
func WithHeader(name string, value []string) Option {
return func(opts *Options) {
if opts.Headers == nil {
opts.Headers = make(drivers.HTTPHeaders)
}
opts.Headers[name] = value
}
}
func WithHeaders(headers drivers.HTTPHeaders) Option {
return func(opts *Options) {
if opts.Headers == nil {
opts.Headers = make(drivers.HTTPHeaders)
}
for k, v := range headers {
opts.Headers[k] = v
}
}
}
func WithCookie(cookie drivers.HTTPCookie) Option {
return func(opts *Options) {
if opts.Cookies == nil {
opts.Cookies = make(drivers.HTTPCookies)
}
opts.Cookies[cookie.Name] = cookie
}
}
func WithCookies(cookies []drivers.HTTPCookie) Option {
return func(opts *Options) {
if opts.Cookies == nil {
opts.Cookies = make(drivers.HTTPCookies)
}
for _, c := range cookies {
opts.Cookies[c.Name] = c
}
} }
} }

18
pkg/drivers/http/page.go
View File

@ -2,24 +2,26 @@ package http
import ( import (
"context" "context"
"hash/fnv"
"github.com/PuerkitoBio/goquery"
"github.com/MontFerret/ferret/pkg/drivers" "github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/common" "github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core" "github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values" "github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/PuerkitoBio/goquery"
"hash/fnv"
) )
type HTMLPage struct { type HTMLPage struct {
document *HTMLDocument document *HTMLDocument
cookies []drivers.HTTPCookie cookies drivers.HTTPCookies
frames *values.Array frames *values.Array
} }
func NewHTMLPage( func NewHTMLPage(
qdoc *goquery.Document, qdoc *goquery.Document,
url string, url string,
cookies []drivers.HTTPCookie, cookies drivers.HTTPCookies,
) (*HTMLPage, error) { ) (*HTMLPage, error) {
doc, err := NewRootHTMLDocument(qdoc, url) doc, err := NewRootHTMLDocument(qdoc, url)
@ -79,7 +81,13 @@ func (p *HTMLPage) Hash() uint64 {
} }
func (p *HTMLPage) Copy() core.Value { func (p *HTMLPage) Copy() core.Value {
page, err := NewHTMLPage(p.document.doc, p.document.GetURL().String(), p.cookies[:]) cookies := make(drivers.HTTPCookies)
for k, v := range p.cookies {
cookies[k] = v
}
page, err := NewHTMLPage(p.document.doc, p.document.GetURL().String(), cookies)
if err != nil { if err != nil {
return values.None return values.None

2
pkg/drivers/params.go
View File

@ -13,7 +13,7 @@ type (
URL string URL string
UserAgent string UserAgent string
KeepCookies bool KeepCookies bool
Cookies []HTTPCookie Cookies HTTPCookies
Headers HTTPHeaders Headers HTTPHeaders
Viewport *Viewport Viewport *Viewport
} }

48
pkg/stdlib/html/document.go
View File

@ -26,7 +26,7 @@ type PageLoadParams struct {
// userAgent (String) - Optional, user agent. // userAgent (String) - Optional, user agent.
// keepCookies (Boolean) - Optional, boolean value indicating whether to use cookies from previous sessions. // keepCookies (Boolean) - Optional, boolean value indicating whether to use cookies from previous sessions.
// i.e. not to open a page in the Incognito mode. // i.e. not to open a page in the Incognito mode.
// cookies (HTTPCookie) - Optional, set of HTTP cookies. // cookies (HTTPCookies) - Optional, set of HTTP cookies.
// headers (HTTPHeaders) - Optional, HTTP headers. // headers (HTTPHeaders) - Optional, HTTP headers.
// viewport (Viewport) - Optional, viewport params. // viewport (Viewport) - Optional, viewport params.
// @returns (HTMLPage) - Returns loaded HTML page. // @returns (HTMLPage) - Returns loaded HTML page.
@ -134,17 +134,30 @@ func newPageLoadParams(url values.String, arg core.Value) (PageLoadParams, error
cookies, exists := obj.Get(values.NewString("cookies")) cookies, exists := obj.Get(values.NewString("cookies"))
if exists { if exists {
if err := core.ValidateType(cookies, types.Array); err != nil { if err := core.ValidateType(cookies, types.Array, types.Object); err != nil {
return res, err return res, err
} }
cookies, err := parseCookies(cookies.(*values.Array)) switch c := cookies.(type) {
case *values.Array:
cookies, err := parseCookieArray(c)
if err != nil { if err != nil {
return res, err return res, err
} }
res.Cookies = cookies res.Cookies = cookies
case *values.Object:
cookies, err := parseCookieObject(c)
if err != nil {
return res, err
}
res.Cookies = cookies
default:
res.Cookies = make(drivers.HTTPCookies)
}
} }
headers, exists := obj.Get(values.NewString("headers")) headers, exists := obj.Get(values.NewString("headers"))
@ -183,11 +196,11 @@ func newPageLoadParams(url values.String, arg core.Value) (PageLoadParams, error
return res, nil return res, nil
} }
func parseCookies(arr *values.Array) ([]drivers.HTTPCookie, error) { func parseCookieObject(obj *values.Object) (drivers.HTTPCookies, error) {
var err error var err error
res := make([]drivers.HTTPCookie, 0, arr.Length()) res := make(drivers.HTTPCookies)
arr.ForEach(func(value core.Value, idx int) bool { obj.ForEach(func(value core.Value, _ string) bool {
cookie, e := parseCookie(value) cookie, e := parseCookie(value)
if e != nil { if e != nil {
@ -196,7 +209,28 @@ func parseCookies(arr *values.Array) ([]drivers.HTTPCookie, error) {
return false return false
} }
res = append(res, cookie) res[cookie.Name] = cookie
return true
})
return res, err
}
func parseCookieArray(arr *values.Array) (drivers.HTTPCookies, error) {
var err error
res := make(drivers.HTTPCookies)
arr.ForEach(func(value core.Value, _ int) bool {
cookie, e := parseCookie(value)
if e != nil {
err = e
return false
}
res[cookie.Name] = cookie
return true return true
}) })