go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2025-10-30 23:37:40 +02:00
Code Issues Releases Activity

Feature/#236 cookies (#242)

Browse Source 
* Added KeepCookies option to CDP driver

* Added LoadDocumentParams

* Added COOKIE_GET and COOKIE_SET methods
This commit is contained in:
Tim Voronov
2019-03-15 19:59:05 -04:00
committed by GitHub
parent 803ae0ea24
commit 71c246dd17
33 changed files with 1305 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@@ -47,6 +47,7 @@ jobs:
- make fmt
- if [[ $(git diff --stat) != '' ]]; then echo 'Invalid formatting!' >&2; exit 1; fi
- stage: compile
go: stable
script:
- make generate
- make compile
@@ -62,5 +63,6 @@ jobs:
after_script:
- killall google-chrome-stable
- stage: bench
go: stable
script:
- make bench

33
Gopkg.lock generated
View File

@@ -90,12 +90,12 @@
version = "v4.20"
[[projects]]
digest = "1:e55506f8670236cf09b6b65cda7d6afa403233a5b75397dfcba3555e484e4b18"
digest = "1:688475ae01f983eceee598c7706119c8cc7649f382e4a186168ea7f9d472727a"
name = "github.com/labstack/echo"
packages = ["."]
pruneopts = "UT"
revision = "c7eb8da9ec73e78c4f38413f3f835e0cd52c7d72"
version = "v3.3.8"
revision = "6d9e043284aea2d07f5fcaf0d3a424eb7d9f6109"
version = "v4.0.0"
[[projects]]
digest = "1:01eb0269028d3c2e21b5b6cd9b1ba81bc4170ab293fcffa84e3aa3a6138a92e8"
@@ -166,20 +166,20 @@
version = "v0.21.0"
[[projects]]
digest = "1:c658e84ad3916da105a761660dcaeb01e63416c8ec7bc62256a9b411a05fcd67"
digest = "1:2fa7b0155cd54479a755c629de26f888a918e13f8857a2c442205d825368e084"
name = "github.com/mattn/go-colorable"
packages = ["."]
pruneopts = "UT"
revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072"
version = "v0.0.9"
revision = "3a70a971f94a22f2fa562ffcc7a0eb45f5daf045"
version = "v0.1.1"
[[projects]]
digest = "1:0981502f9816113c9c8c4ac301583841855c8cf4da8c72f696b3ebedf6d0e4e5"
digest = "1:3bb9c8451d199650bfd303e0068d86f135952fead374ad87c09a9b8a2cc4bd7c"
name = "github.com/mattn/go-isatty"
packages = ["."]
pruneopts = "UT"
revision = "6ca4dbf54d38eea1a992b3c722a76a5d1c4cb25c"
version = "v0.0.4"
revision = "369ecd8cea9851e459abb67eb171853e3986591e"
version = "v0.0.6"
[[projects]]
digest = "1:c805e517269b0ba4c21ded5836019ed7d16953d4026cb7d00041d039c7906be9"
@@ -259,25 +259,25 @@
[[projects]]
branch = "master"
digest = "1:e3d2db9bc633f4635e6418caf0b0734c43821ecd59105d1798458c6ae4d227fd"
digest = "1:398e132d86665f82a3642f675cdadea673d0d1521209ebac3c378141209f99c4"
name = "golang.org/x/crypto"
packages = [
"acme",
"acme/autocert",
]
pruneopts = "UT"
revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908"
revision = "8dd112bcdc25174059e45e07517d9fc663123347"
[[projects]]
branch = "master"
digest = "1:1a1ecfa7b54ca3f7a0115ab5c578d7d6a5d8b605839c549e80260468c42f8be7"
digest = "1:de4815ce3ca5b624af2733716ecd471de1ef50cda8afec39491aab517f73139c"
name = "golang.org/x/net"
packages = [
"html",
"html/atom",
]
pruneopts = "UT"
revision = "915654e7eabcea33ae277abbecf52f0d8b7a9fdc"
revision = "16b79f2e4e95ea23b2bf9903c9809ff7b013ce85"
[[projects]]
branch = "master"
@@ -285,15 +285,15 @@
name = "golang.org/x/sync"
packages = ["errgroup"]
pruneopts = "UT"
revision = "37e7f081c4d4c64e13b10787722085407fe5d15f"
revision = "e225da77a7e68af35c70ccbf71af2b83e6acac3c"
[[projects]]
branch = "master"
digest = "1:91137b48dc3eb34409f731b49f63a5ebf73218168a065e1a93af24eb5b2f99e8"
digest = "1:b95ef12b443f7b5a40ab69e3a02d113f5a7f2b67a32af76eb2fa7bebd52c9eb5"
name = "golang.org/x/sys"
packages = ["unix"]
pruneopts = "UT"
revision = "48ac38b7c8cbedd50b1613c0fccacfc7d88dfcdf"
revision = "e844e0132e93db857c984c24fd4fc86815e43be3"
[solve-meta]
analyzer-name = "dep"
@@ -312,6 +312,7 @@
"github.com/mafredri/cdp/protocol/dom",
"github.com/mafredri/cdp/protocol/emulation",
"github.com/mafredri/cdp/protocol/input",
"github.com/mafredri/cdp/protocol/network",
"github.com/mafredri/cdp/protocol/page",
"github.com/mafredri/cdp/protocol/runtime",
"github.com/mafredri/cdp/protocol/target",

2
Makefile
View File

@@ -29,7 +29,7 @@ cover:
curl -s https://codecov.io/bash | bash
e2e:
go run ${DIR_E2E}/main.go --tests ${DIR_E2E}/tests --pages ${DIR_E2E}/pages
go run ${DIR_E2E}/main.go --tests ${DIR_E2E}/tests --pages ${DIR_E2E}/pages --filter doc_cookie_set*
bench:
go test -run=XXX -bench=. ${DIR_PKG}/...

80
README.md
View File

@@ -462,3 +462,83 @@ func run(q string) ([]byte, error) {
}
```
## Cookies
### Non-incognito mode
By default, ``CDP`` driver execute each query in an incognito mode in order to avoid any collisions related to some persisted cookies from previous queries.
However, sometimes it might not be a desirable behavior and a query needs to be executed within a Chrome tab with earlier persisted cookies.
In order to do that, we need to inform the driver to execute all queries in regular tabs. Here is how to do that:
#### CLI
```sh
ferret --cdp-keep-cookies my-query.fql
```
#### Code
```go
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp"
)
func run(q string) ([]byte, error) {
comp := compiler.New()
program := comp.MustCompile(q)
// create a root context
ctx := context.Background()
// we inform the driver to keep cookies between queries
ctx = drivers.WithContext(
ctx,
cdp.NewDriver(cdp.WithKeepCookies()),
drivers.AsDefault(),
)
return program.Run(ctx)
}
```
#### Query
```
LET doc = DOCUMENT("https://www.google.com", {
driver: "cdp",
keepCookies: true
})
```
### Cookies manipulation
For more precise work, you can set/get/delete cookies manually during and after page load:
```
LET doc = DOCUMENT("https://www.google.com", {
driver: "cdp",
cookies: [
{
name: "foo",
value: "bar"
}
]
})
COOKIES_SET(doc, { name: "baz", value: "qaz"}, { name: "daz", value: "gag" })
COOKIES_DEL(doc, "foo")
LET c = COOKIES_GET(doc, "baz")
FOR cookie IN doc.cookies
RETURN cookie.name
```

21
cli/options.go
View File

@@ -9,11 +9,12 @@ import (
)
type Options struct {
Cdp string
Params map[string]interface{}
Proxy string
UserAgent string
ShowTime bool
Cdp string
Params map[string]interface{}
Proxy string
UserAgent string
ShowTime bool
KeepCookies bool
}
func (opts Options) WithContext(ctx context.Context) (context.Context, context.CancelFunc) {
@@ -28,11 +29,17 @@ func (opts Options) WithContext(ctx context.Context) (context.Context, context.C
drivers.AsDefault(),
)
cdpDriver := cdp.NewDriver(
cdpOpts := []cdp.Option{
cdp.WithAddress(opts.Cdp),
cdp.WithProxy(opts.Proxy),
cdp.WithUserAgent(opts.UserAgent),
)
}
if opts.KeepCookies {
cdpOpts = append(cdpOpts, cdp.WithKeepCookies())
}
cdpDriver := cdp.NewDriver(cdpOpts...)
ctx = drivers.WithContext(
ctx,

12
e2e/server/server.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/labstack/echo"
"net/http"
"path/filepath"
)
@@ -23,6 +24,17 @@ func New(settings Settings) *Server {
e.Debug = false
e.HideBanner = true
e.Use(func(handlerFunc echo.HandlerFunc) echo.HandlerFunc {
return func(ctx echo.Context) error {
ctx.SetCookie(&http.Cookie{
Name: "x-ferret",
Value: "e2e",
HttpOnly: false,
})
return handlerFunc(ctx)
}
})
e.Static("/", settings.Dir)
e.File("/", filepath.Join(settings.Dir, "index.html"))

21
e2e/tests/doc_cookie_del_d.fql Normal file
View File

@@ -0,0 +1,21 @@
LET url = @dynamic
LET doc = DOCUMENT(url, {
driver: "cdp",
cookies: [{
name: "x-e2e",
value: "test"
}, {
name: "x-e2e-2",
value: "test2"
}]
})
COOKIE_DEL(doc, COOKIE_GET(doc, "x-e2e"), "x-e2e-2")
LET cookie1 = COOKIE_GET(doc, "x-e2e")
LET cookie2 = COOKIE_GET(doc, "x-e2e-2")
LET expected = "nonenone"
LET actual = TYPENAME(cookie1) + TYPENAME(cookie2)
RETURN EXPECT(expected, actual)

10
e2e/tests/doc_cookie_get_d.fql Normal file
View File

@@ -0,0 +1,10 @@
LET url = @dynamic
LET doc = DOCUMENT(url, {
driver: "cdp"
})
LET cookiesPath = LENGTH(doc.cookies) > 0 ? "ok" : "false"
LET cookie = COOKIE_GET(doc, "x-ferret")
LET expected = "ok e2e"
RETURN EXPECT(expected, cookiesPath + " " + cookie.value)

14
e2e/tests/doc_cookie_load_d.fql Normal file
View File

@@ -0,0 +1,14 @@
LET url = @dynamic
LET doc = DOCUMENT(url, {
driver: "cdp",
cookies: [{
name: "x-e2e",
value: "test"
}]
})
LET cookiesPath = LENGTH(doc.cookies) > 0 ? "ok" : "false"
LET cookie = COOKIE_GET(doc, "x-e2e")
LET expected = "ok test"
RETURN EXPECT(expected, cookiesPath + " " + cookie.value)

14
e2e/tests/doc_cookie_set_d.fql Normal file
View File

@@ -0,0 +1,14 @@
LET url = @dynamic
LET doc = DOCUMENT(@dynamic, {
driver: "cdp"
})
COOKIE_SET(doc, {
name: "x-e2e",
value: "test"
})
LET cookie = COOKIE_GET(doc, "x-e2e")
LET expected = "test"
RETURN EXPECT(expected, cookie.value)

28
examples/cookies.go Normal file
View File

@@ -0,0 +1,28 @@
package main
import (
"context"
"fmt"
"os"
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp"
)
func run(q string) ([]byte, error) {
comp := compiler.New()
program := comp.MustCompile(q)
// create a root context
ctx := context.Background()
// we inform the driver to keep cookies between queries
ctx = drivers.WithContext(
ctx,
cdp.NewDriver(cdp.WithKeepCookies()),
drivers.AsDefault(),
)
return program.Run(ctx)
}

4
examples/crawler.fql
View File

@@ -1,4 +1,6 @@
LET doc = DOCUMENT('https://www.theverge.com/tech', true)
LET doc = DOCUMENT('https://www.theverge.com/tech', {
driver: "cdp"
})
WAIT_ELEMENT(doc, '.c-compact-river__entry', 5000)
LET articles = ELEMENTS(doc, '.c-entry-box--compact__image-wrapper')
LET links = (

17
main.go
View File

@@ -66,6 +66,12 @@ var (
"launch Chrome",
)
cdpKeepCookies = flag.Bool(
"cdp-keep-cookies",
false,
"keep cookies between queries (i.e. do not open tabs in incognito mode)",
)
proxyAddress = flag.String(
"proxy",
"",
@@ -154,11 +160,12 @@ func main() {
}
opts := cli.Options{
Cdp: cdpConn,
Params: p,
Proxy: *proxyAddress,
UserAgent: *userAgent,
ShowTime: *showTime,
Cdp: cdpConn,
Params: p,
Proxy: *proxyAddress,
UserAgent: *userAgent,
ShowTime: *showTime,
KeepCookies: *cdpKeepCookies,
}
stat, _ := os.Stdin.Stat()

119
pkg/drivers/cdp/document.go
View File

@@ -2,6 +2,7 @@ package cdp
import (
"context"
"encoding/json"
"fmt"
"hash/fnv"
"sync"
@@ -19,6 +20,7 @@ import (
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/input"
"github.com/mafredri/cdp/protocol/network"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/rpcc"
"github.com/pkg/errors"
@@ -49,7 +51,7 @@ func LoadHTMLDocument(
ctx context.Context,
conn *rpcc.Conn,
client *cdp.Client,
url string,
params drivers.LoadDocumentParams,
) (drivers.HTMLDocument, error) {
logger := logging.FromContext(ctx)
@@ -57,13 +59,61 @@ func LoadHTMLDocument(
return nil, core.Error(core.ErrMissedArgument, "connection")
}
if url == "" {
if params.URL == "" {
return nil, core.Error(core.ErrMissedArgument, "url")
}
if params.Cookies != nil {
cookies := make([]network.CookieParam, 0, len(params.Cookies))
for _, c := range params.Cookies {
cookies = append(cookies, fromDriverCookie(params.URL, c))
logger.
Debug().
Timestamp().
Str("cookie", c.Name).
Msg("set cookie")
}
err := client.Network.SetCookies(
ctx,
network.NewSetCookiesArgs(cookies),
)
if err != nil {
return nil, err
}
}
if params.Header != nil {
j, err := json.Marshal(params.Header)
if err != nil {
return nil, err
}
for k := range params.Header {
logger.
Debug().
Timestamp().
Str("header", k).
Msg("set header")
}
err = client.Network.SetExtraHTTPHeaders(
ctx,
network.NewSetExtraHTTPHeadersArgs(network.Headers(j)),
)
if err != nil {
return nil, err
}
}
var err error
if url != BlankPageURL {
if params.URL != BlankPageURL {
err = waitForLoadEvent(ctx, client)
if err != nil {
@@ -109,7 +159,7 @@ func LoadHTMLDocument(
conn,
client,
broker,
values.NewString(url),
values.NewString(params.URL),
rootElement,
), nil
}
@@ -316,6 +366,67 @@ func (doc *HTMLDocument) GetURL() core.Value {
return doc.url
}
func (doc *HTMLDocument) GetCookies(ctx context.Context) (*values.Array, error) {
doc.Lock()
defer doc.Unlock()
repl, err := doc.client.Network.GetAllCookies(ctx)
if err != nil {
return values.NewArray(0), err
}
if repl.Cookies == nil {
return values.NewArray(0), nil
}
cookies := values.NewArray(len(repl.Cookies))
for _, c := range repl.Cookies {
cookies.Push(toDriverCookie(c))
}
return cookies, nil
}
func (doc *HTMLDocument) SetCookies(ctx context.Context, cookies ...drivers.HTTPCookie) error {
doc.Lock()
defer doc.Unlock()
if len(cookies) == 0 {
return nil
}
params := make([]network.CookieParam, 0, len(cookies))
for _, c := range cookies {
params = append(params, fromDriverCookie(doc.url.String(), c))
}
return doc.client.Network.SetCookies(ctx, network.NewSetCookiesArgs(params))
}
func (doc *HTMLDocument) DeleteCookies(ctx context.Context, cookies ...drivers.HTTPCookie) error {
doc.Lock()
defer doc.Unlock()
if len(cookies) == 0 {
return nil
}
var err error
for _, c := range cookies {
err = doc.client.Network.DeleteCookies(ctx, fromDriverCookieDelete(doc.url.String(), c))
if err != nil {
break
}
}
return err
}
func (doc *HTMLDocument) SetURL(ctx context.Context, url values.String) error {
return doc.Navigate(ctx, url)
}

52
pkg/drivers/cdp/driver.go
View File

@@ -7,10 +7,10 @@ import (
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/devtool"
"github.com/mafredri/cdp/protocol/emulation"
"github.com/mafredri/cdp/protocol/network"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/protocol/target"
"github.com/mafredri/cdp/rpcc"
@@ -33,16 +33,16 @@ type Driver struct {
func NewDriver(opts ...Option) *Driver {
drv := new(Driver)
drv.options = newOptions(opts)
drv.dev = devtool.New(drv.options.address)
drv.dev = devtool.New(drv.options.Address)
return drv
}
func (drv *Driver) Name() string {
return DriverName
return drv.options.Name
}
func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (drivers.HTMLDocument, error) {
func (drv *Driver) LoadDocument(ctx context.Context, params drivers.LoadDocumentParams) (drivers.HTMLDocument, error) {
logger := logging.FromContext(ctx)
err := drv.init(ctx)
@@ -52,21 +52,26 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (dr
Error().
Timestamp().
Err(err).
Str("driver", DriverName).
Str("driver", drv.options.Name).
Msg("failed to initialize the driver")
return nil, err
}
url := targetURL.String()
url := params.URL
if url == "" {
url = BlankPageURL
}
// Create a new target belonging to the browser context, similar
// to opening a new tab in an incognito window.
createTargetArgs := target.NewCreateTargetArgs(url).SetBrowserContextID(drv.contextID)
// Create a new target belonging to the browser context
createTargetArgs := target.NewCreateTargetArgs(url)
if drv.options.KeepCookies == false && params.KeepCookies == false {
// Set it to an incognito mode
createTargetArgs.SetBrowserContextID(drv.contextID)
}
createTarget, err := drv.client.Target.CreateTarget(ctx, createTargetArgs)
if err != nil {
@@ -74,7 +79,7 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (dr
Error().
Timestamp().
Err(err).
Str("driver", DriverName).
Str("driver", drv.options.Name).
Msg("failed to create a browser target")
return nil, err
@@ -88,7 +93,7 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (dr
Error().
Timestamp().
Err(err).
Str("driver", DriverName).
Str("driver", drv.options.Name).
Msg("failed to establish a connection")
return nil, err
@@ -117,7 +122,13 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (dr
},
func() error {
ua := common.GetUserAgent(drv.options.userAgent)
var ua string
if params.UserAgent != "" {
ua = common.GetUserAgent(params.UserAgent)
} else {
ua = common.GetUserAgent(drv.options.UserAgent)
}
logger.
Debug().
@@ -135,13 +146,17 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (dr
emulation.NewSetUserAgentOverrideArgs(ua),
)
},
func() error {
return client.Network.Enable(ctx, network.NewEnableArgs())
},
)
if err != nil {
return nil, err
}
return LoadHTMLDocument(ctx, conn, client, url)
return LoadHTMLDocument(ctx, conn, client, params)
}
func (drv *Driver) Close() error {
@@ -184,6 +199,14 @@ func (drv *Driver) init(ctx context.Context) error {
return errors.Wrap(err, "failed to initialize driver")
}
drv.conn = bconn
drv.client = bc
drv.session = sess
if drv.options.KeepCookies {
return nil
}
createCtx, err := bc.Target.CreateBrowserContext(ctx)
if err != nil {
@@ -193,9 +216,6 @@ func (drv *Driver) init(ctx context.Context) error {
return err
}
drv.conn = bconn
drv.client = bc
drv.session = sess
drv.contextID = createCtx.BrowserContextID
}

85
pkg/drivers/cdp/helpers.go
View File

@@ -6,7 +6,9 @@ import (
"errors"
"math"
"strings"
"time"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
"github.com/MontFerret/ferret/pkg/drivers/common"
@@ -14,11 +16,14 @@ import (
"github.com/PuerkitoBio/goquery"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/network"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/protocol/runtime"
"golang.org/x/sync/errgroup"
)
var emptyExpires = time.Time{}
type (
batchFunc = func() error
@@ -402,3 +407,83 @@ func createEventBroker(client *cdp.Client) (*events.EventBroker, error) {
return broker, nil
}
func fromDriverCookie(url string, cookie drivers.HTTPCookie) network.CookieParam {
sameSite := network.CookieSameSiteNotSet
switch cookie.SameSite {
case drivers.SameSiteLaxMode:
sameSite = network.CookieSameSiteLax
case drivers.SameSiteStrictMode:
sameSite = network.CookieSameSiteStrict
default:
sameSite = network.CookieSameSiteNotSet
}
if cookie.Expires == emptyExpires {
cookie.Expires = time.Now().Add(time.Duration(24) + time.Hour)
}
normalizedURL := normalizeCookieURL(url)
return network.CookieParam{
URL: &normalizedURL,
Name: cookie.Name,
Value: cookie.Value,
Secure: &cookie.Secure,
Path: &cookie.Path,
Domain: &cookie.Domain,
HTTPOnly: &cookie.HTTPOnly,
SameSite: sameSite,
Expires: network.TimeSinceEpoch(cookie.Expires.Unix()),
}
}
func fromDriverCookieDelete(url string, cookie drivers.HTTPCookie) *network.DeleteCookiesArgs {
normalizedURL := normalizeCookieURL(url)
return &network.DeleteCookiesArgs{
URL: &normalizedURL,
Name: cookie.Name,
Path: &cookie.Path,
Domain: &cookie.Domain,
}
}
func toDriverCookie(c network.Cookie) drivers.HTTPCookie {
sameSite := drivers.SameSiteDefaultMode
switch c.SameSite {
case network.CookieSameSiteLax:
sameSite = drivers.SameSiteLaxMode
break
case network.CookieSameSiteStrict:
sameSite = drivers.SameSiteStrictMode
break
default:
sameSite = drivers.SameSiteDefaultMode
break
}
return drivers.HTTPCookie{
Name: c.Name,
Value: c.Value,
Path: c.Path,
Domain: c.Domain,
Expires: time.Unix(int64(c.Expires), 0),
SameSite: sameSite,
Secure: c.Secure,
HTTPOnly: c.HTTPOnly,
}
}
func normalizeCookieURL(url string) string {
const httpPrefix = "http://"
const httpsPrefix = "https://"
if strings.HasPrefix(url, httpPrefix) || strings.HasPrefix(url, httpsPrefix) {
return url
}
return httpPrefix + url
}

29
pkg/drivers/cdp/options.go
View File

@@ -2,9 +2,11 @@ package cdp
type (
Options struct {
proxy string
userAgent string
address string
Name string
Proxy string
UserAgent string
Address string
KeepCookies bool
}
Option func(opts *Options)
@@ -14,7 +16,8 @@ const DefaultAddress = "http://127.0.0.1:9222"
func newOptions(setters []Option) *Options {
opts := new(Options)
opts.address = DefaultAddress
opts.Name = DriverName
opts.Address = DefaultAddress
for _, setter := range setters {
setter(opts)
@@ -26,19 +29,31 @@ func newOptions(setters []Option) *Options {
func WithAddress(address string) Option {
return func(opts *Options) {
if address != "" {
opts.address = address
opts.Address = address
}
}
}
func WithProxy(address string) Option {
return func(opts *Options) {
opts.proxy = address
opts.Proxy = address
}
}
func WithUserAgent(value string) Option {
return func(opts *Options) {
opts.userAgent = value
opts.UserAgent = value
}
}
func WithKeepCookies() Option {
return func(opts *Options) {
opts.KeepCookies = true
}
}
func WithCustomName(name string) Option {
return func(opts *Options) {
opts.Name = name
}
}

17
pkg/drivers/common/getter.go
View File

@@ -22,6 +22,23 @@ func GetInDocument(ctx context.Context, doc drivers.HTMLDocument, path []core.Va
switch segment {
case "url", "URL":
return doc.GetURL(), nil
case "cookies":
if len(path) == 1 {
return doc.GetCookies(ctx)
}
switch idx := path[1].(type) {
case values.Int:
cookies, err := doc.GetCookies(ctx)
if err != nil {
return values.None, err
}
return cookies.Get(idx), nil
default:
return values.None, core.TypeError(idx.Type(), types.Int)
}
case "body":
return doc.QuerySelector(ctx, "body"), nil
case "head":

2
pkg/drivers/common/setter.go
View File

@@ -22,6 +22,8 @@ func SetInDocument(ctx context.Context, doc drivers.HTMLDocument, path []core.Va
switch segment {
case "url", "URL":
return doc.SetURL(ctx, values.NewString(value.String()))
case "cookies":
default:
return SetInNode(ctx, doc, path, value)
}

195
pkg/drivers/cookie.go Normal file
View File

@@ -0,0 +1,195 @@
package drivers
import (
"context"
"encoding/json"
"fmt"
"hash/fnv"
"strconv"
"strings"
"time"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
type (
// Polyfill for Go 1.10
SameSite int
// HTTPCookie HTTPCookie object
HTTPCookie struct {
Name string
Value string
Path string
Domain string
Expires time.Time
MaxAge int
Secure bool
HTTPOnly bool
SameSite SameSite
}
)
const (
SameSiteDefaultMode SameSite = iota + 1
SameSiteLaxMode
SameSiteStrictMode
)
func (c HTTPCookie) Type() core.Type {
return HTTPCookieType
}
func (c HTTPCookie) String() string {
return fmt.Sprintf("%s=%s", c.Name, c.Value)
}
func (c HTTPCookie) Compare(other core.Value) int64 {
if other.Type() != HTTPCookieType {
return Compare(HTTPCookieType, other.Type())
}
oc := other.(HTTPCookie)
if c.Name != oc.Name {
return int64(strings.Compare(c.Name, oc.Name))
}
if c.Value != oc.Value {
return int64(strings.Compare(c.Value, oc.Value))
}
if c.Path != oc.Path {
return int64(strings.Compare(c.Path, oc.Path))
}
if c.Domain != oc.Domain {
return int64(strings.Compare(c.Domain, oc.Domain))
}
if c.Expires.After(oc.Expires) {
return 1
} else if c.Expires.Before(oc.Expires) {
return -1
}
if c.MaxAge > oc.MaxAge {
return 1
} else if c.MaxAge < oc.MaxAge {
return -1
}
if c.Secure && !oc.Secure {
return 1
} else if !c.Secure && oc.Secure {
return -1
}
if c.HTTPOnly && !oc.HTTPOnly {
return 1
} else if !c.HTTPOnly && oc.HTTPOnly {
return -1
}
if c.SameSite > oc.SameSite {
return 1
} else if c.SameSite < oc.SameSite {
return -1
}
return 0
}
func (c HTTPCookie) Unwrap() interface{} {
return c.Value
}
func (c HTTPCookie) Hash() uint64 {
h := fnv.New64a()
h.Write([]byte(c.Type().String()))
h.Write([]byte(":"))
h.Write([]byte(c.Name))
h.Write([]byte(c.Value))
h.Write([]byte(c.Path))
h.Write([]byte(c.Domain))
h.Write([]byte(c.Expires.String()))
h.Write([]byte(strconv.Itoa(c.MaxAge)))
h.Write([]byte(fmt.Sprintf("%t", c.Secure)))
h.Write([]byte(fmt.Sprintf("%t", c.HTTPOnly)))
h.Write([]byte(strconv.Itoa(int(c.SameSite))))
return h.Sum64()
}
func (c HTTPCookie) Copy() core.Value {
return *(&c)
}
func (c HTTPCookie) MarshalJSON() ([]byte, error) {
v := map[string]interface{}{
"name": c.Name,
"value": c.Value,
"path": c.Path,
"domain": c.Domain,
"expires": c.Expires,
"max_age": c.MaxAge,
"secure": c.Secure,
"http_only": c.HTTPOnly,
"same_site": c.SameSite,
}
out, err := json.Marshal(v)
if err != nil {
return nil, err
}
return out, err
}
func (c HTTPCookie) GetIn(_ context.Context, path []core.Value) (core.Value, error) {
if len(path) == 0 {
return values.None, nil
}
segment := path[0]
err := core.ValidateType(segment, types.String)
if err != nil {
return values.None, err
}
switch segment.(values.String) {
case "name":
return values.NewString(c.Name), nil
case "value":
return values.NewString(c.Value), nil
case "path":
return values.NewString(c.Path), nil
case "domain":
return values.NewString(c.Domain), nil
case "expires":
return values.NewDateTime(c.Expires), nil
case "maxAge":
return values.NewInt(c.MaxAge), nil
case "secure":
return values.NewBoolean(c.Secure), nil
case "httpOnly":
return values.NewBoolean(c.HTTPOnly), nil
case "sameSite":
switch c.SameSite {
case SameSiteLaxMode:
return values.NewString("Lax"), nil
case SameSiteStrictMode:
return values.NewString("Strict"), nil
default:
return values.EmptyString, nil
}
default:
return values.None, nil
}
}

11
pkg/drivers/driver.go
View File

@@ -6,7 +6,6 @@ import (
"time"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
const DefaultTimeout = time.Second * 30
@@ -19,10 +18,18 @@ type (
drivers map[string]Driver
}
LoadDocumentParams struct {
URL string
UserAgent string
KeepCookies bool
Cookies []HTTPCookie
Header HTTPHeader
}
Driver interface {
io.Closer
Name() string
GetDocument(ctx context.Context, url values.String) (HTMLDocument, error)
LoadDocument(ctx context.Context, params LoadDocumentParams) (HTMLDocument, error)
}
)

135
pkg/drivers/header.go Normal file
View File

@@ -0,0 +1,135 @@
package drivers
import (
"bytes"
"context"
"encoding/json"
"fmt"
"hash/fnv"
"net/textproto"
"sort"
"strings"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// HTTPCookie HTTPCookie object
type HTTPHeader map[string][]string
func (h HTTPHeader) Type() core.Type {
return HTTPHeaderType
}
func (h HTTPHeader) String() string {
var buf bytes.Buffer
for k := range h {
buf.WriteString(fmt.Sprintf("%s=%s;", k, h.Get(k)))
}
return buf.String()
}
func (h HTTPHeader) Compare(other core.Value) int64 {
if other.Type() != HTTPHeaderType {
return Compare(HTTPHeaderType, other.Type())
}
oh := other.(HTTPHeader)
if len(h) > len(oh) {
return 1
} else if len(h) < len(oh) {
return -1
}
for k := range h {
c := strings.Compare(h.Get(k), oh.Get(k))
if c != 0 {
return int64(c)
}
}
return 0
}
func (h HTTPHeader) Unwrap() interface{} {
return h
}
func (h HTTPHeader) Hash() uint64 {
hash := fnv.New64a()
hash.Write([]byte(h.Type().String()))
hash.Write([]byte(":"))
hash.Write([]byte("{"))
keys := make([]string, 0, len(h))
for key := range h {
keys = append(keys, key)
}
// order does not really matter
// but it will give us a consistent hash sum
sort.Strings(keys)
endIndex := len(keys) - 1
for idx, key := range keys {
hash.Write([]byte(key))
hash.Write([]byte(":"))
value := h.Get(key)
hash.Write([]byte(value))
if idx != endIndex {
hash.Write([]byte(","))
}
}
hash.Write([]byte("}"))
return hash.Sum64()
}
func (h HTTPHeader) Copy() core.Value {
return *(&h)
}
func (h HTTPHeader) MarshalJSON() ([]byte, error) {
out, err := json.Marshal(h)
if err != nil {
return nil, err
}
return out, err
}
func (h HTTPHeader) Set(key, value string) {
textproto.MIMEHeader(h).Set(key, value)
}
func (h HTTPHeader) Get(key string) string {
return textproto.MIMEHeader(h).Get(key)
}
func (h HTTPHeader) GetIn(_ context.Context, path []core.Value) (core.Value, error) {
if len(path) == 0 {
return values.None, nil
}
segment := path[0]
err := core.ValidateType(segment, types.String)
if err != nil {
return values.None, err
}
return values.NewString(h.Get(segment.String())), nil
}

44
pkg/drivers/http/document.go
View File

@@ -12,14 +12,16 @@ import (
)
type HTMLDocument struct {
url values.String
docNode *goquery.Document
element drivers.HTMLElement
url values.String
cookies []drivers.HTTPCookie
}
func NewHTMLDocument(
url string,
node *goquery.Document,
url string,
cookies []drivers.HTTPCookie,
) (drivers.HTMLDocument, error) {
if url == "" {
return nil, core.Error(core.ErrMissedArgument, "document url")
@@ -35,7 +37,7 @@ func NewHTMLDocument(
return nil, err
}
return &HTMLDocument{values.NewString(url), node, el}, nil
return &HTMLDocument{node, el, values.NewString(url), cookies}, nil
}
func (doc *HTMLDocument) MarshalJSON() ([]byte, error) {
@@ -82,7 +84,7 @@ func (doc *HTMLDocument) Hash() uint64 {
}
func (doc *HTMLDocument) Copy() core.Value {
cp, err := NewHTMLDocument(string(doc.url), doc.docNode)
cp, err := NewHTMLDocument(doc.docNode, string(doc.url), doc.cookies)
if err != nil {
return values.None
@@ -92,7 +94,17 @@ func (doc *HTMLDocument) Copy() core.Value {
}
func (doc *HTMLDocument) Clone() core.Value {
cp, err := NewHTMLDocument(string(doc.url), goquery.CloneDocument(doc.docNode))
var cookies []drivers.HTTPCookie
if doc.cookies != nil {
cookies = make([]drivers.HTTPCookie, 0, len(doc.cookies))
for i, c := range doc.cookies {
cookies[i] = c
}
}
cp, err := NewHTMLDocument(goquery.CloneDocument(doc.docNode), string(doc.url), cookies)
if err != nil {
return values.None
@@ -161,6 +173,28 @@ func (doc *HTMLDocument) SetURL(_ context.Context, _ values.String) error {
return core.ErrInvalidOperation
}
func (doc *HTMLDocument) GetCookies(_ context.Context) (*values.Array, error) {
if doc.cookies == nil {
return values.NewArray(0), nil
}
arr := values.NewArray(len(doc.cookies))
for _, c := range doc.cookies {
arr.Push(c)
}
return arr, nil
}
func (doc *HTMLDocument) SetCookies(_ context.Context, _ ...drivers.HTTPCookie) error {
return core.ErrNotSupported
}
func (doc *HTMLDocument) DeleteCookies(_ context.Context, _ ...drivers.HTTPCookie) error {
return core.ErrNotSupported
}
func (doc *HTMLDocument) Navigate(_ context.Context, _ values.String) error {
return core.ErrNotSupported
}

57
pkg/drivers/http/driver.go
View File

@@ -11,7 +11,6 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/PuerkitoBio/goquery"
"github.com/corpix/uarand"
"github.com/pkg/errors"
"github.com/sethgrid/pester"
)
@@ -63,39 +62,67 @@ func (drv *Driver) Name() string {
return DriverName
}
func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (drivers.HTMLDocument, error) {
u := targetURL.String()
req, err := http.NewRequest(http.MethodGet, u, nil)
func (drv *Driver) LoadDocument(ctx context.Context, params drivers.LoadDocumentParams) (drivers.HTMLDocument, error) {
req, err := http.NewRequest(http.MethodGet, params.URL, nil)
if err != nil {
return nil, err
}
logger := logging.FromContext(ctx)
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.9,ru;q=0.8")
req.Header.Set("Cache-Control", "no-cache")
req.Header.Set("Pragma", "no-cache")
if params.Header != nil {
for k := range params.Header {
req.Header.Add(k, params.Header.Get(k))
logger.
Debug().
Timestamp().
Str("header", k).
Msg("set header")
}
}
if params.Cookies != nil {
for _, c := range params.Cookies {
req.AddCookie(&http.Cookie{
Name: c.Name,
Value: c.Value,
})
logger.
Debug().
Timestamp().
Str("cookie", c.Name).
Msg("set cookie")
}
}
req = req.WithContext(ctx)
ua := common.GetUserAgent(drv.options.userAgent)
var ua string
if params.UserAgent != "" {
ua = common.GetUserAgent(params.UserAgent)
} else {
ua = common.GetUserAgent(drv.options.userAgent)
}
logger := logging.FromContext(ctx)
logger.
Debug().
Timestamp().
Str("user-agent", ua).
Msg("using User-Agent")
// use custom user agent
if ua != "" {
req.Header.Set("User-Agent", uarand.GetRandom())
}
resp, err := drv.client.Do(req)
if err != nil {
return nil, errors.Wrapf(err, "failed to retrieve a document %s", u)
return nil, errors.Wrapf(err, "failed to retrieve a document %s", params.URL)
}
defer resp.Body.Close()
@@ -103,10 +130,10 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (dr
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse a document %s", u)
return nil, errors.Wrapf(err, "failed to parse a document %s", params.URL)
}
return NewHTMLDocument(u, doc)
return NewHTMLDocument(doc, params.URL, params.Cookies)
}
func (drv *Driver) ParseDocument(_ context.Context, str values.String) (drivers.HTMLDocument, error) {
@@ -118,7 +145,7 @@ func (drv *Driver) ParseDocument(_ context.Context, str values.String) (drivers.
return nil, errors.Wrap(err, "failed to parse a document")
}
return NewHTMLDocument("#string", doc)
return NewHTMLDocument(doc, "#string", nil)
}
func (drv *Driver) Close() error {

4
pkg/drivers/pdf.go
View File

@@ -6,7 +6,7 @@ import "github.com/MontFerret/ferret/pkg/runtime/values"
type PDFParams struct {
// Paper orientation. Defaults to false.
Landscape values.Boolean
// Display header and footer. Defaults to false.
// Display values and footer. Defaults to false.
DisplayHeaderFooter values.Boolean
// Print background graphics. Defaults to false.
PrintBackground values.Boolean
@@ -28,7 +28,7 @@ type PDFParams struct {
PageRanges values.String
// Whether to silently ignore invalid but successfully parsed page ranges, such as '3-2'. Defaults to false.
IgnoreInvalidPageRanges values.Boolean
// HTML template for the print header. Should be valid HTML markup with following classes used to inject printing values into them: - `date`: formatted print date - `title`: document title - `url`: document location - `pageNumber`: current page number - `totalPages`: total pages in the document
// HTML template for the print values. Should be valid HTML markup with following classes used to inject printing values into them: - `date`: formatted print date - `title`: document title - `url`: document location - `pageNumber`: current page number - `totalPages`: total pages in the document
// For example, `<span class=title></span>` would generate span containing the title.
HeaderTemplate values.String
// HTML template for the print footer. Should use the same format as the `headerTemplate`.

8
pkg/drivers/type.go
View File

@@ -3,14 +3,18 @@ package drivers
import "github.com/MontFerret/ferret/pkg/runtime/core"
var (
HTTPHeaderType = core.NewType("HTTPHeader")
HTTPCookieType = core.NewType("HTTPCookie")
HTMLElementType = core.NewType("HTMLElement")
HTMLDocumentType = core.NewType("HTMLDocument")
)
// Comparison table of builtin types
var typeComparisonTable = map[core.Type]uint64{
HTMLElementType: 0,
HTMLDocumentType: 1,
HTTPHeaderType: 0,
HTTPCookieType: 1,
HTMLElementType: 2,
HTMLDocumentType: 3,
}
func Compare(first, second core.Type) int64 {

6
pkg/drivers/value.go
View File

@@ -109,6 +109,12 @@ type (
SetURL(ctx context.Context, url values.String) error
GetCookies(ctx context.Context) (*values.Array, error)
SetCookies(ctx context.Context, cookies ...HTTPCookie) error
DeleteCookies(ctx context.Context, cookies ...HTTPCookie) error
Navigate(ctx context.Context, url values.String) error
NavigateBack(ctx context.Context, skip values.Int) (values.Boolean, error)

10
pkg/runtime/values/array.go
View File

@@ -135,6 +135,16 @@ func (t *Array) ForEach(predicate ArrayPredicate) {
}
}
func (t *Array) Find(predicate ArrayPredicate) (core.Value, Boolean) {
for idx, val := range t.items {
if predicate(val, idx) == true {
return val, True
}
}
return None, False
}
func (t *Array) Get(idx Int) core.Value {
l := len(t.items) - 1

66
pkg/stdlib/html/cookie_del.go Normal file
View File

@@ -0,0 +1,66 @@
package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// CookieSet gets a cookie from a given document by name.
// @param source (HTMLDocument) - Target HTMLDocument.
// @param cookie (...HTTPCookie|String) - Cookie or cookie name to delete.
func CookieDel(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 2, core.MaxArgs)
if err != nil {
return values.None, err
}
err = core.ValidateType(args[0], drivers.HTMLDocumentType)
if err != nil {
return values.None, err
}
doc := args[0].(drivers.HTMLDocument)
inputs := args[1:]
var currentCookies *values.Array
cookies := make([]drivers.HTTPCookie, 0, len(inputs))
for _, c := range inputs {
switch cookie := c.(type) {
case values.String:
if currentCookies == nil {
current, err := doc.GetCookies(ctx)
if err != nil {
return values.None, err
}
currentCookies = current
}
found, isFound := currentCookies.Find(func(value core.Value, _ int) bool {
cv := value.(drivers.HTTPCookie)
return cv.Name == cookie.String()
})
if isFound {
cookies = append(cookies, found.(drivers.HTTPCookie))
}
break
case drivers.HTTPCookie:
cookies = append(cookies, cookie)
break
default:
return values.None, core.TypeError(c.Type(), types.String, drivers.HTTPCookieType)
}
}
return values.None, doc.DeleteCookies(ctx, cookies...)
}

58
pkg/stdlib/html/cookie_get.go Normal file
View File

@@ -0,0 +1,58 @@
package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// CookieSet gets a cookie from a given document by name.
// @param doc (HTMLDocument) - Target HTMLDocument.
// @param name (String) - Cookie or cookie name to delete.
func CookieGet(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 2, 2)
if err != nil {
return values.None, err
}
err = core.ValidateType(args[0], drivers.HTMLDocumentType)
if err != nil {
return values.None, err
}
err = core.ValidateType(args[1], types.String)
if err != nil {
return values.None, err
}
doc := args[0].(drivers.HTMLDocument)
name := args[1].(values.String)
if err != nil {
return values.None, err
}
cookies, err := doc.GetCookies(ctx)
if err != nil {
return values.None, err
}
found, _ := cookies.Find(func(value core.Value, _ int) bool {
cookie, ok := value.(drivers.HTTPCookie)
if !ok {
return ok
}
return cookie.Name == name.String()
})
return found, nil
}

42
pkg/stdlib/html/cookie_set.go Normal file
View File

@@ -0,0 +1,42 @@
package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
// CookieSet sets cookies to a given document
// @param doc (HTMLDocument) - Target document.
// @param cookie... (HTTPCookie) - Target cookies.
func CookieSet(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 2, core.MaxArgs)
if err != nil {
return values.None, err
}
err = core.ValidateType(args[0], drivers.HTMLDocumentType)
if err != nil {
return values.None, err
}
doc := args[0].(drivers.HTMLDocument)
cookies := make([]drivers.HTTPCookie, 0, len(args)-1)
for _, c := range args[1:] {
cookie, err := parseCookie(c)
if err != nil {
return values.None, err
}
cookies = append(cookies, cookie)
}
return values.None, doc.SetCookies(ctx, cookies...)
}

193
pkg/stdlib/html/document.go
View File

@@ -2,6 +2,7 @@ package html
import (
"context"
"strings"
"time"
"github.com/MontFerret/ferret/pkg/drivers"
@@ -12,6 +13,7 @@ import (
)
type DocumentLoadParams struct {
drivers.LoadDocumentParams
Driver string
Timeout time.Duration
}
@@ -43,9 +45,9 @@ func Document(ctx context.Context, args ...core.Value) (core.Value, error) {
var params DocumentLoadParams
if len(args) == 1 {
params = newDefaultDocLoadParams()
params = newDefaultDocLoadParams(url)
} else {
p, err := newDocLoadParams(args[1])
p, err := newDocLoadParams(url, args[1])
if err != nil {
return values.None, err
@@ -63,17 +65,20 @@ func Document(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
return drv.GetDocument(ctx, url)
return drv.LoadDocument(ctx, params.LoadDocumentParams)
}
func newDefaultDocLoadParams() DocumentLoadParams {
func newDefaultDocLoadParams(url values.String) DocumentLoadParams {
return DocumentLoadParams{
LoadDocumentParams: drivers.LoadDocumentParams{
URL: url.String(),
},
Timeout: time.Second * 30,
}
}
func newDocLoadParams(arg core.Value) (DocumentLoadParams, error) {
res := newDefaultDocLoadParams()
func newDocLoadParams(url values.String, arg core.Value) (DocumentLoadParams, error) {
res := newDefaultDocLoadParams(url)
if err := core.ValidateType(arg, types.Boolean, types.String, types.Object); err != nil {
return res, err
@@ -103,6 +108,58 @@ func newDocLoadParams(arg core.Value) (DocumentLoadParams, error) {
res.Timeout = time.Duration(timeout.(values.Int)) + time.Millisecond
}
userAgent, exists := obj.Get(values.NewString("userAgent"))
if exists {
if err := core.ValidateType(userAgent, types.String); err != nil {
return res, err
}
res.UserAgent = userAgent.String()
}
keepCookies, exists := obj.Get(values.NewString("keepCookies"))
if exists {
if err := core.ValidateType(keepCookies, types.Boolean); err != nil {
return res, err
}
res.KeepCookies = bool(keepCookies.(values.Boolean))
}
cookies, exists := obj.Get(values.NewString("cookies"))
if exists {
if err := core.ValidateType(cookies, types.Array); err != nil {
return res, err
}
cookies, err := parseCookies(cookies.(*values.Array))
if err != nil {
return res, err
}
res.Cookies = cookies
}
header, exists := obj.Get(values.NewString("header"))
if exists {
if err := core.ValidateType(header, types.Object); err != nil {
return res, err
}
header, err := parseHeader(header.(*values.Object))
if err != nil {
return res, err
}
res.Header = header
}
break
case types.String:
res.Driver = arg.(values.String).String()
@@ -121,3 +178,127 @@ func newDocLoadParams(arg core.Value) (DocumentLoadParams, error) {
return res, nil
}
func parseCookies(arr *values.Array) ([]drivers.HTTPCookie, error) {
var err error
res := make([]drivers.HTTPCookie, 0, arr.Length())
arr.ForEach(func(value core.Value, idx int) bool {
cookie, e := parseCookie(value)
if e != nil {
err = e
return false
}
res = append(res, cookie)
return true
})
return res, err
}
func parseCookie(value core.Value) (drivers.HTTPCookie, error) {
var err error
if err = core.ValidateType(value, types.Object, drivers.HTTPCookieType); err != nil {
return drivers.HTTPCookie{}, err
}
if value.Type() == drivers.HTTPCookieType {
return value.(drivers.HTTPCookie), nil
}
co := value.(*values.Object)
cookie := drivers.HTTPCookie{
Name: co.MustGet("name").String(),
Value: co.MustGet("value").String(),
Path: co.MustGet("path").String(),
Domain: co.MustGet("domain").String(),
}
maxAge, exists := co.Get("maxAge")
if exists {
if err = core.ValidateType(maxAge, types.Int); err != nil {
return drivers.HTTPCookie{}, err
}
cookie.MaxAge = int(maxAge.(values.Int))
}
expires, exists := co.Get("expires")
if exists {
if err = core.ValidateType(maxAge, types.DateTime, types.String); err != nil {
return drivers.HTTPCookie{}, err
}
if expires.Type() == types.DateTime {
cookie.Expires = expires.(values.DateTime).Unwrap().(time.Time)
} else {
t, err := time.Parse(expires.String(), values.DefaultTimeLayout)
if err != nil {
return drivers.HTTPCookie{}, err
}
cookie.Expires = t
}
}
sameSite, exists := co.Get("sameSite")
if exists {
sameSite := strings.ToLower(sameSite.String())
switch sameSite {
case "lax":
cookie.SameSite = drivers.SameSiteLaxMode
break
case "strict":
cookie.SameSite = drivers.SameSiteStrictMode
break
default:
cookie.SameSite = drivers.SameSiteDefaultMode
break
}
}
httpOnly, exists := co.Get("httpOnly")
if exists {
if err = core.ValidateType(httpOnly, types.Boolean); err != nil {
return drivers.HTTPCookie{}, err
}
cookie.HTTPOnly = bool(httpOnly.(values.Boolean))
}
secure, exists := co.Get("secure")
if exists {
if err = core.ValidateType(secure, types.Boolean); err != nil {
return drivers.HTTPCookie{}, err
}
cookie.Secure = bool(secure.(values.Boolean))
}
return cookie, err
}
func parseHeader(header *values.Object) (drivers.HTTPHeader, error) {
res := make(drivers.HTTPHeader)
header.ForEach(func(value core.Value, key string) bool {
res.Set(key, value.String())
return true
})
return res, nil
}

3
pkg/stdlib/html/lib.go
View File

@@ -17,6 +17,9 @@ func NewLib() map[string]core.Function {
"ATTR_GET": AttributeGet,
"ATTR_REMOVE": AttributeRemove,
"ATTR_SET": AttributeSet,
"COOKIE_DEL": CookieDel,
"COOKIE_GET": CookieGet,
"COOKIE_SET": CookieSet,
"CLICK": Click,
"CLICK_ALL": ClickAll,
"DOCUMENT": Document,