mirror of
https://github.com/MontFerret/ferret.git
synced 2025-03-17 21:18:37 +02:00
Feature/#236 cookies (#242)
* Added KeepCookies option to CDP driver * Added LoadDocumentParams * Added COOKIE_GET and COOKIE_SET methods
This commit is contained in:
parent
803ae0ea24
commit
71c246dd17
@ -47,6 +47,7 @@ jobs:
|
||||
- make fmt
|
||||
- if [[ $(git diff --stat) != '' ]]; then echo 'Invalid formatting!' >&2; exit 1; fi
|
||||
- stage: compile
|
||||
go: stable
|
||||
script:
|
||||
- make generate
|
||||
- make compile
|
||||
@ -62,5 +63,6 @@ jobs:
|
||||
after_script:
|
||||
- killall google-chrome-stable
|
||||
- stage: bench
|
||||
go: stable
|
||||
script:
|
||||
- make bench
|
||||
|
33
Gopkg.lock
generated
33
Gopkg.lock
generated
@ -90,12 +90,12 @@
|
||||
version = "v4.20"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:e55506f8670236cf09b6b65cda7d6afa403233a5b75397dfcba3555e484e4b18"
|
||||
digest = "1:688475ae01f983eceee598c7706119c8cc7649f382e4a186168ea7f9d472727a"
|
||||
name = "github.com/labstack/echo"
|
||||
packages = ["."]
|
||||
pruneopts = "UT"
|
||||
revision = "c7eb8da9ec73e78c4f38413f3f835e0cd52c7d72"
|
||||
version = "v3.3.8"
|
||||
revision = "6d9e043284aea2d07f5fcaf0d3a424eb7d9f6109"
|
||||
version = "v4.0.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:01eb0269028d3c2e21b5b6cd9b1ba81bc4170ab293fcffa84e3aa3a6138a92e8"
|
||||
@ -166,20 +166,20 @@
|
||||
version = "v0.21.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:c658e84ad3916da105a761660dcaeb01e63416c8ec7bc62256a9b411a05fcd67"
|
||||
digest = "1:2fa7b0155cd54479a755c629de26f888a918e13f8857a2c442205d825368e084"
|
||||
name = "github.com/mattn/go-colorable"
|
||||
packages = ["."]
|
||||
pruneopts = "UT"
|
||||
revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072"
|
||||
version = "v0.0.9"
|
||||
revision = "3a70a971f94a22f2fa562ffcc7a0eb45f5daf045"
|
||||
version = "v0.1.1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:0981502f9816113c9c8c4ac301583841855c8cf4da8c72f696b3ebedf6d0e4e5"
|
||||
digest = "1:3bb9c8451d199650bfd303e0068d86f135952fead374ad87c09a9b8a2cc4bd7c"
|
||||
name = "github.com/mattn/go-isatty"
|
||||
packages = ["."]
|
||||
pruneopts = "UT"
|
||||
revision = "6ca4dbf54d38eea1a992b3c722a76a5d1c4cb25c"
|
||||
version = "v0.0.4"
|
||||
revision = "369ecd8cea9851e459abb67eb171853e3986591e"
|
||||
version = "v0.0.6"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:c805e517269b0ba4c21ded5836019ed7d16953d4026cb7d00041d039c7906be9"
|
||||
@ -259,25 +259,25 @@
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:e3d2db9bc633f4635e6418caf0b0734c43821ecd59105d1798458c6ae4d227fd"
|
||||
digest = "1:398e132d86665f82a3642f675cdadea673d0d1521209ebac3c378141209f99c4"
|
||||
name = "golang.org/x/crypto"
|
||||
packages = [
|
||||
"acme",
|
||||
"acme/autocert",
|
||||
]
|
||||
pruneopts = "UT"
|
||||
revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908"
|
||||
revision = "8dd112bcdc25174059e45e07517d9fc663123347"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:1a1ecfa7b54ca3f7a0115ab5c578d7d6a5d8b605839c549e80260468c42f8be7"
|
||||
digest = "1:de4815ce3ca5b624af2733716ecd471de1ef50cda8afec39491aab517f73139c"
|
||||
name = "golang.org/x/net"
|
||||
packages = [
|
||||
"html",
|
||||
"html/atom",
|
||||
]
|
||||
pruneopts = "UT"
|
||||
revision = "915654e7eabcea33ae277abbecf52f0d8b7a9fdc"
|
||||
revision = "16b79f2e4e95ea23b2bf9903c9809ff7b013ce85"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
@ -285,15 +285,15 @@
|
||||
name = "golang.org/x/sync"
|
||||
packages = ["errgroup"]
|
||||
pruneopts = "UT"
|
||||
revision = "37e7f081c4d4c64e13b10787722085407fe5d15f"
|
||||
revision = "e225da77a7e68af35c70ccbf71af2b83e6acac3c"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:91137b48dc3eb34409f731b49f63a5ebf73218168a065e1a93af24eb5b2f99e8"
|
||||
digest = "1:b95ef12b443f7b5a40ab69e3a02d113f5a7f2b67a32af76eb2fa7bebd52c9eb5"
|
||||
name = "golang.org/x/sys"
|
||||
packages = ["unix"]
|
||||
pruneopts = "UT"
|
||||
revision = "48ac38b7c8cbedd50b1613c0fccacfc7d88dfcdf"
|
||||
revision = "e844e0132e93db857c984c24fd4fc86815e43be3"
|
||||
|
||||
[solve-meta]
|
||||
analyzer-name = "dep"
|
||||
@ -312,6 +312,7 @@
|
||||
"github.com/mafredri/cdp/protocol/dom",
|
||||
"github.com/mafredri/cdp/protocol/emulation",
|
||||
"github.com/mafredri/cdp/protocol/input",
|
||||
"github.com/mafredri/cdp/protocol/network",
|
||||
"github.com/mafredri/cdp/protocol/page",
|
||||
"github.com/mafredri/cdp/protocol/runtime",
|
||||
"github.com/mafredri/cdp/protocol/target",
|
||||
|
2
Makefile
2
Makefile
@ -29,7 +29,7 @@ cover:
|
||||
curl -s https://codecov.io/bash | bash
|
||||
|
||||
e2e:
|
||||
go run ${DIR_E2E}/main.go --tests ${DIR_E2E}/tests --pages ${DIR_E2E}/pages
|
||||
go run ${DIR_E2E}/main.go --tests ${DIR_E2E}/tests --pages ${DIR_E2E}/pages --filter doc_cookie_set*
|
||||
|
||||
bench:
|
||||
go test -run=XXX -bench=. ${DIR_PKG}/...
|
||||
|
80
README.md
80
README.md
@ -462,3 +462,83 @@ func run(q string) ([]byte, error) {
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
## Cookies
|
||||
|
||||
### Non-incognito mode
|
||||
|
||||
By default, ``CDP`` driver execute each query in an incognito mode in order to avoid any collisions related to some persisted cookies from previous queries.
|
||||
However, sometimes it might not be a desirable behavior and a query needs to be executed within a Chrome tab with earlier persisted cookies.
|
||||
In order to do that, we need to inform the driver to execute all queries in regular tabs. Here is how to do that:
|
||||
|
||||
#### CLI
|
||||
|
||||
```sh
|
||||
ferret --cdp-keep-cookies my-query.fql
|
||||
```
|
||||
|
||||
#### Code
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/MontFerret/ferret/pkg/compiler"
|
||||
"github.com/MontFerret/ferret/pkg/drivers"
|
||||
"github.com/MontFerret/ferret/pkg/drivers/cdp"
|
||||
)
|
||||
|
||||
func run(q string) ([]byte, error) {
|
||||
comp := compiler.New()
|
||||
program := comp.MustCompile(q)
|
||||
|
||||
// create a root context
|
||||
ctx := context.Background()
|
||||
|
||||
// we inform the driver to keep cookies between queries
|
||||
ctx = drivers.WithContext(
|
||||
ctx,
|
||||
cdp.NewDriver(cdp.WithKeepCookies()),
|
||||
drivers.AsDefault(),
|
||||
)
|
||||
|
||||
return program.Run(ctx)
|
||||
}
|
||||
```
|
||||
|
||||
#### Query
|
||||
```
|
||||
LET doc = DOCUMENT("https://www.google.com", {
|
||||
driver: "cdp",
|
||||
keepCookies: true
|
||||
})
|
||||
```
|
||||
|
||||
### Cookies manipulation
|
||||
For more precise work, you can set/get/delete cookies manually during and after page load:
|
||||
|
||||
```
|
||||
LET doc = DOCUMENT("https://www.google.com", {
|
||||
driver: "cdp",
|
||||
cookies: [
|
||||
{
|
||||
name: "foo",
|
||||
value: "bar"
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
COOKIES_SET(doc, { name: "baz", value: "qaz"}, { name: "daz", value: "gag" })
|
||||
COOKIES_DEL(doc, "foo")
|
||||
|
||||
LET c = COOKIES_GET(doc, "baz")
|
||||
|
||||
FOR cookie IN doc.cookies
|
||||
RETURN cookie.name
|
||||
|
||||
```
|
@ -9,11 +9,12 @@ import (
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
Cdp string
|
||||
Params map[string]interface{}
|
||||
Proxy string
|
||||
UserAgent string
|
||||
ShowTime bool
|
||||
Cdp string
|
||||
Params map[string]interface{}
|
||||
Proxy string
|
||||
UserAgent string
|
||||
ShowTime bool
|
||||
KeepCookies bool
|
||||
}
|
||||
|
||||
func (opts Options) WithContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
@ -28,11 +29,17 @@ func (opts Options) WithContext(ctx context.Context) (context.Context, context.C
|
||||
drivers.AsDefault(),
|
||||
)
|
||||
|
||||
cdpDriver := cdp.NewDriver(
|
||||
cdpOpts := []cdp.Option{
|
||||
cdp.WithAddress(opts.Cdp),
|
||||
cdp.WithProxy(opts.Proxy),
|
||||
cdp.WithUserAgent(opts.UserAgent),
|
||||
)
|
||||
}
|
||||
|
||||
if opts.KeepCookies {
|
||||
cdpOpts = append(cdpOpts, cdp.WithKeepCookies())
|
||||
}
|
||||
|
||||
cdpDriver := cdp.NewDriver(cdpOpts...)
|
||||
|
||||
ctx = drivers.WithContext(
|
||||
ctx,
|
||||
|
@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/labstack/echo"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
@ -23,6 +24,17 @@ func New(settings Settings) *Server {
|
||||
e.Debug = false
|
||||
e.HideBanner = true
|
||||
|
||||
e.Use(func(handlerFunc echo.HandlerFunc) echo.HandlerFunc {
|
||||
return func(ctx echo.Context) error {
|
||||
ctx.SetCookie(&http.Cookie{
|
||||
Name: "x-ferret",
|
||||
Value: "e2e",
|
||||
HttpOnly: false,
|
||||
})
|
||||
|
||||
return handlerFunc(ctx)
|
||||
}
|
||||
})
|
||||
e.Static("/", settings.Dir)
|
||||
e.File("/", filepath.Join(settings.Dir, "index.html"))
|
||||
|
||||
|
21
e2e/tests/doc_cookie_del_d.fql
Normal file
21
e2e/tests/doc_cookie_del_d.fql
Normal file
@ -0,0 +1,21 @@
|
||||
LET url = @dynamic
|
||||
LET doc = DOCUMENT(url, {
|
||||
driver: "cdp",
|
||||
cookies: [{
|
||||
name: "x-e2e",
|
||||
value: "test"
|
||||
}, {
|
||||
name: "x-e2e-2",
|
||||
value: "test2"
|
||||
}]
|
||||
})
|
||||
|
||||
COOKIE_DEL(doc, COOKIE_GET(doc, "x-e2e"), "x-e2e-2")
|
||||
|
||||
LET cookie1 = COOKIE_GET(doc, "x-e2e")
|
||||
LET cookie2 = COOKIE_GET(doc, "x-e2e-2")
|
||||
|
||||
LET expected = "nonenone"
|
||||
LET actual = TYPENAME(cookie1) + TYPENAME(cookie2)
|
||||
|
||||
RETURN EXPECT(expected, actual)
|
10
e2e/tests/doc_cookie_get_d.fql
Normal file
10
e2e/tests/doc_cookie_get_d.fql
Normal file
@ -0,0 +1,10 @@
|
||||
LET url = @dynamic
|
||||
LET doc = DOCUMENT(url, {
|
||||
driver: "cdp"
|
||||
})
|
||||
|
||||
LET cookiesPath = LENGTH(doc.cookies) > 0 ? "ok" : "false"
|
||||
LET cookie = COOKIE_GET(doc, "x-ferret")
|
||||
LET expected = "ok e2e"
|
||||
|
||||
RETURN EXPECT(expected, cookiesPath + " " + cookie.value)
|
14
e2e/tests/doc_cookie_load_d.fql
Normal file
14
e2e/tests/doc_cookie_load_d.fql
Normal file
@ -0,0 +1,14 @@
|
||||
LET url = @dynamic
|
||||
LET doc = DOCUMENT(url, {
|
||||
driver: "cdp",
|
||||
cookies: [{
|
||||
name: "x-e2e",
|
||||
value: "test"
|
||||
}]
|
||||
})
|
||||
|
||||
LET cookiesPath = LENGTH(doc.cookies) > 0 ? "ok" : "false"
|
||||
LET cookie = COOKIE_GET(doc, "x-e2e")
|
||||
LET expected = "ok test"
|
||||
|
||||
RETURN EXPECT(expected, cookiesPath + " " + cookie.value)
|
14
e2e/tests/doc_cookie_set_d.fql
Normal file
14
e2e/tests/doc_cookie_set_d.fql
Normal file
@ -0,0 +1,14 @@
|
||||
LET url = @dynamic
|
||||
LET doc = DOCUMENT(@dynamic, {
|
||||
driver: "cdp"
|
||||
})
|
||||
|
||||
COOKIE_SET(doc, {
|
||||
name: "x-e2e",
|
||||
value: "test"
|
||||
})
|
||||
|
||||
LET cookie = COOKIE_GET(doc, "x-e2e")
|
||||
LET expected = "test"
|
||||
|
||||
RETURN EXPECT(expected, cookie.value)
|
28
examples/cookies.go
Normal file
28
examples/cookies.go
Normal file
@ -0,0 +1,28 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/MontFerret/ferret/pkg/compiler"
|
||||
"github.com/MontFerret/ferret/pkg/drivers"
|
||||
"github.com/MontFerret/ferret/pkg/drivers/cdp"
|
||||
)
|
||||
|
||||
func run(q string) ([]byte, error) {
|
||||
comp := compiler.New()
|
||||
program := comp.MustCompile(q)
|
||||
|
||||
// create a root context
|
||||
ctx := context.Background()
|
||||
|
||||
// we inform the driver to keep cookies between queries
|
||||
ctx = drivers.WithContext(
|
||||
ctx,
|
||||
cdp.NewDriver(cdp.WithKeepCookies()),
|
||||
drivers.AsDefault(),
|
||||
)
|
||||
|
||||
return program.Run(ctx)
|
||||
}
|
@ -1,4 +1,6 @@
|
||||
LET doc = DOCUMENT('https://www.theverge.com/tech', true)
|
||||
LET doc = DOCUMENT('https://www.theverge.com/tech', {
|
||||
driver: "cdp"
|
||||
})
|
||||
WAIT_ELEMENT(doc, '.c-compact-river__entry', 5000)
|
||||
LET articles = ELEMENTS(doc, '.c-entry-box--compact__image-wrapper')
|
||||
LET links = (
|
||||
|
17
main.go
17
main.go
@ -66,6 +66,12 @@ var (
|
||||
"launch Chrome",
|
||||
)
|
||||
|
||||
cdpKeepCookies = flag.Bool(
|
||||
"cdp-keep-cookies",
|
||||
false,
|
||||
"keep cookies between queries (i.e. do not open tabs in incognito mode)",
|
||||
)
|
||||
|
||||
proxyAddress = flag.String(
|
||||
"proxy",
|
||||
"",
|
||||
@ -154,11 +160,12 @@ func main() {
|
||||
}
|
||||
|
||||
opts := cli.Options{
|
||||
Cdp: cdpConn,
|
||||
Params: p,
|
||||
Proxy: *proxyAddress,
|
||||
UserAgent: *userAgent,
|
||||
ShowTime: *showTime,
|
||||
Cdp: cdpConn,
|
||||
Params: p,
|
||||
Proxy: *proxyAddress,
|
||||
UserAgent: *userAgent,
|
||||
ShowTime: *showTime,
|
||||
KeepCookies: *cdpKeepCookies,
|
||||
}
|
||||
|
||||
stat, _ := os.Stdin.Stat()
|
||||
|
@ -2,6 +2,7 @@ package cdp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"sync"
|
||||
@ -19,6 +20,7 @@ import (
|
||||
"github.com/mafredri/cdp"
|
||||
"github.com/mafredri/cdp/protocol/dom"
|
||||
"github.com/mafredri/cdp/protocol/input"
|
||||
"github.com/mafredri/cdp/protocol/network"
|
||||
"github.com/mafredri/cdp/protocol/page"
|
||||
"github.com/mafredri/cdp/rpcc"
|
||||
"github.com/pkg/errors"
|
||||
@ -49,7 +51,7 @@ func LoadHTMLDocument(
|
||||
ctx context.Context,
|
||||
conn *rpcc.Conn,
|
||||
client *cdp.Client,
|
||||
url string,
|
||||
params drivers.LoadDocumentParams,
|
||||
) (drivers.HTMLDocument, error) {
|
||||
logger := logging.FromContext(ctx)
|
||||
|
||||
@ -57,13 +59,61 @@ func LoadHTMLDocument(
|
||||
return nil, core.Error(core.ErrMissedArgument, "connection")
|
||||
}
|
||||
|
||||
if url == "" {
|
||||
if params.URL == "" {
|
||||
return nil, core.Error(core.ErrMissedArgument, "url")
|
||||
}
|
||||
|
||||
if params.Cookies != nil {
|
||||
cookies := make([]network.CookieParam, 0, len(params.Cookies))
|
||||
|
||||
for _, c := range params.Cookies {
|
||||
cookies = append(cookies, fromDriverCookie(params.URL, c))
|
||||
|
||||
logger.
|
||||
Debug().
|
||||
Timestamp().
|
||||
Str("cookie", c.Name).
|
||||
Msg("set cookie")
|
||||
}
|
||||
|
||||
err := client.Network.SetCookies(
|
||||
ctx,
|
||||
network.NewSetCookiesArgs(cookies),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if params.Header != nil {
|
||||
j, err := json.Marshal(params.Header)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for k := range params.Header {
|
||||
logger.
|
||||
Debug().
|
||||
Timestamp().
|
||||
Str("header", k).
|
||||
Msg("set header")
|
||||
}
|
||||
|
||||
err = client.Network.SetExtraHTTPHeaders(
|
||||
ctx,
|
||||
network.NewSetExtraHTTPHeadersArgs(network.Headers(j)),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
if url != BlankPageURL {
|
||||
if params.URL != BlankPageURL {
|
||||
err = waitForLoadEvent(ctx, client)
|
||||
|
||||
if err != nil {
|
||||
@ -109,7 +159,7 @@ func LoadHTMLDocument(
|
||||
conn,
|
||||
client,
|
||||
broker,
|
||||
values.NewString(url),
|
||||
values.NewString(params.URL),
|
||||
rootElement,
|
||||
), nil
|
||||
}
|
||||
@ -316,6 +366,67 @@ func (doc *HTMLDocument) GetURL() core.Value {
|
||||
return doc.url
|
||||
}
|
||||
|
||||
func (doc *HTMLDocument) GetCookies(ctx context.Context) (*values.Array, error) {
|
||||
doc.Lock()
|
||||
defer doc.Unlock()
|
||||
|
||||
repl, err := doc.client.Network.GetAllCookies(ctx)
|
||||
|
||||
if err != nil {
|
||||
return values.NewArray(0), err
|
||||
}
|
||||
|
||||
if repl.Cookies == nil {
|
||||
return values.NewArray(0), nil
|
||||
}
|
||||
|
||||
cookies := values.NewArray(len(repl.Cookies))
|
||||
|
||||
for _, c := range repl.Cookies {
|
||||
cookies.Push(toDriverCookie(c))
|
||||
}
|
||||
|
||||
return cookies, nil
|
||||
}
|
||||
|
||||
func (doc *HTMLDocument) SetCookies(ctx context.Context, cookies ...drivers.HTTPCookie) error {
|
||||
doc.Lock()
|
||||
defer doc.Unlock()
|
||||
|
||||
if len(cookies) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
params := make([]network.CookieParam, 0, len(cookies))
|
||||
|
||||
for _, c := range cookies {
|
||||
params = append(params, fromDriverCookie(doc.url.String(), c))
|
||||
}
|
||||
|
||||
return doc.client.Network.SetCookies(ctx, network.NewSetCookiesArgs(params))
|
||||
}
|
||||
|
||||
func (doc *HTMLDocument) DeleteCookies(ctx context.Context, cookies ...drivers.HTTPCookie) error {
|
||||
doc.Lock()
|
||||
defer doc.Unlock()
|
||||
|
||||
if len(cookies) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
for _, c := range cookies {
|
||||
err = doc.client.Network.DeleteCookies(ctx, fromDriverCookieDelete(doc.url.String(), c))
|
||||
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (doc *HTMLDocument) SetURL(ctx context.Context, url values.String) error {
|
||||
return doc.Navigate(ctx, url)
|
||||
}
|
||||
|
@ -7,10 +7,10 @@ import (
|
||||
"github.com/MontFerret/ferret/pkg/drivers"
|
||||
"github.com/MontFerret/ferret/pkg/drivers/common"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/logging"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
"github.com/mafredri/cdp"
|
||||
"github.com/mafredri/cdp/devtool"
|
||||
"github.com/mafredri/cdp/protocol/emulation"
|
||||
"github.com/mafredri/cdp/protocol/network"
|
||||
"github.com/mafredri/cdp/protocol/page"
|
||||
"github.com/mafredri/cdp/protocol/target"
|
||||
"github.com/mafredri/cdp/rpcc"
|
||||
@ -33,16 +33,16 @@ type Driver struct {
|
||||
func NewDriver(opts ...Option) *Driver {
|
||||
drv := new(Driver)
|
||||
drv.options = newOptions(opts)
|
||||
drv.dev = devtool.New(drv.options.address)
|
||||
drv.dev = devtool.New(drv.options.Address)
|
||||
|
||||
return drv
|
||||
}
|
||||
|
||||
func (drv *Driver) Name() string {
|
||||
return DriverName
|
||||
return drv.options.Name
|
||||
}
|
||||
|
||||
func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (drivers.HTMLDocument, error) {
|
||||
func (drv *Driver) LoadDocument(ctx context.Context, params drivers.LoadDocumentParams) (drivers.HTMLDocument, error) {
|
||||
logger := logging.FromContext(ctx)
|
||||
|
||||
err := drv.init(ctx)
|
||||
@ -52,21 +52,26 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (dr
|
||||
Error().
|
||||
Timestamp().
|
||||
Err(err).
|
||||
Str("driver", DriverName).
|
||||
Str("driver", drv.options.Name).
|
||||
Msg("failed to initialize the driver")
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
url := targetURL.String()
|
||||
url := params.URL
|
||||
|
||||
if url == "" {
|
||||
url = BlankPageURL
|
||||
}
|
||||
|
||||
// Create a new target belonging to the browser context, similar
|
||||
// to opening a new tab in an incognito window.
|
||||
createTargetArgs := target.NewCreateTargetArgs(url).SetBrowserContextID(drv.contextID)
|
||||
// Create a new target belonging to the browser context
|
||||
createTargetArgs := target.NewCreateTargetArgs(url)
|
||||
|
||||
if drv.options.KeepCookies == false && params.KeepCookies == false {
|
||||
// Set it to an incognito mode
|
||||
createTargetArgs.SetBrowserContextID(drv.contextID)
|
||||
}
|
||||
|
||||
createTarget, err := drv.client.Target.CreateTarget(ctx, createTargetArgs)
|
||||
|
||||
if err != nil {
|
||||
@ -74,7 +79,7 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (dr
|
||||
Error().
|
||||
Timestamp().
|
||||
Err(err).
|
||||
Str("driver", DriverName).
|
||||
Str("driver", drv.options.Name).
|
||||
Msg("failed to create a browser target")
|
||||
|
||||
return nil, err
|
||||
@ -88,7 +93,7 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (dr
|
||||
Error().
|
||||
Timestamp().
|
||||
Err(err).
|
||||
Str("driver", DriverName).
|
||||
Str("driver", drv.options.Name).
|
||||
Msg("failed to establish a connection")
|
||||
|
||||
return nil, err
|
||||
@ -117,7 +122,13 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (dr
|
||||
},
|
||||
|
||||
func() error {
|
||||
ua := common.GetUserAgent(drv.options.userAgent)
|
||||
var ua string
|
||||
|
||||
if params.UserAgent != "" {
|
||||
ua = common.GetUserAgent(params.UserAgent)
|
||||
} else {
|
||||
ua = common.GetUserAgent(drv.options.UserAgent)
|
||||
}
|
||||
|
||||
logger.
|
||||
Debug().
|
||||
@ -135,13 +146,17 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (dr
|
||||
emulation.NewSetUserAgentOverrideArgs(ua),
|
||||
)
|
||||
},
|
||||
|
||||
func() error {
|
||||
return client.Network.Enable(ctx, network.NewEnableArgs())
|
||||
},
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return LoadHTMLDocument(ctx, conn, client, url)
|
||||
return LoadHTMLDocument(ctx, conn, client, params)
|
||||
}
|
||||
|
||||
func (drv *Driver) Close() error {
|
||||
@ -184,6 +199,14 @@ func (drv *Driver) init(ctx context.Context) error {
|
||||
return errors.Wrap(err, "failed to initialize driver")
|
||||
}
|
||||
|
||||
drv.conn = bconn
|
||||
drv.client = bc
|
||||
drv.session = sess
|
||||
|
||||
if drv.options.KeepCookies {
|
||||
return nil
|
||||
}
|
||||
|
||||
createCtx, err := bc.Target.CreateBrowserContext(ctx)
|
||||
|
||||
if err != nil {
|
||||
@ -193,9 +216,6 @@ func (drv *Driver) init(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
drv.conn = bconn
|
||||
drv.client = bc
|
||||
drv.session = sess
|
||||
drv.contextID = createCtx.BrowserContextID
|
||||
}
|
||||
|
||||
|
@ -6,7 +6,9 @@ import (
|
||||
"errors"
|
||||
"math"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/MontFerret/ferret/pkg/drivers"
|
||||
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
|
||||
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
|
||||
"github.com/MontFerret/ferret/pkg/drivers/common"
|
||||
@ -14,11 +16,14 @@ import (
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/mafredri/cdp"
|
||||
"github.com/mafredri/cdp/protocol/dom"
|
||||
"github.com/mafredri/cdp/protocol/network"
|
||||
"github.com/mafredri/cdp/protocol/page"
|
||||
"github.com/mafredri/cdp/protocol/runtime"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
var emptyExpires = time.Time{}
|
||||
|
||||
type (
|
||||
batchFunc = func() error
|
||||
|
||||
@ -402,3 +407,83 @@ func createEventBroker(client *cdp.Client) (*events.EventBroker, error) {
|
||||
|
||||
return broker, nil
|
||||
}
|
||||
|
||||
func fromDriverCookie(url string, cookie drivers.HTTPCookie) network.CookieParam {
|
||||
sameSite := network.CookieSameSiteNotSet
|
||||
|
||||
switch cookie.SameSite {
|
||||
case drivers.SameSiteLaxMode:
|
||||
sameSite = network.CookieSameSiteLax
|
||||
case drivers.SameSiteStrictMode:
|
||||
sameSite = network.CookieSameSiteStrict
|
||||
default:
|
||||
sameSite = network.CookieSameSiteNotSet
|
||||
}
|
||||
|
||||
if cookie.Expires == emptyExpires {
|
||||
cookie.Expires = time.Now().Add(time.Duration(24) + time.Hour)
|
||||
}
|
||||
|
||||
normalizedURL := normalizeCookieURL(url)
|
||||
|
||||
return network.CookieParam{
|
||||
URL: &normalizedURL,
|
||||
Name: cookie.Name,
|
||||
Value: cookie.Value,
|
||||
Secure: &cookie.Secure,
|
||||
Path: &cookie.Path,
|
||||
Domain: &cookie.Domain,
|
||||
HTTPOnly: &cookie.HTTPOnly,
|
||||
SameSite: sameSite,
|
||||
Expires: network.TimeSinceEpoch(cookie.Expires.Unix()),
|
||||
}
|
||||
}
|
||||
|
||||
func fromDriverCookieDelete(url string, cookie drivers.HTTPCookie) *network.DeleteCookiesArgs {
|
||||
normalizedURL := normalizeCookieURL(url)
|
||||
|
||||
return &network.DeleteCookiesArgs{
|
||||
URL: &normalizedURL,
|
||||
Name: cookie.Name,
|
||||
Path: &cookie.Path,
|
||||
Domain: &cookie.Domain,
|
||||
}
|
||||
}
|
||||
|
||||
func toDriverCookie(c network.Cookie) drivers.HTTPCookie {
|
||||
sameSite := drivers.SameSiteDefaultMode
|
||||
|
||||
switch c.SameSite {
|
||||
case network.CookieSameSiteLax:
|
||||
sameSite = drivers.SameSiteLaxMode
|
||||
break
|
||||
case network.CookieSameSiteStrict:
|
||||
sameSite = drivers.SameSiteStrictMode
|
||||
break
|
||||
default:
|
||||
sameSite = drivers.SameSiteDefaultMode
|
||||
break
|
||||
}
|
||||
|
||||
return drivers.HTTPCookie{
|
||||
Name: c.Name,
|
||||
Value: c.Value,
|
||||
Path: c.Path,
|
||||
Domain: c.Domain,
|
||||
Expires: time.Unix(int64(c.Expires), 0),
|
||||
SameSite: sameSite,
|
||||
Secure: c.Secure,
|
||||
HTTPOnly: c.HTTPOnly,
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeCookieURL(url string) string {
|
||||
const httpPrefix = "http://"
|
||||
const httpsPrefix = "https://"
|
||||
|
||||
if strings.HasPrefix(url, httpPrefix) || strings.HasPrefix(url, httpsPrefix) {
|
||||
return url
|
||||
}
|
||||
|
||||
return httpPrefix + url
|
||||
}
|
||||
|
@ -2,9 +2,11 @@ package cdp
|
||||
|
||||
type (
|
||||
Options struct {
|
||||
proxy string
|
||||
userAgent string
|
||||
address string
|
||||
Name string
|
||||
Proxy string
|
||||
UserAgent string
|
||||
Address string
|
||||
KeepCookies bool
|
||||
}
|
||||
|
||||
Option func(opts *Options)
|
||||
@ -14,7 +16,8 @@ const DefaultAddress = "http://127.0.0.1:9222"
|
||||
|
||||
func newOptions(setters []Option) *Options {
|
||||
opts := new(Options)
|
||||
opts.address = DefaultAddress
|
||||
opts.Name = DriverName
|
||||
opts.Address = DefaultAddress
|
||||
|
||||
for _, setter := range setters {
|
||||
setter(opts)
|
||||
@ -26,19 +29,31 @@ func newOptions(setters []Option) *Options {
|
||||
func WithAddress(address string) Option {
|
||||
return func(opts *Options) {
|
||||
if address != "" {
|
||||
opts.address = address
|
||||
opts.Address = address
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithProxy(address string) Option {
|
||||
return func(opts *Options) {
|
||||
opts.proxy = address
|
||||
opts.Proxy = address
|
||||
}
|
||||
}
|
||||
|
||||
func WithUserAgent(value string) Option {
|
||||
return func(opts *Options) {
|
||||
opts.userAgent = value
|
||||
opts.UserAgent = value
|
||||
}
|
||||
}
|
||||
|
||||
func WithKeepCookies() Option {
|
||||
return func(opts *Options) {
|
||||
opts.KeepCookies = true
|
||||
}
|
||||
}
|
||||
|
||||
func WithCustomName(name string) Option {
|
||||
return func(opts *Options) {
|
||||
opts.Name = name
|
||||
}
|
||||
}
|
||||
|
@ -22,6 +22,23 @@ func GetInDocument(ctx context.Context, doc drivers.HTMLDocument, path []core.Va
|
||||
switch segment {
|
||||
case "url", "URL":
|
||||
return doc.GetURL(), nil
|
||||
case "cookies":
|
||||
if len(path) == 1 {
|
||||
return doc.GetCookies(ctx)
|
||||
}
|
||||
|
||||
switch idx := path[1].(type) {
|
||||
case values.Int:
|
||||
cookies, err := doc.GetCookies(ctx)
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
return cookies.Get(idx), nil
|
||||
default:
|
||||
return values.None, core.TypeError(idx.Type(), types.Int)
|
||||
}
|
||||
case "body":
|
||||
return doc.QuerySelector(ctx, "body"), nil
|
||||
case "head":
|
||||
|
@ -22,6 +22,8 @@ func SetInDocument(ctx context.Context, doc drivers.HTMLDocument, path []core.Va
|
||||
switch segment {
|
||||
case "url", "URL":
|
||||
return doc.SetURL(ctx, values.NewString(value.String()))
|
||||
case "cookies":
|
||||
|
||||
default:
|
||||
return SetInNode(ctx, doc, path, value)
|
||||
}
|
||||
|
195
pkg/drivers/cookie.go
Normal file
195
pkg/drivers/cookie.go
Normal file
@ -0,0 +1,195 @@
|
||||
package drivers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values/types"
|
||||
)
|
||||
|
||||
type (
|
||||
// Polyfill for Go 1.10
|
||||
SameSite int
|
||||
|
||||
// HTTPCookie HTTPCookie object
|
||||
HTTPCookie struct {
|
||||
Name string
|
||||
Value string
|
||||
Path string
|
||||
Domain string
|
||||
Expires time.Time
|
||||
MaxAge int
|
||||
Secure bool
|
||||
HTTPOnly bool
|
||||
SameSite SameSite
|
||||
}
|
||||
)
|
||||
|
||||
const (
|
||||
SameSiteDefaultMode SameSite = iota + 1
|
||||
SameSiteLaxMode
|
||||
SameSiteStrictMode
|
||||
)
|
||||
|
||||
func (c HTTPCookie) Type() core.Type {
|
||||
return HTTPCookieType
|
||||
}
|
||||
|
||||
func (c HTTPCookie) String() string {
|
||||
return fmt.Sprintf("%s=%s", c.Name, c.Value)
|
||||
}
|
||||
|
||||
func (c HTTPCookie) Compare(other core.Value) int64 {
|
||||
if other.Type() != HTTPCookieType {
|
||||
return Compare(HTTPCookieType, other.Type())
|
||||
}
|
||||
|
||||
oc := other.(HTTPCookie)
|
||||
|
||||
if c.Name != oc.Name {
|
||||
return int64(strings.Compare(c.Name, oc.Name))
|
||||
}
|
||||
|
||||
if c.Value != oc.Value {
|
||||
return int64(strings.Compare(c.Value, oc.Value))
|
||||
}
|
||||
|
||||
if c.Path != oc.Path {
|
||||
return int64(strings.Compare(c.Path, oc.Path))
|
||||
}
|
||||
|
||||
if c.Domain != oc.Domain {
|
||||
return int64(strings.Compare(c.Domain, oc.Domain))
|
||||
}
|
||||
|
||||
if c.Expires.After(oc.Expires) {
|
||||
return 1
|
||||
} else if c.Expires.Before(oc.Expires) {
|
||||
return -1
|
||||
}
|
||||
|
||||
if c.MaxAge > oc.MaxAge {
|
||||
return 1
|
||||
} else if c.MaxAge < oc.MaxAge {
|
||||
return -1
|
||||
}
|
||||
|
||||
if c.Secure && !oc.Secure {
|
||||
return 1
|
||||
} else if !c.Secure && oc.Secure {
|
||||
return -1
|
||||
}
|
||||
|
||||
if c.HTTPOnly && !oc.HTTPOnly {
|
||||
return 1
|
||||
} else if !c.HTTPOnly && oc.HTTPOnly {
|
||||
return -1
|
||||
}
|
||||
|
||||
if c.SameSite > oc.SameSite {
|
||||
return 1
|
||||
} else if c.SameSite < oc.SameSite {
|
||||
return -1
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
func (c HTTPCookie) Unwrap() interface{} {
|
||||
return c.Value
|
||||
}
|
||||
|
||||
func (c HTTPCookie) Hash() uint64 {
|
||||
h := fnv.New64a()
|
||||
|
||||
h.Write([]byte(c.Type().String()))
|
||||
h.Write([]byte(":"))
|
||||
h.Write([]byte(c.Name))
|
||||
h.Write([]byte(c.Value))
|
||||
h.Write([]byte(c.Path))
|
||||
h.Write([]byte(c.Domain))
|
||||
h.Write([]byte(c.Expires.String()))
|
||||
h.Write([]byte(strconv.Itoa(c.MaxAge)))
|
||||
h.Write([]byte(fmt.Sprintf("%t", c.Secure)))
|
||||
h.Write([]byte(fmt.Sprintf("%t", c.HTTPOnly)))
|
||||
h.Write([]byte(strconv.Itoa(int(c.SameSite))))
|
||||
|
||||
return h.Sum64()
|
||||
}
|
||||
|
||||
func (c HTTPCookie) Copy() core.Value {
|
||||
return *(&c)
|
||||
}
|
||||
|
||||
func (c HTTPCookie) MarshalJSON() ([]byte, error) {
|
||||
v := map[string]interface{}{
|
||||
"name": c.Name,
|
||||
"value": c.Value,
|
||||
"path": c.Path,
|
||||
"domain": c.Domain,
|
||||
"expires": c.Expires,
|
||||
"max_age": c.MaxAge,
|
||||
"secure": c.Secure,
|
||||
"http_only": c.HTTPOnly,
|
||||
"same_site": c.SameSite,
|
||||
}
|
||||
|
||||
out, err := json.Marshal(v)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (c HTTPCookie) GetIn(_ context.Context, path []core.Value) (core.Value, error) {
|
||||
if len(path) == 0 {
|
||||
return values.None, nil
|
||||
}
|
||||
|
||||
segment := path[0]
|
||||
|
||||
err := core.ValidateType(segment, types.String)
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
switch segment.(values.String) {
|
||||
case "name":
|
||||
return values.NewString(c.Name), nil
|
||||
case "value":
|
||||
return values.NewString(c.Value), nil
|
||||
case "path":
|
||||
return values.NewString(c.Path), nil
|
||||
case "domain":
|
||||
return values.NewString(c.Domain), nil
|
||||
case "expires":
|
||||
return values.NewDateTime(c.Expires), nil
|
||||
case "maxAge":
|
||||
return values.NewInt(c.MaxAge), nil
|
||||
case "secure":
|
||||
return values.NewBoolean(c.Secure), nil
|
||||
case "httpOnly":
|
||||
return values.NewBoolean(c.HTTPOnly), nil
|
||||
case "sameSite":
|
||||
switch c.SameSite {
|
||||
case SameSiteLaxMode:
|
||||
return values.NewString("Lax"), nil
|
||||
case SameSiteStrictMode:
|
||||
return values.NewString("Strict"), nil
|
||||
default:
|
||||
return values.EmptyString, nil
|
||||
}
|
||||
default:
|
||||
return values.None, nil
|
||||
}
|
||||
}
|
@ -6,7 +6,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
)
|
||||
|
||||
const DefaultTimeout = time.Second * 30
|
||||
@ -19,10 +18,18 @@ type (
|
||||
drivers map[string]Driver
|
||||
}
|
||||
|
||||
LoadDocumentParams struct {
|
||||
URL string
|
||||
UserAgent string
|
||||
KeepCookies bool
|
||||
Cookies []HTTPCookie
|
||||
Header HTTPHeader
|
||||
}
|
||||
|
||||
Driver interface {
|
||||
io.Closer
|
||||
Name() string
|
||||
GetDocument(ctx context.Context, url values.String) (HTMLDocument, error)
|
||||
LoadDocument(ctx context.Context, params LoadDocumentParams) (HTMLDocument, error)
|
||||
}
|
||||
)
|
||||
|
||||
|
135
pkg/drivers/header.go
Normal file
135
pkg/drivers/header.go
Normal file
@ -0,0 +1,135 @@
|
||||
package drivers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"net/textproto"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values/types"
|
||||
)
|
||||
|
||||
// HTTPCookie HTTPCookie object
|
||||
type HTTPHeader map[string][]string
|
||||
|
||||
func (h HTTPHeader) Type() core.Type {
|
||||
return HTTPHeaderType
|
||||
}
|
||||
|
||||
func (h HTTPHeader) String() string {
|
||||
var buf bytes.Buffer
|
||||
|
||||
for k := range h {
|
||||
buf.WriteString(fmt.Sprintf("%s=%s;", k, h.Get(k)))
|
||||
}
|
||||
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func (h HTTPHeader) Compare(other core.Value) int64 {
|
||||
if other.Type() != HTTPHeaderType {
|
||||
return Compare(HTTPHeaderType, other.Type())
|
||||
}
|
||||
|
||||
oh := other.(HTTPHeader)
|
||||
|
||||
if len(h) > len(oh) {
|
||||
return 1
|
||||
} else if len(h) < len(oh) {
|
||||
return -1
|
||||
}
|
||||
|
||||
for k := range h {
|
||||
c := strings.Compare(h.Get(k), oh.Get(k))
|
||||
|
||||
if c != 0 {
|
||||
return int64(c)
|
||||
}
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
func (h HTTPHeader) Unwrap() interface{} {
|
||||
return h
|
||||
}
|
||||
|
||||
func (h HTTPHeader) Hash() uint64 {
|
||||
hash := fnv.New64a()
|
||||
|
||||
hash.Write([]byte(h.Type().String()))
|
||||
hash.Write([]byte(":"))
|
||||
hash.Write([]byte("{"))
|
||||
|
||||
keys := make([]string, 0, len(h))
|
||||
|
||||
for key := range h {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
|
||||
// order does not really matter
|
||||
// but it will give us a consistent hash sum
|
||||
sort.Strings(keys)
|
||||
endIndex := len(keys) - 1
|
||||
|
||||
for idx, key := range keys {
|
||||
hash.Write([]byte(key))
|
||||
hash.Write([]byte(":"))
|
||||
|
||||
value := h.Get(key)
|
||||
|
||||
hash.Write([]byte(value))
|
||||
|
||||
if idx != endIndex {
|
||||
hash.Write([]byte(","))
|
||||
}
|
||||
}
|
||||
|
||||
hash.Write([]byte("}"))
|
||||
|
||||
return hash.Sum64()
|
||||
}
|
||||
|
||||
func (h HTTPHeader) Copy() core.Value {
|
||||
return *(&h)
|
||||
}
|
||||
|
||||
func (h HTTPHeader) MarshalJSON() ([]byte, error) {
|
||||
out, err := json.Marshal(h)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (h HTTPHeader) Set(key, value string) {
|
||||
textproto.MIMEHeader(h).Set(key, value)
|
||||
}
|
||||
|
||||
func (h HTTPHeader) Get(key string) string {
|
||||
return textproto.MIMEHeader(h).Get(key)
|
||||
}
|
||||
|
||||
func (h HTTPHeader) GetIn(_ context.Context, path []core.Value) (core.Value, error) {
|
||||
if len(path) == 0 {
|
||||
return values.None, nil
|
||||
}
|
||||
|
||||
segment := path[0]
|
||||
|
||||
err := core.ValidateType(segment, types.String)
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
return values.NewString(h.Get(segment.String())), nil
|
||||
}
|
@ -12,14 +12,16 @@ import (
|
||||
)
|
||||
|
||||
type HTMLDocument struct {
|
||||
url values.String
|
||||
docNode *goquery.Document
|
||||
element drivers.HTMLElement
|
||||
url values.String
|
||||
cookies []drivers.HTTPCookie
|
||||
}
|
||||
|
||||
func NewHTMLDocument(
|
||||
url string,
|
||||
node *goquery.Document,
|
||||
url string,
|
||||
cookies []drivers.HTTPCookie,
|
||||
) (drivers.HTMLDocument, error) {
|
||||
if url == "" {
|
||||
return nil, core.Error(core.ErrMissedArgument, "document url")
|
||||
@ -35,7 +37,7 @@ func NewHTMLDocument(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &HTMLDocument{values.NewString(url), node, el}, nil
|
||||
return &HTMLDocument{node, el, values.NewString(url), cookies}, nil
|
||||
}
|
||||
|
||||
func (doc *HTMLDocument) MarshalJSON() ([]byte, error) {
|
||||
@ -82,7 +84,7 @@ func (doc *HTMLDocument) Hash() uint64 {
|
||||
}
|
||||
|
||||
func (doc *HTMLDocument) Copy() core.Value {
|
||||
cp, err := NewHTMLDocument(string(doc.url), doc.docNode)
|
||||
cp, err := NewHTMLDocument(doc.docNode, string(doc.url), doc.cookies)
|
||||
|
||||
if err != nil {
|
||||
return values.None
|
||||
@ -92,7 +94,17 @@ func (doc *HTMLDocument) Copy() core.Value {
|
||||
}
|
||||
|
||||
func (doc *HTMLDocument) Clone() core.Value {
|
||||
cp, err := NewHTMLDocument(string(doc.url), goquery.CloneDocument(doc.docNode))
|
||||
var cookies []drivers.HTTPCookie
|
||||
|
||||
if doc.cookies != nil {
|
||||
cookies = make([]drivers.HTTPCookie, 0, len(doc.cookies))
|
||||
|
||||
for i, c := range doc.cookies {
|
||||
cookies[i] = c
|
||||
}
|
||||
}
|
||||
|
||||
cp, err := NewHTMLDocument(goquery.CloneDocument(doc.docNode), string(doc.url), cookies)
|
||||
|
||||
if err != nil {
|
||||
return values.None
|
||||
@ -161,6 +173,28 @@ func (doc *HTMLDocument) SetURL(_ context.Context, _ values.String) error {
|
||||
return core.ErrInvalidOperation
|
||||
}
|
||||
|
||||
func (doc *HTMLDocument) GetCookies(_ context.Context) (*values.Array, error) {
|
||||
if doc.cookies == nil {
|
||||
return values.NewArray(0), nil
|
||||
}
|
||||
|
||||
arr := values.NewArray(len(doc.cookies))
|
||||
|
||||
for _, c := range doc.cookies {
|
||||
arr.Push(c)
|
||||
}
|
||||
|
||||
return arr, nil
|
||||
}
|
||||
|
||||
func (doc *HTMLDocument) SetCookies(_ context.Context, _ ...drivers.HTTPCookie) error {
|
||||
return core.ErrNotSupported
|
||||
}
|
||||
|
||||
func (doc *HTMLDocument) DeleteCookies(_ context.Context, _ ...drivers.HTTPCookie) error {
|
||||
return core.ErrNotSupported
|
||||
}
|
||||
|
||||
func (doc *HTMLDocument) Navigate(_ context.Context, _ values.String) error {
|
||||
return core.ErrNotSupported
|
||||
}
|
||||
|
@ -11,7 +11,6 @@ import (
|
||||
"github.com/MontFerret/ferret/pkg/runtime/logging"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/corpix/uarand"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sethgrid/pester"
|
||||
)
|
||||
@ -63,39 +62,67 @@ func (drv *Driver) Name() string {
|
||||
return DriverName
|
||||
}
|
||||
|
||||
func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (drivers.HTMLDocument, error) {
|
||||
u := targetURL.String()
|
||||
req, err := http.NewRequest(http.MethodGet, u, nil)
|
||||
func (drv *Driver) LoadDocument(ctx context.Context, params drivers.LoadDocumentParams) (drivers.HTMLDocument, error) {
|
||||
req, err := http.NewRequest(http.MethodGet, params.URL, nil)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
logger := logging.FromContext(ctx)
|
||||
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9,ru;q=0.8")
|
||||
req.Header.Set("Cache-Control", "no-cache")
|
||||
req.Header.Set("Pragma", "no-cache")
|
||||
|
||||
if params.Header != nil {
|
||||
for k := range params.Header {
|
||||
req.Header.Add(k, params.Header.Get(k))
|
||||
|
||||
logger.
|
||||
Debug().
|
||||
Timestamp().
|
||||
Str("header", k).
|
||||
Msg("set header")
|
||||
}
|
||||
}
|
||||
|
||||
if params.Cookies != nil {
|
||||
for _, c := range params.Cookies {
|
||||
req.AddCookie(&http.Cookie{
|
||||
Name: c.Name,
|
||||
Value: c.Value,
|
||||
})
|
||||
|
||||
logger.
|
||||
Debug().
|
||||
Timestamp().
|
||||
Str("cookie", c.Name).
|
||||
Msg("set cookie")
|
||||
}
|
||||
}
|
||||
|
||||
req = req.WithContext(ctx)
|
||||
|
||||
ua := common.GetUserAgent(drv.options.userAgent)
|
||||
var ua string
|
||||
|
||||
if params.UserAgent != "" {
|
||||
ua = common.GetUserAgent(params.UserAgent)
|
||||
} else {
|
||||
ua = common.GetUserAgent(drv.options.userAgent)
|
||||
}
|
||||
|
||||
logger := logging.FromContext(ctx)
|
||||
logger.
|
||||
Debug().
|
||||
Timestamp().
|
||||
Str("user-agent", ua).
|
||||
Msg("using User-Agent")
|
||||
|
||||
// use custom user agent
|
||||
if ua != "" {
|
||||
req.Header.Set("User-Agent", uarand.GetRandom())
|
||||
}
|
||||
|
||||
resp, err := drv.client.Do(req)
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to retrieve a document %s", u)
|
||||
return nil, errors.Wrapf(err, "failed to retrieve a document %s", params.URL)
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
@ -103,10 +130,10 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (dr
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to parse a document %s", u)
|
||||
return nil, errors.Wrapf(err, "failed to parse a document %s", params.URL)
|
||||
}
|
||||
|
||||
return NewHTMLDocument(u, doc)
|
||||
return NewHTMLDocument(doc, params.URL, params.Cookies)
|
||||
}
|
||||
|
||||
func (drv *Driver) ParseDocument(_ context.Context, str values.String) (drivers.HTMLDocument, error) {
|
||||
@ -118,7 +145,7 @@ func (drv *Driver) ParseDocument(_ context.Context, str values.String) (drivers.
|
||||
return nil, errors.Wrap(err, "failed to parse a document")
|
||||
}
|
||||
|
||||
return NewHTMLDocument("#string", doc)
|
||||
return NewHTMLDocument(doc, "#string", nil)
|
||||
}
|
||||
|
||||
func (drv *Driver) Close() error {
|
||||
|
@ -6,7 +6,7 @@ import "github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
type PDFParams struct {
|
||||
// Paper orientation. Defaults to false.
|
||||
Landscape values.Boolean
|
||||
// Display header and footer. Defaults to false.
|
||||
// Display values and footer. Defaults to false.
|
||||
DisplayHeaderFooter values.Boolean
|
||||
// Print background graphics. Defaults to false.
|
||||
PrintBackground values.Boolean
|
||||
@ -28,7 +28,7 @@ type PDFParams struct {
|
||||
PageRanges values.String
|
||||
// Whether to silently ignore invalid but successfully parsed page ranges, such as '3-2'. Defaults to false.
|
||||
IgnoreInvalidPageRanges values.Boolean
|
||||
// HTML template for the print header. Should be valid HTML markup with following classes used to inject printing values into them: - `date`: formatted print date - `title`: document title - `url`: document location - `pageNumber`: current page number - `totalPages`: total pages in the document
|
||||
// HTML template for the print values. Should be valid HTML markup with following classes used to inject printing values into them: - `date`: formatted print date - `title`: document title - `url`: document location - `pageNumber`: current page number - `totalPages`: total pages in the document
|
||||
// For example, `<span class=title></span>` would generate span containing the title.
|
||||
HeaderTemplate values.String
|
||||
// HTML template for the print footer. Should use the same format as the `headerTemplate`.
|
||||
|
@ -3,14 +3,18 @@ package drivers
|
||||
import "github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
|
||||
var (
|
||||
HTTPHeaderType = core.NewType("HTTPHeader")
|
||||
HTTPCookieType = core.NewType("HTTPCookie")
|
||||
HTMLElementType = core.NewType("HTMLElement")
|
||||
HTMLDocumentType = core.NewType("HTMLDocument")
|
||||
)
|
||||
|
||||
// Comparison table of builtin types
|
||||
var typeComparisonTable = map[core.Type]uint64{
|
||||
HTMLElementType: 0,
|
||||
HTMLDocumentType: 1,
|
||||
HTTPHeaderType: 0,
|
||||
HTTPCookieType: 1,
|
||||
HTMLElementType: 2,
|
||||
HTMLDocumentType: 3,
|
||||
}
|
||||
|
||||
func Compare(first, second core.Type) int64 {
|
||||
|
@ -109,6 +109,12 @@ type (
|
||||
|
||||
SetURL(ctx context.Context, url values.String) error
|
||||
|
||||
GetCookies(ctx context.Context) (*values.Array, error)
|
||||
|
||||
SetCookies(ctx context.Context, cookies ...HTTPCookie) error
|
||||
|
||||
DeleteCookies(ctx context.Context, cookies ...HTTPCookie) error
|
||||
|
||||
Navigate(ctx context.Context, url values.String) error
|
||||
|
||||
NavigateBack(ctx context.Context, skip values.Int) (values.Boolean, error)
|
||||
|
@ -135,6 +135,16 @@ func (t *Array) ForEach(predicate ArrayPredicate) {
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Array) Find(predicate ArrayPredicate) (core.Value, Boolean) {
|
||||
for idx, val := range t.items {
|
||||
if predicate(val, idx) == true {
|
||||
return val, True
|
||||
}
|
||||
}
|
||||
|
||||
return None, False
|
||||
}
|
||||
|
||||
func (t *Array) Get(idx Int) core.Value {
|
||||
l := len(t.items) - 1
|
||||
|
||||
|
66
pkg/stdlib/html/cookie_del.go
Normal file
66
pkg/stdlib/html/cookie_del.go
Normal file
@ -0,0 +1,66 @@
|
||||
package html
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/MontFerret/ferret/pkg/drivers"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values/types"
|
||||
)
|
||||
|
||||
// CookieSet gets a cookie from a given document by name.
|
||||
// @param source (HTMLDocument) - Target HTMLDocument.
|
||||
// @param cookie (...HTTPCookie|String) - Cookie or cookie name to delete.
|
||||
func CookieDel(ctx context.Context, args ...core.Value) (core.Value, error) {
|
||||
err := core.ValidateArgs(args, 2, core.MaxArgs)
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
err = core.ValidateType(args[0], drivers.HTMLDocumentType)
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
doc := args[0].(drivers.HTMLDocument)
|
||||
inputs := args[1:]
|
||||
var currentCookies *values.Array
|
||||
cookies := make([]drivers.HTTPCookie, 0, len(inputs))
|
||||
|
||||
for _, c := range inputs {
|
||||
switch cookie := c.(type) {
|
||||
case values.String:
|
||||
if currentCookies == nil {
|
||||
current, err := doc.GetCookies(ctx)
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
currentCookies = current
|
||||
}
|
||||
|
||||
found, isFound := currentCookies.Find(func(value core.Value, _ int) bool {
|
||||
cv := value.(drivers.HTTPCookie)
|
||||
|
||||
return cv.Name == cookie.String()
|
||||
})
|
||||
|
||||
if isFound {
|
||||
cookies = append(cookies, found.(drivers.HTTPCookie))
|
||||
}
|
||||
|
||||
break
|
||||
case drivers.HTTPCookie:
|
||||
cookies = append(cookies, cookie)
|
||||
|
||||
break
|
||||
default:
|
||||
return values.None, core.TypeError(c.Type(), types.String, drivers.HTTPCookieType)
|
||||
}
|
||||
}
|
||||
|
||||
return values.None, doc.DeleteCookies(ctx, cookies...)
|
||||
}
|
58
pkg/stdlib/html/cookie_get.go
Normal file
58
pkg/stdlib/html/cookie_get.go
Normal file
@ -0,0 +1,58 @@
|
||||
package html
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/MontFerret/ferret/pkg/drivers"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values/types"
|
||||
)
|
||||
|
||||
// CookieSet gets a cookie from a given document by name.
|
||||
// @param doc (HTMLDocument) - Target HTMLDocument.
|
||||
// @param name (String) - Cookie or cookie name to delete.
|
||||
func CookieGet(ctx context.Context, args ...core.Value) (core.Value, error) {
|
||||
err := core.ValidateArgs(args, 2, 2)
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
err = core.ValidateType(args[0], drivers.HTMLDocumentType)
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
err = core.ValidateType(args[1], types.String)
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
doc := args[0].(drivers.HTMLDocument)
|
||||
name := args[1].(values.String)
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
cookies, err := doc.GetCookies(ctx)
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
found, _ := cookies.Find(func(value core.Value, _ int) bool {
|
||||
cookie, ok := value.(drivers.HTTPCookie)
|
||||
|
||||
if !ok {
|
||||
return ok
|
||||
}
|
||||
|
||||
return cookie.Name == name.String()
|
||||
})
|
||||
|
||||
return found, nil
|
||||
}
|
42
pkg/stdlib/html/cookie_set.go
Normal file
42
pkg/stdlib/html/cookie_set.go
Normal file
@ -0,0 +1,42 @@
|
||||
package html
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/MontFerret/ferret/pkg/drivers"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
)
|
||||
|
||||
// CookieSet sets cookies to a given document
|
||||
// @param doc (HTMLDocument) - Target document.
|
||||
// @param cookie... (HTTPCookie) - Target cookies.
|
||||
func CookieSet(ctx context.Context, args ...core.Value) (core.Value, error) {
|
||||
err := core.ValidateArgs(args, 2, core.MaxArgs)
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
err = core.ValidateType(args[0], drivers.HTMLDocumentType)
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
doc := args[0].(drivers.HTMLDocument)
|
||||
|
||||
cookies := make([]drivers.HTTPCookie, 0, len(args)-1)
|
||||
|
||||
for _, c := range args[1:] {
|
||||
cookie, err := parseCookie(c)
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
cookies = append(cookies, cookie)
|
||||
}
|
||||
|
||||
return values.None, doc.SetCookies(ctx, cookies...)
|
||||
}
|
@ -2,6 +2,7 @@ package html
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/MontFerret/ferret/pkg/drivers"
|
||||
@ -12,6 +13,7 @@ import (
|
||||
)
|
||||
|
||||
type DocumentLoadParams struct {
|
||||
drivers.LoadDocumentParams
|
||||
Driver string
|
||||
Timeout time.Duration
|
||||
}
|
||||
@ -43,9 +45,9 @@ func Document(ctx context.Context, args ...core.Value) (core.Value, error) {
|
||||
var params DocumentLoadParams
|
||||
|
||||
if len(args) == 1 {
|
||||
params = newDefaultDocLoadParams()
|
||||
params = newDefaultDocLoadParams(url)
|
||||
} else {
|
||||
p, err := newDocLoadParams(args[1])
|
||||
p, err := newDocLoadParams(url, args[1])
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
@ -63,17 +65,20 @@ func Document(ctx context.Context, args ...core.Value) (core.Value, error) {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
return drv.GetDocument(ctx, url)
|
||||
return drv.LoadDocument(ctx, params.LoadDocumentParams)
|
||||
}
|
||||
|
||||
func newDefaultDocLoadParams() DocumentLoadParams {
|
||||
func newDefaultDocLoadParams(url values.String) DocumentLoadParams {
|
||||
return DocumentLoadParams{
|
||||
LoadDocumentParams: drivers.LoadDocumentParams{
|
||||
URL: url.String(),
|
||||
},
|
||||
Timeout: time.Second * 30,
|
||||
}
|
||||
}
|
||||
|
||||
func newDocLoadParams(arg core.Value) (DocumentLoadParams, error) {
|
||||
res := newDefaultDocLoadParams()
|
||||
func newDocLoadParams(url values.String, arg core.Value) (DocumentLoadParams, error) {
|
||||
res := newDefaultDocLoadParams(url)
|
||||
|
||||
if err := core.ValidateType(arg, types.Boolean, types.String, types.Object); err != nil {
|
||||
return res, err
|
||||
@ -103,6 +108,58 @@ func newDocLoadParams(arg core.Value) (DocumentLoadParams, error) {
|
||||
res.Timeout = time.Duration(timeout.(values.Int)) + time.Millisecond
|
||||
}
|
||||
|
||||
userAgent, exists := obj.Get(values.NewString("userAgent"))
|
||||
|
||||
if exists {
|
||||
if err := core.ValidateType(userAgent, types.String); err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
res.UserAgent = userAgent.String()
|
||||
}
|
||||
|
||||
keepCookies, exists := obj.Get(values.NewString("keepCookies"))
|
||||
|
||||
if exists {
|
||||
if err := core.ValidateType(keepCookies, types.Boolean); err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
res.KeepCookies = bool(keepCookies.(values.Boolean))
|
||||
}
|
||||
|
||||
cookies, exists := obj.Get(values.NewString("cookies"))
|
||||
|
||||
if exists {
|
||||
if err := core.ValidateType(cookies, types.Array); err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
cookies, err := parseCookies(cookies.(*values.Array))
|
||||
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
res.Cookies = cookies
|
||||
}
|
||||
|
||||
header, exists := obj.Get(values.NewString("header"))
|
||||
|
||||
if exists {
|
||||
if err := core.ValidateType(header, types.Object); err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
header, err := parseHeader(header.(*values.Object))
|
||||
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
res.Header = header
|
||||
}
|
||||
|
||||
break
|
||||
case types.String:
|
||||
res.Driver = arg.(values.String).String()
|
||||
@ -121,3 +178,127 @@ func newDocLoadParams(arg core.Value) (DocumentLoadParams, error) {
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func parseCookies(arr *values.Array) ([]drivers.HTTPCookie, error) {
|
||||
var err error
|
||||
res := make([]drivers.HTTPCookie, 0, arr.Length())
|
||||
|
||||
arr.ForEach(func(value core.Value, idx int) bool {
|
||||
cookie, e := parseCookie(value)
|
||||
|
||||
if e != nil {
|
||||
err = e
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
res = append(res, cookie)
|
||||
|
||||
return true
|
||||
})
|
||||
|
||||
return res, err
|
||||
}
|
||||
|
||||
func parseCookie(value core.Value) (drivers.HTTPCookie, error) {
|
||||
var err error
|
||||
|
||||
if err = core.ValidateType(value, types.Object, drivers.HTTPCookieType); err != nil {
|
||||
return drivers.HTTPCookie{}, err
|
||||
}
|
||||
|
||||
if value.Type() == drivers.HTTPCookieType {
|
||||
return value.(drivers.HTTPCookie), nil
|
||||
}
|
||||
|
||||
co := value.(*values.Object)
|
||||
|
||||
cookie := drivers.HTTPCookie{
|
||||
Name: co.MustGet("name").String(),
|
||||
Value: co.MustGet("value").String(),
|
||||
Path: co.MustGet("path").String(),
|
||||
Domain: co.MustGet("domain").String(),
|
||||
}
|
||||
|
||||
maxAge, exists := co.Get("maxAge")
|
||||
|
||||
if exists {
|
||||
if err = core.ValidateType(maxAge, types.Int); err != nil {
|
||||
return drivers.HTTPCookie{}, err
|
||||
}
|
||||
|
||||
cookie.MaxAge = int(maxAge.(values.Int))
|
||||
}
|
||||
|
||||
expires, exists := co.Get("expires")
|
||||
|
||||
if exists {
|
||||
if err = core.ValidateType(maxAge, types.DateTime, types.String); err != nil {
|
||||
return drivers.HTTPCookie{}, err
|
||||
}
|
||||
|
||||
if expires.Type() == types.DateTime {
|
||||
cookie.Expires = expires.(values.DateTime).Unwrap().(time.Time)
|
||||
} else {
|
||||
t, err := time.Parse(expires.String(), values.DefaultTimeLayout)
|
||||
|
||||
if err != nil {
|
||||
return drivers.HTTPCookie{}, err
|
||||
}
|
||||
|
||||
cookie.Expires = t
|
||||
}
|
||||
}
|
||||
|
||||
sameSite, exists := co.Get("sameSite")
|
||||
|
||||
if exists {
|
||||
sameSite := strings.ToLower(sameSite.String())
|
||||
|
||||
switch sameSite {
|
||||
case "lax":
|
||||
cookie.SameSite = drivers.SameSiteLaxMode
|
||||
break
|
||||
case "strict":
|
||||
cookie.SameSite = drivers.SameSiteStrictMode
|
||||
break
|
||||
default:
|
||||
cookie.SameSite = drivers.SameSiteDefaultMode
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
httpOnly, exists := co.Get("httpOnly")
|
||||
|
||||
if exists {
|
||||
if err = core.ValidateType(httpOnly, types.Boolean); err != nil {
|
||||
return drivers.HTTPCookie{}, err
|
||||
}
|
||||
|
||||
cookie.HTTPOnly = bool(httpOnly.(values.Boolean))
|
||||
}
|
||||
|
||||
secure, exists := co.Get("secure")
|
||||
|
||||
if exists {
|
||||
if err = core.ValidateType(secure, types.Boolean); err != nil {
|
||||
return drivers.HTTPCookie{}, err
|
||||
}
|
||||
|
||||
cookie.Secure = bool(secure.(values.Boolean))
|
||||
}
|
||||
|
||||
return cookie, err
|
||||
}
|
||||
|
||||
func parseHeader(header *values.Object) (drivers.HTTPHeader, error) {
|
||||
res := make(drivers.HTTPHeader)
|
||||
|
||||
header.ForEach(func(value core.Value, key string) bool {
|
||||
res.Set(key, value.String())
|
||||
|
||||
return true
|
||||
})
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
@ -17,6 +17,9 @@ func NewLib() map[string]core.Function {
|
||||
"ATTR_GET": AttributeGet,
|
||||
"ATTR_REMOVE": AttributeRemove,
|
||||
"ATTR_SET": AttributeSet,
|
||||
"COOKIE_DEL": CookieDel,
|
||||
"COOKIE_GET": CookieGet,
|
||||
"COOKIE_SET": CookieSet,
|
||||
"CLICK": Click,
|
||||
"CLICK_ALL": ClickAll,
|
||||
"DOCUMENT": Document,
|
||||
|
Loading…
x
Reference in New Issue
Block a user