go/ferret
go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2025-03-05 15:16:07 +02:00
Code Issues Releases Activity

Merge branch 'master' of https://github.com/MontFerret/ferret

Browse Source
This commit is contained in:
Владимир Фетисов 2019-07-03 22:20:56 +03:00
commit 6203b7cd76
199 changed files with 6708 additions and 4132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/FUNDING.yml vendored Normal file
View File

@ -0,0 +1,5 @@
# These are supported funding model platforms
github: []
patreon: ziflex
open_collective: ferret

5
.github/stale.yml vendored
View File

@ -16,6 +16,7 @@ exemptLabels:
- proposal
- refactoring
- bug
- help wanted
# Set to true to ignore issues in a project (defaults to false)
exemptProjects: false
@ -27,7 +28,7 @@ exemptMilestones: true
exemptAssignees: false
# Label to use when marking as stale
staleLabel: wontfix
staleLabel: stale
# Comment to post when marking as stale. Set to `false` to disable
markComment: >
@ -59,4 +60,4 @@ limitPerRun: 30
# issues:
# exemptLabels:
# - confirmed
# - confirmed

33
.golangci.yml Normal file
View File

@ -0,0 +1,33 @@
# This file contains all available configuration options
# with their default values.
# options for analysis running
run:
# which dirs to skip: they won't be analyzed;
# can use regexp here: generated.*, regexp is applied on full path;
# default value is empty list, but next dirs are always skipped independently
# from this option's value:
# vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
skip-dirs:
- pkg/parser/fql
- pkg/parser/antlr
linters:
disable:
- errcheck
issues:
# List of regexps of issue texts to exclude, empty list by default.
# But independently from this option we use default exclude patterns,
# it can be disabled by `exclude-use-default: false`. To list all
# excluded by default patterns execute `golangci-lint run --help`
exclude:
- '^(G104|G401|G505|G501):'
- '^shadow: declaration of'
# which files to skip: they will be analyzed, but issues from them
# won't be reported. Default value is empty list, but there is
# no need to include all autogenerated files, we confidently recognize
# autogenerated files. If it's not please let us know.
skip-files:
- "*_test.go"

1
.travis.yml
View File

@ -18,6 +18,7 @@ addons:
install:
- go get -u github.com/mgechev/revive
- go get -u github.com/golangci/golangci-lint/cmd/golangci-lint
- sudo curl -o /usr/local/lib/antlr-4.7.1-complete.jar https://www.antlr.org/download/antlr-4.7.1-complete.jar
- export CLASSPATH=".:/usr/local/lib/antlr-4.7.1-complete.jar:$CLASSPATH"
- mkdir $HOME/travis-bin

5
Makefile
View File

@ -30,7 +30,7 @@ cover:
curl -s https://codecov.io/bash | bash
e2e:
go run ${DIR_E2E}/main.go --tests ${DIR_E2E}/tests --pages ${DIR_E2E}/pages --filter doc_cookie_set*
go run ${DIR_E2E}/main.go --tests ${DIR_E2E}/tests --pages ${DIR_E2E}/pages
bench:
go test -run=XXX -bench=. ${DIR_PKG}/...
@ -48,7 +48,8 @@ fmt:
# https://github.com/mgechev/revive
# go get github.com/mgechev/revive
lint:
revive -config revive.toml -formatter friendly -exclude ./pkg/parser/fql/... -exclude ./vendor/... ./...
revive -config revive.toml -formatter friendly -exclude ./pkg/parser/fql/... -exclude ./vendor/... ./... && \
golangci-lint run ./pkg/...
# http://godoc.org/code.google.com/p/go.tools/cmd/vet
# go get code.google.com/p/go.tools/cmd/vet

20
README.md
View File

@ -532,12 +532,26 @@ LET doc = DOCUMENT("https://www.google.com", {
]
})
COOKIES_SET(doc, { name: "baz", value: "qaz"}, { name: "daz", value: "gag" })
COOKIES_DEL(doc, "foo")
COOKIE_SET(doc, { name: "baz", value: "qaz"}, { name: "daz", value: "gag" })
COOKIE_DEL(doc, "foo")
LET c = COOKIES_GET(doc, "baz")
LET c = COOKIE_GET(doc, "baz")
FOR cookie IN doc.cookies
RETURN cookie.name
```
## Sponsors
Support this project by becoming a sponsor. Your logo will show up here with a link to your website.
<p>
<a href="https://opencollective.com/ferret/sponsor/0/website" target="_blank"><img src="https://opencollective.com/ferret/sponsor/0/avatar.svg"></a>
</p>
<p align="center">
<a href="https://opencollective.com/ferret/donate" target="_blank">
<img src="https://opencollective.com/ferret/donate/button@2x.png?color=blue" width="300" />
</a>
</p>

2
cli/browser/browser.go
View File

@ -62,7 +62,7 @@ func (b *Browser) Close() error {
return nil
}
os.RemoveAll(tmpDir)
err = os.RemoveAll(tmpDir)
if err != nil {
return err

2
cli/browser/launcher.go
View File

@ -47,7 +47,7 @@ func Launch(setters ...Option) (*Browser, error) {
temporaryUserDataDir := opts.userDataDir
if temporaryUserDataDir == "" && opts.noUserDataDir == false {
if temporaryUserDataDir == "" && !opts.noUserDataDir {
dirName, err := ioutil.TempDir(os.TempDir(), "ferret_dev_profile-")
if err != nil {

64
e2e/main.go
View File

@ -4,13 +4,15 @@ import (
"context"
"flag"
"fmt"
"github.com/MontFerret/ferret/e2e/runner"
"github.com/MontFerret/ferret/e2e/server"
"github.com/rs/zerolog"
"net"
"os"
"os/signal"
"path/filepath"
"regexp"
"github.com/MontFerret/ferret/e2e/runner"
"github.com/MontFerret/ferret/e2e/server"
"github.com/rs/zerolog"
)
var (
@ -39,6 +41,20 @@ var (
)
)
func getOutboundIP() (net.IP, error) {
conn, err := net.Dial("udp", "8.8.8.8:80")
if err != nil {
return nil, err
}
defer conn.Close()
localAddr := conn.LocalAddr().(*net.UDPAddr)
return localAddr.IP, nil
}
func main() {
flag.Parse()
@ -56,19 +72,6 @@ func main() {
Dir: filepath.Join(*pagesDir, "dynamic"),
})
var filterR *regexp.Regexp
if *filter != "" {
r, err := regexp.Compile(*filter)
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
filterR = r
}
go func() {
if err := static.Start(); err != nil {
logger.Info().Timestamp().Msg("shutting down the static pages server")
@ -81,26 +84,35 @@ func main() {
}
}()
dirname := *testsDir
if dirname == "" {
d, err := filepath.Abs(filepath.Dir(os.Args[0]))
if *testsDir == "" {
_, err := filepath.Abs(filepath.Dir(os.Args[0]))
if err != nil {
logger.Fatal().Timestamp().Err(err).Msg("failed to get testsDir")
return
}
}
dirname = d
var ipAddr string
// we need it in those cases when a Chrome instance is running inside a container
// and it needs an external IP to get access to our static web server
outIP, err := getOutboundIP()
if err != nil {
ipAddr = "0.0.0.0"
logger.Warn().Err(err).Msg("Failed to get outbound IP address")
} else {
ipAddr = outIP.String()
}
r := runner.New(logger, runner.Settings{
StaticServerAddress: fmt.Sprintf("http://0.0.0.0:%d", staticPort),
DynamicServerAddress: fmt.Sprintf("http://0.0.0.0:%d", dynamicPort),
StaticServerAddress: fmt.Sprintf("http://%s:%d", ipAddr, staticPort),
DynamicServerAddress: fmt.Sprintf("http://%s:%d", ipAddr, dynamicPort),
CDPAddress: *cdp,
Dir: *testsDir,
Filter: filterR,
Filter: *filter,
})
ctx, cancel := context.WithCancel(context.Background())
@ -114,7 +126,7 @@ func main() {
}
}()
err := r.Run(ctx)
err = r.Run(ctx)
if err != nil {
os.Exit(1)

30
e2e/pages/dynamic/components/app.js
View File

@ -2,6 +2,7 @@ import Layout from './layout.js';
import IndexPage from './pages/index.js';
import FormsPage from './pages/forms/index.js';
import EventsPage from './pages/events/index.js';
import IframePage from './pages/iframes/index.js';
const e = React.createElement;
const Router = ReactRouter.Router;
@ -10,7 +11,26 @@ const Route = ReactRouter.Route;
const Redirect = ReactRouter.Redirect;
const createBrowserHistory = History.createBrowserHistory;
export default function AppComponent({ redirect = null}) {
export default React.memo(function AppComponent(params = {}) {
let redirectTo;
if (params.redirect) {
let search = '';
Object.keys(params).forEach((key) => {
if (key !== 'redirect') {
search += `${key}=${params[key]}`;
}
});
const to = {
pathname: params.redirect,
search: search ? `?${search}` : '',
};
redirectTo = e(Redirect, { to });
}
return e(Router, { history: createBrowserHistory() },
e(Layout, null, [
e(Switch, null, [
@ -27,8 +47,12 @@ export default function AppComponent({ redirect = null}) {
path: '/events',
component: EventsPage
}),
e(Route, {
path: '/iframe',
component: IframePage
}),
]),
redirect ? e(Redirect, { to: redirect }) : null
redirectTo
])
)
}
})

3
e2e/pages/dynamic/components/layout.js
View File

@ -18,6 +18,9 @@ export default function Layout({ children }) {
]),
e("li", { className: "nav-item"}, [
e(NavLink, { className: "nav-link", to: "/events" }, "Events")
]),
e("li", { className: "nav-item"}, [
e(NavLink, { className: "nav-link", to: "/iframe" }, "iFrame")
])
])
])

26
e2e/pages/dynamic/components/pages/iframes/index.js Normal file
View File

@ -0,0 +1,26 @@
import { parse } from '../../../utils/qs.js';
const e = React.createElement;
export default class IFramePage extends React.Component {
render() {
const search = parse(this.props.location.search);
let redirect;
if (search.src) {
redirect = search.src;
}
return e("div", { id: "iframe" }, [
e("iframe", {
name: 'nested',
style: {
width: '100%',
height: '800px',
},
src: redirect ? `/?redirect=${redirect}` : '/'
}),
])
}
}

6
e2e/pages/dynamic/index.html
View File

@ -11,9 +11,9 @@
</head>
<body class="text-center">
<div id="root"></div>
<script src="https://unpkg.com/react@16.6.1/umd/react.production.min.js"></script>
<script src="https://unpkg.com/react-dom@16.6.1/umd/react-dom.production.min.js"></script>
<script src="https://unpkg.com/history@4.7.2/umd/history.min.js"></script>
<script src="https://unpkg.com/react@16.8.6/umd/react.production.min.js"></script>
<script src="https://unpkg.com/react-dom@16.8.6/umd/react-dom.production.min.js"></script>
<script src="https://unpkg.com/history@4.9.0/umd/history.min.js"></script>
<script src="https://unpkg.com/react-router@4.3.1/umd/react-router.js"></script>
<script src="https://unpkg.com/react-router-dom@4.3.1/umd/react-router-dom.js"></script>
<script src="index.js" type="module"></script>

545
e2e/pages/static/media.html
View File

File diff suppressed because one or more lines are too long

32
e2e/runner/lib.go
View File

@ -3,6 +3,8 @@ package runner
import (
"context"
"fmt"
"io/ioutil"
"net/http"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
@ -10,7 +12,9 @@ import (
func Assertions() map[string]core.Function {
return map[string]core.Function{
"EXPECT": expect,
"EXPECT": expect,
"T::EXPECT": expect,
"T::HTTP::GET": httpGet,
}
}
@ -27,3 +31,29 @@ func expect(_ context.Context, args ...core.Value) (core.Value, error) {
return values.NewString(fmt.Sprintf(`expected "%s", but got "%s"`, args[0], args[1])), nil
}
func httpGet(_ context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 1, 2)
if err != nil {
return values.None, err
}
url := args[0].String()
resp, err := http.Get(url)
if err != nil {
return values.None, err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return values.None, err
}
return values.String(b), nil
}

126
e2e/runner/runner.go
View File

@ -6,7 +6,6 @@ import (
"io/ioutil"
"os"
"path/filepath"
"regexp"
"time"
"github.com/MontFerret/ferret/pkg/compiler"
@ -14,6 +13,8 @@ import (
"github.com/MontFerret/ferret/pkg/drivers/cdp"
"github.com/MontFerret/ferret/pkg/drivers/http"
"github.com/MontFerret/ferret/pkg/runtime"
"github.com/gobwas/glob"
"github.com/pkg/errors"
"github.com/rs/zerolog"
)
@ -24,7 +25,7 @@ type (
DynamicServerAddress string
CDPAddress string
Dir string
Filter *regexp.Regexp
Filter string
}
Result struct {
@ -84,7 +85,7 @@ func (r *Runner) Run(ctx context.Context) error {
Timestamp().
Int("passed", sum.passed).
Int("failed", sum.failed).
Dur("time", sum.duration).
Str("duration", sum.duration.String()).
Msg("Completed")
if sum.failed > 0 {
@ -95,19 +96,7 @@ func (r *Runner) Run(ctx context.Context) error {
}
func (r *Runner) runQueries(ctx context.Context, dir string) ([]Result, error) {
files, err := ioutil.ReadDir(dir)
if err != nil {
r.logger.Error().
Timestamp().
Err(err).
Str("dir", dir).
Msg("failed to read scripts directory")
return nil, err
}
results := make([]Result, 0, len(files))
results := make([]Result, 0, 50)
c := compiler.New()
@ -115,46 +104,69 @@ func (r *Runner) runQueries(ctx context.Context, dir string) ([]Result, error) {
return nil, err
}
// read scripts
for _, f := range files {
n := f.Name()
var filter glob.Glob
var useFilter bool
if r.settings.Filter != nil {
if r.settings.Filter.Match([]byte(n)) != true {
continue
if r.settings.Filter != "" {
f, err := glob.Compile(r.settings.Filter)
if err != nil {
return nil, err
}
filter = f
useFilter = true
}
err := r.traverseDir(ctx, dir, func(name string) error {
if useFilter {
if !filter.Match(name) {
return nil
}
}
fName := filepath.Join(dir, n)
b, err := ioutil.ReadFile(fName)
b, err := ioutil.ReadFile(name)
if err != nil {
results = append(results, Result{
name: fName,
name: name,
err: errors.Wrap(err, "failed to read script file"),
})
continue
return nil
}
r.logger.Info().Timestamp().Str("name", fName).Msg("Running test")
r.logger.Info().Timestamp().Str("name", name).Msg("Running test")
result := r.runQuery(ctx, c, fName, string(b))
select {
case <-ctx.Done():
return context.Canceled
default:
result := r.runQuery(ctx, c, name, string(b))
if result.err == nil {
r.logger.Info().
Timestamp().
Str("file", result.name).
Msg("Test passed")
} else {
r.logger.Error().
Timestamp().
Err(result.err).
Str("file", result.name).
Msg("Test failed")
if result.err == nil {
r.logger.Info().
Timestamp().
Str("file", result.name).
Str("duration", result.duration.String()).
Msg("Test passed")
} else {
r.logger.Error().
Timestamp().
Err(result.err).
Str("file", result.name).
Str("duration", result.duration.String()).
Msg("Test failed")
}
results = append(results, result)
}
results = append(results, result)
return nil
})
if err != nil {
return nil, err
}
return results, nil
@ -180,7 +192,7 @@ func (r *Runner) runQuery(ctx context.Context, c *compiler.FqlCompiler, name, sc
runtime.WithParam("dynamic", r.settings.DynamicServerAddress),
)
duration := time.Now().Sub(start)
duration := time.Since(start)
if err != nil {
return Result{
@ -237,3 +249,35 @@ func (r *Runner) report(results []Result) Summary {
duration: sumDuration,
}
}
func (r *Runner) traverseDir(ctx context.Context, dir string, iteratee func(name string) error) error {
files, err := ioutil.ReadDir(dir)
if err != nil {
r.logger.Error().
Timestamp().
Err(err).
Str("dir", dir).
Msg("failed to read scripts directory")
return err
}
for _, file := range files {
name := filepath.Join(dir, file.Name())
if file.IsDir() {
if err := r.traverseDir(ctx, name, iteratee); err != nil {
return err
}
continue
}
if err := iteratee(name); err != nil {
return err
}
}
return nil
}

42
e2e/server/server.go
View File

@ -2,10 +2,13 @@ package server
import (
"context"
"encoding/json"
"fmt"
"github.com/labstack/echo"
"net/http"
"path/filepath"
"time"
"github.com/labstack/echo"
)
type (
@ -37,6 +40,43 @@ func New(settings Settings) *Server {
})
e.Static("/", settings.Dir)
e.File("/", filepath.Join(settings.Dir, "index.html"))
api := e.Group("/api")
api.GET("/ts", func(ctx echo.Context) error {
var headers string
if len(ctx.Request().Header) > 0 {
b, err := json.Marshal(ctx.Request().Header)
if err != nil {
return err
}
headers = string(b)
}
ts := time.Now().Format("2006-01-02 15:04:05")
return ctx.HTML(http.StatusOK, fmt.Sprintf(`
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
</head>
<body>
<span id="timestamp">%s</span>
<span id="headers">%s</span>
</body>
</html>
`, ts, headers))
})
api.GET("/ping", func(ctx echo.Context) error {
return ctx.JSON(http.StatusOK, echo.Map{
"header": ctx.Request().Header,
"url": ctx.Request().URL,
"data": "pong",
"ts": time.Now(),
})
})
return &Server{e, settings}
}

12
e2e/tests/doc_inner_html_1_arg_d.fql
View File

@ -1,7 +1,7 @@
LET url = @dynamic
LET doc = DOCUMENT(url, true)
LET expected = `<!DOCTYPE html><html lang="en"><head>
LET expected = `<head>
<meta charset="utf-8">
<meta http-equiv="x-ua-compatible" content="ie=edge">
<title>Ferret E2E SPA</title>
@ -11,16 +11,16 @@ LET expected = `<!DOCTYPE html><html lang="en"><head>
<link rel="stylesheet" href="index.css">
</head>
<body class="text-center">
<div id="root"><div id="layout"><nav class="navbar navbar-expand-md navbar-dark bg-dark mb-4" id="navbar"><a class="navbar-brand active" aria-current="page" href="/">Ferret</a><button class="navbar-toggler" type="button"><span class="navbar-toggler-icon"></span></button><div class="collapse navbar-collapse"><ul class="navbar-nav mr-auto"><li class="nav-item"><a class="nav-link" href="/forms">Forms</a></li><li class="nav-item"><a class="nav-link" href="/navigation">Navigation</a></li><li class="nav-item"><a class="nav-link" href="/events">Events</a></li></ul></div></nav><main class="container"><div class="jumbotron" data-type="page" id="index"><div><h1>Welcome to Ferret E2E test page!</h1></div><div><p class="lead">It has several pages for testing different possibilities of the library</p></div></div></main></div></div>
<script src="https://unpkg.com/react@16.6.1/umd/react.production.min.js"></script>
<script src="https://unpkg.com/react-dom@16.6.1/umd/react-dom.production.min.js"></script>
<script src="https://unpkg.com/history@4.7.2/umd/history.min.js"></script>
<div id="root"><div id="layout"><nav class="navbar navbar-expand-md navbar-dark bg-dark mb-4" id="navbar"><a class="navbar-brand active" aria-current="page" href="/">Ferret</a><button class="navbar-toggler" type="button"><span class="navbar-toggler-icon"></span></button><div class="collapse navbar-collapse"><ul class="navbar-nav mr-auto"><li class="nav-item"><a class="nav-link" href="/forms">Forms</a></li><li class="nav-item"><a class="nav-link" href="/navigation">Navigation</a></li><li class="nav-item"><a class="nav-link" href="/events">Events</a></li><li class="nav-item"><a class="nav-link" href="/iframe">iFrame</a></li></ul></div></nav><main class="container"><div class="jumbotron" data-type="page" id="index"><div><h1>Welcome to Ferret E2E test page!</h1></div><div><p class="lead">It has several pages for testing different possibilities of the library</p></div></div></main></div></div>
<script src="https://unpkg.com/react@16.8.6/umd/react.production.min.js"></script>
<script src="https://unpkg.com/react-dom@16.8.6/umd/react-dom.production.min.js"></script>
<script src="https://unpkg.com/history@4.9.0/umd/history.min.js"></script>
<script src="https://unpkg.com/react-router@4.3.1/umd/react-router.js"></script>
<script src="https://unpkg.com/react-router-dom@4.3.1/umd/react-router-dom.js"></script>
<script src="index.js" type="module"></script>
</body></html>`
</body>`
LET actual = INNER_HTML(doc)
LET r1 = '(\s|\")'

4
e2e/tests/doc_inner_text_1_arg_d.fql
View File

@ -1,11 +1,11 @@
LET url = @dynamic
LET doc = DOCUMENT(url, true)
LET expected = `Ferret E2E SPA
Ferret
LET expected = `Ferret
Forms
Navigation
Events
iFrame
Welcome to Ferret E2E test page!
It has several pages for testing different possibilities of the library
`

12
e2e/tests/dynamic/doc/iframes/element_exists.fql Normal file
View File

@ -0,0 +1,12 @@
LET url = @dynamic + "?redirect=/iframe"
LET page = DOCUMENT(url, { driver: 'cdp' })
LET doc = page.frames[1]
LET expectedP = TRUE
LET actualP = ELEMENT_EXISTS(doc, '.text-center')
LET expectedN = FALSE
LET actualN = ELEMENT_EXISTS(doc, '.foo-bar')
RETURN EXPECT(expectedP + expectedN, actualP + expectedN)

12
e2e/tests/dynamic/doc/iframes/hover.fql Normal file
View File

@ -0,0 +1,12 @@
LET url = @dynamic + "?redirect=/iframe&src=/events"
LET page = DOCUMENT(url, { driver: 'cdp' })
LET doc = page.frames[1]
WAIT_ELEMENT(doc, "#page-events")
HOVER(doc, "#hoverable-btn")
WAIT_ELEMENT(doc, "#hoverable-content")
LET output = INNER_TEXT(doc, "#hoverable-content")
RETURN EXPECT(output, "Lorem ipsum dolor sit amet.")

11
e2e/tests/dynamic/doc/iframes/input.fql Normal file
View File

@ -0,0 +1,11 @@
LET url = @dynamic + "?redirect=/iframe&src=/forms"
LET page = DOCUMENT(url, true)
LET doc = page.frames[1]
WAIT_ELEMENT(doc, "form")
LET output = ELEMENT(doc, "#text_output")
INPUT(doc, "#text_input", "foo")
RETURN EXPECT(output.innerText, "foo")

4
e2e/tests/dynamic/doc/iframes/length.fql Normal file
View File

@ -0,0 +1,4 @@
LET url = @dynamic + "?redirect=/iframe"
LET doc = DOCUMENT(url, { driver: 'cdp' })
RETURN EXPECT(2, LENGTH(doc.frames))

15
e2e/tests/dynamic/doc/iframes/wait_class.fql Normal file
View File

@ -0,0 +1,15 @@
LET url = @dynamic + "?redirect=/iframe&src=/events"
LET page = DOCUMENT(url, true)
LET doc = page.frames[1]
WAIT_ELEMENT(doc, "#page-events")
// with fixed timeout
CLICK(doc, "#wait-class-btn")
WAIT_CLASS(doc, "#wait-class-content", "alert-success")
// with random timeout
CLICK(doc, "#wait-class-random-btn")
WAIT_CLASS(doc, "#wait-class-random-content", "alert-success", 10000)
RETURN ""

6
e2e/tests/dynamic/doc/xpath/count.fql Normal file
View File

@ -0,0 +1,6 @@
LET url = @dynamic
LET page = DOCUMENT(url, true)
LET actual = XPATH(page, "count(//body)")
RETURN EXPECT(1, actual)

6
e2e/tests/dynamic/doc/xpath/query.fql Normal file
View File

@ -0,0 +1,6 @@
LET url = @dynamic + "?redirect=/forms"
LET page = DOCUMENT(url, true)
LET actual = XPATH(page, "//div[contains(@class, 'form-group')]")
RETURN EXPECT(4, LENGTH(actual))

14
e2e/tests/dynamic/element/iframes/hover.fql Normal file
View File

@ -0,0 +1,14 @@
LET url = @dynamic + "?redirect=/iframe&src=/events"
LET page = DOCUMENT(url, { driver: 'cdp' })
LET doc = page.frames[1]
WAIT_ELEMENT(doc, "#page-events")
LET input = ELEMENT(doc, "#hoverable-btn")
HOVER(input)
WAIT_ELEMENT(doc, "#hoverable-content")
LET output = ELEMENT(doc, "#hoverable-content")
RETURN EXPECT(output.innerText, "Lorem ipsum dolor sit amet.")

12
e2e/tests/dynamic/element/iframes/input.fql Normal file
View File

@ -0,0 +1,12 @@
LET url = @dynamic + "?redirect=/iframe&src=/forms"
LET page = DOCUMENT(url, { driver: 'cdp' })
LET doc = page.frames[1]
WAIT_ELEMENT(doc, "form")
LET input = ELEMENT(doc, "#text_input")
LET output = ELEMENT(doc, "#text_output")
INPUT(input, "foo")
RETURN EXPECT(output.innerText, "foo")

11
e2e/tests/dynamic/element/iframes/select_single.fql Normal file
View File

@ -0,0 +1,11 @@
LET url = @dynamic + "?redirect=/iframe&src=/forms"
LET page = DOCUMENT(url, { driver: 'cdp' })
LET doc = page.frames[1]
WAIT_ELEMENT(doc, "form")
LET input = ELEMENT(doc, "#select_input")
LET output = ELEMENT(doc, "#select_output")
LET result = SELECT(input, ["4"])
RETURN EXPECT(output.innerText, "4") + EXPECT(JSON_STRINGIFY(result), '["4"]')

23
e2e/tests/dynamic/element/iframes/wait_class.fql Normal file
View File

@ -0,0 +1,23 @@
LET url = @dynamic + "?redirect=/iframe&src=/events"
LET page = DOCUMENT(url, { driver: 'cdp' })
LET doc = page.frames[1]
WAIT_ELEMENT(doc, "#page-events")
// with fixed timeout
LET b1 = ELEMENT(doc, "#wait-class-btn")
LET c1 = ELEMENT(doc, "#wait-class-content")
WAIT(2000)
CLICK(b1)
WAIT_CLASS(c1, "alert-success")
// with random timeout
LET b2 = ELEMENT(doc, "#wait-class-random-btn")
LET c2 = ELEMENT(doc, "#wait-class-random-content")
CLICK(b2)
WAIT_CLASS(c2, "alert-success", 10000)
RETURN ""

7
e2e/tests/dynamic/element/xpath/count.fql Normal file
View File

@ -0,0 +1,7 @@
LET url = @dynamic
LET page = DOCUMENT(url, true)
LET el = ELEMENT(page, 'main')
LET actual = XPATH(el, "count(//p)")
RETURN EXPECT(1, actual)

7
e2e/tests/dynamic/element/xpath/query.fql Normal file
View File

@ -0,0 +1,7 @@
LET url = @dynamic + "?redirect=/forms"
LET page = DOCUMENT(url, true)
LET element = ELEMENT(page, '#page-form')
LET actual = XPATH(element, "//div[contains(@class, 'form-group')]")
RETURN EXPECT(4, LENGTH(actual))

2
e2e/tests/el_inner_html_1_arg_d.fql
View File

@ -2,7 +2,7 @@ LET url = @dynamic
LET doc = DOCUMENT(url, true)
LET el = ELEMENT(doc, "#root")
LET expected = `<div id="layout"><nav class="navbar navbar-expand-md navbar-dark bg-dark mb-4" id="navbar"><a class="navbar-brand active" aria-current="page" href="/">Ferret</a><button class="navbar-toggler" type="button"><span class="navbar-toggler-icon"></span></button><div class="collapse navbar-collapse"><ul class="navbar-nav mr-auto"><li class="nav-item"><a class="nav-link" href="/forms">Forms</a></li><li class="nav-item"><a class="nav-link" href="/navigation">Navigation</a></li><li class="nav-item"><a class="nav-link" href="/events">Events</a></li></ul></div></nav><main class="container"><div class="jumbotron" data-type="page" id="index"><div><h1>Welcome to Ferret E2E test page!</h1></div><div><p class="lead">It has several pages for testing different possibilities of the library</p></div></div></main></div>`
LET expected = `<div id="layout"><nav class="navbar navbar-expand-md navbar-dark bg-dark mb-4" id="navbar"><a class="navbar-brand active" aria-current="page" href="/">Ferret</a><button class="navbar-toggler" type="button"><span class="navbar-toggler-icon"></span></button><div class="collapse navbar-collapse"><ul class="navbar-nav mr-auto"><li class="nav-item"><a class="nav-link" href="/forms">Forms</a></li><li class="nav-item"><a class="nav-link" href="/navigation">Navigation</a></li><li class="nav-item"><a class="nav-link" href="/events">Events</a></li><li class="nav-item"><a class="nav-link" href="/iframe">iFrame</a></li></ul></div></nav><main class="container"><div class="jumbotron" data-type="page" id="index"><div><h1>Welcome to Ferret E2E test page!</h1></div><div><p class="lead">It has several pages for testing different possibilities of the library</p></div></div></main></div>`
LET actual = INNER_HTML(el)
LET r1 = '(\s|\")'

11
e2e/tests/el_value_d.fql
View File

@ -1,12 +1,9 @@
LET url = @static + '/value.html'
LET url = @dynamic + "?redirect=/forms"
LET doc = DOCUMENT(url, true)
LET expected = ["068728","068728","816410","52024413","698690","210583","049700","826394","354369","135911","700285","557242","278832","357701","313034","959368","703500","842750","777175","378061","072489","383005","843393","59912263","464535","229710","230550","767964","758862","944384","025449","010245","844935","038760","013450","124139","211145","758761","448667","488966"]
LET el = ELEMENT(doc, "#select_input")
LET actual = (
FOR tr IN ELEMENTS(doc, '#listings_table > tbody > tr')
LET elem = ELEMENT(tr, 'td > input')
RETURN elem.value
)
LET expected = "1"
LET actual = el.value
RETURN EXPECT(actual, expected)

5
e2e/tests/page_load_d.fql
View File

@ -1,4 +1,7 @@
LET url = @dynamic
LET doc = DOCUMENT(url, true)
RETURN EXPECT(doc.url, url)
LET expected = url + '/'
LET actual = doc.url
RETURN EXPECT(expected, actual)

10
e2e/tests/static/doc/ua.fql Normal file
View File

@ -0,0 +1,10 @@
LET url = @static + '/api/ts'
LET ua = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) brave/0.7.10 Chrome/47.0.2526.110 Brave/0.36.5 Safari/537.36"
LET page = DOCUMENT(url, {
userAgent: ua
})
LET h = ELEMENT(page, "#headers")
LET headers = JSON_PARSE(h.innerText)
RETURN T::EXPECT(ua, headers["User-Agent"][0])

6
e2e/tests/static/doc/xpath/count.fql Normal file
View File

@ -0,0 +1,6 @@
LET url = @static + '/overview.html'
LET page = DOCUMENT(url)
LET actual = XPATH(page, "count(//body)")
RETURN EXPECT(1, actual)

6
e2e/tests/static/doc/xpath/query.fql Normal file
View File

@ -0,0 +1,6 @@
LET url = @static + '/value.html'
LET page = DOCUMENT(url)
LET actual = XPATH(page, "//tr[contains(@class, 'odd')]")
RETURN EXPECT(20, LENGTH(actual))

7
e2e/tests/static/element/xpath/count.fql Normal file
View File

@ -0,0 +1,7 @@
LET url = @static + '/value.html'
LET page = DOCUMENT(url)
LET el = ELEMENT(page, '#listings_table')
LET actual = XPATH(el, "count(//tr)")
RETURN EXPECT(41, actual)

7
e2e/tests/static/element/xpath/query.fql Normal file
View File

@ -0,0 +1,7 @@
LET url = @static + '/value.html'
LET page = DOCUMENT(url, true)
LET element = ELEMENT(page, '.tablesorter')
LET actual = XPATH(element, "//input[contains(@type, 'hidden')]")
RETURN EXPECT(40, LENGTH(actual))

2
examples/cookies.go
View File

@ -2,8 +2,6 @@ package main
import (
"context"
"fmt"
"os"
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/drivers"

3
examples/extensible.go
View File

@ -10,6 +10,7 @@ import (
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
func main() {
@ -37,7 +38,7 @@ func getStrings() ([]string, error) {
}
// this is another helper functions allowing to do type validation
err = core.ValidateType(args[0], core.StringType)
err = core.ValidateType(args[0], types.String)
if err != nil {
return values.None, err

5
examples/iframes.fql Normal file
View File

@ -0,0 +1,5 @@
LET page = DOCUMENT("https://www.w3schools.com/html/html_iframe.asp", { driver: "cdp" })
LET c2 = page.frames[1].head.innerHTML
RETURN c2

11
examples/pagination.fql
View File

@ -4,14 +4,15 @@ INPUT(amazon, '#twotabsearchtextbox', @criteria)
CLICK(amazon, '.nav-search-submit input[type="submit"]')
WAIT_NAVIGATION(amazon)
LET resultListSelector = '#s-results-list-atf'
LET resultItemSelector = '.s-result-item.celwidget'
LET nextBtnSelector = '#pagnNextLink'
LET resultListSelector = 'div.s-result-list'
LET resultItemSelector = 'div.s-result-item'
LET nextBtnSelector = 'ul.a-pagination .a-last a'
LET vendorSelector1 = 'div > div:nth-child(3) > div:nth-child(2) > span:nth-child(2)'
LET vendorSelector2 = 'div > div:nth-child(5) > div:nth-child(2) > span:nth-child(2)'
LET priceWholeSelector = 'span.sx-price-whole'
LET priceFracSelector = 'sup.sx-price-fractional'
LET pages = TO_INT(INNER_TEXT(amazon, '#pagn > span.pagnDisabled'))
LET pagers = ELEMENTS(amazon, 'ul.a-pagination li.a-disabled')
LET pages = LENGTH(pagers) > 0 ? TO_INT(INNER_TEXT(LAST(pagers))) : 0
LET result = (
FOR pageNum IN 1..pages
@ -19,6 +20,8 @@ LET result = (
LET wait = clicked ? WAIT_NAVIGATION(amazon) : false
LET waitSelector = wait ? WAIT_ELEMENT(amazon, resultListSelector) : false
PRINT("page:", pageNum, "clicked", clicked)
LET items = (
FOR el IN ELEMENTS(amazon, resultItemSelector)
LET priceWholeTxt = INNER_TEXT(el, priceWholeSelector)

3
examples/screenshot.fql Normal file
View File

@ -0,0 +1,3 @@
LET data = SCREENSHOT("https://github.com/MontFerret/ferret/raw/master/assets/logo.png")
RETURN { type: "png", data }

60
go.mod
View File

@ -3,74 +3,42 @@ module github.com/MontFerret/ferret
go 1.12
require (
github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 // indirect
github.com/BurntSushi/toml v0.3.1 // indirect
github.com/PuerkitoBio/goquery v1.5.0
github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 // indirect
github.com/antchfx/htmlquery v1.0.0
github.com/antchfx/xpath v1.0.0
github.com/antlr/antlr4 v0.0.0-20190325153624-837aa60e2c47
github.com/chzyer/logex v1.1.10 // indirect
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
github.com/coreos/etcd v3.3.12+incompatible // indirect
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 // indirect
github.com/corpix/uarand v0.0.0
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/derekparker/trie v0.0.0-20190322172448-1ce4922c7ad9
github.com/fatih/color v1.7.0 // indirect
github.com/go-critic/go-critic v0.3.4 // indirect
github.com/go-ole/go-ole v1.2.4 // indirect
github.com/go-toolsmith/astcast v1.0.0 // indirect
github.com/go-toolsmith/astcopy v1.0.0 // indirect
github.com/go-toolsmith/astfmt v1.0.0 // indirect
github.com/go-toolsmith/astp v1.0.0 // indirect
github.com/go-toolsmith/pkgload v1.0.0 // indirect
github.com/go-toolsmith/typep v1.0.0 // indirect
github.com/gobwas/glob v0.2.3
github.com/gofrs/uuid v3.2.0+incompatible
github.com/gogo/protobuf v1.2.1 // indirect
github.com/golang/mock v1.2.0 // indirect
github.com/golang/protobuf v1.3.1 // indirect
github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect
github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 // indirect
github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d // indirect
github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 // indirect
github.com/golangci/golangci-lint v1.15.0 // indirect
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb // indirect
github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 // indirect
github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 // indirect
github.com/google/go-cmp v0.2.0 // indirect
github.com/gopherjs/gopherjs v0.0.0-20190328170749-bb2674552d8f // indirect
github.com/gorilla/css v1.0.0
github.com/gorilla/websocket v1.4.0 // indirect
github.com/kisielk/errcheck v1.2.0 // indirect
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/kr/pty v1.1.4 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/labstack/echo v3.3.10+incompatible
github.com/labstack/gommon v0.2.8 // indirect
github.com/mafredri/cdp v0.23.1
github.com/mafredri/cdp v0.23.4
github.com/mattn/go-colorable v0.1.1 // indirect
github.com/mattn/go-isatty v0.0.7 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mozilla/tls-observatory v0.0.0-20190313211306-43961c0c7a1f // indirect
github.com/natefinch/lumberjack v2.0.0+incompatible
github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d // indirect
github.com/onsi/ginkgo v1.8.0 // indirect
github.com/onsi/gomega v1.5.0 // indirect
github.com/pkg/errors v0.8.1
github.com/rogpeppe/go-internal v1.2.2 // indirect
github.com/rs/zerolog v1.14.3
github.com/ryanuber/go-glob v1.0.0 // indirect
github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0
github.com/shirou/gopsutil v2.18.12+incompatible // indirect
github.com/shurcooL/go v0.0.0-20190121191506-3fef8c783dec // indirect
github.com/sirupsen/logrus v1.4.0 // indirect
github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac // indirect
github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff
github.com/spf13/afero v1.2.2 // indirect
github.com/spf13/cobra v0.0.3 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/viper v1.3.2 // indirect
github.com/stretchr/testify v1.3.0 // indirect
github.com/ugorji/go/codec v0.0.0-20190320090025-2dc34c0b8780 // indirect
github.com/valyala/fasttemplate v1.0.1 // indirect
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c // indirect
golang.org/x/net v0.0.0-20190328230028-74de082e2cca
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6
golang.org/x/sync v0.0.0-20190423024810-112230192c58
golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc // indirect
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 // indirect
mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe // indirect
sourcegraph.com/sqs/pbtypes v1.0.0 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/yaml.v2 v2.2.2 // indirect
)

210
go.sum
View File

@ -1,249 +1,105 @@
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/OpenPeeDeeP/depguard v0.0.0-20180806142446-a69c782687b2/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/antchfx/htmlquery v1.0.0 h1:O5IXz8fZF3B3MW+B33MZWbTHBlYmcfw0BAxgErHuaMA=
github.com/antchfx/htmlquery v1.0.0/go.mod h1:MS9yksVSQXls00iXkiMqXr0J+umL/AmxXKuP28SUJM8=
github.com/antchfx/xpath v1.0.0 h1:Q5gFgh2O40VTSwMOVbFE7nFNRBu3tS21Tn0KAWeEjtk=
github.com/antchfx/xpath v1.0.0/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
github.com/antlr/antlr4 v0.0.0-20190325153624-837aa60e2c47 h1:Lp5nUoQzppfVmfZadpzAytNyb5IMtxyOJLzoQS5dExg=
github.com/antlr/antlr4 v0.0.0-20190325153624-837aa60e2c47/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/corpix/uarand v0.0.0 h1:mNbzro1GwUcZ1hmO2rWXytkR3JBxNxxctzjyuhO+Aig=
github.com/corpix/uarand v0.0.0/go.mod h1:JSm890tOkDN+M1jqN8pUGDKnzJrsVbJwSMHBY4zwz7M=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/derekparker/trie v0.0.0-20190322172448-1ce4922c7ad9 h1:aSaTVlEXc2QKl4fzXU1tMYCjlrSc2mA4DZtiVfckQHo=
github.com/derekparker/trie v0.0.0-20190322172448-1ce4922c7ad9/go.mod h1:D6ICZm05D9VN1n/8iOtBxLpXtoGp6HDFUJ1RNVieOSE=
github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/go-critic/go-critic v0.0.0-20181204210945-ee9bf5809ead/go.mod h1:3MzXZKJdeXqdU9cj+rvZdNiN7SZ8V9OjybF8loZDmHU=
github.com/go-critic/go-critic v0.3.4/go.mod h1:AHR42Lk/E/aOznsrYdMYeIQS5RH10HZHSqP+rD6AJrc=
github.com/go-lintpack/lintpack v0.5.1/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
github.com/go-toolsmith/astcast v0.0.0-20181028201508-b7a89ed70af1/go.mod h1:TEo3Ghaj7PsZawQHxT/oBvo4HK/sl1RcuUHDKTTju+o=
github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
github.com/go-toolsmith/astcopy v0.0.0-20180903214859-79b422d080c4/go.mod h1:c9CPdq2AzM8oPomdlPniEfPAC6g1s7NqZzODt8y6ib8=
github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
github.com/go-toolsmith/strparse v0.0.0-20180903215201-830b6daa1241/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
github.com/go-toolsmith/typep v0.0.0-20181030061450-d63dc7650676/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
github.com/golangci/errcheck v0.0.0-20181003203344-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
github.com/golangci/go-tools v0.0.0-20180109140146-35a9f45a5db0/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
github.com/golangci/gofmt v0.0.0-20181105071733-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
github.com/golangci/golangci-lint v1.15.0/go.mod h1:iEsyA2h6yMxPzFAlb/Q9UuXBrXIDtXkbUoukuqUAX/8=
github.com/golangci/gosec v0.0.0-20180901114220-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb/go.mod h1:ON/c2UR0VAAv6ZEAFKhjCLplESSmRFfZcDLASbI1GWo=
github.com/golangci/govet v0.0.0-20180818181408-44ddbe260190/go.mod h1:pPwb+AK755h3/r73avHz5bEN6sa51/2HEZlLaV53hCo=
github.com/golangci/ineffassign v0.0.0-20180808204949-2ee8f2867dde/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
github.com/golangci/lint-1 v0.0.0-20180610141402-4bf9709227d1/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20190328170749-bb2674552d8f h1:4Gslotqbs16iAg+1KR/XdabIfq8TlAWHdwS5QJFksLc=
github.com/gopherjs/gopherjs v0.0.0-20190328170749-bb2674552d8f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg=
github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s=
github.com/labstack/gommon v0.2.8 h1:JvRqmeZcfrHC5u6uVleB4NxxNbzx6gpbJiQknDbKQu0=
github.com/labstack/gommon v0.2.8/go.mod h1:/tj9csK2iPSBvn+3NLM9e52usepMtrd5ilFYA+wQNJ4=
github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/mafredri/cdp v0.23.1 h1:aqW20I/3CzR8/8VEj+d4zV97l3GU7VdCgi8OTGeJKkA=
github.com/mafredri/cdp v0.23.1/go.mod h1:hgdiA0yp1uqhSaDOHJWPgXpMbh+LAfUdD9vbN2AM8gE=
github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mafredri/cdp v0.23.4 h1:ffp4qq6slfCL4rFWBDeRHapkLE776gER4tX5Z3LS8CY=
github.com/mafredri/cdp v0.23.4/go.mod h1:hgdiA0yp1uqhSaDOHJWPgXpMbh+LAfUdD9vbN2AM8gE=
github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
github.com/mozilla/tls-observatory v0.0.0-20190313211306-43961c0c7a1f/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.2.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/zerolog v1.13.0 h1:hSNcYHyxDWycfePW7pUI8swuFkcSMPKh3E63Pokg1Hk=
github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
github.com/rs/zerolog v1.14.0 h1:F2F6pGdMrQHGPwr05uwcQNSiWnX5PD76SWw/mYvRBXs=
github.com/rs/zerolog v1.14.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
github.com/rs/zerolog v1.14.3 h1:4EGfSkR2hJDB0s3oFfrlPqjU1e4WLncergLil3nEKW0=
github.com/rs/zerolog v1.14.3/go.mod h1:3WXPzbXEEliJ+a6UFE4vhIxV8qR1EML6ngzP9ug4eYg=
github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0 h1:X9XMOYjxEfAYSy3xK1DzO5dMkkWhs9E9UCcS1IERx2k=
github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0/go.mod h1:Ad7IjTpvzZO8Fl0vh9AzQ+j/jYZfyp2diGwI8m5q+ns=
github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
github.com/shurcooL/go v0.0.0-20190121191506-3fef8c783dec/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac h1:wbW+Bybf9pXxnCFAOWZTqkRjAc7rAIwo2e1ArUhiHxg=
github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff h1:86HlEv0yBCry9syNuylzqznKXDK11p6D0DT596yNMys=
github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs=
github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/ugorji/go/codec v0.0.0-20190320090025-2dc34c0b8780/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
golang.org/x/crypto v0.0.0-20180505025534-4ec37c66abab/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190328230028-74de082e2cca h1:hyA6yiAgbUwuWqtscNvWAI7U1CtlaD1KilQ6iudt1aI=
golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc h1:4gbWbmmPFp4ySWICouJl6emP0MyS31yy9SrTlAGFT+g=
golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181205014116-22934f0fdb62/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190125232054-379209517ffe/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
mvdan.cc/unparam v0.0.0-20190124213536-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe/go.mod h1:BnhuWBAqxH3+J5bDybdxgw5ZfS+DsVd4iylsKQePN8o=
sourcegraph.com/sourcegraph/go-diff v0.5.1-0.20190210232911-dee78e514455/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4=

37
pkg/compiler/compiler.go
View File

@ -1,6 +1,7 @@
package compiler
import (
"regexp"
"strings"
"github.com/MontFerret/ferret/pkg/parser"
@ -10,6 +11,8 @@ import (
"github.com/pkg/errors"
)
var fnNameValidation = regexp.MustCompile("^[a-zA-Z]+[a-zA-Z0-9_]*(::[a-zA-Z]+[a-zA-Z0-9_]*)*$")
type FqlCompiler struct {
funcs map[string]core.Function
}
@ -38,6 +41,11 @@ func (c *FqlCompiler) RegisterFunction(name string, fun core.Function) error {
return errors.Errorf("function already exists: %s", name)
}
// validation the name
if !fnNameValidation.MatchString(name) {
return errors.Errorf("invalid function name: %s", name)
}
c.funcs[strings.ToUpper(name)] = fun
return nil
@ -57,6 +65,28 @@ func (c *FqlCompiler) RegisterFunctions(funcs map[string]core.Function) error {
return nil
}
func (c *FqlCompiler) RegisteredFunctions() []string {
res := make([]string, 0, len(c.funcs))
for k := range c.funcs {
res = append(res, k)
}
return res
}
func (c *FqlCompiler) RegisteredFunctionsNS(namespace string) []string {
res := make([]string, 0, len(c.funcs))
for k := range c.funcs {
if strings.HasPrefix(k, namespace) {
res = append(res, k)
}
}
return res
}
func (c *FqlCompiler) Compile(query string) (program *runtime.Program, err error) {
if query == "" {
return nil, ErrEmptyQuery
@ -103,10 +133,3 @@ func (c *FqlCompiler) MustCompile(query string) *runtime.Program {
return program
}
func (c *FqlCompiler) RegisteredFunctions() (funcs []string) {
for k := range c.funcs {
funcs = append(funcs, k)
}
return
}

94
pkg/compiler/compiler_func_ns_test.go Normal file
View File

@ -0,0 +1,94 @@
package compiler_test
import (
"context"
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func TestFunctionNSCall(t *testing.T) {
Convey("Should compile RETURN T::SPY", t, func() {
c := compiler.New()
var counter int
err := c.RegisterFunction("T::SPY", func(_ context.Context, _ ...core.Value) (core.Value, error) {
counter++
return values.None, nil
})
So(err, ShouldBeNil)
p, err := c.Compile(`
RETURN T::SPY()
`)
So(err, ShouldBeNil)
_, err = p.Run(context.Background())
So(err, ShouldBeNil)
So(counter, ShouldEqual, 1)
})
Convey("Should compile RETURN T::UTILS::SPY", t, func() {
c := compiler.New()
var counter int
err := c.RegisterFunction("T::UTILS::SPY", func(_ context.Context, _ ...core.Value) (core.Value, error) {
counter++
return values.None, nil
})
So(err, ShouldBeNil)
p, err := c.Compile(`
RETURN T::UTILS::SPY()
`)
So(err, ShouldBeNil)
_, err = p.Run(context.Background())
So(err, ShouldBeNil)
So(counter, ShouldEqual, 1)
})
Convey("Should NOT compile RETURN T:UTILS::SPY", t, func() {
c := compiler.New()
var counter int
err := c.RegisterFunction("T::UTILS::SPY", func(_ context.Context, _ ...core.Value) (core.Value, error) {
counter++
return values.None, nil
})
So(err, ShouldBeNil)
_, err = c.Compile(`
RETURN T:UTILS::SPY()
`)
So(err, ShouldNotBeNil)
})
Convey("Should NOT register RETURN T:UTILS::SPY", t, func() {
c := compiler.New()
var counter int
err := c.RegisterFunction("T::UTILS:SPY", func(_ context.Context, _ ...core.Value) (core.Value, error) {
counter++
return values.None, nil
})
So(err, ShouldNotBeNil)
})
}

6
pkg/compiler/compiler_str_test.go
View File

@ -24,13 +24,13 @@ BAR
})
Convey("Should be possible to use multi line string with nested strings", t, func() {
out := compiler.New().
compiler.New().
MustCompile(fmt.Sprintf(`
RETURN %s<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Title</title>
<title>GetTitle</title>
</head>
<body>
Hello world
@ -43,7 +43,7 @@ RETURN %s<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Title</title>
<title>GetTitle</title>
</head>
<body>
Hello world

11
pkg/compiler/compiler_test.go
View File

@ -1,11 +0,0 @@
package compiler_test
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
func NOOP(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, nil
}

22
pkg/compiler/visitor.go
View File

@ -426,7 +426,7 @@ func (v *visitor) doVisitCollectClause(ctx *fql.CollectClauseContext, scope *sco
collectSelectors := groupingCtx.AllCollectSelector()
// group selectors
if collectSelectors != nil && len(collectSelectors) > 0 {
if len(collectSelectors) > 0 {
selectors = make([]*clauses.CollectSelector, 0, len(collectSelectors))
for _, cs := range collectSelectors {
@ -481,10 +481,6 @@ func (v *visitor) doVisitCollectClause(ctx *fql.CollectClauseContext, scope *sco
projectionSelectorExp := literals.NewObjectLiteralWith(propExp)
if err != nil {
return nil, err
}
selector, err := clauses.NewCollectSelector(projectionIdentifier.GetText(), projectionSelectorExp)
if err != nil {
@ -1049,12 +1045,20 @@ func (v *visitor) doVisitFunctionCallExpression(context *fql.FunctionCallExpress
}
}
funcName := context.Identifier().GetText()
var name string
fun, exists := v.funcs[funcName]
funcNS := context.Namespace()
if funcNS != nil {
name += funcNS.GetText()
}
name += context.Identifier().GetText()
fun, exists := v.funcs[name]
if !exists {
return nil, core.Error(core.ErrNotFound, fmt.Sprintf("function: '%s'", funcName))
return nil, core.Error(core.ErrNotFound, fmt.Sprintf("function: '%s'", name))
}
return expressions.NewFunctionCallExpression(
@ -1413,7 +1417,7 @@ func (v *visitor) doVisitChildren(node antlr.RuleNode, scope *scope) ([]core.Exp
children := node.GetChildren()
if children == nil {
return make([]core.Expression, 0, 0), nil
return make([]core.Expression, 0), nil
}
result := make([]core.Expression, 0, len(children))

1004
pkg/drivers/cdp/document.go
View File

File diff suppressed because it is too large Load Diff

1
pkg/drivers/cdp/document_test.go Normal file
View File

@ -0,0 +1 @@
package cdp

93
pkg/drivers/cdp/driver.go
View File

@ -4,24 +4,21 @@ import (
"context"
"sync"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/devtool"
"github.com/mafredri/cdp/protocol/emulation"
"github.com/mafredri/cdp/protocol/network"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/protocol/target"
"github.com/mafredri/cdp/rpcc"
"github.com/mafredri/cdp/session"
"github.com/pkg/errors"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/logging"
)
const DriverName = "cdp"
type Driver struct {
sync.Mutex
mu sync.Mutex
dev *devtool.DevTools
conn *rpcc.Conn
client *cdp.Client
@ -42,7 +39,7 @@ func (drv *Driver) Name() string {
return drv.options.Name
}
func (drv *Driver) LoadDocument(ctx context.Context, params drivers.LoadDocumentParams) (drivers.HTMLDocument, error) {
func (drv *Driver) Open(ctx context.Context, params drivers.OpenPageParams) (drivers.HTMLPage, error) {
logger := logging.FromContext(ctx)
err := drv.init(ctx)
@ -58,20 +55,15 @@ func (drv *Driver) LoadDocument(ctx context.Context, params drivers.LoadDocument
return nil, err
}
url := params.URL
// Args for a new target belonging to the browser context
createTargetArgs := target.NewCreateTargetArgs(BlankPageURL)
if url == "" {
url = BlankPageURL
}
// Create a new target belonging to the browser context
createTargetArgs := target.NewCreateTargetArgs(url)
if drv.options.KeepCookies == false && params.KeepCookies == false {
if !drv.options.KeepCookies && !params.KeepCookies {
// Set it to an incognito mode
createTargetArgs.SetBrowserContextID(drv.contextID)
}
// New target
createTarget, err := drv.client.Target.CreateTarget(ctx, createTargetArgs)
if err != nil {
@ -99,69 +91,16 @@ func (drv *Driver) LoadDocument(ctx context.Context, params drivers.LoadDocument
return nil, err
}
client := cdp.NewClient(conn)
err = runBatch(
func() error {
return client.Page.Enable(ctx)
},
func() error {
return client.Page.SetLifecycleEventsEnabled(
ctx,
page.NewSetLifecycleEventsEnabledArgs(true),
)
},
func() error {
return client.DOM.Enable(ctx)
},
func() error {
return client.Runtime.Enable(ctx)
},
func() error {
var ua string
if params.UserAgent != "" {
ua = common.GetUserAgent(params.UserAgent)
} else {
ua = common.GetUserAgent(drv.options.UserAgent)
}
logger.
Debug().
Timestamp().
Str("user-agent", ua).
Msg("using User-Agent")
// do not use custom user agent
if ua == "" {
return nil
}
return client.Emulation.SetUserAgentOverride(
ctx,
emulation.NewSetUserAgentOverrideArgs(ua),
)
},
func() error {
return client.Network.Enable(ctx, network.NewEnableArgs())
},
)
if err != nil {
return nil, err
if params.UserAgent == "" {
params.UserAgent = drv.options.UserAgent
}
return LoadHTMLDocument(ctx, conn, client, params)
return LoadHTMLPage(ctx, conn, params)
}
func (drv *Driver) Close() error {
drv.Lock()
defer drv.Unlock()
drv.mu.Lock()
defer drv.mu.Unlock()
if drv.session != nil {
drv.session.Close()
@ -173,8 +112,8 @@ func (drv *Driver) Close() error {
}
func (drv *Driver) init(ctx context.Context) error {
drv.Lock()
defer drv.Unlock()
drv.mu.Lock()
defer drv.mu.Unlock()
if drv.session == nil {
ver, err := drv.dev.Version(ctx)

481
pkg/drivers/cdp/element.go
View File

@ -4,35 +4,35 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/MontFerret/ferret/pkg/drivers/cdp/templates"
"github.com/pkg/errors"
"golang.org/x/net/html"
"hash/fnv"
"strconv"
"strings"
"sync"
"time"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
"github.com/MontFerret/ferret/pkg/drivers/cdp/input"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
"github.com/gofrs/uuid"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/input"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/rs/zerolog"
)
var emptyNodeID = dom.NodeID(0)
var emptyBackendID = dom.BackendNodeID(0)
type (
HTMLElementIdentity struct {
nodeID dom.NodeID
backendID dom.BackendNodeID
objectID runtime.RemoteObjectID
nodeID dom.NodeID
objectID runtime.RemoteObjectID
}
HTMLElement struct {
@ -40,40 +40,37 @@ type (
logger *zerolog.Logger
client *cdp.Client
events *events.EventBroker
input *input.Manager
exec *eval.ExecutionContext
connected values.Boolean
id *HTMLElementIdentity
nodeType values.Int
id HTMLElementIdentity
nodeType html.NodeType
nodeName values.String
innerHTML values.String
innerText *common.LazyValue
value core.Value
attributes *common.LazyValue
style *common.LazyValue
children []*HTMLElementIdentity
children []HTMLElementIdentity
loadedChildren *common.LazyValue
}
)
func LoadElement(
func LoadHTMLElement(
ctx context.Context,
logger *zerolog.Logger,
client *cdp.Client,
broker *events.EventBroker,
input *input.Manager,
exec *eval.ExecutionContext,
nodeID dom.NodeID,
backendID dom.BackendNodeID,
) (*HTMLElement, error) {
if client == nil {
return nil, core.Error(core.ErrMissedArgument, "client")
}
// getting a remote object that represents the current DOM Node
var args *dom.ResolveNodeArgs
if backendID > 0 {
args = dom.NewResolveNodeArgs().SetBackendNodeID(backendID)
} else {
args = dom.NewResolveNodeArgs().SetNodeID(nodeID)
}
args := dom.NewResolveNodeArgs().SetNodeID(nodeID).SetExecutionContextID(exec.ID())
obj, err := client.DOM.ResolveNode(ctx, args)
@ -85,34 +82,46 @@ func LoadElement(
return nil, core.Error(core.ErrNotFound, fmt.Sprintf("element %d", nodeID))
}
objectID := *obj.Object.ObjectID
id := HTMLElementIdentity{}
id.nodeID = nodeID
id.objectID = *obj.Object.ObjectID
return LoadHTMLElementWithID(
ctx,
logger,
client,
broker,
input,
exec,
id,
)
}
func LoadHTMLElementWithID(
ctx context.Context,
logger *zerolog.Logger,
client *cdp.Client,
broker *events.EventBroker,
input *input.Manager,
exec *eval.ExecutionContext,
id HTMLElementIdentity,
) (*HTMLElement, error) {
node, err := client.DOM.DescribeNode(
ctx,
dom.
NewDescribeNodeArgs().
SetObjectID(objectID).
SetObjectID(id.objectID).
SetDepth(1),
)
if err != nil {
return nil, core.Error(err, strconv.Itoa(int(nodeID)))
return nil, core.Error(err, strconv.Itoa(int(id.nodeID)))
}
id := new(HTMLElementIdentity)
id.nodeID = nodeID
id.objectID = objectID
if backendID > 0 {
id.backendID = backendID
} else {
id.backendID = node.Node.BackendNodeID
}
innerHTML, err := loadInnerHTML(ctx, client, id)
innerHTML, err := loadInnerHTML(ctx, client, exec, id, common.ToHTMLType(node.Node.NodeType))
if err != nil {
return nil, core.Error(err, strconv.Itoa(int(nodeID)))
return nil, core.Error(err, strconv.Itoa(int(id.nodeID)))
}
var val string
@ -125,6 +134,8 @@ func LoadElement(
logger,
client,
broker,
input,
exec,
id,
node.Node.NodeType,
node.Node.NodeName,
@ -138,20 +149,24 @@ func NewHTMLElement(
logger *zerolog.Logger,
client *cdp.Client,
broker *events.EventBroker,
id *HTMLElementIdentity,
input *input.Manager,
exec *eval.ExecutionContext,
id HTMLElementIdentity,
nodeType int,
nodeName string,
value string,
innerHTML values.String,
children []*HTMLElementIdentity,
children []HTMLElementIdentity,
) *HTMLElement {
el := new(HTMLElement)
el.logger = logger
el.client = client
el.events = broker
el.input = input
el.exec = exec
el.connected = values.True
el.id = id
el.nodeType = values.NewInt(nodeType)
el.nodeType = common.ToHTMLType(nodeType)
el.nodeName = values.NewString(nodeName)
el.innerHTML = innerHTML
el.innerText = common.NewLazyValue(el.loadInnerText)
@ -207,7 +222,7 @@ func (el *HTMLElement) MarshalJSON() ([]byte, error) {
}
func (el *HTMLElement) String() string {
return el.InnerHTML(context.Background()).String()
return el.GetInnerHTML(context.Background()).String()
}
func (el *HTMLElement) Compare(other core.Value) int64 {
@ -217,7 +232,7 @@ func (el *HTMLElement) Compare(other core.Value) int64 {
ctx := context.Background()
return el.InnerHTML(ctx).Compare(other.InnerHTML(ctx))
return el.GetInnerHTML(ctx).Compare(other.GetInnerHTML(ctx))
default:
return drivers.Compare(el.Type(), other.Type())
}
@ -257,11 +272,11 @@ func (el *HTMLElement) SetIn(ctx context.Context, path []core.Value, value core.
}
func (el *HTMLElement) GetValue(ctx context.Context) core.Value {
if !el.IsConnected() {
if el.IsDetached() {
return el.value
}
val, err := eval.Property(ctx, el.client, el.id.objectID, "value")
val, err := el.exec.ReadProperty(ctx, el.id.objectID, "value")
if err != nil {
el.logError(err).Msg("failed to get node value")
@ -275,7 +290,7 @@ func (el *HTMLElement) GetValue(ctx context.Context) core.Value {
}
func (el *HTMLElement) SetValue(ctx context.Context, value core.Value) error {
if !el.IsConnected() {
if el.IsDetached() {
// TODO: Return an error
return nil
}
@ -283,11 +298,11 @@ func (el *HTMLElement) SetValue(ctx context.Context, value core.Value) error {
return el.client.DOM.SetNodeValue(ctx, dom.NewSetNodeValueArgs(el.id.nodeID, value.String()))
}
func (el *HTMLElement) NodeType() values.Int {
return el.nodeType
func (el *HTMLElement) GetNodeType() values.Int {
return values.NewInt(common.FromHTMLType(el.nodeType))
}
func (el *HTMLElement) NodeName() values.String {
func (el *HTMLElement) GetNodeName() values.String {
return el.nodeName
}
@ -472,7 +487,7 @@ func (el *HTMLElement) GetChildNode(ctx context.Context, idx values.Int) core.Va
}
func (el *HTMLElement) QuerySelector(ctx context.Context, selector values.String) core.Value {
if !el.IsConnected() {
if el.IsDetached() {
return values.None
}
@ -496,7 +511,7 @@ func (el *HTMLElement) QuerySelector(ctx context.Context, selector values.String
return values.None
}
res, err := LoadElement(ctx, el.logger, el.client, el.events, found.NodeID, emptyBackendID)
res, err := LoadHTMLElement(ctx, el.logger, el.client, el.events, el.input, el.exec, found.NodeID)
if err != nil {
el.logError(err).
@ -510,7 +525,7 @@ func (el *HTMLElement) QuerySelector(ctx context.Context, selector values.String
}
func (el *HTMLElement) QuerySelectorAll(ctx context.Context, selector values.String) core.Value {
if !el.IsConnected() {
if el.IsDetached() {
return values.NewArray(0)
}
@ -537,7 +552,7 @@ func (el *HTMLElement) QuerySelectorAll(ctx context.Context, selector values.Str
continue
}
childEl, err := LoadElement(ctx, el.logger, el.client, el.events, id, emptyBackendID)
childEl, err := LoadHTMLElement(ctx, el.logger, el.client, el.events, el.input, el.exec, id)
if err != nil {
el.logError(err).
@ -562,7 +577,154 @@ func (el *HTMLElement) QuerySelectorAll(ctx context.Context, selector values.Str
return arr
}
func (el *HTMLElement) InnerText(ctx context.Context) values.String {
func (el *HTMLElement) XPath(ctx context.Context, expression values.String) (result core.Value, err error) {
exp, err := expression.MarshalJSON()
if err != nil {
return values.None, err
}
out, err := el.exec.CallFunction(ctx, templates.XPath(),
runtime.CallArgument{
ObjectID: &el.id.objectID,
},
runtime.CallArgument{
Value: json.RawMessage(exp),
},
)
if err != nil {
return values.None, err
}
typeName := out.Type
// checking whether it's actually an array
if typeName == "object" {
isArrayRes, err := el.exec.CallFunction(ctx, `
(target) => Array.isArray(target)
`,
runtime.CallArgument{
ObjectID: out.ObjectID,
},
)
if err != nil {
return values.None, err
}
isArray, err := eval.Unmarshal(&isArrayRes)
if err != nil {
return values.None, err
}
if isArray == values.True {
typeName = "array"
}
}
switch typeName {
case "string", "number", "boolean":
return eval.Unmarshal(&out)
case "array":
if out.ObjectID == nil {
return values.None, nil
}
props, err := el.client.Runtime.GetProperties(ctx, runtime.NewGetPropertiesArgs(*out.ObjectID).SetOwnProperties(true))
if err != nil {
return values.None, err
}
if props.ExceptionDetails != nil {
exception := *props.ExceptionDetails
return values.None, errors.New(exception.Text)
}
result := values.NewArray(len(props.Result))
defer func() {
if err != nil {
result.ForEach(func(value core.Value, idx int) bool {
el, ok := value.(*HTMLElement)
if ok {
el.Close()
}
return true
})
}
}()
for _, descr := range props.Result {
if !descr.Enumerable {
continue
}
if descr.Value == nil {
continue
}
repl, err := el.client.DOM.RequestNode(ctx, dom.NewRequestNodeArgs(*descr.Value.ObjectID))
if err != nil {
return values.None, err
}
el, err := LoadHTMLElementWithID(
ctx,
el.logger,
el.client,
el.events,
el.input,
el.exec,
HTMLElementIdentity{
nodeID: repl.NodeID,
objectID: *descr.Value.ObjectID,
},
)
if err != nil {
return values.None, err
}
result.Push(el)
}
return result, nil
case "object":
if out.ObjectID == nil {
return values.None, nil
}
repl, err := el.client.DOM.RequestNode(ctx, dom.NewRequestNodeArgs(*out.ObjectID))
if err != nil {
return values.None, err
}
return LoadHTMLElementWithID(
ctx,
el.logger,
el.client,
el.events,
el.input,
el.exec,
HTMLElementIdentity{
nodeID: repl.NodeID,
objectID: *out.ObjectID,
},
)
default:
return values.None, nil
}
}
func (el *HTMLElement) GetInnerText(ctx context.Context) values.String {
val, err := el.innerText.Read(ctx)
if err != nil {
@ -577,7 +739,7 @@ func (el *HTMLElement) InnerText(ctx context.Context) values.String {
}
func (el *HTMLElement) InnerTextBySelector(ctx context.Context, selector values.String) values.String {
if !el.IsConnected() {
if el.IsDetached() {
return values.EmptyString
}
@ -624,7 +786,7 @@ func (el *HTMLElement) InnerTextBySelector(ctx context.Context, selector values.
objID := *obj.Object.ObjectID
text, err := eval.Property(ctx, el.client, objID, "innerText")
text, err := el.exec.ReadProperty(ctx, objID, "innerText")
if err != nil {
el.logError(err).
@ -679,7 +841,7 @@ func (el *HTMLElement) InnerTextBySelectorAll(ctx context.Context, selector valu
objID := *obj.Object.ObjectID
text, err := eval.Property(ctx, el.client, objID, "innerText")
text, err := el.exec.ReadProperty(ctx, objID, "innerText")
if err != nil {
el.logError(err).
@ -696,7 +858,7 @@ func (el *HTMLElement) InnerTextBySelectorAll(ctx context.Context, selector valu
return arr
}
func (el *HTMLElement) InnerHTML(_ context.Context) values.String {
func (el *HTMLElement) GetInnerHTML(_ context.Context) values.String {
el.mu.Lock()
defer el.mu.Unlock()
@ -704,7 +866,7 @@ func (el *HTMLElement) InnerHTML(_ context.Context) values.String {
}
func (el *HTMLElement) InnerHTMLBySelector(ctx context.Context, selector values.String) values.String {
if !el.IsConnected() {
if el.IsDetached() {
return values.EmptyString
}
@ -719,13 +881,12 @@ func (el *HTMLElement) InnerHTMLBySelector(ctx context.Context, selector values.
return values.EmptyString
}
text, err := loadInnerHTML(ctx, el.client, &HTMLElementIdentity{
nodeID: found.NodeID,
})
text, err := loadInnerHTMLByNodeID(ctx, el.client, el.exec, found.NodeID)
if err != nil {
el.logError(err).
Str("selector", selector.String()).
Int("childNodeID", int(found.NodeID)).
Msg("failed to load inner HTML for found child el")
return values.EmptyString
@ -750,13 +911,12 @@ func (el *HTMLElement) InnerHTMLBySelectorAll(ctx context.Context, selector valu
arr := values.NewArray(len(res.NodeIDs))
for _, id := range res.NodeIDs {
text, err := loadInnerHTML(ctx, el.client, &HTMLElementIdentity{
nodeID: id,
})
text, err := loadInnerHTMLByNodeID(ctx, el.client, el.exec, id)
if err != nil {
el.logError(err).
Str("selector", selector.String()).
Int("childNodeID", int(id)).
Msg("failed to load inner HTML for found child el")
// return what we have
@ -770,7 +930,7 @@ func (el *HTMLElement) InnerHTMLBySelectorAll(ctx context.Context, selector valu
}
func (el *HTMLElement) CountBySelector(ctx context.Context, selector values.String) values.Int {
if !el.IsConnected() {
if el.IsDetached() {
return values.ZeroInt
}
@ -790,7 +950,7 @@ func (el *HTMLElement) CountBySelector(ctx context.Context, selector values.Stri
}
func (el *HTMLElement) ExistsBySelector(ctx context.Context, selector values.String) values.Boolean {
if !el.IsConnected() {
if el.IsDetached() {
return values.False
}
@ -887,183 +1047,43 @@ func (el *HTMLElement) WaitForStyle(ctx context.Context, name values.String, val
}
func (el *HTMLElement) Click(ctx context.Context) (values.Boolean, error) {
return events.DispatchEvent(ctx, el.client, el.id.objectID, "click")
if err := el.input.ClickByNodeID(ctx, el.id.nodeID); err != nil {
return values.False, err
}
return values.True, nil
}
func (el *HTMLElement) Input(ctx context.Context, value core.Value, delay values.Int) error {
if err := el.client.DOM.Focus(ctx, dom.NewFocusArgs().SetObjectID(el.id.objectID)); err != nil {
el.logError(err).Msg("failed to focus")
return err
if el.GetNodeName() != "INPUT" {
return core.Error(core.ErrInvalidOperation, "element is not an <input> element.")
}
delayMs := time.Duration(delay)
time.Sleep(delayMs * time.Millisecond)
valStr := value.String()
for _, ch := range valStr {
for _, ev := range []string{"keyDown", "keyUp"} {
ke := input.NewDispatchKeyEventArgs(ev).SetText(string(ch))
if err := el.client.Input.DispatchKeyEvent(ctx, ke); err != nil {
el.logError(err).Str("value", value.String()).Msg("failed to input a value")
return err
}
time.Sleep(delayMs * time.Millisecond)
}
}
return nil
return el.input.TypeByNodeID(ctx, el.id.nodeID, value, delay)
}
func (el *HTMLElement) Select(ctx context.Context, value *values.Array) (*values.Array, error) {
var attrID = "data-ferret-select"
if el.NodeName() != "SELECT" {
return nil, core.Error(core.ErrInvalidOperation, "element is not a <select> element.")
}
id, err := uuid.NewV4()
if err != nil {
return nil, err
}
err = el.client.DOM.SetAttributeValue(ctx, dom.NewSetAttributeValueArgs(el.id.nodeID, attrID, id.String()))
if err != nil {
return nil, err
}
res, err := eval.Eval(
ctx,
el.client,
fmt.Sprintf(`
var el = document.querySelector('[%s="%s"]');
if (el == null) {
return [];
}
var values = %s;
if (el.nodeName.toLowerCase() !== 'select') {
throw new Error('element is not a <select> element.');
}
var options = Array.from(el.options);
el.value = undefined;
for (var option of options) {
option.selected = values.includes(option.value);
if (option.selected && !el.multiple) {
break;
}
}
el.dispatchEvent(new Event('input', { 'bubbles': true }));
el.dispatchEvent(new Event('change', { 'bubbles': true }));
return options.filter(option => option.selected).map(option => option.value);
`,
attrID,
id.String(),
value.String(),
),
true,
false,
)
if err != nil {
return nil, err
}
err = el.client.DOM.RemoveAttribute(ctx, dom.NewRemoveAttributeArgs(el.id.nodeID, attrID))
if err != nil {
return nil, err
}
arr, ok := res.(*values.Array)
if ok {
return arr, nil
}
return nil, core.TypeError(types.Array, res.Type())
return el.input.SelectByNodeID(ctx, el.id.nodeID, value)
}
func (el *HTMLElement) ScrollIntoView(ctx context.Context) error {
var attrID = "data-ferret-scroll"
id, err := uuid.NewV4()
if err != nil {
return err
}
err = el.client.DOM.SetAttributeValue(ctx, dom.NewSetAttributeValueArgs(el.id.nodeID, attrID, id.String()))
if err != nil {
return err
}
_, err = eval.Eval(
ctx,
el.client,
fmt.Sprintf(`
var el = document.querySelector('[%s="%s"]');
if (el == null) {
throw new Error('element not found');
}
el.scrollIntoView({
behavior: 'instant',
inline: 'center',
block: 'center'
});
`,
attrID,
id.String(),
), false, false)
if err != nil {
return err
}
err = el.client.DOM.RemoveAttribute(ctx, dom.NewRemoveAttributeArgs(el.id.nodeID, attrID))
return err
return el.input.ScrollIntoViewByNodeID(ctx, el.id.nodeID)
}
func (el *HTMLElement) Hover(ctx context.Context) error {
err := el.ScrollIntoView(ctx)
if err != nil {
return err
}
q, err := getClickablePoint(ctx, el.client, el.id)
if err != nil {
return err
}
return el.client.Input.DispatchMouseEvent(
ctx,
input.NewDispatchMouseEventArgs("mouseMoved", q.X, q.Y),
)
return el.input.MoveMouseByNodeID(ctx, el.id.nodeID)
}
func (el *HTMLElement) IsConnected() values.Boolean {
func (el *HTMLElement) IsDetached() values.Boolean {
el.mu.Lock()
defer el.mu.Unlock()
return el.connected
return !el.connected
}
func (el *HTMLElement) loadInnerText(ctx context.Context) (core.Value, error) {
if el.IsConnected() {
text, err := loadInnerText(ctx, el.client, el.id)
if !el.IsDetached() {
text, err := loadInnerText(ctx, el.client, el.exec, el.id, el.nodeType)
if err == nil {
return text, nil
@ -1074,7 +1094,7 @@ func (el *HTMLElement) loadInnerText(ctx context.Context) (core.Value, error) {
// and just parse cached innerHTML
}
h := el.InnerHTML(ctx)
h := el.GetInnerHTML(ctx)
if h == values.EmptyString {
return h, nil
@ -1102,20 +1122,21 @@ func (el *HTMLElement) loadAttrs(ctx context.Context) (core.Value, error) {
}
func (el *HTMLElement) loadChildren(ctx context.Context) (core.Value, error) {
if !el.IsConnected() {
if el.IsDetached() {
return values.NewArray(0), nil
}
loaded := values.NewArray(len(el.children))
for _, childID := range el.children {
child, err := LoadElement(
child, err := LoadHTMLElement(
ctx,
el.logger,
el.client,
el.events,
el.input,
el.exec,
childID.nodeID,
childID.backendID,
)
if err != nil {
@ -1285,21 +1306,20 @@ func (el *HTMLElement) handleChildInserted(ctx context.Context, message interfac
return
}
nextIdentity := &HTMLElementIdentity{
nodeID: reply.Node.NodeID,
backendID: reply.Node.BackendNodeID,
nextIdentity := HTMLElementIdentity{
nodeID: reply.Node.NodeID,
}
arr := el.children
el.children = append(arr[:targetIDx], append([]*HTMLElementIdentity{nextIdentity}, arr[targetIDx:]...)...)
el.children = append(arr[:targetIDx], append([]HTMLElementIdentity{nextIdentity}, arr[targetIDx:]...)...)
if !el.loadedChildren.Ready() {
return
}
el.loadedChildren.Write(ctx, func(v core.Value, err error) {
el.loadedChildren.Write(ctx, func(v core.Value, _ error) {
loadedArr := v.(*values.Array)
loadedEl, err := LoadElement(ctx, el.logger, el.client, el.events, nextID, emptyBackendID)
loadedEl, err := LoadHTMLElement(ctx, el.logger, el.client, el.events, el.input, el.exec, nextID)
if err != nil {
el.logError(err).Msg("failed to load an inserted element")
@ -1309,7 +1329,7 @@ func (el *HTMLElement) handleChildInserted(ctx context.Context, message interfac
loadedArr.Insert(values.NewInt(targetIDx), loadedEl)
newInnerHTML, err := loadInnerHTML(ctx, el.client, el.id)
newInnerHTML, err := loadInnerHTML(ctx, el.client, el.exec, el.id, el.nodeType)
if err != nil {
el.logError(err).Msg("failed to update element")
@ -1371,7 +1391,7 @@ func (el *HTMLElement) handleChildRemoved(ctx context.Context, message interface
loadedArr := v.(*values.Array)
loadedArr.RemoveAt(values.NewInt(targetIDx))
newInnerHTML, err := loadInnerHTML(ctx, el.client, el.id)
newInnerHTML, err := loadInnerHTML(ctx, el.client, el.exec, el.id, el.nodeType)
if err != nil {
el.logger.Error().
@ -1393,7 +1413,6 @@ func (el *HTMLElement) logError(err error) *zerolog.Event {
Error().
Timestamp().
Int("nodeID", int(el.id.nodeID)).
Int("backendID", int(el.id.backendID)).
Str("objectID", string(el.id.objectID)).
Err(err)
}

265
pkg/drivers/cdp/eval/context.go Normal file
View File

@ -0,0 +1,265 @@
package eval
import (
"context"
"fmt"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/pkg/errors"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
const EmptyExecutionContextID = runtime.ExecutionContextID(-1)
type ExecutionContext struct {
client *cdp.Client
frame page.Frame
contextID runtime.ExecutionContextID
}
func NewExecutionContext(client *cdp.Client, frame page.Frame, contextID runtime.ExecutionContextID) *ExecutionContext {
ec := new(ExecutionContext)
ec.client = client
ec.frame = frame
ec.contextID = contextID
return ec
}
func (ec *ExecutionContext) ID() runtime.ExecutionContextID {
return ec.contextID
}
func (ec *ExecutionContext) Eval(ctx context.Context, exp string) error {
_, err := ec.evalWithValueInternal(
ctx,
runtime.
NewEvaluateArgs(PrepareEval(exp)),
)
return err
}
func (ec *ExecutionContext) EvalWithValue(ctx context.Context, exp string) (core.Value, error) {
return ec.evalWithValueInternal(
ctx,
runtime.
NewEvaluateArgs(PrepareEval(exp)).
SetReturnByValue(true),
)
}
func (ec *ExecutionContext) EvalAsync(ctx context.Context, exp string) (core.Value, error) {
return ec.evalWithValueInternal(
ctx,
runtime.
NewEvaluateArgs(PrepareEval(exp)).
SetReturnByValue(true).
SetAwaitPromise(true),
)
}
func (ec *ExecutionContext) ResolveRemoteObject(ctx context.Context, exp string) (runtime.RemoteObject, error) {
res, err := ec.evalInternal(ctx, runtime.NewEvaluateArgs(PrepareEval(exp)))
if err != nil {
return runtime.RemoteObject{}, err
}
if res.ObjectID == nil {
return runtime.RemoteObject{}, errors.Wrap(core.ErrUnexpected, "unable to resolve remote object")
}
return res, nil
}
func (ec *ExecutionContext) CallMethod(
ctx context.Context,
objectID runtime.RemoteObjectID,
methodName string,
args []runtime.CallArgument,
) (*runtime.RemoteObject, error) {
callArgs := runtime.NewCallFunctionOnArgs(methodName).
SetObjectID(objectID).
SetArguments(args)
if ec.contextID != EmptyExecutionContextID {
callArgs.SetExecutionContextID(ec.contextID)
}
found, err := ec.client.Runtime.CallFunctionOn(
ctx,
callArgs,
)
if err != nil {
return nil, err
}
if found.ExceptionDetails != nil {
return nil, found.ExceptionDetails
}
if found.Result.ObjectID == nil {
return nil, nil
}
return &found.Result, nil
}
func (ec *ExecutionContext) ReadProperty(
ctx context.Context,
objectID runtime.RemoteObjectID,
propName string,
) (core.Value, error) {
res, err := ec.client.Runtime.GetProperties(
ctx,
runtime.NewGetPropertiesArgs(objectID),
)
if err != nil {
return values.None, err
}
if res.ExceptionDetails != nil {
return values.None, res.ExceptionDetails
}
// all props
if propName == "" {
arr := values.NewArray(len(res.Result))
for _, prop := range res.Result {
val, err := Unmarshal(prop.Value)
if err != nil {
return values.None, err
}
arr.Push(val)
}
return arr, nil
}
for _, prop := range res.Result {
if prop.Name == propName {
return Unmarshal(prop.Value)
}
}
return values.None, nil
}
func (ec *ExecutionContext) DispatchEvent(
ctx context.Context,
objectID runtime.RemoteObjectID,
eventName string,
) (values.Boolean, error) {
args := runtime.NewEvaluateArgs(PrepareEval(fmt.Sprintf(`
return new window.MouseEvent('%s', { bubbles: true, cancelable: true })
`, eventName)))
if ec.contextID != EmptyExecutionContextID {
args.SetContextID(ec.contextID)
}
evt, err := ec.client.Runtime.Evaluate(ctx, args)
if err != nil {
return values.False, nil
}
if evt.ExceptionDetails != nil {
return values.False, evt.ExceptionDetails
}
if evt.Result.ObjectID == nil {
return values.False, nil
}
evtID := evt.Result.ObjectID
// release the event object
defer ec.client.Runtime.ReleaseObject(ctx, runtime.NewReleaseObjectArgs(*evtID))
_, err = ec.CallMethod(
ctx,
objectID,
"dispatchEvent",
[]runtime.CallArgument{
{
ObjectID: evt.Result.ObjectID,
},
},
)
if err != nil {
return values.False, err
}
return values.True, nil
}
func (ec *ExecutionContext) CallFunction(ctx context.Context, declaration string, args ...runtime.CallArgument) (runtime.RemoteObject, error) {
cfArgs := runtime.NewCallFunctionOnArgs(declaration).SetArguments(args)
if ec.contextID != EmptyExecutionContextID {
cfArgs.SetExecutionContextID(ec.contextID)
}
repl, err := ec.client.Runtime.CallFunctionOn(ctx, cfArgs)
if err != nil {
return runtime.RemoteObject{}, err
}
if repl.ExceptionDetails != nil {
exception := *repl.ExceptionDetails
return runtime.RemoteObject{}, errors.New(exception.Error())
}
return repl.Result, nil
}
func (ec *ExecutionContext) evalWithValueInternal(ctx context.Context, args *runtime.EvaluateArgs) (core.Value, error) {
obj, err := ec.evalInternal(ctx, args)
if err != nil {
return values.None, err
}
if obj.Type != "undefined" && obj.Type != "null" {
return values.Unmarshal(obj.Value)
}
return Unmarshal(&obj)
}
func (ec *ExecutionContext) evalInternal(ctx context.Context, args *runtime.EvaluateArgs) (runtime.RemoteObject, error) {
if ec.contextID != EmptyExecutionContextID {
args.SetContextID(ec.contextID)
}
out, err := ec.client.Runtime.Evaluate(ctx, args)
if err != nil {
return runtime.RemoteObject{}, err
}
if out.ExceptionDetails != nil {
ex := out.ExceptionDetails
return runtime.RemoteObject{}, core.Error(
core.ErrUnexpected,
fmt.Sprintf("%s: %s", ex.Text, *ex.Exception.Description),
)
}
return out.Result, nil
}

160
pkg/drivers/cdp/eval/eval.go
View File

@ -1,160 +0,0 @@
package eval
import (
"context"
"encoding/json"
"fmt"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/runtime"
)
func PrepareEval(exp string) string {
return fmt.Sprintf("((function () {%s})())", exp)
}
func Eval(ctx context.Context, client *cdp.Client, exp string, ret bool, async bool) (core.Value, error) {
args := runtime.
NewEvaluateArgs(PrepareEval(exp)).
SetReturnByValue(ret).
SetAwaitPromise(async)
out, err := client.Runtime.Evaluate(ctx, args)
if err != nil {
return values.None, err
}
if out.ExceptionDetails != nil {
ex := out.ExceptionDetails
return values.None, core.Error(
core.ErrUnexpected,
fmt.Sprintf("%s: %s", ex.Text, *ex.Exception.Description),
)
}
if out.Result.Type != "undefined" {
return values.Unmarshal(out.Result.Value)
}
return Unmarshal(&out.Result)
}
func Property(
ctx context.Context,
client *cdp.Client,
objectID runtime.RemoteObjectID,
propName string,
) (core.Value, error) {
res, err := client.Runtime.GetProperties(
ctx,
runtime.NewGetPropertiesArgs(objectID),
)
if err != nil {
return values.None, err
}
if res.ExceptionDetails != nil {
return values.None, res.ExceptionDetails
}
// all props
if propName == "" {
var arr *values.Array
arr = values.NewArray(len(res.Result))
for _, prop := range res.Result {
val, err := Unmarshal(prop.Value)
if err != nil {
return values.None, err
}
arr.Push(val)
}
return arr, nil
}
for _, prop := range res.Result {
if prop.Name == propName {
return Unmarshal(prop.Value)
}
}
return values.None, nil
}
func Method(
ctx context.Context,
client *cdp.Client,
objectID runtime.RemoteObjectID,
methodName string,
args []runtime.CallArgument,
) (*runtime.RemoteObject, error) {
found, err := client.Runtime.CallFunctionOn(
ctx,
runtime.NewCallFunctionOnArgs(methodName).
SetObjectID(objectID).
SetArguments(args),
)
if err != nil {
return nil, err
}
if found.ExceptionDetails != nil {
return nil, found.ExceptionDetails
}
if found.Result.ObjectID == nil {
return nil, nil
}
return &found.Result, nil
}
func MethodQuerySelector(
ctx context.Context,
client *cdp.Client,
objectID runtime.RemoteObjectID,
selector string,
) (runtime.RemoteObjectID, error) {
bytes, err := json.Marshal(selector)
if err != nil {
return "", err
}
obj, err := Method(ctx, client, objectID, "querySelector", []runtime.CallArgument{
{
Value: json.RawMessage(bytes),
},
})
if err != nil {
return "", err
}
if obj.ObjectID == nil {
return "", nil
}
return *obj.ObjectID, nil
}
func Unmarshal(obj *runtime.RemoteObject) (core.Value, error) {
if obj == nil {
return values.None, nil
}
if obj.Type != "undefined" {
return values.Unmarshal(obj.Value)
}
return values.None, nil
}

35
pkg/drivers/cdp/eval/helpers.go Normal file
View File

@ -0,0 +1,35 @@
package eval
import (
"fmt"
"strconv"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/mafredri/cdp/protocol/runtime"
)
func PrepareEval(exp string) string {
return fmt.Sprintf("((function () {%s})())", exp)
}
func Unmarshal(obj *runtime.RemoteObject) (core.Value, error) {
if obj == nil {
return values.None, nil
}
switch obj.Type {
case "string":
str, err := strconv.Unquote(string(obj.Value))
if err != nil {
return values.None, err
}
return values.NewString(str), nil
case "undefined", "null":
return values.None, nil
default:
return values.Unmarshal(obj.Value)
}
}

3
pkg/drivers/cdp/eval/param.go
View File

@ -1,9 +1,10 @@
package eval
import (
"strconv"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"strconv"
)
func Param(input core.Value) string {

17
pkg/drivers/cdp/events/broker.go
View File

@ -5,11 +5,11 @@ import (
"reflect"
"sync"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/page"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
)
type (
@ -179,6 +179,16 @@ func (broker *EventBroker) Close() error {
return nil
}
func (broker *EventBroker) StopAndClose() error {
err := broker.Stop()
if err != nil {
return err
}
return broker.Close()
}
func (broker *EventBroker) runLoop(ctx context.Context) {
for {
select {
@ -265,6 +275,7 @@ func (broker *EventBroker) emit(ctx context.Context, event Event, message interf
listeners, ok := broker.listeners[event]
if !ok {
broker.mu.Unlock()
return
}

2
pkg/drivers/cdp/events/broker_test.go
View File

@ -269,7 +269,7 @@ func TestEventBroker(t *testing.T) {
var listener events.EventListener
listener = func(ctx context.Context, message interface{}) {
counter += 1
counter++
b.RemoveEventListener(events.EventLoad, listener)
}

56
pkg/drivers/cdp/events/dispatch.go
View File

@ -1,56 +0,0 @@
package events
import (
"context"
"fmt"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/runtime"
)
func DispatchEvent(
ctx context.Context,
client *cdp.Client,
objectID runtime.RemoteObjectID,
eventName string,
) (values.Boolean, error) {
evt, err := client.Runtime.Evaluate(ctx, runtime.NewEvaluateArgs(eval.PrepareEval(fmt.Sprintf(`
return new window.MouseEvent('%s', { bubbles: true, cancelable: true })
`, eventName))))
if err != nil {
return values.False, nil
}
if evt.ExceptionDetails != nil {
return values.False, evt.ExceptionDetails
}
if evt.Result.ObjectID == nil {
return values.False, nil
}
evtID := evt.Result.ObjectID
// release the event object
defer client.Runtime.ReleaseObject(ctx, runtime.NewReleaseObjectArgs(*evtID))
_, err = eval.Method(
ctx,
client,
objectID,
"dispatchEvent",
[]runtime.CallArgument{
{
ObjectID: evt.Result.ObjectID,
},
},
)
if err != nil {
return values.False, err
}
return values.True, nil
}

126
pkg/drivers/cdp/events/helpers.go Normal file
View File

@ -0,0 +1,126 @@
package events
import (
"context"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/page"
"github.com/pkg/errors"
)
func WaitForLoadEvent(ctx context.Context, client *cdp.Client) error {
loadEventFired, err := client.Page.LoadEventFired(ctx)
if err != nil {
return errors.Wrap(err, "failed to create load event hook")
}
_, err = loadEventFired.Recv()
if err != nil {
return err
}
return loadEventFired.Close()
}
func CreateEventBroker(client *cdp.Client) (*EventBroker, error) {
var err error
var onLoad page.LoadEventFiredClient
var onReload dom.DocumentUpdatedClient
var onAttrModified dom.AttributeModifiedClient
var onAttrRemoved dom.AttributeRemovedClient
var onChildCountUpdated dom.ChildNodeCountUpdatedClient
var onChildNodeInserted dom.ChildNodeInsertedClient
var onChildNodeRemoved dom.ChildNodeRemovedClient
ctx := context.Background()
onLoad, err = client.Page.LoadEventFired(ctx)
if err != nil {
return nil, err
}
onReload, err = client.DOM.DocumentUpdated(ctx)
if err != nil {
onLoad.Close()
return nil, err
}
onAttrModified, err = client.DOM.AttributeModified(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
return nil, err
}
onAttrRemoved, err = client.DOM.AttributeRemoved(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
return nil, err
}
onChildCountUpdated, err = client.DOM.ChildNodeCountUpdated(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
onAttrRemoved.Close()
return nil, err
}
onChildNodeInserted, err = client.DOM.ChildNodeInserted(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
onAttrRemoved.Close()
onChildCountUpdated.Close()
return nil, err
}
onChildNodeRemoved, err = client.DOM.ChildNodeRemoved(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
onAttrRemoved.Close()
onChildCountUpdated.Close()
onChildNodeInserted.Close()
return nil, err
}
broker := NewEventBroker(
onLoad,
onReload,
onAttrModified,
onAttrRemoved,
onChildCountUpdated,
onChildNodeInserted,
onChildNodeRemoved,
)
err = broker.Start()
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
onAttrRemoved.Close()
onChildCountUpdated.Close()
onChildNodeInserted.Close()
onChildNodeRemoved.Close()
return nil, err
}
return broker, nil
}

11
pkg/drivers/cdp/events/wait.go
View File

@ -2,12 +2,12 @@ package events
import (
"context"
"time"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/mafredri/cdp"
"time"
)
type (
@ -58,18 +58,15 @@ func (task *WaitTask) Run(ctx context.Context) (core.Value, error) {
}
func NewEvalWaitTask(
client *cdp.Client,
ec *eval.ExecutionContext,
predicate string,
polling time.Duration,
) *WaitTask {
return NewWaitTask(
func(ctx context.Context) (core.Value, error) {
return eval.Eval(
return ec.EvalWithValue(
ctx,
client,
predicate,
true,
false,
)
},
polling,

444
pkg/drivers/cdp/helpers.go
View File

@ -4,16 +4,15 @@ import (
"bytes"
"context"
"errors"
"math"
"golang.org/x/net/html"
"strings"
"time"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/PuerkitoBio/goquery"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
@ -27,11 +26,6 @@ var emptyExpires = time.Time{}
type (
batchFunc = func() error
Quad struct {
X float64
Y float64
}
)
func runBatch(funcs ...batchFunc) error {
@ -44,101 +38,6 @@ func runBatch(funcs ...batchFunc) error {
return eg.Wait()
}
func getRootElement(ctx context.Context, client *cdp.Client) (*dom.GetDocumentReply, error) {
d, err := client.DOM.GetDocument(ctx, dom.NewGetDocumentArgs().SetDepth(1))
if err != nil {
return nil, err
}
return d, nil
}
func fromProtocolQuad(quad dom.Quad) []Quad {
return []Quad{
{
X: quad[0],
Y: quad[1],
},
{
X: quad[2],
Y: quad[3],
},
{
X: quad[4],
Y: quad[5],
},
{
X: quad[6],
Y: quad[7],
},
}
}
func computeQuadArea(quads []Quad) float64 {
var area float64
for i := range quads {
p1 := quads[i]
p2 := quads[(i+1)%len(quads)]
area += (p1.X*p2.Y - p2.X*p1.Y) / 2
}
return math.Abs(area)
}
func getClickablePoint(ctx context.Context, client *cdp.Client, id *HTMLElementIdentity) (Quad, error) {
qargs := dom.NewGetContentQuadsArgs()
switch {
case id.objectID != "":
qargs.SetObjectID(id.objectID)
case id.backendID != 0:
qargs.SetBackendNodeID(id.backendID)
default:
qargs.SetNodeID(id.nodeID)
}
res, err := client.DOM.GetContentQuads(ctx, qargs)
if err != nil {
return Quad{}, err
}
if res.Quads == nil || len(res.Quads) == 0 {
return Quad{}, errors.New("node is either not visible or not an HTMLElement")
}
quads := make([][]Quad, 0, len(res.Quads))
for _, q := range res.Quads {
quad := fromProtocolQuad(q)
if computeQuadArea(quad) > 1 {
quads = append(quads, quad)
}
}
if len(quads) == 0 {
return Quad{}, errors.New("node is either not visible or not an HTMLElement")
}
// Return the middle point of the first quad.
quad := quads[0]
var x float64
var y float64
for _, q := range quad {
x += q.X
y += q.Y
}
return Quad{
X: x / 4,
Y: y / 4,
}, nil
}
func parseAttrs(attrs []string) *values.Object {
var attr values.String
@ -167,41 +66,79 @@ func parseAttrs(attrs []string) *values.Object {
return res
}
func loadInnerHTML(ctx context.Context, client *cdp.Client, id *HTMLElementIdentity) (values.String, error) {
var objID runtime.RemoteObjectID
func loadInnerHTML(ctx context.Context, client *cdp.Client, exec *eval.ExecutionContext, id HTMLElementIdentity, nodeType html.NodeType) (values.String, error) {
// not a document
if nodeType != html.DocumentNode {
var objID runtime.RemoteObjectID
switch {
case id.objectID != "":
objID = id.objectID
case id.backendID > 0:
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetBackendNodeID(id.backendID))
if id.objectID != "" {
objID = id.objectID
} else {
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.nodeID))
if err != nil {
return "", err
}
if repl.Object.ObjectID == nil {
return "", errors.New("unable to resolve node")
}
objID = *repl.Object.ObjectID
}
res, err := exec.ReadProperty(ctx, objID, "innerHTML")
if err != nil {
return "", err
}
if repl.Object.ObjectID == nil {
return "", errors.New("unable to resolve node")
}
objID = *repl.Object.ObjectID
default:
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.nodeID))
if err != nil {
return "", err
}
if repl.Object.ObjectID == nil {
return "", errors.New("unable to resolve node")
}
objID = *repl.Object.ObjectID
return values.NewString(res.String()), nil
}
repl, err := exec.EvalWithValue(ctx, "return document.documentElement.innerHTML")
if err != nil {
return "", err
}
return values.NewString(repl.String()), nil
}
func loadInnerHTMLByNodeID(ctx context.Context, client *cdp.Client, exec *eval.ExecutionContext, nodeID dom.NodeID) (values.String, error) {
node, err := client.DOM.DescribeNode(ctx, dom.NewDescribeNodeArgs().SetNodeID(nodeID))
if err != nil {
return values.EmptyString, err
}
return loadInnerHTML(ctx, client, exec, HTMLElementIdentity{
nodeID: nodeID,
}, common.ToHTMLType(node.Node.NodeType))
}
func loadInnerText(ctx context.Context, client *cdp.Client, exec *eval.ExecutionContext, id HTMLElementIdentity, nodeType html.NodeType) (values.String, error) {
// not a document
if id.nodeID != 1 {
res, err := eval.Property(ctx, client, objID, "innerHTML")
if nodeType != html.DocumentNode {
var objID runtime.RemoteObjectID
if id.objectID != "" {
objID = id.objectID
} else {
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.nodeID))
if err != nil {
return "", err
}
if repl.Object.ObjectID == nil {
return "", errors.New("unable to resolve node")
}
objID = *repl.Object.ObjectID
}
res, err := exec.ReadProperty(ctx, objID, "innerText")
if err != nil {
return "", err
@ -210,66 +147,26 @@ func loadInnerHTML(ctx context.Context, client *cdp.Client, id *HTMLElementIdent
return values.NewString(res.String()), err
}
repl, err := client.DOM.GetOuterHTML(ctx, dom.NewGetOuterHTMLArgs().SetObjectID(objID))
repl, err := exec.EvalWithValue(ctx, "return document.documentElement.innerText")
if err != nil {
return "", err
}
return values.NewString(repl.OuterHTML), nil
return values.NewString(repl.String()), nil
}
func loadInnerText(ctx context.Context, client *cdp.Client, id *HTMLElementIdentity) (values.String, error) {
var objID runtime.RemoteObjectID
switch {
case id.objectID != "":
objID = id.objectID
case id.backendID > 0:
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetBackendNodeID(id.backendID))
if err != nil {
return "", err
}
if repl.Object.ObjectID == nil {
return "", errors.New("unable to resolve node")
}
objID = *repl.Object.ObjectID
default:
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.nodeID))
if err != nil {
return "", err
}
if repl.Object.ObjectID == nil {
return "", errors.New("unable to resolve node")
}
objID = *repl.Object.ObjectID
}
// not a document
if id.nodeID != 1 {
res, err := eval.Property(ctx, client, objID, "innerText")
if err != nil {
return "", err
}
return values.NewString(res.String()), err
}
repl, err := client.DOM.GetOuterHTML(ctx, dom.NewGetOuterHTMLArgs().SetObjectID(objID))
if err != nil {
return "", err
}
return parseInnerText(repl.OuterHTML)
}
//func loadInnerTextByNodeID(ctx context.Context, client *cdp.Client, exec *eval.ExecutionContext, nodeID dom.NodeID) (values.String, error) {
// node, err := client.DOM.DescribeNode(ctx, dom.NewDescribeNodeArgs().SetNodeID(nodeID))
//
// if err != nil {
// return values.EmptyString, err
// }
//
// return loadInnerText(ctx, client, exec, HTMLElementIdentity{
// nodeID: nodeID,
// }, common.ToHTMLType(node.Node.NodeType))
//}
func parseInnerText(innerHTML string) (values.String, error) {
buff := bytes.NewBuffer([]byte(innerHTML))
@ -283,135 +180,19 @@ func parseInnerText(innerHTML string) (values.String, error) {
return values.NewString(parsed.Text()), nil
}
func createChildrenArray(nodes []dom.Node) []*HTMLElementIdentity {
children := make([]*HTMLElementIdentity, len(nodes))
func createChildrenArray(nodes []dom.Node) []HTMLElementIdentity {
children := make([]HTMLElementIdentity, len(nodes))
for idx, child := range nodes {
children[idx] = &HTMLElementIdentity{
nodeID: child.NodeID,
backendID: child.BackendNodeID,
child := child
children[idx] = HTMLElementIdentity{
nodeID: child.NodeID,
}
}
return children
}
func waitForLoadEvent(ctx context.Context, client *cdp.Client) error {
loadEventFired, err := client.Page.LoadEventFired(ctx)
if err != nil {
return err
}
_, err = loadEventFired.Recv()
if err != nil {
return err
}
return loadEventFired.Close()
}
func createEventBroker(client *cdp.Client) (*events.EventBroker, error) {
var err error
var onLoad page.LoadEventFiredClient
var onReload dom.DocumentUpdatedClient
var onAttrModified dom.AttributeModifiedClient
var onAttrRemoved dom.AttributeRemovedClient
var onChildCountUpdated dom.ChildNodeCountUpdatedClient
var onChildNodeInserted dom.ChildNodeInsertedClient
var onChildNodeRemoved dom.ChildNodeRemovedClient
ctx := context.Background()
onLoad, err = client.Page.LoadEventFired(ctx)
if err != nil {
return nil, err
}
onReload, err = client.DOM.DocumentUpdated(ctx)
if err != nil {
onLoad.Close()
return nil, err
}
onAttrModified, err = client.DOM.AttributeModified(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
return nil, err
}
onAttrRemoved, err = client.DOM.AttributeRemoved(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
return nil, err
}
onChildCountUpdated, err = client.DOM.ChildNodeCountUpdated(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
onAttrRemoved.Close()
return nil, err
}
onChildNodeInserted, err = client.DOM.ChildNodeInserted(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
onAttrRemoved.Close()
onChildCountUpdated.Close()
return nil, err
}
onChildNodeRemoved, err = client.DOM.ChildNodeRemoved(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
onAttrRemoved.Close()
onChildCountUpdated.Close()
onChildNodeInserted.Close()
return nil, err
}
broker := events.NewEventBroker(
onLoad,
onReload,
onAttrModified,
onAttrRemoved,
onChildCountUpdated,
onChildNodeInserted,
onChildNodeRemoved,
)
err = broker.Start()
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
onAttrRemoved.Close()
onChildCountUpdated.Close()
onChildNodeInserted.Close()
onChildNodeRemoved.Close()
return nil, err
}
return broker, nil
}
func fromDriverCookie(url string, cookie drivers.HTTPCookie) network.CookieParam {
sameSite := network.CookieSameSiteNotSet
@ -485,9 +266,58 @@ func normalizeCookieURL(url string) string {
return httpPrefix + url
}
func randomDuration(delay values.Int) time.Duration {
max, min := core.NumberBoundaries(float64(int64(delay)))
value := core.Random(max, min)
func resolveFrame(ctx context.Context, client *cdp.Client, frame page.Frame) (dom.Node, runtime.ExecutionContextID, error) {
worldRepl, err := client.Page.CreateIsolatedWorld(ctx, page.NewCreateIsolatedWorldArgs(frame.ID))
return time.Duration(int64(value))
if err != nil {
return dom.Node{}, -1, err
}
evalRes, err := client.Runtime.Evaluate(
ctx,
runtime.NewEvaluateArgs(eval.PrepareEval("return document")).
SetContextID(worldRepl.ExecutionContextID),
)
if err != nil {
return dom.Node{}, -1, err
}
if evalRes.ExceptionDetails != nil {
exception := *evalRes.ExceptionDetails
return dom.Node{}, -1, errors.New(exception.Text)
}
if evalRes.Result.ObjectID == nil {
return dom.Node{}, -1, errors.New("failed to resolve frame document")
}
req, err := client.DOM.RequestNode(ctx, dom.NewRequestNodeArgs(*evalRes.Result.ObjectID))
if err != nil {
return dom.Node{}, -1, err
}
if req.NodeID == 0 {
return dom.Node{}, -1, errors.New("framed document is resolved with empty node id")
}
desc, err := client.DOM.DescribeNode(
ctx,
dom.
NewDescribeNodeArgs().
SetNodeID(req.NodeID).
SetDepth(1),
)
if err != nil {
return dom.Node{}, -1, err
}
// Returned node, by some reason, does not contain the NodeID
// So, we have to set it manually
desc.Node.NodeID = req.NodeID
return desc.Node, worldRepl.ExecutionContextID, nil
}

14
pkg/drivers/cdp/input/helpers.go Normal file
View File

@ -0,0 +1,14 @@
package input
import (
"time"
"github.com/MontFerret/ferret/pkg/runtime/core"
)
func randomDuration(delay int) time.Duration {
max, min := core.NumberBoundaries(float64(delay))
value := core.Random(max, min)
return time.Duration(int64(value))
}

52
pkg/drivers/cdp/input/keyboard.go Normal file
View File

@ -0,0 +1,52 @@
package input
import (
"context"
"time"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/input"
)
type Keyboard struct {
client *cdp.Client
}
func NewKeyboard(client *cdp.Client) *Keyboard {
return &Keyboard{client}
}
func (k *Keyboard) Down(ctx context.Context, char string) error {
return k.client.Input.DispatchKeyEvent(
ctx,
input.NewDispatchKeyEventArgs("keyDown").
SetText(char),
)
}
func (k *Keyboard) Up(ctx context.Context, char string) error {
return k.client.Input.DispatchKeyEvent(
ctx,
input.NewDispatchKeyEventArgs("keyUp").
SetText(char),
)
}
func (k *Keyboard) Type(ctx context.Context, text string, delay int) error {
for _, ch := range text {
ch := string(ch)
if err := k.Down(ctx, ch); err != nil {
return err
}
releaseDelay := randomDuration(delay)
time.Sleep(releaseDelay)
if err := k.Up(ctx, ch); err != nil {
return err
}
}
return nil
}

305
pkg/drivers/cdp/input/manager.go Normal file
View File

@ -0,0 +1,305 @@
package input
import (
"context"
"fmt"
"time"
"github.com/gofrs/uuid"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
type Manager struct {
client *cdp.Client
exec *eval.ExecutionContext
keyboard *Keyboard
mouse *Mouse
}
func NewManager(
client *cdp.Client,
exec *eval.ExecutionContext,
keyboard *Keyboard,
mouse *Mouse,
) *Manager {
return &Manager{
client,
exec,
keyboard,
mouse,
}
}
func (m *Manager) Keyboard() *Keyboard {
return m.keyboard
}
func (m *Manager) Mouse() *Mouse {
return m.mouse
}
func (m *Manager) Scroll(ctx context.Context, x, y values.Float) error {
return m.exec.Eval(ctx, fmt.Sprintf(`
window.scrollBy({
top: %s,
left: %s,
behavior: 'instant'
});
`,
eval.ParamFloat(float64(x)),
eval.ParamFloat(float64(y)),
))
}
func (m *Manager) ScrollIntoViewBySelector(ctx context.Context, selector values.String) error {
return m.exec.Eval(ctx, fmt.Sprintf(`
var el = document.querySelector(%s);
if (el == null) {
throw new Error("element not found");
}
el.scrollIntoView({
behavior: 'instant'
});
return true;
`, eval.ParamString(selector.String()),
))
}
func (m *Manager) ScrollIntoViewByNodeID(ctx context.Context, nodeID dom.NodeID) error {
var attrID = "data-ferret-scroll"
id, err := uuid.NewV4()
if err != nil {
return err
}
err = m.client.DOM.SetAttributeValue(ctx, dom.NewSetAttributeValueArgs(nodeID, attrID, id.String()))
if err != nil {
return err
}
err = m.exec.Eval(
ctx,
fmt.Sprintf(`
var el = document.querySelector('[%s="%s"]');
if (el == null) {
throw new Error('element not found');
}
el.scrollIntoView({
behavior: 'instant',
inline: 'center',
block: 'center'
});
`,
attrID,
id.String(),
))
if err != nil {
return err
}
err = m.client.DOM.RemoveAttribute(ctx, dom.NewRemoveAttributeArgs(nodeID, attrID))
return err
}
func (m *Manager) ScrollTop(ctx context.Context) error {
return m.exec.Eval(ctx, `
window.scrollTo({
left: 0,
top: 0,
behavior: 'instant'
});
`)
}
func (m *Manager) ScrollBottom(ctx context.Context) error {
return m.exec.Eval(ctx, `
window.scrollTo({
left: 0,
top: window.document.body.scrollHeight,
behavior: 'instant'
});
`)
}
func (m *Manager) MoveMouseBySelector(ctx context.Context, parentNodeID dom.NodeID, selector values.String) error {
found, err := m.client.DOM.QuerySelector(ctx, dom.NewQuerySelectorArgs(parentNodeID, selector.String()))
if err != nil {
return err
}
return m.MoveMouseByNodeID(ctx, found.NodeID)
}
func (m *Manager) MoveMouseByNodeID(ctx context.Context, nodeID dom.NodeID) error {
err := m.ScrollIntoViewByNodeID(ctx, nodeID)
if err != nil {
return err
}
q, err := GetClickablePointByNodeID(ctx, m.client, nodeID)
if err != nil {
return err
}
return m.mouse.Move(ctx, q.X, q.Y)
}
func (m *Manager) MoveMouse(ctx context.Context, x, y values.Float) error {
return m.mouse.Move(ctx, float64(x), float64(y))
}
func (m *Manager) ClickBySelector(ctx context.Context, parentNodeID dom.NodeID, selector values.String) error {
found, err := m.client.DOM.QuerySelector(ctx, dom.NewQuerySelectorArgs(parentNodeID, selector.String()))
if err != nil {
return err
}
return m.ClickByNodeID(ctx, found.NodeID)
}
func (m *Manager) ClickByNodeID(ctx context.Context, nodeID dom.NodeID) error {
if err := m.ScrollIntoViewByNodeID(ctx, nodeID); err != nil {
return err
}
points, err := GetClickablePointByNodeID(ctx, m.client, nodeID)
if err != nil {
return err
}
if err := m.mouse.Click(ctx, points.X, points.Y, 50); err != nil {
return nil
}
return nil
}
func (m *Manager) TypeBySelector(ctx context.Context, parentNodeID dom.NodeID, selector values.String, text core.Value, delay values.Int) error {
found, err := m.client.DOM.QuerySelector(ctx, dom.NewQuerySelectorArgs(parentNodeID, selector.String()))
if err != nil {
return err
}
return m.TypeByNodeID(ctx, found.NodeID, text, delay)
}
func (m *Manager) TypeByNodeID(ctx context.Context, nodeID dom.NodeID, text core.Value, delay values.Int) error {
if err := m.ScrollIntoViewByNodeID(ctx, nodeID); err != nil {
return err
}
if err := m.client.DOM.Focus(ctx, dom.NewFocusArgs().SetNodeID(nodeID)); err != nil {
return err
}
_, min := core.NumberBoundaries(float64(delay))
beforeTypeDelay := time.Duration(min)
time.Sleep(beforeTypeDelay * time.Millisecond)
return m.keyboard.Type(ctx, text.String(), int(delay))
}
func (m *Manager) SelectBySelector(ctx context.Context, parentNodeID dom.NodeID, selector values.String, value *values.Array) (*values.Array, error) {
found, err := m.client.DOM.QuerySelector(ctx, dom.NewQuerySelectorArgs(parentNodeID, selector.String()))
if err != nil {
return nil, err
}
return m.SelectByNodeID(ctx, found.NodeID, value)
}
func (m *Manager) SelectByNodeID(ctx context.Context, nodeID dom.NodeID, value *values.Array) (*values.Array, error) {
if err := m.ScrollIntoViewByNodeID(ctx, nodeID); err != nil {
return nil, err
}
if err := m.client.DOM.Focus(ctx, dom.NewFocusArgs().SetNodeID(nodeID)); err != nil {
return nil, err
}
var attrID = "data-ferret-select"
id, err := uuid.NewV4()
if err != nil {
return nil, err
}
err = m.client.DOM.SetAttributeValue(ctx, dom.NewSetAttributeValueArgs(nodeID, attrID, id.String()))
if err != nil {
return nil, err
}
res, err := m.exec.EvalWithValue(
ctx,
fmt.Sprintf(`
var el = document.querySelector('[%s="%s"]');
if (el == null) {
return [];
}
var values = %s;
if (el.nodeName.toLowerCase() !== 'select') {
throw new Error('element is not a <select> element.');
}
var options = Array.from(el.options);
el.value = undefined;
for (var option of options) {
option.selected = values.includes(option.value);
if (option.selected && !el.multiple) {
break;
}
}
el.dispatchEvent(new Event('input', { 'bubbles': true }));
el.dispatchEvent(new Event('change', { 'bubbles': true }));
return options.filter(option => option.selected).map(option => option.value);
`,
attrID,
id.String(),
value.String(),
),
)
if err != nil {
return nil, err
}
err = m.client.DOM.RemoveAttribute(ctx, dom.NewRemoveAttributeArgs(nodeID, attrID))
if err != nil {
return nil, err
}
arr, ok := res.(*values.Array)
if ok {
return arr, nil
}
return nil, core.TypeError(types.Array, res.Type())
}

83
pkg/drivers/cdp/input/mouse.go Normal file
View File

@ -0,0 +1,83 @@
package input
import (
"context"
"time"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/input"
)
type Mouse struct {
client *cdp.Client
x float64
y float64
}
func NewMouse(client *cdp.Client) *Mouse {
return &Mouse{client, 0, 0}
}
func (m *Mouse) Click(ctx context.Context, x, y float64, delay int) error {
if err := m.Move(ctx, x, y); err != nil {
return err
}
if err := m.Down(ctx, "left"); err != nil {
return err
}
releaseDelay := randomDuration(delay)
time.Sleep(releaseDelay * time.Millisecond)
return m.Up(ctx, "left")
}
func (m *Mouse) Down(ctx context.Context, button string) error {
return m.client.Input.DispatchMouseEvent(
ctx,
input.NewDispatchMouseEventArgs("mousePressed", m.x, m.y).
SetClickCount(1).
SetButton(button),
)
}
func (m *Mouse) Up(ctx context.Context, button string) error {
return m.client.Input.DispatchMouseEvent(
ctx,
input.NewDispatchMouseEventArgs("mouseReleased", m.x, m.y).
SetClickCount(1).
SetButton(button),
)
}
func (m *Mouse) Move(ctx context.Context, x, y float64) error {
return m.MoveBySteps(ctx, x, y, 1)
}
func (m *Mouse) MoveBySteps(ctx context.Context, x, y float64, steps int) error {
fromX := m.x
fromY := m.y
for i := 0; i <= steps; i++ {
iFloat := float64(i)
stepFloat := float64(steps)
toX := fromX + (x-fromX)*(iFloat/stepFloat)
toY := fromY + (y-fromY)*(iFloat/stepFloat)
err := m.client.Input.DispatchMouseEvent(
ctx,
input.NewDispatchMouseEventArgs("mouseMoved", toX, toY),
)
if err != nil {
return err
}
}
m.x = x
m.y = y
return nil
}

124
pkg/drivers/cdp/input/quad.go Normal file
View File

@ -0,0 +1,124 @@
package input
import (
"context"
"math"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/pkg/errors"
)
type Quad struct {
X float64
Y float64
}
func fromProtocolQuad(quad dom.Quad) []Quad {
return []Quad{
{
X: quad[0],
Y: quad[1],
},
{
X: quad[2],
Y: quad[3],
},
{
X: quad[4],
Y: quad[5],
},
{
X: quad[6],
Y: quad[7],
},
}
}
func computeQuadArea(quads []Quad) float64 {
var area float64
for i := range quads {
p1 := quads[i]
p2 := quads[(i+1)%len(quads)]
area += (p1.X*p2.Y - p2.X*p1.Y) / 2
}
return math.Abs(area)
}
func intersectQuadWithViewport(quad []Quad, width, height float64) []Quad {
quads := make([]Quad, 0, len(quad))
for _, point := range quad {
quads = append(quads, Quad{
X: math.Min(math.Max(point.X, 0), width),
Y: math.Min(math.Max(point.Y, 0), height),
})
}
return quads
}
func getClickablePoint(ctx context.Context, client *cdp.Client, qargs *dom.GetContentQuadsArgs) (Quad, error) {
contentQuadsReply, err := client.DOM.GetContentQuads(ctx, qargs)
if err != nil {
return Quad{}, err
}
if contentQuadsReply.Quads == nil || len(contentQuadsReply.Quads) == 0 {
return Quad{}, errors.New("node is either not visible or not an HTMLElement")
}
layoutMetricsReply, err := client.Page.GetLayoutMetrics(ctx)
if err != nil {
return Quad{}, err
}
clientWidth := layoutMetricsReply.LayoutViewport.ClientWidth
clientHeight := layoutMetricsReply.LayoutViewport.ClientHeight
quads := make([][]Quad, 0, len(contentQuadsReply.Quads))
for _, q := range contentQuadsReply.Quads {
quad := intersectQuadWithViewport(fromProtocolQuad(q), float64(clientWidth), float64(clientHeight))
if computeQuadArea(quad) > 1 {
quads = append(quads, quad)
}
}
if len(quads) == 0 {
return Quad{}, errors.New("node is either not visible or not an HTMLElement")
}
// Return the middle point of the first quad.
quad := quads[0]
var x float64
var y float64
for _, q := range quad {
x += q.X
y += q.Y
}
return Quad{
X: x / 4,
Y: y / 4,
}, nil
}
func GetClickablePointByNodeID(ctx context.Context, client *cdp.Client, nodeID dom.NodeID) (Quad, error) {
return getClickablePoint(ctx, client, dom.NewGetContentQuadsArgs().SetNodeID(nodeID))
}
func GetClickablePointByObjectID(ctx context.Context, client *cdp.Client, objectID runtime.RemoteObjectID) (Quad, error) {
return getClickablePoint(ctx, client, dom.NewGetContentQuadsArgs().SetObjectID(objectID))
}
func GetClickablePointByBackendID(ctx context.Context, client *cdp.Client, backendID dom.BackendNodeID) (Quad, error) {
return getClickablePoint(ctx, client, dom.NewGetContentQuadsArgs().SetBackendNodeID(backendID))
}

778
pkg/drivers/cdp/page.go Normal file
View File

@ -0,0 +1,778 @@
package cdp
import (
"context"
"encoding/json"
"hash/fnv"
"sync"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/emulation"
"github.com/mafredri/cdp/protocol/network"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/rpcc"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
"github.com/MontFerret/ferret/pkg/drivers/cdp/input"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type HTMLPage struct {
mu sync.Mutex
closed values.Boolean
logger *zerolog.Logger
conn *rpcc.Conn
client *cdp.Client
events *events.EventBroker
mouse *input.Mouse
keyboard *input.Keyboard
document *HTMLDocument
frames *common.LazyValue
}
func handleLoadError(logger *zerolog.Logger, client *cdp.Client) {
err := client.Page.Close(context.Background())
if err != nil {
logger.Warn().Timestamp().Err(err).Msg("failed to close document on load error")
}
}
func LoadHTMLPage(
ctx context.Context,
conn *rpcc.Conn,
params drivers.OpenPageParams,
) (*HTMLPage, error) {
logger := logging.FromContext(ctx)
if conn == nil {
return nil, core.Error(core.ErrMissedArgument, "connection")
}
client := cdp.NewClient(conn)
err := runBatch(
func() error {
return client.Page.Enable(ctx)
},
func() error {
return client.Page.SetLifecycleEventsEnabled(
ctx,
page.NewSetLifecycleEventsEnabledArgs(true),
)
},
func() error {
return client.DOM.Enable(ctx)
},
func() error {
return client.Runtime.Enable(ctx)
},
func() error {
ua := common.GetUserAgent(params.UserAgent)
logger.
Debug().
Timestamp().
Str("user-agent", ua).
Msg("using User-Agent")
// do not use custom user agent
if ua == "" {
return nil
}
return client.Emulation.SetUserAgentOverride(
ctx,
emulation.NewSetUserAgentOverrideArgs(ua),
)
},
func() error {
return client.Network.Enable(ctx, network.NewEnableArgs())
},
)
if err != nil {
return nil, err
}
err = client.Page.SetBypassCSP(ctx, page.NewSetBypassCSPArgs(true))
if err != nil {
return nil, err
}
if params.Cookies != nil {
cookies := make([]network.CookieParam, 0, len(params.Cookies))
for _, c := range params.Cookies {
cookies = append(cookies, fromDriverCookie(params.URL, c))
logger.
Debug().
Timestamp().
Str("cookie", c.Name).
Msg("set cookie")
}
err = client.Network.SetCookies(
ctx,
network.NewSetCookiesArgs(cookies),
)
if err != nil {
return nil, errors.Wrap(err, "failed to set cookies")
}
}
if params.Header != nil {
j, err := json.Marshal(params.Header)
if err != nil {
return nil, err
}
for k := range params.Header {
logger.
Debug().
Timestamp().
Str("header", k).
Msg("set header")
}
err = client.Network.SetExtraHTTPHeaders(
ctx,
network.NewSetExtraHTTPHeadersArgs(network.Headers(j)),
)
if err != nil {
return nil, errors.Wrap(err, "failed to set headers")
}
}
if params.URL != BlankPageURL && params.URL != "" {
repl, err := client.Page.Navigate(ctx, page.NewNavigateArgs(params.URL))
if err != nil {
return nil, errors.Wrap(err, "failed to load the page")
}
if repl.ErrorText != nil {
return nil, errors.Wrapf(errors.New(*repl.ErrorText), "failed to load the page: %s", params.URL)
}
err = events.WaitForLoadEvent(ctx, client)
if err != nil {
handleLoadError(logger, client)
return nil, errors.Wrap(err, "failed to load the page")
}
}
broker, err := events.CreateEventBroker(client)
if err != nil {
handleLoadError(logger, client)
return nil, errors.Wrap(err, "failed to create event events")
}
mouse := input.NewMouse(client)
keyboard := input.NewKeyboard(client)
doc, err := LoadRootHTMLDocument(ctx, logger, client, broker, mouse, keyboard)
if err != nil {
broker.StopAndClose()
handleLoadError(logger, client)
return nil, errors.Wrap(err, "failed to load root element")
}
return NewHTMLPage(
logger,
conn,
client,
broker,
mouse,
keyboard,
doc,
), nil
}
func NewHTMLPage(
logger *zerolog.Logger,
conn *rpcc.Conn,
client *cdp.Client,
broker *events.EventBroker,
mouse *input.Mouse,
keyboard *input.Keyboard,
document *HTMLDocument,
) *HTMLPage {
p := new(HTMLPage)
p.closed = values.False
p.logger = logger
p.conn = conn
p.client = client
p.events = broker
p.mouse = mouse
p.keyboard = keyboard
p.document = document
p.frames = common.NewLazyValue(p.unfoldFrames)
broker.AddEventListener(events.EventLoad, p.handlePageLoad)
broker.AddEventListener(events.EventError, p.handleError)
return p
}
func (p *HTMLPage) MarshalJSON() ([]byte, error) {
p.mu.Lock()
defer p.mu.Unlock()
return p.document.MarshalJSON()
}
func (p *HTMLPage) Type() core.Type {
return drivers.HTMLPageType
}
func (p *HTMLPage) String() string {
p.mu.Lock()
defer p.mu.Unlock()
return p.document.GetURL().String()
}
func (p *HTMLPage) Compare(other core.Value) int64 {
p.mu.Lock()
defer p.mu.Unlock()
tc := drivers.Compare(p.Type(), other.Type())
if tc != 0 {
return tc
}
cdpPage, ok := other.(*HTMLPage)
if !ok {
return 1
}
return p.document.GetURL().Compare(cdpPage.GetURL())
}
func (p *HTMLPage) Unwrap() interface{} {
p.mu.Lock()
defer p.mu.Unlock()
return p
}
func (p *HTMLPage) Hash() uint64 {
p.mu.Lock()
defer p.mu.Unlock()
h := fnv.New64a()
h.Write([]byte("CDP"))
h.Write([]byte(p.Type().String()))
h.Write([]byte(":"))
h.Write([]byte(p.document.GetURL()))
return h.Sum64()
}
func (p *HTMLPage) Copy() core.Value {
return values.None
}
func (p *HTMLPage) GetIn(ctx context.Context, path []core.Value) (core.Value, error) {
return common.GetInPage(ctx, p, path)
}
func (p *HTMLPage) SetIn(ctx context.Context, path []core.Value, value core.Value) error {
return common.SetInPage(ctx, p, path, value)
}
func (p *HTMLPage) Iterate(ctx context.Context) (core.Iterator, error) {
p.mu.Lock()
defer p.mu.Unlock()
return p.document.Iterate(ctx)
}
func (p *HTMLPage) Length() values.Int {
p.mu.Lock()
defer p.mu.Unlock()
return p.document.Length()
}
func (p *HTMLPage) Close() error {
p.mu.Lock()
defer p.mu.Unlock()
p.closed = values.True
err := p.events.Stop()
if err != nil {
p.logger.Warn().
Timestamp().
Str("url", p.document.GetURL().String()).
Err(err).
Msg("failed to stop event events")
}
err = p.events.Close()
if err != nil {
p.logger.Warn().
Timestamp().
Str("url", p.document.GetURL().String()).
Err(err).
Msg("failed to close event events")
}
err = p.document.Close()
if err != nil {
p.logger.Warn().
Timestamp().
Str("url", p.document.GetURL().String()).
Err(err).
Msg("failed to close root document")
}
err = p.client.Page.Close(context.Background())
if err != nil {
p.logger.Warn().
Timestamp().
Str("url", p.document.GetURL().String()).
Err(err).
Msg("failed to close browser page")
}
return p.conn.Close()
}
func (p *HTMLPage) IsClosed() values.Boolean {
p.mu.Lock()
defer p.mu.Unlock()
return p.closed
}
func (p *HTMLPage) GetURL() values.String {
p.mu.Lock()
defer p.mu.Unlock()
return p.document.GetURL()
}
func (p *HTMLPage) GetMainFrame() drivers.HTMLDocument {
p.mu.Lock()
defer p.mu.Unlock()
return p.document
}
func (p *HTMLPage) GetFrames(ctx context.Context) (*values.Array, error) {
p.mu.Lock()
defer p.mu.Unlock()
res, err := p.frames.Read(ctx)
if err != nil {
return nil, err
}
return res.(*values.Array).Clone().(*values.Array), nil
}
func (p *HTMLPage) GetFrame(ctx context.Context, idx values.Int) (core.Value, error) {
p.mu.Lock()
defer p.mu.Unlock()
res, err := p.frames.Read(ctx)
if err != nil {
return nil, err
}
return res.(*values.Array).Get(idx), nil
}
func (p *HTMLPage) GetCookies(ctx context.Context) (*values.Array, error) {
p.mu.Lock()
defer p.mu.Unlock()
repl, err := p.client.Network.GetAllCookies(ctx)
if err != nil {
return values.NewArray(0), err
}
if repl.Cookies == nil {
return values.NewArray(0), nil
}
cookies := values.NewArray(len(repl.Cookies))
for _, c := range repl.Cookies {
cookies.Push(toDriverCookie(c))
}
return cookies, nil
}
func (p *HTMLPage) SetCookies(ctx context.Context, cookies ...drivers.HTTPCookie) error {
p.mu.Lock()
defer p.mu.Unlock()
if len(cookies) == 0 {
return nil
}
params := make([]network.CookieParam, 0, len(cookies))
for _, c := range cookies {
params = append(params, fromDriverCookie(p.document.GetURL().String(), c))
}
return p.client.Network.SetCookies(ctx, network.NewSetCookiesArgs(params))
}
func (p *HTMLPage) DeleteCookies(ctx context.Context, cookies ...drivers.HTTPCookie) error {
p.mu.Lock()
defer p.mu.Unlock()
if len(cookies) == 0 {
return nil
}
var err error
for _, c := range cookies {
err = p.client.Network.DeleteCookies(ctx, fromDriverCookieDelete(p.document.GetURL().String(), c))
if err != nil {
break
}
}
return err
}
func (p *HTMLPage) PrintToPDF(ctx context.Context, params drivers.PDFParams) (values.Binary, error) {
p.mu.Lock()
defer p.mu.Unlock()
args := page.NewPrintToPDFArgs()
args.
SetLandscape(bool(params.Landscape)).
SetDisplayHeaderFooter(bool(params.DisplayHeaderFooter)).
SetPrintBackground(bool(params.PrintBackground)).
SetIgnoreInvalidPageRanges(bool(params.IgnoreInvalidPageRanges)).
SetPreferCSSPageSize(bool(params.PreferCSSPageSize))
if params.Scale > 0 {
args.SetScale(float64(params.Scale))
}
if params.PaperWidth > 0 {
args.SetPaperWidth(float64(params.PaperWidth))
}
if params.PaperHeight > 0 {
args.SetPaperHeight(float64(params.PaperHeight))
}
if params.MarginTop > 0 {
args.SetMarginTop(float64(params.MarginTop))
}
if params.MarginBottom > 0 {
args.SetMarginBottom(float64(params.MarginBottom))
}
if params.MarginRight > 0 {
args.SetMarginRight(float64(params.MarginRight))
}
if params.MarginLeft > 0 {
args.SetMarginLeft(float64(params.MarginLeft))
}
if params.PageRanges != values.EmptyString {
args.SetPageRanges(string(params.PageRanges))
}
if params.HeaderTemplate != values.EmptyString {
args.SetHeaderTemplate(string(params.HeaderTemplate))
}
if params.FooterTemplate != values.EmptyString {
args.SetFooterTemplate(string(params.FooterTemplate))
}
reply, err := p.client.Page.PrintToPDF(ctx, args)
if err != nil {
return values.NewBinary([]byte{}), err
}
return values.NewBinary(reply.Data), nil
}
func (p *HTMLPage) CaptureScreenshot(ctx context.Context, params drivers.ScreenshotParams) (values.Binary, error) {
p.mu.Lock()
defer p.mu.Unlock()
metrics, err := p.client.Page.GetLayoutMetrics(ctx)
if err != nil {
return values.NewBinary(nil), err
}
if params.Format == drivers.ScreenshotFormatJPEG && params.Quality < 0 && params.Quality > 100 {
params.Quality = 100
}
if params.X < 0 {
params.X = 0
}
if params.Y < 0 {
params.Y = 0
}
if params.Width <= 0 {
params.Width = values.Float(metrics.LayoutViewport.ClientWidth) - params.X
}
if params.Height <= 0 {
params.Height = values.Float(metrics.LayoutViewport.ClientHeight) - params.Y
}
clip := page.Viewport{
X: float64(params.X),
Y: float64(params.Y),
Width: float64(params.Width),
Height: float64(params.Height),
Scale: 1.0,
}
format := string(params.Format)
quality := int(params.Quality)
args := page.CaptureScreenshotArgs{
Format: &format,
Quality: &quality,
Clip: &clip,
}
reply, err := p.client.Page.CaptureScreenshot(ctx, &args)
if err != nil {
return values.NewBinary([]byte{}), err
}
return values.NewBinary(reply.Data), nil
}
func (p *HTMLPage) Navigate(ctx context.Context, url values.String) error {
p.mu.Lock()
defer p.mu.Unlock()
if url == "" {
url = BlankPageURL
}
repl, err := p.client.Page.Navigate(ctx, page.NewNavigateArgs(url.String()))
if err != nil {
return err
}
if repl.ErrorText != nil {
return errors.New(*repl.ErrorText)
}
return p.WaitForNavigation(ctx)
}
func (p *HTMLPage) NavigateBack(ctx context.Context, skip values.Int) (values.Boolean, error) {
p.mu.Lock()
defer p.mu.Unlock()
history, err := p.client.Page.GetNavigationHistory(ctx)
if err != nil {
return values.False, err
}
// we are in the beginning
if history.CurrentIndex == 0 {
return values.False, nil
}
if skip < 1 {
skip = 1
}
to := history.CurrentIndex - int(skip)
if to < 0 {
// TODO: Return error?
return values.False, nil
}
prev := history.Entries[to]
err = p.client.Page.NavigateToHistoryEntry(ctx, page.NewNavigateToHistoryEntryArgs(prev.ID))
if err != nil {
return values.False, err
}
err = p.WaitForNavigation(ctx)
if err != nil {
return values.False, err
}
return values.True, nil
}
func (p *HTMLPage) NavigateForward(ctx context.Context, skip values.Int) (values.Boolean, error) {
p.mu.Lock()
defer p.mu.Unlock()
history, err := p.client.Page.GetNavigationHistory(ctx)
if err != nil {
return values.False, err
}
length := len(history.Entries)
lastIndex := length - 1
// nowhere to go forward
if history.CurrentIndex == lastIndex {
return values.False, nil
}
if skip < 1 {
skip = 1
}
to := int(skip) + history.CurrentIndex
if to > lastIndex {
// TODO: Return error?
return values.False, nil
}
next := history.Entries[to]
err = p.client.Page.NavigateToHistoryEntry(ctx, page.NewNavigateToHistoryEntryArgs(next.ID))
if err != nil {
return values.False, err
}
err = p.WaitForNavigation(ctx)
if err != nil {
return values.False, err
}
return values.True, nil
}
func (p *HTMLPage) WaitForNavigation(ctx context.Context) error {
onEvent := make(chan struct{})
var once sync.Once
listener := func(_ context.Context, _ interface{}) {
once.Do(func() {
close(onEvent)
})
}
defer p.events.RemoveEventListener(events.EventLoad, listener)
p.events.AddEventListener(events.EventLoad, listener)
select {
case <-onEvent:
return nil
case <-ctx.Done():
return core.ErrTimeout
}
}
func (p *HTMLPage) handlePageLoad(ctx context.Context, _ interface{}) {
p.mu.Lock()
defer p.mu.Unlock()
nextDoc, err := LoadRootHTMLDocument(ctx, p.logger, p.client, p.events, p.mouse, p.keyboard)
if err != nil {
p.logger.Error().
Timestamp().
Err(err).
Msg("failed to load new root document after page load")
return
}
// close the prev document
err = p.document.Close()
if err != nil {
p.logger.Error().
Timestamp().
Err(err).
Msgf("failed to close root document: %s", p.document.GetURL())
}
// set the new root document
p.document = nextDoc
// reset all loaded frames
p.frames.Reset()
}
func (p *HTMLPage) handleError(_ context.Context, val interface{}) {
err, ok := val.(error)
if !ok {
return
}
p.logger.Error().
Timestamp().
Err(err).
Msg("unexpected error")
}
func (p *HTMLPage) unfoldFrames(ctx context.Context) (core.Value, error) {
res := values.NewArray(10)
err := common.CollectFrames(ctx, res, p.document)
if err != nil {
return nil, err
}
return res, nil
}

2
pkg/drivers/cdp/templates/attributes.go
View File

@ -2,7 +2,9 @@ package templates
import (
"fmt"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/runtime/values"
)

4
pkg/drivers/cdp/templates/wait_by_selector_all.go
View File

@ -2,11 +2,11 @@ package templates
import (
"fmt"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/drivers"
)
func WaitBySelectorAll(selector values.String, when drivers.WaitEvent, value core.Value, check string) string {

66
pkg/drivers/cdp/templates/xpath.go Normal file
View File

@ -0,0 +1,66 @@
package templates
const xPathTemplate = `
(element, expression) => {
const out = document.evaluate(
expression,
element,
null,
XPathResult.ANY_TYPE
);
let result;
switch (out.resultType) {
case XPathResult.UNORDERED_NODE_ITERATOR_TYPE:
case XPathResult.ORDERED_NODE_ITERATOR_TYPE: {
result = [];
let item;
while ((item = out.iterateNext())) {
result.push(item);
}
break;
}
case XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE:
case XPathResult.ORDERED_NODE_SNAPSHOT_TYPE: {
result = [];
for (let i = 0; i < out.snapshotLength; i++) {
const item = out.snapshotItem(i);
if (item != null) {
result.push(item);
}
}
break;
}
case XPathResult.NUMBER_TYPE: {
result = out.numberValue;
break;
}
case XPathResult.STRING_TYPE: {
result = out.stringValue;
break;
}
case XPathResult.BOOLEAN_TYPE: {
result = out.booleanValue;
break;
}
case XPathResult.ANY_UNORDERED_NODE_TYPE:
case XPathResult.FIRST_ORDERED_NODE_TYPE: {
result = out.singleNodeValue;
break;
}
default: {
break;
}
}
return result;
}
`
func XPath() string {
return xPathTemplate
}

33
pkg/drivers/common/frames.go Normal file
View File

@ -0,0 +1,33 @@
package common
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
func CollectFrames(ctx context.Context, receiver *values.Array, doc drivers.HTMLDocument) error {
receiver.Push(doc)
children, err := doc.GetChildDocuments(ctx)
if err != nil {
return err
}
children.ForEach(func(value core.Value, idx int) bool {
childDoc, ok := value.(drivers.HTMLDocument)
if !ok {
err = core.TypeError(value.Type(), drivers.HTMLDocumentType)
return false
}
return CollectFrames(ctx, receiver, childDoc) == nil
})
return nil
}

145
pkg/drivers/common/getter.go
View File

@ -9,9 +9,81 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
func GetInPage(ctx context.Context, page drivers.HTMLPage, path []core.Value) (core.Value, error) {
if len(path) == 0 {
return page, nil
}
segment := path[0]
if segment.Type() == types.String {
segment := segment.(values.String)
switch segment {
case "mainFrame", "document":
return GetInDocument(ctx, page.GetMainFrame(), path[1:])
case "frames":
if len(path) == 1 {
return page.GetFrames(ctx)
}
idx := path[1]
if !values.IsNumber(idx) {
return values.None, core.TypeError(idx.Type(), types.Int, types.Float)
}
value, err := page.GetFrame(ctx, values.ToInt(idx))
if err != nil {
return values.None, err
}
if len(path) == 2 {
return value, nil
}
frame, err := drivers.ToDocument(value)
if err != nil {
return values.None, err
}
return GetInDocument(ctx, frame, path[2:])
case "url", "URL":
return page.GetMainFrame().GetURL(), nil
case "cookies":
if len(path) == 1 {
return page.GetCookies(ctx)
}
switch idx := path[1].(type) {
case values.Int:
cookies, err := page.GetCookies(ctx)
if err != nil {
return values.None, err
}
return cookies.Get(idx), nil
default:
return values.None, core.TypeError(idx.Type(), types.Int)
}
case "isClosed":
return page.IsClosed(), nil
case "title":
return page.GetMainFrame().GetTitle(), nil
default:
return GetInDocument(ctx, page.GetMainFrame(), path)
}
}
return GetInDocument(ctx, page.GetMainFrame(), path)
}
func GetInDocument(ctx context.Context, doc drivers.HTMLDocument, path []core.Value) (core.Value, error) {
if path == nil || len(path) == 0 {
return values.None, nil
if len(path) == 0 {
return doc, nil
}
segment := path[0]
@ -22,38 +94,53 @@ func GetInDocument(ctx context.Context, doc drivers.HTMLDocument, path []core.Va
switch segment {
case "url", "URL":
return doc.GetURL(), nil
case "cookies":
case "title":
return doc.GetTitle(), nil
case "parent":
parent := doc.GetParentDocument()
if parent == nil {
return values.None, nil
}
if len(path) == 1 {
return doc.GetCookies(ctx)
return parent, nil
}
switch idx := path[1].(type) {
case values.Int:
cookies, err := doc.GetCookies(ctx)
return GetInDocument(ctx, parent, path[1:])
case "body", "head":
out := doc.QuerySelector(ctx, segment)
if err != nil {
return values.None, err
}
return cookies.Get(idx), nil
default:
return values.None, core.TypeError(idx.Type(), types.Int)
if out == values.None {
return out, nil
}
case "body":
return doc.QuerySelector(ctx, "body"), nil
case "head":
return doc.QuerySelector(ctx, "head"), nil
if len(path) == 1 {
return out, nil
}
el, err := drivers.ToElement(out)
if err != nil {
return values.None, err
}
return GetInElement(ctx, el, path[1:])
case "innerHTML":
return doc.GetElement().GetInnerHTML(ctx), nil
case "innerText":
return doc.GetElement().GetInnerText(ctx), nil
default:
return GetInNode(ctx, doc.DocumentElement(), path)
return GetInNode(ctx, doc.GetElement(), path)
}
}
return GetInNode(ctx, doc.DocumentElement(), path)
return GetInNode(ctx, doc.GetElement(), path)
}
func GetInElement(ctx context.Context, el drivers.HTMLElement, path []core.Value) (core.Value, error) {
if path == nil || len(path) == 0 {
return values.None, nil
if len(path) == 0 {
return el, nil
}
segment := path[0]
@ -63,9 +150,9 @@ func GetInElement(ctx context.Context, el drivers.HTMLElement, path []core.Value
switch segment {
case "innerText":
return el.InnerText(ctx), nil
return el.GetInnerText(ctx), nil
case "innerHTML":
return el.InnerHTML(ctx), nil
return el.GetInnerHTML(ctx), nil
case "value":
return el.GetValue(ctx), nil
case "attributes":
@ -97,8 +184,8 @@ func GetInElement(ctx context.Context, el drivers.HTMLElement, path []core.Value
}
func GetInNode(ctx context.Context, node drivers.HTMLNode, path []core.Value) (core.Value, error) {
if path == nil || len(path) == 0 {
return values.None, nil
if len(path) == 0 {
return node, nil
}
nt := node.Type()
@ -118,10 +205,12 @@ func GetInNode(ctx context.Context, node drivers.HTMLNode, path []core.Value) (c
segment := segment.(values.String)
switch segment {
case "isDetached":
return node.IsDetached(), nil
case "nodeType":
return node.NodeType(), nil
return node.GetNodeType(), nil
case "nodeName":
return node.NodeName(), nil
return node.GetNodeName(), nil
case "children":
children := node.GetChildNodes(ctx)

18
pkg/drivers/common/lazy.go
View File

@ -12,7 +12,7 @@ type (
LazyValueFactory func(ctx context.Context) (core.Value, error)
LazyValue struct {
sync.Mutex
mu sync.Mutex
factory LazyValueFactory
ready bool
value core.Value
@ -32,8 +32,8 @@ func NewLazyValue(factory LazyValueFactory) *LazyValue {
// Ready indicates whether the value is ready.
// @returns (Boolean) - Boolean value indicating whether the value is ready.
func (lv *LazyValue) Ready() bool {
lv.Lock()
defer lv.Unlock()
lv.mu.Lock()
defer lv.mu.Unlock()
return lv.ready
}
@ -42,8 +42,8 @@ func (lv *LazyValue) Ready() bool {
// Not thread safe. Should not mutated.
// @returns (Value) - Underlying value if successfully loaded, otherwise error
func (lv *LazyValue) Read(ctx context.Context) (core.Value, error) {
lv.Lock()
defer lv.Unlock()
lv.mu.Lock()
defer lv.mu.Unlock()
if !lv.ready {
lv.load(ctx)
@ -56,8 +56,8 @@ func (lv *LazyValue) Read(ctx context.Context) (core.Value, error) {
// Loads a value if it's not ready.
// Thread safe.
func (lv *LazyValue) Write(ctx context.Context, writer func(v core.Value, err error)) {
lv.Lock()
defer lv.Unlock()
lv.mu.Lock()
defer lv.mu.Unlock()
if !lv.ready {
lv.load(ctx)
@ -69,8 +69,8 @@ func (lv *LazyValue) Write(ctx context.Context, writer func(v core.Value, err er
// Reset resets the storage.
// Next call of Read will trigger the factory function again.
func (lv *LazyValue) Reset() {
lv.Lock()
defer lv.Unlock()
lv.mu.Lock()
defer lv.mu.Unlock()
lv.ready = false
lv.value = values.None

25
pkg/drivers/common/setter.go
View File

@ -9,31 +9,24 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
func SetInDocument(ctx context.Context, doc drivers.HTMLDocument, path []core.Value, value core.Value) error {
if path == nil || len(path) == 0 {
func SetInPage(ctx context.Context, page drivers.HTMLPage, path []core.Value, value core.Value) error {
if len(path) == 0 {
return nil
}
segment := path[0]
return SetInDocument(ctx, page.GetMainFrame(), path, value)
}
if segment.Type() == types.String {
segment := segment.(values.String)
switch segment {
case "url", "URL":
return doc.SetURL(ctx, values.NewString(value.String()))
case "cookies":
default:
return SetInNode(ctx, doc, path, value)
}
func SetInDocument(ctx context.Context, doc drivers.HTMLDocument, path []core.Value, value core.Value) error {
if len(path) == 0 {
return nil
}
return SetInNode(ctx, doc, path, value)
}
func SetInElement(ctx context.Context, el drivers.HTMLElement, path []core.Value, value core.Value) error {
if path == nil || len(path) == 0 {
if len(path) == 0 {
return nil
}
@ -115,7 +108,7 @@ func SetInElement(ctx context.Context, el drivers.HTMLElement, path []core.Value
}
func SetInNode(_ context.Context, _ drivers.HTMLNode, path []core.Value, _ core.Value) error {
if path == nil || len(path) == 0 {
if len(path) == 0 {
return nil
}

5
pkg/drivers/common/styles.go
View File

@ -3,12 +3,13 @@ package common
import (
"bytes"
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
"strconv"
"strings"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/gorilla/css/scanner"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
func DeserializeStyles(input values.String) (*values.Object, error) {

23
pkg/drivers/common/types.go
View File

@ -1,8 +1,10 @@
package common
import "golang.org/x/net/html"
import (
"golang.org/x/net/html"
)
func ToHTMLType(nt html.NodeType) int {
func FromHTMLType(nt html.NodeType) int {
switch nt {
case html.DocumentNode:
return 9
@ -18,3 +20,20 @@ func ToHTMLType(nt html.NodeType) int {
return 0
}
func ToHTMLType(input int) html.NodeType {
switch input {
case 1:
return html.ElementNode
case 3:
return html.TextNode
case 8:
return html.CommentNode
case 9:
return html.DocumentNode
case 10:
return html.DoctypeNode
default:
return html.ErrorNode
}
}

24
pkg/drivers/cookie.go
View File

@ -38,6 +38,17 @@ const (
SameSiteStrictMode
)
func (s SameSite) String() string {
switch s {
case SameSiteLaxMode:
return "Lax"
case SameSiteStrictMode:
return "Strict"
default:
return ""
}
}
func (c HTTPCookie) Type() core.Type {
return HTTPCookieType
}
@ -119,7 +130,7 @@ func (c HTTPCookie) Hash() uint64 {
h.Write([]byte(strconv.Itoa(c.MaxAge)))
h.Write([]byte(fmt.Sprintf("%t", c.Secure)))
h.Write([]byte(fmt.Sprintf("%t", c.HTTPOnly)))
h.Write([]byte(strconv.Itoa(int(c.SameSite))))
h.Write([]byte(c.SameSite.String()))
return h.Sum64()
}
@ -138,7 +149,7 @@ func (c HTTPCookie) MarshalJSON() ([]byte, error) {
"max_age": c.MaxAge,
"secure": c.Secure,
"http_only": c.HTTPOnly,
"same_site": c.SameSite,
"same_site": c.SameSite.String(),
}
out, err := json.Marshal(v)
@ -181,14 +192,7 @@ func (c HTTPCookie) GetIn(_ context.Context, path []core.Value) (core.Value, err
case "httpOnly":
return values.NewBoolean(c.HTTPOnly), nil
case "sameSite":
switch c.SameSite {
case SameSiteLaxMode:
return values.NewString("Lax"), nil
case SameSiteStrictMode:
return values.NewString("Strict"), nil
default:
return values.EmptyString, nil
}
return values.NewString(c.SameSite.String()), nil
default:
return values.None, nil
}

33
pkg/drivers/cookie_test.go Normal file
View File

@ -0,0 +1,33 @@
package drivers_test
import (
"testing"
. "github.com/smartystreets/goconvey/convey"
"github.com/MontFerret/ferret/pkg/drivers"
)
func TestHTTPCookie(t *testing.T) {
Convey("HTTPCookie", t, func() {
Convey(".MarshalJSON", func() {
Convey("Should serialize cookie values", func() {
cookie := &drivers.HTTPCookie{}
cookie.Name = "test_cookie"
cookie.Value = "test_value"
cookie.Domain = "montferret.dev"
cookie.HTTPOnly = true
cookie.MaxAge = 320
cookie.Path = "/"
cookie.SameSite = drivers.SameSiteLaxMode
cookie.Secure = true
out, err := cookie.MarshalJSON()
So(err, ShouldBeNil)
So(string(out), ShouldEqual, `{"domain":"montferret.dev","expires":"0001-01-01T00:00:00Z","http_only":true,"max_age":320,"name":"test_cookie","path":"/","same_site":"Lax","secure":true,"value":"test_value"}`)
})
})
})
}

4
pkg/drivers/driver.go
View File

@ -18,7 +18,7 @@ type (
drivers map[string]Driver
}
LoadDocumentParams struct {
OpenPageParams struct {
URL string
UserAgent string
KeepCookies bool
@ -29,7 +29,7 @@ type (
Driver interface {
io.Closer
Name() string
LoadDocument(ctx context.Context, params LoadDocumentParams) (HTMLDocument, error)
Open(ctx context.Context, params OpenPageParams) (HTMLPage, error)
}
)

4
pkg/drivers/header.go
View File

@ -15,7 +15,7 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// HTTPCookie HTTPCookie object
// HTTPHeader HTTP header object
type HTTPHeader map[string][]string
func (h HTTPHeader) Type() core.Type {
@ -101,7 +101,7 @@ func (h HTTPHeader) Copy() core.Value {
}
func (h HTTPHeader) MarshalJSON() ([]byte, error) {
out, err := json.Marshal(h)
out, err := json.Marshal(map[string][]string(h))
if err != nil {
return nil, err

26
pkg/drivers/header_test.go Normal file
View File

@ -0,0 +1,26 @@
package drivers_test
import (
"github.com/MontFerret/ferret/pkg/drivers"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func TestHTTPHeader(t *testing.T) {
Convey("HTTPHeader", t, func() {
Convey(".MarshalJSON", func() {
Convey("Should serialize header values", func() {
headers := make(drivers.HTTPHeader)
headers["content-encoding"] = []string{"gzip"}
headers["content-type"] = []string{"text/html", "charset=utf-8"}
out, err := headers.MarshalJSON()
So(err, ShouldBeNil)
So(string(out), ShouldEqual, `{"content-encoding":["gzip"],"content-type":["text/html","charset=utf-8"]}`)
})
})
})
}

45
pkg/drivers/helpers.go
View File

@ -2,8 +2,53 @@ package drivers
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
)
func WithDefaultTimeout(ctx context.Context) (context.Context, context.CancelFunc) {
return context.WithTimeout(ctx, DefaultTimeout)
}
func ToPage(value core.Value) (HTMLPage, error) {
err := core.ValidateType(value, HTMLPageType)
if err != nil {
return nil, err
}
return value.(HTMLPage), nil
}
func ToDocument(value core.Value) (HTMLDocument, error) {
switch v := value.(type) {
case HTMLPage:
return v.GetMainFrame(), nil
case HTMLDocument:
return v, nil
default:
return nil, core.TypeError(
value.Type(),
HTMLPageType,
HTMLDocumentType,
)
}
}
func ToElement(value core.Value) (HTMLElement, error) {
switch v := value.(type) {
case HTMLPage:
return v.GetMainFrame().GetElement(), nil
case HTMLDocument:
return v.GetElement(), nil
case HTMLElement:
return v, nil
default:
return nil, core.TypeError(
value.Type(),
HTMLPageType,
HTMLDocumentType,
HTMLElementType,
)
}
}

120
pkg/drivers/http/document.go
View File

@ -12,17 +12,25 @@ import (
)
type HTMLDocument struct {
docNode *goquery.Document
element drivers.HTMLElement
url values.String
cookies []drivers.HTTPCookie
doc *goquery.Document
element drivers.HTMLElement
url values.String
parent drivers.HTMLDocument
children *values.Array
}
func NewRootHTMLDocument(
node *goquery.Document,
url string,
) (*HTMLDocument, error) {
return NewHTMLDocument(node, url, nil)
}
func NewHTMLDocument(
node *goquery.Document,
url string,
cookies []drivers.HTTPCookie,
) (drivers.HTMLDocument, error) {
parent drivers.HTMLDocument,
) (*HTMLDocument, error) {
if url == "" {
return nil, core.Error(core.ErrMissedArgument, "document url")
}
@ -37,7 +45,21 @@ func NewHTMLDocument(
return nil, err
}
return &HTMLDocument{node, el, values.NewString(url), cookies}, nil
doc := new(HTMLDocument)
doc.doc = node
doc.element = el
doc.parent = parent
doc.url = values.NewString(url)
doc.children = values.NewArray(10)
frames := node.Find("iframe")
frames.Each(func(i int, selection *goquery.Selection) {
child, _ := NewHTMLDocument(goquery.NewDocumentFromNode(selection.Nodes[0]), selection.AttrOr("src", url), doc)
doc.children.Push(child)
})
return doc, nil
}
func (doc *HTMLDocument) MarshalJSON() ([]byte, error) {
@ -49,7 +71,7 @@ func (doc *HTMLDocument) Type() core.Type {
}
func (doc *HTMLDocument) String() string {
str, err := doc.docNode.Html()
str, err := doc.doc.Html()
if err != nil {
return ""
@ -70,7 +92,7 @@ func (doc *HTMLDocument) Compare(other core.Value) int64 {
}
func (doc *HTMLDocument) Unwrap() interface{} {
return doc.docNode
return doc.doc
}
func (doc *HTMLDocument) Hash() uint64 {
@ -84,7 +106,7 @@ func (doc *HTMLDocument) Hash() uint64 {
}
func (doc *HTMLDocument) Copy() core.Value {
cp, err := NewHTMLDocument(doc.docNode, string(doc.url), doc.cookies)
cp, err := NewHTMLDocument(doc.doc, string(doc.url), doc.parent)
if err != nil {
return values.None
@ -94,27 +116,17 @@ func (doc *HTMLDocument) Copy() core.Value {
}
func (doc *HTMLDocument) Clone() core.Value {
var cookies []drivers.HTTPCookie
if doc.cookies != nil {
cookies = make([]drivers.HTTPCookie, 0, len(doc.cookies))
for i, c := range doc.cookies {
cookies[i] = c
}
}
cp, err := NewHTMLDocument(goquery.CloneDocument(doc.docNode), string(doc.url), cookies)
cloned, err := NewHTMLDocument(doc.doc, doc.url.String(), doc.parent)
if err != nil {
return values.None
}
return cp
return cloned
}
func (doc *HTMLDocument) Length() values.Int {
return values.NewInt(doc.docNode.Length())
return values.NewInt(doc.doc.Length())
}
func (doc *HTMLDocument) Iterate(_ context.Context) (core.Iterator, error) {
@ -129,11 +141,11 @@ func (doc *HTMLDocument) SetIn(ctx context.Context, path []core.Value, value cor
return common.SetInDocument(ctx, doc, path, value)
}
func (doc *HTMLDocument) NodeType() values.Int {
func (doc *HTMLDocument) GetNodeType() values.Int {
return 9
}
func (doc *HTMLDocument) NodeName() values.String {
func (doc *HTMLDocument) GetNodeName() values.String {
return "#document"
}
@ -161,50 +173,38 @@ func (doc *HTMLDocument) ExistsBySelector(ctx context.Context, selector values.S
return doc.element.ExistsBySelector(ctx, selector)
}
func (doc *HTMLDocument) DocumentElement() drivers.HTMLElement {
return doc.element
func (doc *HTMLDocument) XPath(ctx context.Context, expression values.String) (core.Value, error) {
return doc.element.XPath(ctx, expression)
}
func (doc *HTMLDocument) GetURL() core.Value {
func (doc *HTMLDocument) IsDetached() values.Boolean {
return values.False
}
func (doc *HTMLDocument) GetTitle() values.String {
title := doc.doc.Find("head > title")
return values.NewString(title.Text())
}
func (doc *HTMLDocument) GetChildDocuments(_ context.Context) (*values.Array, error) {
return doc.children.Clone().(*values.Array), nil
}
func (doc *HTMLDocument) GetURL() values.String {
return doc.url
}
func (doc *HTMLDocument) SetURL(_ context.Context, _ values.String) error {
return core.ErrInvalidOperation
func (doc *HTMLDocument) GetElement() drivers.HTMLElement {
return doc.element
}
func (doc *HTMLDocument) GetCookies(_ context.Context) (*values.Array, error) {
if doc.cookies == nil {
return values.NewArray(0), nil
}
arr := values.NewArray(len(doc.cookies))
for _, c := range doc.cookies {
arr.Push(c)
}
return arr, nil
func (doc *HTMLDocument) GetName() values.String {
return ""
}
func (doc *HTMLDocument) SetCookies(_ context.Context, _ ...drivers.HTTPCookie) error {
return core.ErrNotSupported
}
func (doc *HTMLDocument) DeleteCookies(_ context.Context, _ ...drivers.HTTPCookie) error {
return core.ErrNotSupported
}
func (doc *HTMLDocument) Navigate(_ context.Context, _ values.String) error {
return core.ErrNotSupported
}
func (doc *HTMLDocument) NavigateBack(_ context.Context, _ values.Int) (values.Boolean, error) {
return false, core.ErrNotSupported
}
func (doc *HTMLDocument) NavigateForward(_ context.Context, _ values.Int) (values.Boolean, error) {
return false, core.ErrNotSupported
func (doc *HTMLDocument) GetParentDocument() drivers.HTMLDocument {
return doc.parent
}
func (doc *HTMLDocument) ClickBySelector(_ context.Context, _ values.String) (values.Boolean, error) {

8
pkg/drivers/http/document_test.go
View File

@ -2,10 +2,12 @@ package http_test
import (
"bytes"
"testing"
"github.com/MontFerret/ferret/pkg/drivers/http"
"github.com/PuerkitoBio/goquery"
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func TestDocument(t *testing.T) {
@ -218,7 +220,7 @@ func TestDocument(t *testing.T) {
</footer>
<svg xmlns="http://www.w3.org/2000/svg" width="348" height="225" viewBox="0 0 348 225" preserveAspectRatio="none" style="display: none; visibility: hidden; position: absolute; top: -100%; left: -100%;"><defs><style type="text/css"></style></defs><text x="0" y="17" style="font-weight:bold;font-size:17pt;font-family:Arial, Helvetica, Open Sans, sans-serif">Thumbnail</text></svg></body></html>
`
Convey(".NodeType", t, func() {
Convey(".GetNodeType", t, func() {
Convey("Should serialize a boolean value", func() {
buff := bytes.NewBuffer([]byte(doc))
@ -232,7 +234,7 @@ func TestDocument(t *testing.T) {
So(err, ShouldBeNil)
So(el.NodeType(), ShouldEqual, 9)
So(el.GetNodeType(), ShouldEqual, 9)
})
})
}

16
pkg/drivers/http/driver.go
View File

@ -62,7 +62,7 @@ func (drv *Driver) Name() string {
return DriverName
}
func (drv *Driver) LoadDocument(ctx context.Context, params drivers.LoadDocumentParams) (drivers.HTMLDocument, error) {
func (drv *Driver) Open(ctx context.Context, params drivers.OpenPageParams) (drivers.HTMLPage, error) {
req, err := http.NewRequest(http.MethodGet, params.URL, nil)
if err != nil {
@ -119,6 +119,10 @@ func (drv *Driver) LoadDocument(ctx context.Context, params drivers.LoadDocument
Str("user-agent", ua).
Msg("using User-Agent")
if ua != "" {
req.Header.Set("User-Agent", ua)
}
resp, err := drv.client.Do(req)
if err != nil {
@ -127,16 +131,20 @@ func (drv *Driver) LoadDocument(ctx context.Context, params drivers.LoadDocument
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, errors.New(resp.Status)
}
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse a document %s", params.URL)
}
return NewHTMLDocument(doc, params.URL, params.Cookies)
return NewHTMLPage(doc, params.URL, params.Cookies)
}
func (drv *Driver) ParseDocument(_ context.Context, str values.String) (drivers.HTMLDocument, error) {
func (drv *Driver) Parse(_ context.Context, str values.String) (drivers.HTMLPage, error) {
buf := bytes.NewBuffer([]byte(str))
doc, err := goquery.NewDocumentFromReader(buf)
@ -145,7 +153,7 @@ func (drv *Driver) ParseDocument(_ context.Context, str values.String) (drivers.
return nil, errors.Wrap(err, "failed to parse a document")
}
return NewHTMLDocument(doc, "#string", nil)
return NewHTMLPage(doc, "#blank", nil)
}
func (drv *Driver) Close() error {

71
pkg/drivers/http/element.go
View File

@ -3,6 +3,8 @@ package http
import (
"context"
"encoding/json"
"fmt"
"github.com/antchfx/htmlquery"
"hash/fnv"
"strings"
@ -10,7 +12,9 @@ import (
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/PuerkitoBio/goquery"
"github.com/antchfx/xpath"
)
type HTMLElement struct {
@ -29,7 +33,7 @@ func NewHTMLElement(node *goquery.Selection) (drivers.HTMLElement, error) {
}
func (el *HTMLElement) MarshalJSON() ([]byte, error) {
return json.Marshal(el.InnerText(context.Background()).String())
return json.Marshal(el.GetInnerText(context.Background()).String())
}
func (el *HTMLElement) Type() core.Type {
@ -37,7 +41,7 @@ func (el *HTMLElement) Type() core.Type {
}
func (el *HTMLElement) String() string {
return el.InnerHTML(context.Background()).String()
return el.GetInnerHTML(context.Background()).String()
}
func (el *HTMLElement) Compare(other core.Value) int64 {
@ -48,7 +52,7 @@ func (el *HTMLElement) Compare(other core.Value) int64 {
ctx, fn := drivers.WithDefaultTimeout(context.Background())
defer fn()
return el.InnerHTML(ctx).Compare(other.InnerHTML(ctx))
return el.GetInnerHTML(ctx).Compare(other.GetInnerHTML(ctx))
default:
return drivers.Compare(el.Type(), other.Type())
}
@ -80,21 +84,25 @@ func (el *HTMLElement) Copy() core.Value {
return c
}
func (el *HTMLElement) NodeType() values.Int {
func (el *HTMLElement) IsDetached() values.Boolean {
return values.True
}
func (el *HTMLElement) GetNodeType() values.Int {
nodes := el.selection.Nodes
if len(nodes) == 0 {
return 0
}
return values.NewInt(common.ToHTMLType(nodes[0].Type))
return values.NewInt(common.FromHTMLType(nodes[0].Type))
}
func (el *HTMLElement) Close() error {
return nil
}
func (el *HTMLElement) NodeName() values.String {
func (el *HTMLElement) GetNodeName() values.String {
return values.NewString(goquery.NodeName(el.selection))
}
@ -122,11 +130,11 @@ func (el *HTMLElement) SetValue(_ context.Context, value core.Value) error {
return nil
}
func (el *HTMLElement) InnerText(_ context.Context) values.String {
func (el *HTMLElement) GetInnerText(_ context.Context) values.String {
return values.NewString(el.selection.Text())
}
func (el *HTMLElement) InnerHTML(_ context.Context) values.String {
func (el *HTMLElement) GetInnerHTML(_ context.Context) values.String {
h, err := el.selection.Html()
if err != nil {
@ -304,6 +312,53 @@ func (el *HTMLElement) QuerySelectorAll(_ context.Context, selector values.Strin
return arr
}
func (el *HTMLElement) XPath(_ context.Context, expression values.String) (core.Value, error) {
h, err := outerHTML(el.selection)
if err != nil {
return values.None, err
}
exp, err := xpath.Compile(expression.String())
if err != nil {
return values.None, err
}
rootNode, err := htmlquery.Parse(strings.NewReader(h))
if err != nil {
return values.None, err
}
fmt.Println(htmlquery.OutputHTML(rootNode, true))
out := exp.Evaluate(htmlquery.CreateXPathNavigator(rootNode))
switch res := out.(type) {
case *xpath.NodeIterator:
items := values.NewArray(10)
for {
if !res.MoveNext() {
break
}
item, err := parseXPathNode(res.Current().(*htmlquery.NodeNavigator))
if err != nil {
return values.None, err
}
items.Push(item)
}
return items, nil
default:
return values.Parse(res), nil
}
}
func (el *HTMLElement) InnerHTMLBySelector(_ context.Context, selector values.String) values.String {
selection := el.selection.Find(selector.String())

16
pkg/drivers/http/element_test.go
View File

@ -244,7 +244,7 @@ func TestElement(t *testing.T) {
<svg xmlns="http://www.w3.org/2000/svg" width="348" height="225" viewBox="0 0 348 225" preserveAspectRatio="none" style="display: none; visibility: hidden; position: absolute; top: -100%; left: -100%;"><defs><style type="text/css"></style></defs><text x="0" y="17" style="font-weight:bold;font-size:17pt;font-family:Arial, Helvetica, Open Sans, sans-serif">Thumbnail</text></svg></body></html>
`
Convey(".NodeType", t, func() {
Convey(".GetNodeType", t, func() {
buff := bytes.NewBuffer([]byte(doc))
buff.Write([]byte(doc))
@ -257,10 +257,10 @@ func TestElement(t *testing.T) {
So(err, ShouldBeNil)
So(el.NodeType(), ShouldEqual, 1)
So(el.GetNodeType(), ShouldEqual, 1)
})
Convey(".NodeName", t, func() {
Convey(".GetNodeName", t, func() {
buff := bytes.NewBuffer([]byte(doc))
buff.Write([]byte(doc))
@ -273,7 +273,7 @@ func TestElement(t *testing.T) {
So(err, ShouldBeNil)
So(el.NodeName(), ShouldEqual, "body")
So(el.GetNodeName(), ShouldEqual, "body")
})
Convey(".Length", t, func() {
@ -327,7 +327,7 @@ func TestElement(t *testing.T) {
So(v, ShouldEqual, "find")
})
Convey(".InnerText", t, func() {
Convey(".GetInnerText", t, func() {
buff := bytes.NewBuffer([]byte(`
<html>
<head></head>
@ -349,7 +349,7 @@ func TestElement(t *testing.T) {
So(err, ShouldBeNil)
v := el.InnerText(context.Background())
v := el.GetInnerText(context.Background())
So(v, ShouldEqual, "Ferret")
})
@ -376,7 +376,7 @@ func TestElement(t *testing.T) {
So(err, ShouldBeNil)
v := el.InnerHTML(context.Background())
v := el.GetInnerHTML(context.Background())
So(v, ShouldEqual, "<h2>Ferret</h2>")
})
@ -396,7 +396,7 @@ func TestElement(t *testing.T) {
So(found, ShouldNotEqual, values.None)
v := found.(drivers.HTMLNode).NodeName()
v := found.(drivers.HTMLNode).GetNodeName()
So(err, ShouldBeNil)

47
pkg/drivers/http/helpers.go Normal file
View File

@ -0,0 +1,47 @@
package http
import (
"bytes"
"golang.org/x/net/html"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/PuerkitoBio/goquery"
"github.com/antchfx/htmlquery"
"github.com/antchfx/xpath"
)
func parseXPathNode(nav *htmlquery.NodeNavigator) (core.Value, error) {
node := nav.Current()
if node == nil {
return values.None, nil
}
switch nav.NodeType() {
case xpath.ElementNode:
return NewHTMLElement(&goquery.Selection{Nodes: []*html.Node{node}})
case xpath.RootNode:
url := htmlquery.SelectAttr(node, "url")
return NewHTMLDocument(goquery.NewDocumentFromNode(node), url, nil)
default:
return values.Parse(node.Data), nil
}
}
func outerHTML(s *goquery.Selection) (string, error) {
var buf bytes.Buffer
if len(s.Nodes) > 0 {
c := s.Nodes[0]
err := html.Render(&buf, c)
if err != nil {
return "", err
}
}
return buf.String(), nil
}

199
pkg/drivers/http/page.go Normal file
View File

@ -0,0 +1,199 @@
package http
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/PuerkitoBio/goquery"
"hash/fnv"
)
type HTMLPage struct {
document *HTMLDocument
cookies []drivers.HTTPCookie
frames *values.Array
}
func NewHTMLPage(
qdoc *goquery.Document,
url string,
cookies []drivers.HTTPCookie,
) (*HTMLPage, error) {
doc, err := NewRootHTMLDocument(qdoc, url)
if err != nil {
return nil, err
}
p := new(HTMLPage)
p.document = doc
p.cookies = cookies
p.frames = nil
return p, nil
}
func (p *HTMLPage) MarshalJSON() ([]byte, error) {
return p.document.MarshalJSON()
}
func (p *HTMLPage) Type() core.Type {
return drivers.HTMLPageType
}
func (p *HTMLPage) String() string {
return p.document.GetURL().String()
}
func (p *HTMLPage) Compare(other core.Value) int64 {
tc := drivers.Compare(p.Type(), other.Type())
if tc != 0 {
return tc
}
httpPage, ok := other.(*HTMLPage)
if !ok {
return 1
}
return p.document.GetURL().Compare(httpPage.GetURL())
}
func (p *HTMLPage) Unwrap() interface{} {
return p.document
}
func (p *HTMLPage) Hash() uint64 {
h := fnv.New64a()
h.Write([]byte("HTTP"))
h.Write([]byte(p.Type().String()))
h.Write([]byte(":"))
h.Write([]byte(p.document.GetURL()))
return h.Sum64()
}
func (p *HTMLPage) Copy() core.Value {
page, err := NewHTMLPage(p.document.doc, p.document.GetURL().String(), p.cookies[:])
if err != nil {
return values.None
}
return page
}
func (p *HTMLPage) Iterate(ctx context.Context) (core.Iterator, error) {
return p.document.Iterate(ctx)
}
func (p *HTMLPage) GetIn(ctx context.Context, path []core.Value) (core.Value, error) {
return common.GetInPage(ctx, p, path)
}
func (p *HTMLPage) SetIn(ctx context.Context, path []core.Value, value core.Value) error {
return common.SetInPage(ctx, p, path, value)
}
func (p *HTMLPage) Length() values.Int {
return p.document.Length()
}
func (p *HTMLPage) Close() error {
return nil
}
func (p *HTMLPage) IsClosed() values.Boolean {
return values.True
}
func (p *HTMLPage) GetURL() values.String {
return p.document.GetURL()
}
func (p *HTMLPage) GetMainFrame() drivers.HTMLDocument {
return p.document
}
func (p *HTMLPage) GetFrames(ctx context.Context) (*values.Array, error) {
if p.frames == nil {
arr := values.NewArray(10)
err := common.CollectFrames(ctx, arr, p.document)
if err != nil {
return values.NewArray(0), err
}
p.frames = arr
}
return p.frames, nil
}
func (p *HTMLPage) GetFrame(ctx context.Context, idx values.Int) (core.Value, error) {
if p.frames == nil {
arr := values.NewArray(10)
err := common.CollectFrames(ctx, arr, p.document)
if err != nil {
return values.None, err
}
p.frames = arr
}
return p.frames.Get(idx), nil
}
func (p *HTMLPage) GetCookies(_ context.Context) (*values.Array, error) {
if p.cookies == nil {
return values.NewArray(0), nil
}
arr := values.NewArray(len(p.cookies))
for _, c := range p.cookies {
arr.Push(c)
}
return arr, nil
}
func (p *HTMLPage) SetCookies(_ context.Context, _ ...drivers.HTTPCookie) error {
return core.ErrNotSupported
}
func (p *HTMLPage) DeleteCookies(_ context.Context, _ ...drivers.HTTPCookie) error {
return core.ErrNotSupported
}
func (p *HTMLPage) PrintToPDF(_ context.Context, _ drivers.PDFParams) (values.Binary, error) {
return nil, core.ErrNotSupported
}
func (p *HTMLPage) CaptureScreenshot(_ context.Context, _ drivers.ScreenshotParams) (values.Binary, error) {
return nil, core.ErrNotSupported
}
func (p *HTMLPage) WaitForNavigation(_ context.Context) error {
return core.ErrNotSupported
}
func (p *HTMLPage) Navigate(_ context.Context, _ values.String) error {
return core.ErrNotSupported
}
func (p *HTMLPage) NavigateBack(_ context.Context, _ values.Int) (values.Boolean, error) {
return false, core.ErrNotSupported
}
func (p *HTMLPage) NavigateForward(_ context.Context, _ values.Int) (values.Boolean, error) {
return false, core.ErrNotSupported
}

2
pkg/drivers/type.go
View File

@ -7,6 +7,7 @@ var (
HTTPCookieType = core.NewType("HTTPCookie")
HTMLElementType = core.NewType("HTMLElement")
HTMLDocumentType = core.NewType("HTMLDocument")
HTMLPageType = core.NewType("HTMLPageType")
)
// Comparison table of builtin types
@ -15,6 +16,7 @@ var typeComparisonTable = map[core.Type]uint64{
HTTPCookieType: 1,
HTMLElementType: 2,
HTMLDocumentType: 3,
HTMLPageType: 4,
}
func Compare(first, second core.Type) int64 {

79
pkg/drivers/value.go
View File

@ -24,9 +24,11 @@ type (
collections.Measurable
io.Closer
NodeType() values.Int
IsDetached() values.Boolean
NodeName() values.String
GetNodeType() values.Int
GetNodeName() values.String
GetChildNodes(ctx context.Context) core.Value
@ -39,15 +41,17 @@ type (
CountBySelector(ctx context.Context, selector values.String) values.Int
ExistsBySelector(ctx context.Context, selector values.String) values.Boolean
XPath(ctx context.Context, expression values.String) (core.Value, error)
}
// HTMLElement is the most general base interface which most objects in a Document implement.
// HTMLElement is the most general base interface which most objects in a GetMainFrame implement.
HTMLElement interface {
HTMLNode
InnerText(ctx context.Context) values.String
GetInnerText(ctx context.Context) values.String
InnerHTML(ctx context.Context) values.String
GetInnerHTML(ctx context.Context) values.String
GetValue(ctx context.Context) core.Value
@ -98,28 +102,20 @@ type (
WaitForClass(ctx context.Context, class values.String, when WaitEvent) error
}
// The Document interface represents any web page loaded in the browser
// and serves as an entry point into the web page's content, which is the DOM tree.
HTMLDocument interface {
HTMLNode
DocumentElement() HTMLElement
GetTitle() values.String
GetURL() core.Value
GetElement() HTMLElement
SetURL(ctx context.Context, url values.String) error
GetURL() values.String
GetCookies(ctx context.Context) (*values.Array, error)
GetName() values.String
SetCookies(ctx context.Context, cookies ...HTTPCookie) error
GetParentDocument() HTMLDocument
DeleteCookies(ctx context.Context, cookies ...HTTPCookie) error
Navigate(ctx context.Context, url values.String) error
NavigateBack(ctx context.Context, skip values.Int) (values.Boolean, error)
NavigateForward(ctx context.Context, skip values.Int) (values.Boolean, error)
GetChildDocuments(ctx context.Context) (*values.Array, error)
ClickBySelector(ctx context.Context, selector values.String) (values.Boolean, error)
@ -129,10 +125,6 @@ type (
SelectBySelector(ctx context.Context, selector values.String, value *values.Array) (*values.Array, error)
PrintToPDF(ctx context.Context, params PDFParams) (values.Binary, error)
CaptureScreenshot(ctx context.Context, params ScreenshotParams) (values.Binary, error)
ScrollTop(ctx context.Context) error
ScrollBottom(ctx context.Context) error
@ -145,8 +137,6 @@ type (
MoveMouseBySelector(ctx context.Context, selector values.String) error
WaitForNavigation(ctx context.Context) error
WaitForElement(ctx context.Context, selector values.String, when WaitEvent) error
WaitForAttributeBySelector(ctx context.Context, selector, name values.String, value core.Value, when WaitEvent) error
@ -161,6 +151,45 @@ type (
WaitForClassBySelectorAll(ctx context.Context, selector, class values.String, when WaitEvent) error
}
// HTMLPage interface represents any web page loaded in the browser
// and serves as an entry point into the web page's content
HTMLPage interface {
core.Value
core.Iterable
core.Getter
core.Setter
collections.Measurable
io.Closer
IsClosed() values.Boolean
GetURL() values.String
GetMainFrame() HTMLDocument
GetFrames(ctx context.Context) (*values.Array, error)
GetFrame(ctx context.Context, idx values.Int) (core.Value, error)
GetCookies(ctx context.Context) (*values.Array, error)
SetCookies(ctx context.Context, cookies ...HTTPCookie) error
DeleteCookies(ctx context.Context, cookies ...HTTPCookie) error
PrintToPDF(ctx context.Context, params PDFParams) (values.Binary, error)
CaptureScreenshot(ctx context.Context, params ScreenshotParams) (values.Binary, error)
WaitForNavigation(ctx context.Context) error
Navigate(ctx context.Context, url values.String) error
NavigateBack(ctx context.Context, skip values.Int) (values.Boolean, error)
NavigateForward(ctx context.Context, skip values.Int) (values.Boolean, error)
}
)
const (

5
pkg/parser/antlr/FqlLexer.g4
View File

@ -86,6 +86,8 @@ FloatLiteral
| DecimalIntegerLiteral ExponentPart?
;
NamespaceSegment: Identifier NamespaceSeparator;
// Fragments
fragment HexDigit
: [0-9a-fA-F]
@ -105,4 +107,5 @@ fragment Digit
: '0'..'9'
;
fragment DQSring: '"' ( '\\'. | '""' | ~('"'| '\\') )* '"';
fragment SQString: '\'' ('\\'. | '\'\'' | ~('\'' | '\\'))* '\'';
fragment SQString: '\'' ('\\'. | '\'\'' | ~('\'' | '\\'))* '\'';
fragment NamespaceSeparator: '::';

1
pkg/parser/antlr/FqlLexer.tokens
View File

@ -60,6 +60,7 @@ StringLiteral=59
TemplateStringLiteral=60
IntegerLiteral=61
FloatLiteral=62
NamespaceSegment=63
':'=5
';'=6
'.'=7

6
pkg/parser/antlr/FqlParser.g4
View File

@ -207,8 +207,12 @@ expressionGroup
: OpenParen expression CloseParen
;
namespace
: (NamespaceSegment)*
;
functionCallExpression
: Identifier arguments
: namespace Identifier arguments
;
arguments

6
pkg/parser/fql/FqlLexer.interp
View File

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show More