go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2025-10-30 23:37:40 +02:00
Code Issues Releases Activity

Feature/#220 iframe support (#315)

Browse Source 
* Refactored Virtual DOM structure
* Added new E2E tests
* Updated E2E Test Runner
This commit is contained in:
Tim Voronov
2019-06-19 17:58:56 -04:00
committed by GitHub
parent 8c07516ed1
commit d7b923e4c3
103 changed files with 2815 additions and 1629 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
.golangci.yml Normal file
View File

@@ -0,0 +1,33 @@
# This file contains all available configuration options
# with their default values.
# options for analysis running
run:
# which dirs to skip: they won't be analyzed;
# can use regexp here: generated.*, regexp is applied on full path;
# default value is empty list, but next dirs are always skipped independently
# from this option's value:
# vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
skip-dirs:
- pkg/parser/fql
- pkg/parser/antlr
linters:
disable:
- errcheck
issues:
# List of regexps of issue texts to exclude, empty list by default.
# But independently from this option we use default exclude patterns,
# it can be disabled by `exclude-use-default: false`. To list all
# excluded by default patterns execute `golangci-lint run --help`
exclude:
- '^(G104|G401|G505|G501):'
- '^shadow: declaration of'
# which files to skip: they will be analyzed, but issues from them
# won't be reported. Default value is empty list, but there is
# no need to include all autogenerated files, we confidently recognize
# autogenerated files. If it's not please let us know.
skip-files:
- "*_test.go"

1
.travis.yml
View File

@@ -18,6 +18,7 @@ addons:
install:
- go get -u github.com/mgechev/revive
- go get -u github.com/golangci/golangci-lint/cmd/golangci-lint
- sudo curl -o /usr/local/lib/antlr-4.7.1-complete.jar https://www.antlr.org/download/antlr-4.7.1-complete.jar
- export CLASSPATH=".:/usr/local/lib/antlr-4.7.1-complete.jar:$CLASSPATH"
- mkdir $HOME/travis-bin

5
Makefile
View File

@@ -30,7 +30,7 @@ cover:
curl -s https://codecov.io/bash | bash
e2e:
go run ${DIR_E2E}/main.go --tests ${DIR_E2E}/tests --pages ${DIR_E2E}/pages --filter doc_cookie_set*
go run ${DIR_E2E}/main.go --tests ${DIR_E2E}/tests --pages ${DIR_E2E}/pages
bench:
go test -run=XXX -bench=. ${DIR_PKG}/...
@@ -48,7 +48,8 @@ fmt:
# https://github.com/mgechev/revive
# go get github.com/mgechev/revive
lint:
revive -config revive.toml -formatter friendly -exclude ./pkg/parser/fql/... -exclude ./vendor/... ./...
revive -config revive.toml -formatter friendly -exclude ./pkg/parser/fql/... -exclude ./vendor/... ./... && \
golangci-lint run ./pkg/...
# http://godoc.org/code.google.com/p/go.tools/cmd/vet
# go get code.google.com/p/go.tools/cmd/vet

58
e2e/main.go
View File

@@ -4,13 +4,15 @@ import (
"context"
"flag"
"fmt"
"github.com/MontFerret/ferret/e2e/runner"
"github.com/MontFerret/ferret/e2e/server"
"github.com/rs/zerolog"
"net"
"os"
"os/signal"
"path/filepath"
"regexp"
"github.com/MontFerret/ferret/e2e/runner"
"github.com/MontFerret/ferret/e2e/server"
"github.com/rs/zerolog"
)
var (
@@ -39,6 +41,20 @@ var (
)
)
func getOutboundIP() (net.IP, error) {
conn, err := net.Dial("udp", "8.8.8.8:80")
if err != nil {
return nil, err
}
defer conn.Close()
localAddr := conn.LocalAddr().(*net.UDPAddr)
return localAddr.IP, nil
}
func main() {
flag.Parse()
@@ -56,19 +72,6 @@ func main() {
Dir: filepath.Join(*pagesDir, "dynamic"),
})
var filterR *regexp.Regexp
if *filter != "" {
r, err := regexp.Compile(*filter)
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
filterR = r
}
go func() {
if err := static.Start(); err != nil {
logger.Info().Timestamp().Msg("shutting down the static pages server")
@@ -91,12 +94,25 @@ func main() {
}
}
var ipAddr string
// we need it in those cases when a Chrome instance is running inside a container
// and it needs an external IP to get access to our static web server
outIP, err := getOutboundIP()
if err != nil {
ipAddr = "0.0.0.0"
logger.Warn().Err(err).Msg("Failed to get outbound IP address")
} else {
ipAddr = outIP.String()
}
r := runner.New(logger, runner.Settings{
StaticServerAddress: fmt.Sprintf("http://0.0.0.0:%d", staticPort),
DynamicServerAddress: fmt.Sprintf("http://0.0.0.0:%d", dynamicPort),
StaticServerAddress: fmt.Sprintf("http://%s:%d", ipAddr, staticPort),
DynamicServerAddress: fmt.Sprintf("http://%s:%d", ipAddr, dynamicPort),
CDPAddress: *cdp,
Dir: *testsDir,
Filter: filterR,
Filter: *filter,
})
ctx, cancel := context.WithCancel(context.Background())
@@ -110,7 +126,7 @@ func main() {
}
}()
err := r.Run(ctx)
err = r.Run(ctx)
if err != nil {
os.Exit(1)

30
e2e/pages/dynamic/components/app.js
View File

@@ -2,6 +2,7 @@ import Layout from './layout.js';
import IndexPage from './pages/index.js';
import FormsPage from './pages/forms/index.js';
import EventsPage from './pages/events/index.js';
import IframePage from './pages/iframes/index.js';
const e = React.createElement;
const Router = ReactRouter.Router;
@@ -10,7 +11,26 @@ const Route = ReactRouter.Route;
const Redirect = ReactRouter.Redirect;
const createBrowserHistory = History.createBrowserHistory;
export default function AppComponent({ redirect = null}) {
export default React.memo(function AppComponent(params = {}) {
let redirectTo;
if (params.redirect) {
let search = '';
Object.keys(params).forEach((key) => {
if (key !== 'redirect') {
search += `${key}=${params[key]}`;
}
});
const to = {
pathname: params.redirect,
search: search ? `?${search}` : '',
};
redirectTo = e(Redirect, { to });
}
return e(Router, { history: createBrowserHistory() },
e(Layout, null, [
e(Switch, null, [
@@ -27,8 +47,12 @@ export default function AppComponent({ redirect = null}) {
path: '/events',
component: EventsPage
}),
e(Route, {
path: '/iframe',
component: IframePage
}),
]),
redirect ? e(Redirect, { to: redirect }) : null
redirectTo
])
)
}
})

3
e2e/pages/dynamic/components/layout.js
View File

@@ -18,6 +18,9 @@ export default function Layout({ children }) {
]),
e("li", { className: "nav-item"}, [
e(NavLink, { className: "nav-link", to: "/events" }, "Events")
]),
e("li", { className: "nav-item"}, [
e(NavLink, { className: "nav-link", to: "/iframe" }, "iFrame")
])
])
])

26
e2e/pages/dynamic/components/pages/iframes/index.js Normal file
View File

@@ -0,0 +1,26 @@
import { parse } from '../../../utils/qs.js';
const e = React.createElement;
export default class IFramePage extends React.Component {
render() {
const search = parse(this.props.location.search);
let redirect;
if (search.src) {
redirect = search.src;
}
return e("div", { id: "iframe" }, [
e("iframe", {
name: 'nested',
style: {
width: '100%',
height: '800px',
},
src: redirect ? `/?redirect=${redirect}` : '/'
}),
])
}
}

6
e2e/pages/dynamic/index.html
View File

@@ -11,9 +11,9 @@
</head>
<body class="text-center">
<div id="root"></div>
<script src="https://unpkg.com/react@16.6.1/umd/react.production.min.js"></script>
<script src="https://unpkg.com/react-dom@16.6.1/umd/react-dom.production.min.js"></script>
<script src="https://unpkg.com/history@4.7.2/umd/history.min.js"></script>
<script src="https://unpkg.com/react@16.8.6/umd/react.production.min.js"></script>
<script src="https://unpkg.com/react-dom@16.8.6/umd/react-dom.production.min.js"></script>
<script src="https://unpkg.com/history@4.9.0/umd/history.min.js"></script>
<script src="https://unpkg.com/react-router@4.3.1/umd/react-router.js"></script>
<script src="https://unpkg.com/react-router-dom@4.3.1/umd/react-router-dom.js"></script>
<script src="index.js" type="module"></script>

115
e2e/runner/runner.go
View File

@@ -6,7 +6,6 @@ import (
"io/ioutil"
"os"
"path/filepath"
"regexp"
"time"
"github.com/MontFerret/ferret/pkg/compiler"
@@ -14,6 +13,7 @@ import (
"github.com/MontFerret/ferret/pkg/drivers/cdp"
"github.com/MontFerret/ferret/pkg/drivers/http"
"github.com/MontFerret/ferret/pkg/runtime"
"github.com/pkg/errors"
"github.com/rs/zerolog"
)
@@ -24,7 +24,7 @@ type (
DynamicServerAddress string
CDPAddress string
Dir string
Filter *regexp.Regexp
Filter string
}
Result struct {
@@ -84,7 +84,7 @@ func (r *Runner) Run(ctx context.Context) error {
Timestamp().
Int("passed", sum.passed).
Int("failed", sum.failed).
Dur("time", sum.duration).
Str("duration", sum.duration.String()).
Msg("Completed")
if sum.failed > 0 {
@@ -95,19 +95,7 @@ func (r *Runner) Run(ctx context.Context) error {
}
func (r *Runner) runQueries(ctx context.Context, dir string) ([]Result, error) {
files, err := ioutil.ReadDir(dir)
if err != nil {
r.logger.Error().
Timestamp().
Err(err).
Str("dir", dir).
Msg("failed to read scripts directory")
return nil, err
}
results := make([]Result, 0, len(files))
results := make([]Result, 0, 50)
c := compiler.New()
@@ -115,46 +103,61 @@ func (r *Runner) runQueries(ctx context.Context, dir string) ([]Result, error) {
return nil, err
}
// read scripts
for _, f := range files {
n := f.Name()
err := r.traverseDir(ctx, dir, func(name string) error {
if r.settings.Filter != "" {
matched, err := filepath.Match(r.settings.Filter, name)
if r.settings.Filter != nil {
if !r.settings.Filter.Match([]byte(n)) {
continue
if err != nil {
return err
}
if !matched {
return nil
}
}
fName := filepath.Join(dir, n)
b, err := ioutil.ReadFile(fName)
b, err := ioutil.ReadFile(name)
if err != nil {
results = append(results, Result{
name: fName,
name: name,
err: errors.Wrap(err, "failed to read script file"),
})
continue
return nil
}
r.logger.Info().Timestamp().Str("name", fName).Msg("Running test")
r.logger.Info().Timestamp().Str("name", name).Msg("Running test")
result := r.runQuery(ctx, c, fName, string(b))
select {
case <-ctx.Done():
return context.Canceled
default:
result := r.runQuery(ctx, c, name, string(b))
if result.err == nil {
r.logger.Info().
Timestamp().
Str("file", result.name).
Msg("Test passed")
} else {
r.logger.Error().
Timestamp().
Err(result.err).
Str("file", result.name).
Msg("Test failed")
if result.err == nil {
r.logger.Info().
Timestamp().
Str("file", result.name).
Str("duration", result.duration.String()).
Msg("Test passed")
} else {
r.logger.Error().
Timestamp().
Err(result.err).
Str("file", result.name).
Str("duration", result.duration.String()).
Msg("Test failed")
}
results = append(results, result)
}
results = append(results, result)
return nil
})
if err != nil {
return nil, err
}
return results, nil
@@ -237,3 +240,35 @@ func (r *Runner) report(results []Result) Summary {
duration: sumDuration,
}
}
func (r *Runner) traverseDir(ctx context.Context, dir string, iteratee func(name string) error) error {
files, err := ioutil.ReadDir(dir)
if err != nil {
r.logger.Error().
Timestamp().
Err(err).
Str("dir", dir).
Msg("failed to read scripts directory")
return err
}
for _, file := range files {
name := filepath.Join(dir, file.Name())
if file.IsDir() {
if err := r.traverseDir(ctx, name, iteratee); err != nil {
return err
}
continue
}
if err := iteratee(name); err != nil {
return err
}
}
return nil
}

12
e2e/tests/doc_inner_html_1_arg_d.fql
View File

@@ -1,7 +1,7 @@
LET url = @dynamic
LET doc = DOCUMENT(url, true)
LET expected = `<!DOCTYPE html><html lang="en"><head>
LET expected = `<head>
<meta charset="utf-8">
<meta http-equiv="x-ua-compatible" content="ie=edge">
<title>Ferret E2E SPA</title>
@@ -11,16 +11,16 @@ LET expected = `<!DOCTYPE html><html lang="en"><head>
<link rel="stylesheet" href="index.css">
</head>
<body class="text-center">
<div id="root"><div id="layout"><nav class="navbar navbar-expand-md navbar-dark bg-dark mb-4" id="navbar"><a class="navbar-brand active" aria-current="page" href="/">Ferret</a><button class="navbar-toggler" type="button"><span class="navbar-toggler-icon"></span></button><div class="collapse navbar-collapse"><ul class="navbar-nav mr-auto"><li class="nav-item"><a class="nav-link" href="/forms">Forms</a></li><li class="nav-item"><a class="nav-link" href="/navigation">Navigation</a></li><li class="nav-item"><a class="nav-link" href="/events">Events</a></li></ul></div></nav><main class="container"><div class="jumbotron" data-type="page" id="index"><div><h1>Welcome to Ferret E2E test page!</h1></div><div><p class="lead">It has several pages for testing different possibilities of the library</p></div></div></main></div></div>
<script src="https://unpkg.com/react@16.6.1/umd/react.production.min.js"></script>
<script src="https://unpkg.com/react-dom@16.6.1/umd/react-dom.production.min.js"></script>
<script src="https://unpkg.com/history@4.7.2/umd/history.min.js"></script>
<div id="root"><div id="layout"><nav class="navbar navbar-expand-md navbar-dark bg-dark mb-4" id="navbar"><a class="navbar-brand active" aria-current="page" href="/">Ferret</a><button class="navbar-toggler" type="button"><span class="navbar-toggler-icon"></span></button><div class="collapse navbar-collapse"><ul class="navbar-nav mr-auto"><li class="nav-item"><a class="nav-link" href="/forms">Forms</a></li><li class="nav-item"><a class="nav-link" href="/navigation">Navigation</a></li><li class="nav-item"><a class="nav-link" href="/events">Events</a></li><li class="nav-item"><a class="nav-link" href="/iframe">iFrame</a></li></ul></div></nav><main class="container"><div class="jumbotron" data-type="page" id="index"><div><h1>Welcome to Ferret E2E test page!</h1></div><div><p class="lead">It has several pages for testing different possibilities of the library</p></div></div></main></div></div>
<script src="https://unpkg.com/react@16.8.6/umd/react.production.min.js"></script>
<script src="https://unpkg.com/react-dom@16.8.6/umd/react-dom.production.min.js"></script>
<script src="https://unpkg.com/history@4.9.0/umd/history.min.js"></script>
<script src="https://unpkg.com/react-router@4.3.1/umd/react-router.js"></script>
<script src="https://unpkg.com/react-router-dom@4.3.1/umd/react-router-dom.js"></script>
<script src="index.js" type="module"></script>
</body></html>`
</body>`
LET actual = INNER_HTML(doc)
LET r1 = '(\s|\")'

4
e2e/tests/doc_inner_text_1_arg_d.fql
View File

@@ -1,11 +1,11 @@
LET url = @dynamic
LET doc = DOCUMENT(url, true)
LET expected = `Ferret E2E SPA
Ferret
LET expected = `Ferret
Forms
Navigation
Events
iFrame
Welcome to Ferret E2E test page!
It has several pages for testing different possibilities of the library
`

12
e2e/tests/dynamic/doc/iframes/element_exists.fql Normal file
View File

@@ -0,0 +1,12 @@
LET url = @dynamic + "?redirect=/iframe"
LET page = DOCUMENT(url, { driver: 'cdp' })
LET doc = page.frames[1]
LET expectedP = TRUE
LET actualP = ELEMENT_EXISTS(doc, '.text-center')
LET expectedN = FALSE
LET actualN = ELEMENT_EXISTS(doc, '.foo-bar')
RETURN EXPECT(expectedP + expectedN, actualP + expectedN)

12
e2e/tests/dynamic/doc/iframes/hover.fql Normal file
View File

@@ -0,0 +1,12 @@
LET url = @dynamic + "?redirect=/iframe&src=/events"
LET page = DOCUMENT(url, { driver: 'cdp' })
LET doc = page.frames[1]
WAIT_ELEMENT(doc, "#page-events")
HOVER(doc, "#hoverable-btn")
WAIT_ELEMENT(doc, "#hoverable-content")
LET output = INNER_TEXT(doc, "#hoverable-content")
RETURN EXPECT(output, "Lorem ipsum dolor sit amet.")

11
e2e/tests/dynamic/doc/iframes/input.fql Normal file
View File

@@ -0,0 +1,11 @@
LET url = @dynamic + "?redirect=/iframe&src=/forms"
LET page = DOCUMENT(url, true)
LET doc = page.frames[1]
WAIT_ELEMENT(doc, "form")
LET output = ELEMENT(doc, "#text_output")
INPUT(doc, "#text_input", "foo")
RETURN EXPECT(output.innerText, "foo")

4
e2e/tests/dynamic/doc/iframes/length.fql Normal file
View File

@@ -0,0 +1,4 @@
LET url = @dynamic + "?redirect=/iframe"
LET doc = DOCUMENT(url, { driver: 'cdp' })
RETURN EXPECT(2, LENGTH(doc.frames))

15
e2e/tests/dynamic/doc/iframes/wait_class.fql Normal file
View File

@@ -0,0 +1,15 @@
LET url = @dynamic + "?redirect=/iframe&src=/events"
LET page = DOCUMENT(url, true)
LET doc = page.frames[1]
WAIT_ELEMENT(doc, "#page-events")
// with fixed timeout
CLICK(doc, "#wait-class-btn")
WAIT_CLASS(doc, "#wait-class-content", "alert-success")
// with random timeout
CLICK(doc, "#wait-class-random-btn")
WAIT_CLASS(doc, "#wait-class-random-content", "alert-success", 10000)
RETURN ""

14
e2e/tests/dynamic/element/iframes/hover.fql Normal file
View File

@@ -0,0 +1,14 @@
LET url = @dynamic + "?redirect=/iframe&src=/events"
LET page = DOCUMENT(url, { driver: 'cdp' })
LET doc = page.frames[1]
WAIT_ELEMENT(doc, "#page-events")
LET input = ELEMENT(doc, "#hoverable-btn")
HOVER(input)
WAIT_ELEMENT(doc, "#hoverable-content")
LET output = ELEMENT(doc, "#hoverable-content")
RETURN EXPECT(output.innerText, "Lorem ipsum dolor sit amet.")

12
e2e/tests/dynamic/element/iframes/input.fql Normal file
View File

@@ -0,0 +1,12 @@
LET url = @dynamic + "?redirect=/iframe&src=/forms"
LET page = DOCUMENT(url, { driver: 'cdp' })
LET doc = page.frames[1]
WAIT_ELEMENT(doc, "form")
LET input = ELEMENT(doc, "#text_input")
LET output = ELEMENT(doc, "#text_output")
INPUT(input, "foo")
RETURN EXPECT(output.innerText, "foo")

11
e2e/tests/dynamic/element/iframes/select_single.fql Normal file
View File

@@ -0,0 +1,11 @@
LET url = @dynamic + "?redirect=/iframe&src=/forms"
LET page = DOCUMENT(url, { driver: 'cdp' })
LET doc = page.frames[1]
WAIT_ELEMENT(doc, "form")
LET input = ELEMENT(doc, "#select_input")
LET output = ELEMENT(doc, "#select_output")
LET result = SELECT(input, ["4"])
RETURN EXPECT(output.innerText, "4") + EXPECT(JSON_STRINGIFY(result), '["4"]')

23
e2e/tests/dynamic/element/iframes/wait_class.fql Normal file
View File

@@ -0,0 +1,23 @@
LET url = @dynamic + "?redirect=/iframe&src=/events"
LET page = DOCUMENT(url, { driver: 'cdp' })
LET doc = page.frames[1]
WAIT_ELEMENT(doc, "#page-events")
// with fixed timeout
LET b1 = ELEMENT(doc, "#wait-class-btn")
LET c1 = ELEMENT(doc, "#wait-class-content")
WAIT(2000)
CLICK(b1)
WAIT_CLASS(c1, "alert-success")
// with random timeout
LET b2 = ELEMENT(doc, "#wait-class-random-btn")
LET c2 = ELEMENT(doc, "#wait-class-random-content")
CLICK(b2)
WAIT_CLASS(c2, "alert-success", 10000)
RETURN ""

2
e2e/tests/el_inner_html_1_arg_d.fql
View File

@@ -2,7 +2,7 @@ LET url = @dynamic
LET doc = DOCUMENT(url, true)
LET el = ELEMENT(doc, "#root")
LET expected = `<div id="layout"><nav class="navbar navbar-expand-md navbar-dark bg-dark mb-4" id="navbar"><a class="navbar-brand active" aria-current="page" href="/">Ferret</a><button class="navbar-toggler" type="button"><span class="navbar-toggler-icon"></span></button><div class="collapse navbar-collapse"><ul class="navbar-nav mr-auto"><li class="nav-item"><a class="nav-link" href="/forms">Forms</a></li><li class="nav-item"><a class="nav-link" href="/navigation">Navigation</a></li><li class="nav-item"><a class="nav-link" href="/events">Events</a></li></ul></div></nav><main class="container"><div class="jumbotron" data-type="page" id="index"><div><h1>Welcome to Ferret E2E test page!</h1></div><div><p class="lead">It has several pages for testing different possibilities of the library</p></div></div></main></div>`
LET expected = `<div id="layout"><nav class="navbar navbar-expand-md navbar-dark bg-dark mb-4" id="navbar"><a class="navbar-brand active" aria-current="page" href="/">Ferret</a><button class="navbar-toggler" type="button"><span class="navbar-toggler-icon"></span></button><div class="collapse navbar-collapse"><ul class="navbar-nav mr-auto"><li class="nav-item"><a class="nav-link" href="/forms">Forms</a></li><li class="nav-item"><a class="nav-link" href="/navigation">Navigation</a></li><li class="nav-item"><a class="nav-link" href="/events">Events</a></li><li class="nav-item"><a class="nav-link" href="/iframe">iFrame</a></li></ul></div></nav><main class="container"><div class="jumbotron" data-type="page" id="index"><div><h1>Welcome to Ferret E2E test page!</h1></div><div><p class="lead">It has several pages for testing different possibilities of the library</p></div></div></main></div>`
LET actual = INNER_HTML(el)
LET r1 = '(\s|\")'

11
e2e/tests/el_value_d.fql
View File

@@ -1,12 +1,9 @@
LET url = @static + '/value.html'
LET url = @dynamic + "?redirect=/forms"
LET doc = DOCUMENT(url, true)
LET expected = ["068728","068728","816410","52024413","698690","210583","049700","826394","354369","135911","700285","557242","278832","357701","313034","959368","703500","842750","777175","378061","072489","383005","843393","59912263","464535","229710","230550","767964","758862","944384","025449","010245","844935","038760","013450","124139","211145","758761","448667","488966"]
LET el = ELEMENT(doc, "#select_input")
LET actual = (
FOR tr IN ELEMENTS(doc, '#listings_table > tbody > tr')
LET elem = ELEMENT(tr, 'td > input')
RETURN elem.value
)
LET expected = "1"
LET actual = el.value
RETURN EXPECT(actual, expected)

5
e2e/tests/page_load_d.fql
View File

@@ -1,4 +1,7 @@
LET url = @dynamic
LET doc = DOCUMENT(url, true)
RETURN EXPECT(doc.url, url)
LET expected = url + '/'
LET actual = doc.url
RETURN EXPECT(expected, actual)

5
examples/iframes.fql Normal file
View File

@@ -0,0 +1,5 @@
LET page = DOCUMENT("https://www.w3schools.com/html/html_iframe.asp", { driver: "cdp" })
LET c2 = page.frames[1].head.innerHTML
RETURN c2

11
examples/pagination.fql
View File

@@ -4,14 +4,15 @@ INPUT(amazon, '#twotabsearchtextbox', @criteria)
CLICK(amazon, '.nav-search-submit input[type="submit"]')
WAIT_NAVIGATION(amazon)
LET resultListSelector = '#s-results-list-atf'
LET resultItemSelector = '.s-result-item.celwidget'
LET nextBtnSelector = '#pagnNextLink'
LET resultListSelector = 'div.s-result-list'
LET resultItemSelector = 'div.s-result-item'
LET nextBtnSelector = 'ul.a-pagination .a-last a'
LET vendorSelector1 = 'div > div:nth-child(3) > div:nth-child(2) > span:nth-child(2)'
LET vendorSelector2 = 'div > div:nth-child(5) > div:nth-child(2) > span:nth-child(2)'
LET priceWholeSelector = 'span.sx-price-whole'
LET priceFracSelector = 'sup.sx-price-fractional'
LET pages = TO_INT(INNER_TEXT(amazon, '#pagn > span.pagnDisabled'))
LET pagers = ELEMENTS(amazon, 'ul.a-pagination li.a-disabled')
LET pages = LENGTH(pagers) > 0 ? TO_INT(INNER_TEXT(LAST(pagers))) : 0
LET result = (
FOR pageNum IN 1..pages
@@ -19,6 +20,8 @@ LET result = (
LET wait = clicked ? WAIT_NAVIGATION(amazon) : false
LET waitSelector = wait ? WAIT_ELEMENT(amazon, resultListSelector) : false
PRINT("page:", pageNum, "clicked", clicked)
LET items = (
FOR el IN ELEMENTS(amazon, resultItemSelector)
LET priceWholeTxt = INNER_TEXT(el, priceWholeSelector)

3
examples/screenshot.fql Normal file
View File

@@ -0,0 +1,3 @@
LET data = SCREENSHOT("https://github.com/MontFerret/ferret/raw/master/assets/logo.png")
RETURN { type: "png", data }

6
pkg/compiler/compiler_str_test.go
View File

@@ -24,13 +24,13 @@ BAR
})
Convey("Should be possible to use multi line string with nested strings", t, func() {
out := compiler.New().
compiler.New().
MustCompile(fmt.Sprintf(`
RETURN %s<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Title</title>
<title>GetTitle</title>
</head>
<body>
Hello world
@@ -43,7 +43,7 @@ RETURN %s<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Title</title>
<title>GetTitle</title>
</head>
<body>
Hello world

830
pkg/drivers/cdp/document.go
View File

File diff suppressed because it is too large Load Diff

1
pkg/drivers/cdp/document_test.go Normal file
View File

@@ -0,0 +1 @@
package cdp

91
pkg/drivers/cdp/driver.go
View File

@@ -4,24 +4,21 @@ import (
"context"
"sync"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/devtool"
"github.com/mafredri/cdp/protocol/emulation"
"github.com/mafredri/cdp/protocol/network"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/protocol/target"
"github.com/mafredri/cdp/rpcc"
"github.com/mafredri/cdp/session"
"github.com/pkg/errors"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/logging"
)
const DriverName = "cdp"
type Driver struct {
sync.Mutex
mu sync.Mutex
dev *devtool.DevTools
conn *rpcc.Conn
client *cdp.Client
@@ -42,7 +39,7 @@ func (drv *Driver) Name() string {
return drv.options.Name
}
func (drv *Driver) LoadDocument(ctx context.Context, params drivers.LoadDocumentParams) (drivers.HTMLDocument, error) {
func (drv *Driver) Open(ctx context.Context, params drivers.OpenPageParams) (drivers.HTMLPage, error) {
logger := logging.FromContext(ctx)
err := drv.init(ctx)
@@ -58,20 +55,15 @@ func (drv *Driver) LoadDocument(ctx context.Context, params drivers.LoadDocument
return nil, err
}
url := params.URL
if url == "" {
url = BlankPageURL
}
// Create a new target belonging to the browser context
createTargetArgs := target.NewCreateTargetArgs(url)
// Args for a new target belonging to the browser context
createTargetArgs := target.NewCreateTargetArgs(BlankPageURL)
if !drv.options.KeepCookies && !params.KeepCookies {
// Set it to an incognito mode
createTargetArgs.SetBrowserContextID(drv.contextID)
}
// New target
createTarget, err := drv.client.Target.CreateTarget(ctx, createTargetArgs)
if err != nil {
@@ -99,69 +91,16 @@ func (drv *Driver) LoadDocument(ctx context.Context, params drivers.LoadDocument
return nil, err
}
client := cdp.NewClient(conn)
err = runBatch(
func() error {
return client.Page.Enable(ctx)
},
func() error {
return client.Page.SetLifecycleEventsEnabled(
ctx,
page.NewSetLifecycleEventsEnabledArgs(true),
)
},
func() error {
return client.DOM.Enable(ctx)
},
func() error {
return client.Runtime.Enable(ctx)
},
func() error {
var ua string
if params.UserAgent != "" {
ua = common.GetUserAgent(params.UserAgent)
} else {
ua = common.GetUserAgent(drv.options.UserAgent)
}
logger.
Debug().
Timestamp().
Str("user-agent", ua).
Msg("using User-Agent")
// do not use custom user agent
if ua == "" {
return nil
}
return client.Emulation.SetUserAgentOverride(
ctx,
emulation.NewSetUserAgentOverrideArgs(ua),
)
},
func() error {
return client.Network.Enable(ctx, network.NewEnableArgs())
},
)
if err != nil {
return nil, err
if params.UserAgent == "" {
params.UserAgent = drv.options.UserAgent
}
return LoadHTMLDocument(ctx, conn, client, params)
return LoadHTMLPage(ctx, conn, params)
}
func (drv *Driver) Close() error {
drv.Lock()
defer drv.Unlock()
drv.mu.Lock()
defer drv.mu.Unlock()
if drv.session != nil {
drv.session.Close()
@@ -173,8 +112,8 @@ func (drv *Driver) Close() error {
}
func (drv *Driver) init(ctx context.Context) error {
drv.Lock()
defer drv.Unlock()
drv.mu.Lock()
defer drv.mu.Unlock()
if drv.session == nil {
ver, err := drv.dev.Version(ctx)

158
pkg/drivers/cdp/element.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"golang.org/x/net/html"
"hash/fnv"
"strconv"
"strings"
@@ -40,25 +41,27 @@ type (
logger *zerolog.Logger
client *cdp.Client
events *events.EventBroker
exec *eval.ExecutionContext
connected values.Boolean
id *HTMLElementIdentity
nodeType values.Int
id HTMLElementIdentity
nodeType html.NodeType
nodeName values.String
innerHTML values.String
innerText *common.LazyValue
value core.Value
attributes *common.LazyValue
style *common.LazyValue
children []*HTMLElementIdentity
children []HTMLElementIdentity
loadedChildren *common.LazyValue
}
)
func LoadElement(
func LoadHTMLElement(
ctx context.Context,
logger *zerolog.Logger,
client *cdp.Client,
broker *events.EventBroker,
exec *eval.ExecutionContext,
nodeID dom.NodeID,
backendID dom.BackendNodeID,
) (*HTMLElement, error) {
@@ -99,7 +102,7 @@ func LoadElement(
return nil, core.Error(err, strconv.Itoa(int(nodeID)))
}
id := new(HTMLElementIdentity)
id := HTMLElementIdentity{}
id.nodeID = nodeID
id.objectID = objectID
@@ -109,7 +112,7 @@ func LoadElement(
id.backendID = node.Node.BackendNodeID
}
innerHTML, err := loadInnerHTML(ctx, client, id)
innerHTML, err := loadInnerHTML(ctx, client, exec, id, common.ToHTMLType(node.Node.NodeType))
if err != nil {
return nil, core.Error(err, strconv.Itoa(int(nodeID)))
@@ -125,6 +128,7 @@ func LoadElement(
logger,
client,
broker,
exec,
id,
node.Node.NodeType,
node.Node.NodeName,
@@ -138,20 +142,22 @@ func NewHTMLElement(
logger *zerolog.Logger,
client *cdp.Client,
broker *events.EventBroker,
id *HTMLElementIdentity,
exec *eval.ExecutionContext,
id HTMLElementIdentity,
nodeType int,
nodeName string,
value string,
innerHTML values.String,
children []*HTMLElementIdentity,
children []HTMLElementIdentity,
) *HTMLElement {
el := new(HTMLElement)
el.logger = logger
el.client = client
el.events = broker
el.exec = exec
el.connected = values.True
el.id = id
el.nodeType = values.NewInt(nodeType)
el.nodeType = common.ToHTMLType(nodeType)
el.nodeName = values.NewString(nodeName)
el.innerHTML = innerHTML
el.innerText = common.NewLazyValue(el.loadInnerText)
@@ -207,7 +213,7 @@ func (el *HTMLElement) MarshalJSON() ([]byte, error) {
}
func (el *HTMLElement) String() string {
return el.InnerHTML(context.Background()).String()
return el.GetInnerHTML(context.Background()).String()
}
func (el *HTMLElement) Compare(other core.Value) int64 {
@@ -217,7 +223,7 @@ func (el *HTMLElement) Compare(other core.Value) int64 {
ctx := context.Background()
return el.InnerHTML(ctx).Compare(other.InnerHTML(ctx))
return el.GetInnerHTML(ctx).Compare(other.GetInnerHTML(ctx))
default:
return drivers.Compare(el.Type(), other.Type())
}
@@ -257,11 +263,11 @@ func (el *HTMLElement) SetIn(ctx context.Context, path []core.Value, value core.
}
func (el *HTMLElement) GetValue(ctx context.Context) core.Value {
if !el.IsConnected() {
if el.IsDetached() {
return el.value
}
val, err := eval.Property(ctx, el.client, el.id.objectID, "value")
val, err := el.exec.ReadProperty(ctx, el.id.objectID, "value")
if err != nil {
el.logError(err).Msg("failed to get node value")
@@ -275,7 +281,7 @@ func (el *HTMLElement) GetValue(ctx context.Context) core.Value {
}
func (el *HTMLElement) SetValue(ctx context.Context, value core.Value) error {
if !el.IsConnected() {
if el.IsDetached() {
// TODO: Return an error
return nil
}
@@ -283,11 +289,11 @@ func (el *HTMLElement) SetValue(ctx context.Context, value core.Value) error {
return el.client.DOM.SetNodeValue(ctx, dom.NewSetNodeValueArgs(el.id.nodeID, value.String()))
}
func (el *HTMLElement) NodeType() values.Int {
return el.nodeType
func (el *HTMLElement) GetNodeType() values.Int {
return values.NewInt(common.FromHTMLType(el.nodeType))
}
func (el *HTMLElement) NodeName() values.String {
func (el *HTMLElement) GetNodeName() values.String {
return el.nodeName
}
@@ -472,7 +478,7 @@ func (el *HTMLElement) GetChildNode(ctx context.Context, idx values.Int) core.Va
}
func (el *HTMLElement) QuerySelector(ctx context.Context, selector values.String) core.Value {
if !el.IsConnected() {
if el.IsDetached() {
return values.None
}
@@ -496,7 +502,7 @@ func (el *HTMLElement) QuerySelector(ctx context.Context, selector values.String
return values.None
}
res, err := LoadElement(ctx, el.logger, el.client, el.events, found.NodeID, emptyBackendID)
res, err := LoadHTMLElement(ctx, el.logger, el.client, el.events, el.exec, found.NodeID, emptyBackendID)
if err != nil {
el.logError(err).
@@ -510,7 +516,7 @@ func (el *HTMLElement) QuerySelector(ctx context.Context, selector values.String
}
func (el *HTMLElement) QuerySelectorAll(ctx context.Context, selector values.String) core.Value {
if !el.IsConnected() {
if el.IsDetached() {
return values.NewArray(0)
}
@@ -537,7 +543,7 @@ func (el *HTMLElement) QuerySelectorAll(ctx context.Context, selector values.Str
continue
}
childEl, err := LoadElement(ctx, el.logger, el.client, el.events, id, emptyBackendID)
childEl, err := LoadHTMLElement(ctx, el.logger, el.client, el.events, el.exec, id, emptyBackendID)
if err != nil {
el.logError(err).
@@ -562,7 +568,7 @@ func (el *HTMLElement) QuerySelectorAll(ctx context.Context, selector values.Str
return arr
}
func (el *HTMLElement) InnerText(ctx context.Context) values.String {
func (el *HTMLElement) GetInnerText(ctx context.Context) values.String {
val, err := el.innerText.Read(ctx)
if err != nil {
@@ -577,7 +583,7 @@ func (el *HTMLElement) InnerText(ctx context.Context) values.String {
}
func (el *HTMLElement) InnerTextBySelector(ctx context.Context, selector values.String) values.String {
if !el.IsConnected() {
if el.IsDetached() {
return values.EmptyString
}
@@ -624,7 +630,7 @@ func (el *HTMLElement) InnerTextBySelector(ctx context.Context, selector values.
objID := *obj.Object.ObjectID
text, err := eval.Property(ctx, el.client, objID, "innerText")
text, err := el.exec.ReadProperty(ctx, objID, "innerText")
if err != nil {
el.logError(err).
@@ -679,7 +685,7 @@ func (el *HTMLElement) InnerTextBySelectorAll(ctx context.Context, selector valu
objID := *obj.Object.ObjectID
text, err := eval.Property(ctx, el.client, objID, "innerText")
text, err := el.exec.ReadProperty(ctx, objID, "innerText")
if err != nil {
el.logError(err).
@@ -696,7 +702,7 @@ func (el *HTMLElement) InnerTextBySelectorAll(ctx context.Context, selector valu
return arr
}
func (el *HTMLElement) InnerHTML(_ context.Context) values.String {
func (el *HTMLElement) GetInnerHTML(_ context.Context) values.String {
el.mu.Lock()
defer el.mu.Unlock()
@@ -704,7 +710,7 @@ func (el *HTMLElement) InnerHTML(_ context.Context) values.String {
}
func (el *HTMLElement) InnerHTMLBySelector(ctx context.Context, selector values.String) values.String {
if !el.IsConnected() {
if el.IsDetached() {
return values.EmptyString
}
@@ -719,13 +725,12 @@ func (el *HTMLElement) InnerHTMLBySelector(ctx context.Context, selector values.
return values.EmptyString
}
text, err := loadInnerHTML(ctx, el.client, &HTMLElementIdentity{
nodeID: found.NodeID,
})
text, err := loadInnerHTMLByNodeID(ctx, el.client, el.exec, found.NodeID)
if err != nil {
el.logError(err).
Str("selector", selector.String()).
Int("childNodeID", int(found.NodeID)).
Msg("failed to load inner HTML for found child el")
return values.EmptyString
@@ -750,13 +755,12 @@ func (el *HTMLElement) InnerHTMLBySelectorAll(ctx context.Context, selector valu
arr := values.NewArray(len(res.NodeIDs))
for _, id := range res.NodeIDs {
text, err := loadInnerHTML(ctx, el.client, &HTMLElementIdentity{
nodeID: id,
})
text, err := loadInnerHTMLByNodeID(ctx, el.client, el.exec, id)
if err != nil {
el.logError(err).
Str("selector", selector.String()).
Int("childNodeID", int(id)).
Msg("failed to load inner HTML for found child el")
// return what we have
@@ -770,7 +774,7 @@ func (el *HTMLElement) InnerHTMLBySelectorAll(ctx context.Context, selector valu
}
func (el *HTMLElement) CountBySelector(ctx context.Context, selector values.String) values.Int {
if !el.IsConnected() {
if el.IsDetached() {
return values.ZeroInt
}
@@ -790,7 +794,7 @@ func (el *HTMLElement) CountBySelector(ctx context.Context, selector values.Stri
}
func (el *HTMLElement) ExistsBySelector(ctx context.Context, selector values.String) values.Boolean {
if !el.IsConnected() {
if el.IsDetached() {
return values.False
}
@@ -887,7 +891,50 @@ func (el *HTMLElement) WaitForStyle(ctx context.Context, name values.String, val
}
func (el *HTMLElement) Click(ctx context.Context) (values.Boolean, error) {
return events.DispatchEvent(ctx, el.client, el.id.objectID, "click")
if err := el.ScrollIntoView(ctx); err != nil {
return values.False, err
}
points, err := getClickablePoint(ctx, el.client, el.id)
if err != nil {
return values.False, err
}
moveArgs := input.NewDispatchMouseEventArgs("mouseMoved", points.X, points.Y)
if err := el.client.Input.DispatchMouseEvent(ctx, moveArgs); err != nil {
return values.False, err
}
beforePressDelay := time.Duration(core.Random(100, 50))
time.Sleep(beforePressDelay)
btn := "left"
clickCount := 1
downArgs := input.NewDispatchMouseEventArgs("mousePressed", points.X, points.Y)
downArgs.ClickCount = &clickCount
downArgs.Button = &btn
if err := el.client.Input.DispatchMouseEvent(ctx, downArgs); err != nil {
return values.False, err
}
beforeReleaseDelay := time.Duration(core.Random(50, 25))
time.Sleep(beforeReleaseDelay * time.Millisecond)
upArgs := input.NewDispatchMouseEventArgs("mouseReleased", points.X, points.Y)
upArgs.ClickCount = &clickCount
upArgs.Button = &btn
if err := el.client.Input.DispatchMouseEvent(ctx, upArgs); err != nil {
return values.False, err
}
return values.True, nil
}
func (el *HTMLElement) Input(ctx context.Context, value core.Value, delay values.Int) error {
@@ -923,7 +970,7 @@ func (el *HTMLElement) Input(ctx context.Context, value core.Value, delay values
func (el *HTMLElement) Select(ctx context.Context, value *values.Array) (*values.Array, error) {
var attrID = "data-ferret-select"
if el.NodeName() != "SELECT" {
if el.GetNodeName() != "SELECT" {
return nil, core.Error(core.ErrInvalidOperation, "element is not a <select> element.")
}
@@ -939,9 +986,8 @@ func (el *HTMLElement) Select(ctx context.Context, value *values.Array) (*values
return nil, err
}
res, err := eval.Eval(
res, err := el.exec.EvalWithReturn(
ctx,
el.client,
fmt.Sprintf(`
var el = document.querySelector('[%s="%s"]');
if (el == null) {
@@ -969,8 +1015,6 @@ func (el *HTMLElement) Select(ctx context.Context, value *values.Array) (*values
id.String(),
value.String(),
),
true,
false,
)
if err != nil {
@@ -1007,9 +1051,8 @@ func (el *HTMLElement) ScrollIntoView(ctx context.Context) error {
return err
}
_, err = eval.Eval(
err = el.exec.Eval(
ctx,
el.client,
fmt.Sprintf(`
var el = document.querySelector('[%s="%s"]');
if (el == null) {
@@ -1024,7 +1067,7 @@ func (el *HTMLElement) ScrollIntoView(ctx context.Context) error {
`,
attrID,
id.String(),
), false, false)
))
if err != nil {
return err
@@ -1054,16 +1097,16 @@ func (el *HTMLElement) Hover(ctx context.Context) error {
)
}
func (el *HTMLElement) IsConnected() values.Boolean {
func (el *HTMLElement) IsDetached() values.Boolean {
el.mu.Lock()
defer el.mu.Unlock()
return el.connected
return !el.connected
}
func (el *HTMLElement) loadInnerText(ctx context.Context) (core.Value, error) {
if el.IsConnected() {
text, err := loadInnerText(ctx, el.client, el.id)
if !el.IsDetached() {
text, err := loadInnerText(ctx, el.client, el.exec, el.id, el.nodeType)
if err == nil {
return text, nil
@@ -1074,7 +1117,7 @@ func (el *HTMLElement) loadInnerText(ctx context.Context) (core.Value, error) {
// and just parse cached innerHTML
}
h := el.InnerHTML(ctx)
h := el.GetInnerHTML(ctx)
if h == values.EmptyString {
return h, nil
@@ -1102,18 +1145,19 @@ func (el *HTMLElement) loadAttrs(ctx context.Context) (core.Value, error) {
}
func (el *HTMLElement) loadChildren(ctx context.Context) (core.Value, error) {
if !el.IsConnected() {
if el.IsDetached() {
return values.NewArray(0), nil
}
loaded := values.NewArray(len(el.children))
for _, childID := range el.children {
child, err := LoadElement(
child, err := LoadHTMLElement(
ctx,
el.logger,
el.client,
el.events,
el.exec,
childID.nodeID,
childID.backendID,
)
@@ -1285,21 +1329,21 @@ func (el *HTMLElement) handleChildInserted(ctx context.Context, message interfac
return
}
nextIdentity := &HTMLElementIdentity{
nextIdentity := HTMLElementIdentity{
nodeID: reply.Node.NodeID,
backendID: reply.Node.BackendNodeID,
}
arr := el.children
el.children = append(arr[:targetIDx], append([]*HTMLElementIdentity{nextIdentity}, arr[targetIDx:]...)...)
el.children = append(arr[:targetIDx], append([]HTMLElementIdentity{nextIdentity}, arr[targetIDx:]...)...)
if !el.loadedChildren.Ready() {
return
}
el.loadedChildren.Write(ctx, func(v core.Value, err error) {
el.loadedChildren.Write(ctx, func(v core.Value, _ error) {
loadedArr := v.(*values.Array)
loadedEl, err := LoadElement(ctx, el.logger, el.client, el.events, nextID, emptyBackendID)
loadedEl, err := LoadHTMLElement(ctx, el.logger, el.client, el.events, el.exec, nextID, emptyBackendID)
if err != nil {
el.logError(err).Msg("failed to load an inserted element")
@@ -1309,7 +1353,7 @@ func (el *HTMLElement) handleChildInserted(ctx context.Context, message interfac
loadedArr.Insert(values.NewInt(targetIDx), loadedEl)
newInnerHTML, err := loadInnerHTML(ctx, el.client, el.id)
newInnerHTML, err := loadInnerHTML(ctx, el.client, el.exec, el.id, el.nodeType)
if err != nil {
el.logError(err).Msg("failed to update element")
@@ -1371,7 +1415,7 @@ func (el *HTMLElement) handleChildRemoved(ctx context.Context, message interface
loadedArr := v.(*values.Array)
loadedArr.RemoveAt(values.NewInt(targetIDx))
newInnerHTML, err := loadInnerHTML(ctx, el.client, el.id)
newInnerHTML, err := loadInnerHTML(ctx, el.client, el.exec, el.id, el.nodeType)
if err != nil {
el.logger.Error().

217
pkg/drivers/cdp/eval/context.go Normal file
View File

@@ -0,0 +1,217 @@
package eval
import (
"context"
"fmt"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
const EmptyExecutionContextID = runtime.ExecutionContextID(-1)
type ExecutionContext struct {
client *cdp.Client
frame page.Frame
contextID runtime.ExecutionContextID
}
func NewExecutionContext(client *cdp.Client, frame page.Frame, contextID runtime.ExecutionContextID) *ExecutionContext {
ec := new(ExecutionContext)
ec.client = client
ec.frame = frame
ec.contextID = contextID
return ec
}
func (ec *ExecutionContext) ID() runtime.ExecutionContextID {
return ec.contextID
}
func (ec *ExecutionContext) Eval(ctx context.Context, exp string) error {
_, err := ec.eval(
ctx,
runtime.
NewEvaluateArgs(PrepareEval(exp)),
)
return err
}
func (ec *ExecutionContext) EvalWithReturn(ctx context.Context, exp string) (core.Value, error) {
return ec.eval(
ctx,
runtime.
NewEvaluateArgs(PrepareEval(exp)).
SetReturnByValue(true),
)
}
func (ec *ExecutionContext) EvalAsync(ctx context.Context, exp string) (core.Value, error) {
return ec.eval(
ctx,
runtime.
NewEvaluateArgs(PrepareEval(exp)).
SetReturnByValue(true).
SetAwaitPromise(true),
)
}
func (ec *ExecutionContext) eval(ctx context.Context, args *runtime.EvaluateArgs) (core.Value, error) {
if ec.contextID != EmptyExecutionContextID {
args.SetContextID(ec.contextID)
}
out, err := ec.client.Runtime.Evaluate(ctx, args)
if err != nil {
return values.None, err
}
if out.ExceptionDetails != nil {
ex := out.ExceptionDetails
return values.None, core.Error(
core.ErrUnexpected,
fmt.Sprintf("%s: %s", ex.Text, *ex.Exception.Description),
)
}
if out.Result.Type != "undefined" && out.Result.Type != "null" {
return values.Unmarshal(out.Result.Value)
}
return Unmarshal(&out.Result)
}
func (ec *ExecutionContext) CallMethod(
ctx context.Context,
objectID runtime.RemoteObjectID,
methodName string,
args []runtime.CallArgument,
) (*runtime.RemoteObject, error) {
callArgs := runtime.NewCallFunctionOnArgs(methodName).
SetObjectID(objectID).
SetArguments(args)
if ec.contextID != EmptyExecutionContextID {
callArgs.SetExecutionContextID(ec.contextID)
}
found, err := ec.client.Runtime.CallFunctionOn(
ctx,
callArgs,
)
if err != nil {
return nil, err
}
if found.ExceptionDetails != nil {
return nil, found.ExceptionDetails
}
if found.Result.ObjectID == nil {
return nil, nil
}
return &found.Result, nil
}
func (ec *ExecutionContext) ReadProperty(
ctx context.Context,
objectID runtime.RemoteObjectID,
propName string,
) (core.Value, error) {
res, err := ec.client.Runtime.GetProperties(
ctx,
runtime.NewGetPropertiesArgs(objectID),
)
if err != nil {
return values.None, err
}
if res.ExceptionDetails != nil {
return values.None, res.ExceptionDetails
}
// all props
if propName == "" {
arr := values.NewArray(len(res.Result))
for _, prop := range res.Result {
val, err := Unmarshal(prop.Value)
if err != nil {
return values.None, err
}
arr.Push(val)
}
return arr, nil
}
for _, prop := range res.Result {
if prop.Name == propName {
return Unmarshal(prop.Value)
}
}
return values.None, nil
}
func (ec *ExecutionContext) DispatchEvent(
ctx context.Context,
objectID runtime.RemoteObjectID,
eventName string,
) (values.Boolean, error) {
args := runtime.NewEvaluateArgs(PrepareEval(fmt.Sprintf(`
return new window.MouseEvent('%s', { bubbles: true, cancelable: true })
`, eventName)))
if ec.contextID != EmptyExecutionContextID {
args.SetContextID(ec.contextID)
}
evt, err := ec.client.Runtime.Evaluate(ctx, args)
if err != nil {
return values.False, nil
}
if evt.ExceptionDetails != nil {
return values.False, evt.ExceptionDetails
}
if evt.Result.ObjectID == nil {
return values.False, nil
}
evtID := evt.Result.ObjectID
// release the event object
defer ec.client.Runtime.ReleaseObject(ctx, runtime.NewReleaseObjectArgs(*evtID))
_, err = ec.CallMethod(
ctx,
objectID,
"dispatchEvent",
[]runtime.CallArgument{
{
ObjectID: evt.Result.ObjectID,
},
},
)
if err != nil {
return values.False, err
}
return values.True, nil
}

159
pkg/drivers/cdp/eval/eval.go
View File

@@ -1,159 +0,0 @@
package eval
import (
"context"
"encoding/json"
"fmt"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/runtime"
)
func PrepareEval(exp string) string {
return fmt.Sprintf("((function () {%s})())", exp)
}
func Eval(ctx context.Context, client *cdp.Client, exp string, ret bool, async bool) (core.Value, error) {
args := runtime.
NewEvaluateArgs(PrepareEval(exp)).
SetReturnByValue(ret).
SetAwaitPromise(async)
out, err := client.Runtime.Evaluate(ctx, args)
if err != nil {
return values.None, err
}
if out.ExceptionDetails != nil {
ex := out.ExceptionDetails
return values.None, core.Error(
core.ErrUnexpected,
fmt.Sprintf("%s: %s", ex.Text, *ex.Exception.Description),
)
}
if out.Result.Type != "undefined" {
return values.Unmarshal(out.Result.Value)
}
return Unmarshal(&out.Result)
}
func Property(
ctx context.Context,
client *cdp.Client,
objectID runtime.RemoteObjectID,
propName string,
) (core.Value, error) {
res, err := client.Runtime.GetProperties(
ctx,
runtime.NewGetPropertiesArgs(objectID),
)
if err != nil {
return values.None, err
}
if res.ExceptionDetails != nil {
return values.None, res.ExceptionDetails
}
// all props
if propName == "" {
arr := values.NewArray(len(res.Result))
for _, prop := range res.Result {
val, err := Unmarshal(prop.Value)
if err != nil {
return values.None, err
}
arr.Push(val)
}
return arr, nil
}
for _, prop := range res.Result {
if prop.Name == propName {
return Unmarshal(prop.Value)
}
}
return values.None, nil
}
func Method(
ctx context.Context,
client *cdp.Client,
objectID runtime.RemoteObjectID,
methodName string,
args []runtime.CallArgument,
) (*runtime.RemoteObject, error) {
found, err := client.Runtime.CallFunctionOn(
ctx,
runtime.NewCallFunctionOnArgs(methodName).
SetObjectID(objectID).
SetArguments(args),
)
if err != nil {
return nil, err
}
if found.ExceptionDetails != nil {
return nil, found.ExceptionDetails
}
if found.Result.ObjectID == nil {
return nil, nil
}
return &found.Result, nil
}
func MethodQuerySelector(
ctx context.Context,
client *cdp.Client,
objectID runtime.RemoteObjectID,
selector string,
) (runtime.RemoteObjectID, error) {
bytes, err := json.Marshal(selector)
if err != nil {
return "", err
}
obj, err := Method(ctx, client, objectID, "querySelector", []runtime.CallArgument{
{
Value: json.RawMessage(bytes),
},
})
if err != nil {
return "", err
}
if obj.ObjectID == nil {
return "", nil
}
return *obj.ObjectID, nil
}
func Unmarshal(obj *runtime.RemoteObject) (core.Value, error) {
if obj == nil {
return values.None, nil
}
if obj.Type != "undefined" {
return values.Unmarshal(obj.Value)
}
return values.None, nil
}

35
pkg/drivers/cdp/eval/helpers.go Normal file
View File

@@ -0,0 +1,35 @@
package eval
import (
"fmt"
"strconv"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/mafredri/cdp/protocol/runtime"
)
func PrepareEval(exp string) string {
return fmt.Sprintf("((function () {%s})())", exp)
}
func Unmarshal(obj *runtime.RemoteObject) (core.Value, error) {
if obj == nil {
return values.None, nil
}
switch obj.Type {
case "string":
str, err := strconv.Unquote(string(obj.Value))
if err != nil {
return values.None, err
}
return values.NewString(str), nil
case "undefined", "null":
return values.None, nil
default:
return values.Unmarshal(obj.Value)
}
}

11
pkg/drivers/cdp/events/broker.go
View File

@@ -179,6 +179,16 @@ func (broker *EventBroker) Close() error {
return nil
}
func (broker *EventBroker) StopAndClose() error {
err := broker.Stop()
if err != nil {
return err
}
return broker.Close()
}
func (broker *EventBroker) runLoop(ctx context.Context) {
for {
select {
@@ -265,6 +275,7 @@ func (broker *EventBroker) emit(ctx context.Context, event Event, message interf
listeners, ok := broker.listeners[event]
if !ok {
broker.mu.Unlock()
return
}

56
pkg/drivers/cdp/events/dispatch.go
View File

@@ -1,56 +0,0 @@
package events
import (
"context"
"fmt"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/runtime"
)
func DispatchEvent(
ctx context.Context,
client *cdp.Client,
objectID runtime.RemoteObjectID,
eventName string,
) (values.Boolean, error) {
evt, err := client.Runtime.Evaluate(ctx, runtime.NewEvaluateArgs(eval.PrepareEval(fmt.Sprintf(`
return new window.MouseEvent('%s', { bubbles: true, cancelable: true })
`, eventName))))
if err != nil {
return values.False, nil
}
if evt.ExceptionDetails != nil {
return values.False, evt.ExceptionDetails
}
if evt.Result.ObjectID == nil {
return values.False, nil
}
evtID := evt.Result.ObjectID
// release the event object
defer client.Runtime.ReleaseObject(ctx, runtime.NewReleaseObjectArgs(*evtID))
_, err = eval.Method(
ctx,
client,
objectID,
"dispatchEvent",
[]runtime.CallArgument{
{
ObjectID: evt.Result.ObjectID,
},
},
)
if err != nil {
return values.False, err
}
return values.True, nil
}

126
pkg/drivers/cdp/events/helpers.go Normal file
View File

@@ -0,0 +1,126 @@
package events
import (
"context"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/page"
"github.com/pkg/errors"
"github.com/mafredri/cdp"
)
func WaitForLoadEvent(ctx context.Context, client *cdp.Client) error {
loadEventFired, err := client.Page.LoadEventFired(ctx)
if err != nil {
return errors.Wrap(err, "failed to create load event hook")
}
_, err = loadEventFired.Recv()
if err != nil {
return err
}
return loadEventFired.Close()
}
func CreateEventBroker(client *cdp.Client) (*EventBroker, error) {
var err error
var onLoad page.LoadEventFiredClient
var onReload dom.DocumentUpdatedClient
var onAttrModified dom.AttributeModifiedClient
var onAttrRemoved dom.AttributeRemovedClient
var onChildCountUpdated dom.ChildNodeCountUpdatedClient
var onChildNodeInserted dom.ChildNodeInsertedClient
var onChildNodeRemoved dom.ChildNodeRemovedClient
ctx := context.Background()
onLoad, err = client.Page.LoadEventFired(ctx)
if err != nil {
return nil, err
}
onReload, err = client.DOM.DocumentUpdated(ctx)
if err != nil {
onLoad.Close()
return nil, err
}
onAttrModified, err = client.DOM.AttributeModified(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
return nil, err
}
onAttrRemoved, err = client.DOM.AttributeRemoved(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
return nil, err
}
onChildCountUpdated, err = client.DOM.ChildNodeCountUpdated(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
onAttrRemoved.Close()
return nil, err
}
onChildNodeInserted, err = client.DOM.ChildNodeInserted(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
onAttrRemoved.Close()
onChildCountUpdated.Close()
return nil, err
}
onChildNodeRemoved, err = client.DOM.ChildNodeRemoved(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
onAttrRemoved.Close()
onChildCountUpdated.Close()
onChildNodeInserted.Close()
return nil, err
}
broker := NewEventBroker(
onLoad,
onReload,
onAttrModified,
onAttrRemoved,
onChildCountUpdated,
onChildNodeInserted,
onChildNodeRemoved,
)
err = broker.Start()
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
onAttrRemoved.Close()
onChildCountUpdated.Close()
onChildNodeInserted.Close()
onChildNodeRemoved.Close()
return nil, err
}
return broker, nil
}

11
pkg/drivers/cdp/events/wait.go
View File

@@ -2,12 +2,12 @@ package events
import (
"context"
"time"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/mafredri/cdp"
"time"
)
type (
@@ -58,18 +58,15 @@ func (task *WaitTask) Run(ctx context.Context) (core.Value, error) {
}
func NewEvalWaitTask(
client *cdp.Client,
ec *eval.ExecutionContext,
predicate string,
polling time.Duration,
) *WaitTask {
return NewWaitTask(
func(ctx context.Context) (core.Value, error) {
return eval.Eval(
return ec.EvalWithReturn(
ctx,
client,
predicate,
true,
false,
)
},
polling,

407
pkg/drivers/cdp/helpers.go
View File

@@ -4,13 +4,13 @@ import (
"bytes"
"context"
"errors"
"golang.org/x/net/html"
"math"
"strings"
"time"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
@@ -44,16 +44,6 @@ func runBatch(funcs ...batchFunc) error {
return eg.Wait()
}
func getRootElement(ctx context.Context, client *cdp.Client) (*dom.GetDocumentReply, error) {
d, err := client.DOM.GetDocument(ctx, dom.NewGetDocumentArgs().SetDepth(1))
if err != nil {
return nil, err
}
return d, nil
}
func fromProtocolQuad(quad dom.Quad) []Quad {
return []Quad{
{
@@ -87,7 +77,20 @@ func computeQuadArea(quads []Quad) float64 {
return math.Abs(area)
}
func getClickablePoint(ctx context.Context, client *cdp.Client, id *HTMLElementIdentity) (Quad, error) {
func intersectQuadWithViewport(quad []Quad, width, height float64) []Quad {
quads := make([]Quad, 0, len(quad))
for _, point := range quad {
quads = append(quads, Quad{
X: math.Min(math.Max(point.X, 0), width),
Y: math.Min(math.Max(point.Y, 0), height),
})
}
return quads
}
func getClickablePoint(ctx context.Context, client *cdp.Client, id HTMLElementIdentity) (Quad, error) {
qargs := dom.NewGetContentQuadsArgs()
switch {
@@ -99,20 +102,29 @@ func getClickablePoint(ctx context.Context, client *cdp.Client, id *HTMLElementI
qargs.SetNodeID(id.nodeID)
}
res, err := client.DOM.GetContentQuads(ctx, qargs)
contentQuadsReply, err := client.DOM.GetContentQuads(ctx, qargs)
if err != nil {
return Quad{}, err
}
if res.Quads == nil || len(res.Quads) == 0 {
if contentQuadsReply.Quads == nil || len(contentQuadsReply.Quads) == 0 {
return Quad{}, errors.New("node is either not visible or not an HTMLElement")
}
quads := make([][]Quad, 0, len(res.Quads))
layoutMetricsReply, err := client.Page.GetLayoutMetrics(ctx)
for _, q := range res.Quads {
quad := fromProtocolQuad(q)
if err != nil {
return Quad{}, err
}
clientWidth := layoutMetricsReply.LayoutViewport.ClientWidth
clientHeight := layoutMetricsReply.LayoutViewport.ClientHeight
quads := make([][]Quad, 0, len(contentQuadsReply.Quads))
for _, q := range contentQuadsReply.Quads {
quad := intersectQuadWithViewport(fromProtocolQuad(q), float64(clientWidth), float64(clientHeight))
if computeQuadArea(quad) > 1 {
quads = append(quads, quad)
@@ -167,41 +179,105 @@ func parseAttrs(attrs []string) *values.Object {
return res
}
func loadInnerHTML(ctx context.Context, client *cdp.Client, id *HTMLElementIdentity) (values.String, error) {
var objID runtime.RemoteObjectID
func loadInnerHTML(ctx context.Context, client *cdp.Client, exec *eval.ExecutionContext, id HTMLElementIdentity, nodeType html.NodeType) (values.String, error) {
// not a document
if nodeType != html.DocumentNode {
var objID runtime.RemoteObjectID
switch {
case id.objectID != "":
objID = id.objectID
case id.backendID > 0:
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetBackendNodeID(id.backendID))
switch {
case id.objectID != "":
objID = id.objectID
case id.backendID > 0:
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetBackendNodeID(id.backendID))
if err != nil {
return "", err
}
if repl.Object.ObjectID == nil {
return "", errors.New("unable to resolve node")
}
objID = *repl.Object.ObjectID
default:
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.nodeID))
if err != nil {
return "", err
}
if repl.Object.ObjectID == nil {
return "", errors.New("unable to resolve node")
}
objID = *repl.Object.ObjectID
}
res, err := exec.ReadProperty(ctx, objID, "innerHTML")
if err != nil {
return "", err
}
if repl.Object.ObjectID == nil {
return "", errors.New("unable to resolve node")
}
objID = *repl.Object.ObjectID
default:
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.nodeID))
if err != nil {
return "", err
}
if repl.Object.ObjectID == nil {
return "", errors.New("unable to resolve node")
}
objID = *repl.Object.ObjectID
return values.NewString(res.String()), nil
}
repl, err := exec.EvalWithReturn(ctx, "return document.documentElement.innerHTML")
if err != nil {
return "", err
}
return values.NewString(repl.String()), nil
}
func loadInnerHTMLByNodeID(ctx context.Context, client *cdp.Client, exec *eval.ExecutionContext, nodeID dom.NodeID) (values.String, error) {
node, err := client.DOM.DescribeNode(ctx, dom.NewDescribeNodeArgs().SetNodeID(nodeID))
if err != nil {
return values.EmptyString, err
}
return loadInnerHTML(ctx, client, exec, HTMLElementIdentity{
nodeID: nodeID,
}, common.ToHTMLType(node.Node.NodeType))
}
func loadInnerText(ctx context.Context, client *cdp.Client, exec *eval.ExecutionContext, id HTMLElementIdentity, nodeType html.NodeType) (values.String, error) {
// not a document
if id.nodeID != 1 {
res, err := eval.Property(ctx, client, objID, "innerHTML")
if nodeType != html.DocumentNode {
var objID runtime.RemoteObjectID
switch {
case id.objectID != "":
objID = id.objectID
case id.backendID > 0:
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetBackendNodeID(id.backendID))
if err != nil {
return "", err
}
if repl.Object.ObjectID == nil {
return "", errors.New("unable to resolve node")
}
objID = *repl.Object.ObjectID
default:
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.nodeID))
if err != nil {
return "", err
}
if repl.Object.ObjectID == nil {
return "", errors.New("unable to resolve node")
}
objID = *repl.Object.ObjectID
}
res, err := exec.ReadProperty(ctx, objID, "innerText")
if err != nil {
return "", err
@@ -210,66 +286,26 @@ func loadInnerHTML(ctx context.Context, client *cdp.Client, id *HTMLElementIdent
return values.NewString(res.String()), err
}
repl, err := client.DOM.GetOuterHTML(ctx, dom.NewGetOuterHTMLArgs().SetObjectID(objID))
repl, err := exec.EvalWithReturn(ctx, "return document.documentElement.innerText")
if err != nil {
return "", err
}
return values.NewString(repl.OuterHTML), nil
return values.NewString(repl.String()), nil
}
func loadInnerText(ctx context.Context, client *cdp.Client, id *HTMLElementIdentity) (values.String, error) {
var objID runtime.RemoteObjectID
switch {
case id.objectID != "":
objID = id.objectID
case id.backendID > 0:
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetBackendNodeID(id.backendID))
if err != nil {
return "", err
}
if repl.Object.ObjectID == nil {
return "", errors.New("unable to resolve node")
}
objID = *repl.Object.ObjectID
default:
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.nodeID))
if err != nil {
return "", err
}
if repl.Object.ObjectID == nil {
return "", errors.New("unable to resolve node")
}
objID = *repl.Object.ObjectID
}
// not a document
if id.nodeID != 1 {
res, err := eval.Property(ctx, client, objID, "innerText")
if err != nil {
return "", err
}
return values.NewString(res.String()), err
}
repl, err := client.DOM.GetOuterHTML(ctx, dom.NewGetOuterHTMLArgs().SetObjectID(objID))
if err != nil {
return "", err
}
return parseInnerText(repl.OuterHTML)
}
//func loadInnerTextByNodeID(ctx context.Context, client *cdp.Client, exec *eval.ExecutionContext, nodeID dom.NodeID) (values.String, error) {
// node, err := client.DOM.DescribeNode(ctx, dom.NewDescribeNodeArgs().SetNodeID(nodeID))
//
// if err != nil {
// return values.EmptyString, err
// }
//
// return loadInnerText(ctx, client, exec, HTMLElementIdentity{
// nodeID: nodeID,
// }, common.ToHTMLType(node.Node.NodeType))
//}
func parseInnerText(innerHTML string) (values.String, error) {
buff := bytes.NewBuffer([]byte(innerHTML))
@@ -283,11 +319,12 @@ func parseInnerText(innerHTML string) (values.String, error) {
return values.NewString(parsed.Text()), nil
}
func createChildrenArray(nodes []dom.Node) []*HTMLElementIdentity {
children := make([]*HTMLElementIdentity, len(nodes))
func createChildrenArray(nodes []dom.Node) []HTMLElementIdentity {
children := make([]HTMLElementIdentity, len(nodes))
for idx, child := range nodes {
children[idx] = &HTMLElementIdentity{
child := child
children[idx] = HTMLElementIdentity{
nodeID: child.NodeID,
backendID: child.BackendNodeID,
}
@@ -296,122 +333,6 @@ func createChildrenArray(nodes []dom.Node) []*HTMLElementIdentity {
return children
}
func waitForLoadEvent(ctx context.Context, client *cdp.Client) error {
loadEventFired, err := client.Page.LoadEventFired(ctx)
if err != nil {
return err
}
_, err = loadEventFired.Recv()
if err != nil {
return err
}
return loadEventFired.Close()
}
func createEventBroker(client *cdp.Client) (*events.EventBroker, error) {
var err error
var onLoad page.LoadEventFiredClient
var onReload dom.DocumentUpdatedClient
var onAttrModified dom.AttributeModifiedClient
var onAttrRemoved dom.AttributeRemovedClient
var onChildCountUpdated dom.ChildNodeCountUpdatedClient
var onChildNodeInserted dom.ChildNodeInsertedClient
var onChildNodeRemoved dom.ChildNodeRemovedClient
ctx := context.Background()
onLoad, err = client.Page.LoadEventFired(ctx)
if err != nil {
return nil, err
}
onReload, err = client.DOM.DocumentUpdated(ctx)
if err != nil {
onLoad.Close()
return nil, err
}
onAttrModified, err = client.DOM.AttributeModified(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
return nil, err
}
onAttrRemoved, err = client.DOM.AttributeRemoved(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
return nil, err
}
onChildCountUpdated, err = client.DOM.ChildNodeCountUpdated(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
onAttrRemoved.Close()
return nil, err
}
onChildNodeInserted, err = client.DOM.ChildNodeInserted(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
onAttrRemoved.Close()
onChildCountUpdated.Close()
return nil, err
}
onChildNodeRemoved, err = client.DOM.ChildNodeRemoved(ctx)
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
onAttrRemoved.Close()
onChildCountUpdated.Close()
onChildNodeInserted.Close()
return nil, err
}
broker := events.NewEventBroker(
onLoad,
onReload,
onAttrModified,
onAttrRemoved,
onChildCountUpdated,
onChildNodeInserted,
onChildNodeRemoved,
)
err = broker.Start()
if err != nil {
onLoad.Close()
onReload.Close()
onAttrModified.Close()
onAttrRemoved.Close()
onChildCountUpdated.Close()
onChildNodeInserted.Close()
onChildNodeRemoved.Close()
return nil, err
}
return broker, nil
}
func fromDriverCookie(url string, cookie drivers.HTTPCookie) network.CookieParam {
sameSite := network.CookieSameSiteNotSet
@@ -491,3 +412,59 @@ func randomDuration(delay values.Int) time.Duration {
return time.Duration(int64(value))
}
func resolveFrame(ctx context.Context, client *cdp.Client, frame page.Frame) (dom.Node, runtime.ExecutionContextID, error) {
worldRepl, err := client.Page.CreateIsolatedWorld(ctx, page.NewCreateIsolatedWorldArgs(frame.ID))
if err != nil {
return dom.Node{}, -1, err
}
evalRes, err := client.Runtime.Evaluate(
ctx,
runtime.NewEvaluateArgs(eval.PrepareEval("return document")).
SetContextID(worldRepl.ExecutionContextID),
)
if err != nil {
return dom.Node{}, -1, err
}
if evalRes.ExceptionDetails != nil {
exception := *evalRes.ExceptionDetails
return dom.Node{}, -1, errors.New(exception.Text)
}
if evalRes.Result.ObjectID == nil {
return dom.Node{}, -1, errors.New("failed to resolve frame document")
}
req, err := client.DOM.RequestNode(ctx, dom.NewRequestNodeArgs(*evalRes.Result.ObjectID))
if err != nil {
return dom.Node{}, -1, err
}
if req.NodeID == 0 {
return dom.Node{}, -1, errors.New("framed document is resolved with empty node id")
}
desc, err := client.DOM.DescribeNode(
ctx,
dom.
NewDescribeNodeArgs().
SetNodeID(req.NodeID).
SetDepth(1),
)
if err != nil {
return dom.Node{}, -1, err
}
// Returned node, by some reason, does not contain the NodeID
// So, we have to set it manually
desc.Node.NodeID = req.NodeID
return desc.Node, worldRepl.ExecutionContextID, nil
}

751
pkg/drivers/cdp/page.go Normal file
View File

@@ -0,0 +1,751 @@
package cdp
import (
"context"
"encoding/json"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/mafredri/cdp/protocol/emulation"
"github.com/mafredri/cdp/protocol/page"
"hash/fnv"
"sync"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/network"
"github.com/mafredri/cdp/rpcc"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type HTMLPage struct {
mu sync.Mutex
closed values.Boolean
logger *zerolog.Logger
conn *rpcc.Conn
client *cdp.Client
events *events.EventBroker
document *HTMLDocument
frames *common.LazyValue
}
func handleLoadError(logger *zerolog.Logger, client *cdp.Client) {
err := client.Page.Close(context.Background())
if err != nil {
logger.Warn().Timestamp().Err(err).Msg("failed to close document on load error")
}
}
func LoadHTMLPage(
ctx context.Context,
conn *rpcc.Conn,
params drivers.OpenPageParams,
) (*HTMLPage, error) {
logger := logging.FromContext(ctx)
if conn == nil {
return nil, core.Error(core.ErrMissedArgument, "connection")
}
client := cdp.NewClient(conn)
err := runBatch(
func() error {
return client.Page.Enable(ctx)
},
func() error {
return client.Page.SetLifecycleEventsEnabled(
ctx,
page.NewSetLifecycleEventsEnabledArgs(true),
)
},
func() error {
return client.DOM.Enable(ctx)
},
func() error {
return client.Runtime.Enable(ctx)
},
func() error {
ua := common.GetUserAgent(params.UserAgent)
logger.
Debug().
Timestamp().
Str("user-agent", ua).
Msg("using User-Agent")
// do not use custom user agent
if ua == "" {
return nil
}
return client.Emulation.SetUserAgentOverride(
ctx,
emulation.NewSetUserAgentOverrideArgs(ua),
)
},
func() error {
return client.Network.Enable(ctx, network.NewEnableArgs())
},
)
if err != nil {
return nil, err
}
err = client.Page.SetBypassCSP(ctx, page.NewSetBypassCSPArgs(true))
if err != nil {
return nil, err
}
if params.Cookies != nil {
cookies := make([]network.CookieParam, 0, len(params.Cookies))
for _, c := range params.Cookies {
cookies = append(cookies, fromDriverCookie(params.URL, c))
logger.
Debug().
Timestamp().
Str("cookie", c.Name).
Msg("set cookie")
}
err = client.Network.SetCookies(
ctx,
network.NewSetCookiesArgs(cookies),
)
if err != nil {
return nil, errors.Wrap(err, "failed to set cookies")
}
}
if params.Header != nil {
j, err := json.Marshal(params.Header)
if err != nil {
return nil, err
}
for k := range params.Header {
logger.
Debug().
Timestamp().
Str("header", k).
Msg("set header")
}
err = client.Network.SetExtraHTTPHeaders(
ctx,
network.NewSetExtraHTTPHeadersArgs(network.Headers(j)),
)
if err != nil {
return nil, errors.Wrap(err, "failed to set headers")
}
}
if params.URL != BlankPageURL && params.URL != "" {
repl, err := client.Page.Navigate(ctx, page.NewNavigateArgs(params.URL))
if err != nil {
return nil, errors.Wrap(err, "failed to load the page")
}
if repl.ErrorText != nil {
return nil, errors.Wrapf(errors.New(*repl.ErrorText), "failed to load the page: %s", params.URL)
}
err = events.WaitForLoadEvent(ctx, client)
if err != nil {
handleLoadError(logger, client)
return nil, errors.Wrap(err, "failed to load the page")
}
}
broker, err := events.CreateEventBroker(client)
if err != nil {
handleLoadError(logger, client)
return nil, errors.Wrap(err, "failed to create event events")
}
doc, err := LoadRootHTMLDocument(ctx, logger, client, broker)
if err != nil {
broker.StopAndClose()
handleLoadError(logger, client)
return nil, errors.Wrap(err, "failed to load root element")
}
return NewHTMLPage(
logger,
conn,
client,
broker,
doc,
), nil
}
func NewHTMLPage(
logger *zerolog.Logger,
conn *rpcc.Conn,
client *cdp.Client,
broker *events.EventBroker,
document *HTMLDocument,
) *HTMLPage {
p := new(HTMLPage)
p.closed = values.False
p.logger = logger
p.conn = conn
p.client = client
p.events = broker
p.document = document
p.frames = common.NewLazyValue(p.unfoldFrames)
broker.AddEventListener(events.EventLoad, p.handlePageLoad)
broker.AddEventListener(events.EventError, p.handleError)
return p
}
func (p *HTMLPage) MarshalJSON() ([]byte, error) {
p.mu.Lock()
defer p.mu.Unlock()
return p.document.MarshalJSON()
}
func (p *HTMLPage) Type() core.Type {
return drivers.HTMLPageType
}
func (p *HTMLPage) String() string {
p.mu.Lock()
defer p.mu.Unlock()
return p.document.GetURL().String()
}
func (p *HTMLPage) Compare(other core.Value) int64 {
p.mu.Lock()
defer p.mu.Unlock()
tc := drivers.Compare(p.Type(), other.Type())
if tc != 0 {
return tc
}
cdpPage, ok := other.(*HTMLPage)
if !ok {
return 1
}
return p.document.GetURL().Compare(cdpPage.GetURL())
}
func (p *HTMLPage) Unwrap() interface{} {
p.mu.Lock()
defer p.mu.Unlock()
return p
}
func (p *HTMLPage) Hash() uint64 {
p.mu.Lock()
defer p.mu.Unlock()
h := fnv.New64a()
h.Write([]byte("CDP"))
h.Write([]byte(p.Type().String()))
h.Write([]byte(":"))
h.Write([]byte(p.document.GetURL()))
return h.Sum64()
}
func (p *HTMLPage) Copy() core.Value {
return values.None
}
func (p *HTMLPage) GetIn(ctx context.Context, path []core.Value) (core.Value, error) {
return common.GetInPage(ctx, p, path)
}
func (p *HTMLPage) SetIn(ctx context.Context, path []core.Value, value core.Value) error {
return common.SetInPage(ctx, p, path, value)
}
func (p *HTMLPage) Iterate(ctx context.Context) (core.Iterator, error) {
p.mu.Lock()
defer p.mu.Unlock()
return p.document.Iterate(ctx)
}
func (p *HTMLPage) Length() values.Int {
p.mu.Lock()
defer p.mu.Unlock()
return p.document.Length()
}
func (p *HTMLPage) Close() error {
p.mu.Lock()
defer p.mu.Unlock()
p.closed = values.True
err := p.events.Stop()
if err != nil {
p.logger.Warn().
Timestamp().
Str("url", p.document.GetURL().String()).
Err(err).
Msg("failed to stop event events")
}
err = p.events.Close()
if err != nil {
p.logger.Warn().
Timestamp().
Str("url", p.document.GetURL().String()).
Err(err).
Msg("failed to close event events")
}
err = p.document.Close()
if err != nil {
p.logger.Warn().
Timestamp().
Str("url", p.document.GetURL().String()).
Err(err).
Msg("failed to close root document")
}
err = p.client.Page.Close(context.Background())
if err != nil {
p.logger.Warn().
Timestamp().
Str("url", p.document.GetURL().String()).
Err(err).
Msg("failed to close browser page")
}
return p.conn.Close()
}
func (p *HTMLPage) IsClosed() values.Boolean {
p.mu.Lock()
defer p.mu.Unlock()
return p.closed
}
func (p *HTMLPage) GetURL() values.String {
p.mu.Lock()
defer p.mu.Unlock()
return p.document.GetURL()
}
func (p *HTMLPage) GetMainFrame() drivers.HTMLDocument {
p.mu.Lock()
defer p.mu.Unlock()
return p.document
}
func (p *HTMLPage) GetFrames(ctx context.Context) (*values.Array, error) {
p.mu.Lock()
defer p.mu.Unlock()
res, err := p.frames.Read(ctx)
if err != nil {
return nil, err
}
return res.(*values.Array).Clone().(*values.Array), nil
}
func (p *HTMLPage) GetFrame(ctx context.Context, idx values.Int) (core.Value, error) {
p.mu.Lock()
defer p.mu.Unlock()
res, err := p.frames.Read(ctx)
if err != nil {
return nil, err
}
return res.(*values.Array).Get(idx), nil
}
func (p *HTMLPage) GetCookies(ctx context.Context) (*values.Array, error) {
p.mu.Lock()
defer p.mu.Unlock()
repl, err := p.client.Network.GetAllCookies(ctx)
if err != nil {
return values.NewArray(0), err
}
if repl.Cookies == nil {
return values.NewArray(0), nil
}
cookies := values.NewArray(len(repl.Cookies))
for _, c := range repl.Cookies {
cookies.Push(toDriverCookie(c))
}
return cookies, nil
}
func (p *HTMLPage) SetCookies(ctx context.Context, cookies ...drivers.HTTPCookie) error {
p.mu.Lock()
defer p.mu.Unlock()
if len(cookies) == 0 {
return nil
}
params := make([]network.CookieParam, 0, len(cookies))
for _, c := range cookies {
params = append(params, fromDriverCookie(p.document.GetURL().String(), c))
}
return p.client.Network.SetCookies(ctx, network.NewSetCookiesArgs(params))
}
func (p *HTMLPage) DeleteCookies(ctx context.Context, cookies ...drivers.HTTPCookie) error {
p.mu.Lock()
defer p.mu.Unlock()
if len(cookies) == 0 {
return nil
}
var err error
for _, c := range cookies {
err = p.client.Network.DeleteCookies(ctx, fromDriverCookieDelete(p.document.GetURL().String(), c))
if err != nil {
break
}
}
return err
}
func (p *HTMLPage) PrintToPDF(ctx context.Context, params drivers.PDFParams) (values.Binary, error) {
args := page.NewPrintToPDFArgs()
args.
SetLandscape(bool(params.Landscape)).
SetDisplayHeaderFooter(bool(params.DisplayHeaderFooter)).
SetPrintBackground(bool(params.PrintBackground)).
SetIgnoreInvalidPageRanges(bool(params.IgnoreInvalidPageRanges)).
SetPreferCSSPageSize(bool(params.PreferCSSPageSize))
if params.Scale > 0 {
args.SetScale(float64(params.Scale))
}
if params.PaperWidth > 0 {
args.SetPaperWidth(float64(params.PaperWidth))
}
if params.PaperHeight > 0 {
args.SetPaperHeight(float64(params.PaperHeight))
}
if params.MarginTop > 0 {
args.SetMarginTop(float64(params.MarginTop))
}
if params.MarginBottom > 0 {
args.SetMarginBottom(float64(params.MarginBottom))
}
if params.MarginRight > 0 {
args.SetMarginRight(float64(params.MarginRight))
}
if params.MarginLeft > 0 {
args.SetMarginLeft(float64(params.MarginLeft))
}
if params.PageRanges != values.EmptyString {
args.SetPageRanges(string(params.PageRanges))
}
if params.HeaderTemplate != values.EmptyString {
args.SetHeaderTemplate(string(params.HeaderTemplate))
}
if params.FooterTemplate != values.EmptyString {
args.SetFooterTemplate(string(params.FooterTemplate))
}
reply, err := p.client.Page.PrintToPDF(ctx, args)
if err != nil {
return values.NewBinary([]byte{}), err
}
return values.NewBinary(reply.Data), nil
}
func (p *HTMLPage) CaptureScreenshot(ctx context.Context, params drivers.ScreenshotParams) (values.Binary, error) {
metrics, err := p.client.Page.GetLayoutMetrics(ctx)
if err != nil {
return values.NewBinary(nil), err
}
if params.Format == drivers.ScreenshotFormatJPEG && params.Quality < 0 && params.Quality > 100 {
params.Quality = 100
}
if params.X < 0 {
params.X = 0
}
if params.Y < 0 {
params.Y = 0
}
if params.Width <= 0 {
params.Width = values.Float(metrics.LayoutViewport.ClientWidth) - params.X
}
if params.Height <= 0 {
params.Height = values.Float(metrics.LayoutViewport.ClientHeight) - params.Y
}
clip := page.Viewport{
X: float64(params.X),
Y: float64(params.Y),
Width: float64(params.Width),
Height: float64(params.Height),
Scale: 1.0,
}
format := string(params.Format)
quality := int(params.Quality)
args := page.CaptureScreenshotArgs{
Format: &format,
Quality: &quality,
Clip: &clip,
}
reply, err := p.client.Page.CaptureScreenshot(ctx, &args)
if err != nil {
return values.NewBinary([]byte{}), err
}
return values.NewBinary(reply.Data), nil
}
func (p *HTMLPage) Navigate(ctx context.Context, url values.String) error {
if url == "" {
url = BlankPageURL
}
repl, err := p.client.Page.Navigate(ctx, page.NewNavigateArgs(url.String()))
if err != nil {
return err
}
if repl.ErrorText != nil {
return errors.New(*repl.ErrorText)
}
return p.WaitForNavigation(ctx)
}
func (p *HTMLPage) NavigateBack(ctx context.Context, skip values.Int) (values.Boolean, error) {
history, err := p.client.Page.GetNavigationHistory(ctx)
if err != nil {
return values.False, err
}
// we are in the beginning
if history.CurrentIndex == 0 {
return values.False, nil
}
if skip < 1 {
skip = 1
}
to := history.CurrentIndex - int(skip)
if to < 0 {
// TODO: Return error?
return values.False, nil
}
prev := history.Entries[to]
err = p.client.Page.NavigateToHistoryEntry(ctx, page.NewNavigateToHistoryEntryArgs(prev.ID))
if err != nil {
return values.False, err
}
err = p.WaitForNavigation(ctx)
if err != nil {
return values.False, err
}
return values.True, nil
}
func (p *HTMLPage) NavigateForward(ctx context.Context, skip values.Int) (values.Boolean, error) {
history, err := p.client.Page.GetNavigationHistory(ctx)
if err != nil {
return values.False, err
}
length := len(history.Entries)
lastIndex := length - 1
// nowhere to go forward
if history.CurrentIndex == lastIndex {
return values.False, nil
}
if skip < 1 {
skip = 1
}
to := int(skip) + history.CurrentIndex
if to > lastIndex {
// TODO: Return error?
return values.False, nil
}
next := history.Entries[to]
err = p.client.Page.NavigateToHistoryEntry(ctx, page.NewNavigateToHistoryEntryArgs(next.ID))
if err != nil {
return values.False, err
}
err = p.WaitForNavigation(ctx)
if err != nil {
return values.False, err
}
return values.True, nil
}
func (p *HTMLPage) WaitForNavigation(ctx context.Context) error {
onEvent := make(chan struct{})
var once sync.Once
listener := func(_ context.Context, _ interface{}) {
once.Do(func() {
close(onEvent)
})
}
defer p.events.RemoveEventListener(events.EventLoad, listener)
p.events.AddEventListener(events.EventLoad, listener)
select {
case <-onEvent:
return nil
case <-ctx.Done():
return core.ErrTimeout
}
}
func (p *HTMLPage) handlePageLoad(ctx context.Context, _ interface{}) {
p.mu.Lock()
defer p.mu.Unlock()
nextDoc, err := LoadRootHTMLDocument(ctx, p.logger, p.client, p.events)
if err != nil {
p.logger.Error().
Timestamp().
Err(err).
Msg("failed to load new root document after page load")
return
}
// close the prev document
err = p.document.Close()
if err != nil {
p.logger.Error().
Timestamp().
Err(err).
Msgf("failed to close root document: %s", p.document.GetURL())
}
// set the new root document
p.document = nextDoc
// reset all loaded frames
p.frames.Reset()
}
func (p *HTMLPage) handleError(_ context.Context, val interface{}) {
err, ok := val.(error)
if !ok {
return
}
p.logger.Error().
Timestamp().
Err(err).
Msg("unexpected error")
}
func (p *HTMLPage) unfoldFrames(ctx context.Context) (core.Value, error) {
res := values.NewArray(10)
err := common.CollectFrames(ctx, res, p.document)
if err != nil {
return nil, err
}
return res, nil
}

32
pkg/drivers/common/frames.go Normal file
View File

@@ -0,0 +1,32 @@
package common
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
func CollectFrames(ctx context.Context, receiver *values.Array, doc drivers.HTMLDocument) error {
receiver.Push(doc)
children, err := doc.GetChildDocuments(ctx)
if err != nil {
return err
}
children.ForEach(func(value core.Value, idx int) bool {
childDoc, ok := value.(drivers.HTMLDocument)
if !ok {
err = core.TypeError(value.Type(), drivers.HTMLDocumentType)
return false
}
return CollectFrames(ctx, receiver, childDoc) == nil
})
return nil
}

135
pkg/drivers/common/getter.go
View File

@@ -9,9 +9,81 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
func GetInPage(ctx context.Context, page drivers.HTMLPage, path []core.Value) (core.Value, error) {
if len(path) == 0 {
return page, nil
}
segment := path[0]
if segment.Type() == types.String {
segment := segment.(values.String)
switch segment {
case "mainFrame", "document":
return GetInDocument(ctx, page.GetMainFrame(), path[1:])
case "frames":
if len(path) == 1 {
return page.GetFrames(ctx)
}
idx := path[1]
if !values.IsNumber(idx) {
return values.None, core.TypeError(idx.Type(), types.Int, types.Float)
}
value, err := page.GetFrame(ctx, values.ToInt(idx))
if err != nil {
return values.None, err
}
if len(path) == 2 {
return value, nil
}
frame, err := drivers.ToDocument(value)
if err != nil {
return values.None, err
}
return GetInDocument(ctx, frame, path[2:])
case "url", "URL":
return page.GetMainFrame().GetURL(), nil
case "cookies":
if len(path) == 1 {
return page.GetCookies(ctx)
}
switch idx := path[1].(type) {
case values.Int:
cookies, err := page.GetCookies(ctx)
if err != nil {
return values.None, err
}
return cookies.Get(idx), nil
default:
return values.None, core.TypeError(idx.Type(), types.Int)
}
case "isClosed":
return page.IsClosed(), nil
case "title":
return page.GetMainFrame().GetTitle(), nil
default:
return GetInDocument(ctx, page.GetMainFrame(), path)
}
}
return GetInDocument(ctx, page.GetMainFrame(), path)
}
func GetInDocument(ctx context.Context, doc drivers.HTMLDocument, path []core.Value) (core.Value, error) {
if len(path) == 0 {
return values.None, nil
return doc, nil
}
segment := path[0]
@@ -22,38 +94,49 @@ func GetInDocument(ctx context.Context, doc drivers.HTMLDocument, path []core.Va
switch segment {
case "url", "URL":
return doc.GetURL(), nil
case "cookies":
case "title":
return doc.GetTitle(), nil
case "parent":
parent := doc.GetParentDocument()
if parent == nil {
return values.None, nil
}
if len(path) == 1 {
return doc.GetCookies(ctx)
return parent, nil
}
switch idx := path[1].(type) {
case values.Int:
cookies, err := doc.GetCookies(ctx)
return GetInDocument(ctx, parent, path[1:])
case "body", "head":
out := doc.QuerySelector(ctx, segment)
if err != nil {
return values.None, err
}
return cookies.Get(idx), nil
default:
return values.None, core.TypeError(idx.Type(), types.Int)
if out == values.None {
return out, nil
}
case "body":
return doc.QuerySelector(ctx, "body"), nil
case "head":
return doc.QuerySelector(ctx, "head"), nil
if len(path) == 1 {
return out, nil
}
el, err := drivers.ToElement(out)
if err != nil {
return values.None, err
}
return GetInElement(ctx, el, path[1:])
default:
return GetInNode(ctx, doc.DocumentElement(), path)
return GetInNode(ctx, doc.GetElement(), path)
}
}
return GetInNode(ctx, doc.DocumentElement(), path)
return GetInNode(ctx, doc.GetElement(), path)
}
func GetInElement(ctx context.Context, el drivers.HTMLElement, path []core.Value) (core.Value, error) {
if len(path) == 0 {
return values.None, nil
return el, nil
}
segment := path[0]
@@ -63,9 +146,9 @@ func GetInElement(ctx context.Context, el drivers.HTMLElement, path []core.Value
switch segment {
case "innerText":
return el.InnerText(ctx), nil
return el.GetInnerText(ctx), nil
case "innerHTML":
return el.InnerHTML(ctx), nil
return el.GetInnerHTML(ctx), nil
case "value":
return el.GetValue(ctx), nil
case "attributes":
@@ -98,7 +181,7 @@ func GetInElement(ctx context.Context, el drivers.HTMLElement, path []core.Value
func GetInNode(ctx context.Context, node drivers.HTMLNode, path []core.Value) (core.Value, error) {
if len(path) == 0 {
return values.None, nil
return node, nil
}
nt := node.Type()
@@ -118,10 +201,12 @@ func GetInNode(ctx context.Context, node drivers.HTMLNode, path []core.Value) (c
segment := segment.(values.String)
switch segment {
case "isDetached":
return node.IsDetached(), nil
case "nodeType":
return node.NodeType(), nil
return node.GetNodeType(), nil
case "nodeName":
return node.NodeName(), nil
return node.GetNodeName(), nil
case "children":
children := node.GetChildNodes(ctx)

18
pkg/drivers/common/lazy.go
View File

@@ -12,7 +12,7 @@ type (
LazyValueFactory func(ctx context.Context) (core.Value, error)
LazyValue struct {
sync.Mutex
mu sync.Mutex
factory LazyValueFactory
ready bool
value core.Value
@@ -32,8 +32,8 @@ func NewLazyValue(factory LazyValueFactory) *LazyValue {
// Ready indicates whether the value is ready.
// @returns (Boolean) - Boolean value indicating whether the value is ready.
func (lv *LazyValue) Ready() bool {
lv.Lock()
defer lv.Unlock()
lv.mu.Lock()
defer lv.mu.Unlock()
return lv.ready
}
@@ -42,8 +42,8 @@ func (lv *LazyValue) Ready() bool {
// Not thread safe. Should not mutated.
// @returns (Value) - Underlying value if successfully loaded, otherwise error
func (lv *LazyValue) Read(ctx context.Context) (core.Value, error) {
lv.Lock()
defer lv.Unlock()
lv.mu.Lock()
defer lv.mu.Unlock()
if !lv.ready {
lv.load(ctx)
@@ -56,8 +56,8 @@ func (lv *LazyValue) Read(ctx context.Context) (core.Value, error) {
// Loads a value if it's not ready.
// Thread safe.
func (lv *LazyValue) Write(ctx context.Context, writer func(v core.Value, err error)) {
lv.Lock()
defer lv.Unlock()
lv.mu.Lock()
defer lv.mu.Unlock()
if !lv.ready {
lv.load(ctx)
@@ -69,8 +69,8 @@ func (lv *LazyValue) Write(ctx context.Context, writer func(v core.Value, err er
// Reset resets the storage.
// Next call of Read will trigger the factory function again.
func (lv *LazyValue) Reset() {
lv.Lock()
defer lv.Unlock()
lv.mu.Lock()
defer lv.mu.Unlock()
lv.ready = false
lv.value = values.None

19
pkg/drivers/common/setter.go
View File

@@ -9,24 +9,17 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
func SetInDocument(ctx context.Context, doc drivers.HTMLDocument, path []core.Value, value core.Value) error {
func SetInPage(ctx context.Context, page drivers.HTMLPage, path []core.Value, value core.Value) error {
if len(path) == 0 {
return nil
}
segment := path[0]
return SetInDocument(ctx, page.GetMainFrame(), path, value)
}
if segment.Type() == types.String {
segment := segment.(values.String)
switch segment {
case "url", "URL":
return doc.SetURL(ctx, values.NewString(value.String()))
case "cookies":
default:
return SetInNode(ctx, doc, path, value)
}
func SetInDocument(ctx context.Context, doc drivers.HTMLDocument, path []core.Value, value core.Value) error {
if len(path) == 0 {
return nil
}
return SetInNode(ctx, doc, path, value)

23
pkg/drivers/common/types.go
View File

@@ -1,8 +1,10 @@
package common
import "golang.org/x/net/html"
import (
"golang.org/x/net/html"
)
func ToHTMLType(nt html.NodeType) int {
func FromHTMLType(nt html.NodeType) int {
switch nt {
case html.DocumentNode:
return 9
@@ -18,3 +20,20 @@ func ToHTMLType(nt html.NodeType) int {
return 0
}
func ToHTMLType(input int) html.NodeType {
switch input {
case 1:
return html.ElementNode
case 3:
return html.TextNode
case 8:
return html.CommentNode
case 9:
return html.DocumentNode
case 10:
return html.DoctypeNode
default:
return html.ErrorNode
}
}

4
pkg/drivers/driver.go
View File

@@ -18,7 +18,7 @@ type (
drivers map[string]Driver
}
LoadDocumentParams struct {
OpenPageParams struct {
URL string
UserAgent string
KeepCookies bool
@@ -29,7 +29,7 @@ type (
Driver interface {
io.Closer
Name() string
LoadDocument(ctx context.Context, params LoadDocumentParams) (HTMLDocument, error)
Open(ctx context.Context, params OpenPageParams) (HTMLPage, error)
}
)

44
pkg/drivers/helpers.go
View File

@@ -2,8 +2,52 @@ package drivers
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
)
func WithDefaultTimeout(ctx context.Context) (context.Context, context.CancelFunc) {
return context.WithTimeout(ctx, DefaultTimeout)
}
func ToPage(value core.Value) (HTMLPage, error) {
err := core.ValidateType(value, HTMLPageType)
if err != nil {
return nil, err
}
return value.(HTMLPage), nil
}
func ToDocument(value core.Value) (HTMLDocument, error) {
switch v := value.(type) {
case HTMLPage:
return v.GetMainFrame(), nil
case HTMLDocument:
return v, nil
default:
return nil, core.TypeError(
value.Type(),
HTMLPageType,
HTMLDocumentType,
)
}
}
func ToElement(value core.Value) (HTMLElement, error) {
switch v := value.(type) {
case HTMLPage:
return v.GetMainFrame().GetElement(), nil
case HTMLDocument:
return v.GetElement(), nil
case HTMLElement:
return v, nil
default:
return nil, core.TypeError(
value.Type(),
HTMLPageType,
HTMLDocumentType,
HTMLElementType,
)
}
}

113
pkg/drivers/http/document.go
View File

@@ -12,17 +12,25 @@ import (
)
type HTMLDocument struct {
docNode *goquery.Document
element drivers.HTMLElement
url values.String
cookies []drivers.HTTPCookie
doc *goquery.Document
element drivers.HTMLElement
url values.String
parent drivers.HTMLDocument
children *values.Array
}
func NewRootHTMLDocument(
node *goquery.Document,
url string,
) (*HTMLDocument, error) {
return NewHTMLDocument(node, url, nil)
}
func NewHTMLDocument(
node *goquery.Document,
url string,
cookies []drivers.HTTPCookie,
) (drivers.HTMLDocument, error) {
parent drivers.HTMLDocument,
) (*HTMLDocument, error) {
if url == "" {
return nil, core.Error(core.ErrMissedArgument, "document url")
}
@@ -37,7 +45,21 @@ func NewHTMLDocument(
return nil, err
}
return &HTMLDocument{node, el, values.NewString(url), cookies}, nil
doc := new(HTMLDocument)
doc.doc = node
doc.element = el
doc.parent = parent
doc.url = values.NewString(url)
doc.children = values.NewArray(10)
frames := node.Find("iframe")
frames.Each(func(i int, selection *goquery.Selection) {
child, _ := NewHTMLDocument(goquery.NewDocumentFromNode(selection.Nodes[0]), selection.AttrOr("src", url), doc)
doc.children.Push(child)
})
return doc, nil
}
func (doc *HTMLDocument) MarshalJSON() ([]byte, error) {
@@ -49,7 +71,7 @@ func (doc *HTMLDocument) Type() core.Type {
}
func (doc *HTMLDocument) String() string {
str, err := doc.docNode.Html()
str, err := doc.doc.Html()
if err != nil {
return ""
@@ -70,7 +92,7 @@ func (doc *HTMLDocument) Compare(other core.Value) int64 {
}
func (doc *HTMLDocument) Unwrap() interface{} {
return doc.docNode
return doc.doc
}
func (doc *HTMLDocument) Hash() uint64 {
@@ -84,7 +106,7 @@ func (doc *HTMLDocument) Hash() uint64 {
}
func (doc *HTMLDocument) Copy() core.Value {
cp, err := NewHTMLDocument(doc.docNode, string(doc.url), doc.cookies)
cp, err := NewHTMLDocument(doc.doc, string(doc.url), doc.parent)
if err != nil {
return values.None
@@ -94,24 +116,17 @@ func (doc *HTMLDocument) Copy() core.Value {
}
func (doc *HTMLDocument) Clone() core.Value {
var cookies []drivers.HTTPCookie
if doc.cookies != nil {
cookies = make([]drivers.HTTPCookie, len(doc.cookies))
copy(cookies, doc.cookies)
}
cp, err := NewHTMLDocument(goquery.CloneDocument(doc.docNode), string(doc.url), cookies)
cloned, err := NewHTMLDocument(doc.doc, doc.url.String(), doc.parent)
if err != nil {
return values.None
}
return cp
return cloned
}
func (doc *HTMLDocument) Length() values.Int {
return values.NewInt(doc.docNode.Length())
return values.NewInt(doc.doc.Length())
}
func (doc *HTMLDocument) Iterate(_ context.Context) (core.Iterator, error) {
@@ -126,11 +141,11 @@ func (doc *HTMLDocument) SetIn(ctx context.Context, path []core.Value, value cor
return common.SetInDocument(ctx, doc, path, value)
}
func (doc *HTMLDocument) NodeType() values.Int {
func (doc *HTMLDocument) GetNodeType() values.Int {
return 9
}
func (doc *HTMLDocument) NodeName() values.String {
func (doc *HTMLDocument) GetNodeName() values.String {
return "#document"
}
@@ -158,50 +173,34 @@ func (doc *HTMLDocument) ExistsBySelector(ctx context.Context, selector values.S
return doc.element.ExistsBySelector(ctx, selector)
}
func (doc *HTMLDocument) DocumentElement() drivers.HTMLElement {
return doc.element
func (doc *HTMLDocument) IsDetached() values.Boolean {
return values.False
}
func (doc *HTMLDocument) GetURL() core.Value {
func (doc *HTMLDocument) GetTitle() values.String {
title := doc.doc.Find("head > title")
return values.NewString(title.Text())
}
func (doc *HTMLDocument) GetChildDocuments(_ context.Context) (*values.Array, error) {
return doc.children.Clone().(*values.Array), nil
}
func (doc *HTMLDocument) GetURL() values.String {
return doc.url
}
func (doc *HTMLDocument) SetURL(_ context.Context, _ values.String) error {
return core.ErrInvalidOperation
func (doc *HTMLDocument) GetElement() drivers.HTMLElement {
return doc.element
}
func (doc *HTMLDocument) GetCookies(_ context.Context) (*values.Array, error) {
if doc.cookies == nil {
return values.NewArray(0), nil
}
arr := values.NewArray(len(doc.cookies))
for _, c := range doc.cookies {
arr.Push(c)
}
return arr, nil
func (doc *HTMLDocument) GetName() values.String {
return ""
}
func (doc *HTMLDocument) SetCookies(_ context.Context, _ ...drivers.HTTPCookie) error {
return core.ErrNotSupported
}
func (doc *HTMLDocument) DeleteCookies(_ context.Context, _ ...drivers.HTTPCookie) error {
return core.ErrNotSupported
}
func (doc *HTMLDocument) Navigate(_ context.Context, _ values.String) error {
return core.ErrNotSupported
}
func (doc *HTMLDocument) NavigateBack(_ context.Context, _ values.Int) (values.Boolean, error) {
return false, core.ErrNotSupported
}
func (doc *HTMLDocument) NavigateForward(_ context.Context, _ values.Int) (values.Boolean, error) {
return false, core.ErrNotSupported
func (doc *HTMLDocument) GetParentDocument() drivers.HTMLDocument {
return doc.parent
}
func (doc *HTMLDocument) ClickBySelector(_ context.Context, _ values.String) (values.Boolean, error) {

4
pkg/drivers/http/document_test.go
View File

@@ -220,7 +220,7 @@ func TestDocument(t *testing.T) {
</footer>
<svg xmlns="http://www.w3.org/2000/svg" width="348" height="225" viewBox="0 0 348 225" preserveAspectRatio="none" style="display: none; visibility: hidden; position: absolute; top: -100%; left: -100%;"><defs><style type="text/css"></style></defs><text x="0" y="17" style="font-weight:bold;font-size:17pt;font-family:Arial, Helvetica, Open Sans, sans-serif">Thumbnail</text></svg></body></html>
`
Convey(".NodeType", t, func() {
Convey(".GetNodeType", t, func() {
Convey("Should serialize a boolean value", func() {
buff := bytes.NewBuffer([]byte(doc))
@@ -234,7 +234,7 @@ func TestDocument(t *testing.T) {
So(err, ShouldBeNil)
So(el.NodeType(), ShouldEqual, 9)
So(el.GetNodeType(), ShouldEqual, 9)
})
})
}

8
pkg/drivers/http/driver.go
View File

@@ -62,7 +62,7 @@ func (drv *Driver) Name() string {
return DriverName
}
func (drv *Driver) LoadDocument(ctx context.Context, params drivers.LoadDocumentParams) (drivers.HTMLDocument, error) {
func (drv *Driver) Open(ctx context.Context, params drivers.OpenPageParams) (drivers.HTMLPage, error) {
req, err := http.NewRequest(http.MethodGet, params.URL, nil)
if err != nil {
@@ -133,10 +133,10 @@ func (drv *Driver) LoadDocument(ctx context.Context, params drivers.LoadDocument
return nil, errors.Wrapf(err, "failed to parse a document %s", params.URL)
}
return NewHTMLDocument(doc, params.URL, params.Cookies)
return NewHTMLPage(doc, params.URL, params.Cookies)
}
func (drv *Driver) ParseDocument(_ context.Context, str values.String) (drivers.HTMLDocument, error) {
func (drv *Driver) Parse(_ context.Context, str values.String) (drivers.HTMLPage, error) {
buf := bytes.NewBuffer([]byte(str))
doc, err := goquery.NewDocumentFromReader(buf)
@@ -145,7 +145,7 @@ func (drv *Driver) ParseDocument(_ context.Context, str values.String) (drivers.
return nil, errors.Wrap(err, "failed to parse a document")
}
return NewHTMLDocument(doc, "#string", nil)
return NewHTMLPage(doc, "#blank", nil)
}
func (drv *Driver) Close() error {

20
pkg/drivers/http/element.go
View File

@@ -29,7 +29,7 @@ func NewHTMLElement(node *goquery.Selection) (drivers.HTMLElement, error) {
}
func (el *HTMLElement) MarshalJSON() ([]byte, error) {
return json.Marshal(el.InnerText(context.Background()).String())
return json.Marshal(el.GetInnerText(context.Background()).String())
}
func (el *HTMLElement) Type() core.Type {
@@ -37,7 +37,7 @@ func (el *HTMLElement) Type() core.Type {
}
func (el *HTMLElement) String() string {
return el.InnerHTML(context.Background()).String()
return el.GetInnerHTML(context.Background()).String()
}
func (el *HTMLElement) Compare(other core.Value) int64 {
@@ -48,7 +48,7 @@ func (el *HTMLElement) Compare(other core.Value) int64 {
ctx, fn := drivers.WithDefaultTimeout(context.Background())
defer fn()
return el.InnerHTML(ctx).Compare(other.InnerHTML(ctx))
return el.GetInnerHTML(ctx).Compare(other.GetInnerHTML(ctx))
default:
return drivers.Compare(el.Type(), other.Type())
}
@@ -80,21 +80,25 @@ func (el *HTMLElement) Copy() core.Value {
return c
}
func (el *HTMLElement) NodeType() values.Int {
func (el *HTMLElement) IsDetached() values.Boolean {
return values.True
}
func (el *HTMLElement) GetNodeType() values.Int {
nodes := el.selection.Nodes
if len(nodes) == 0 {
return 0
}
return values.NewInt(common.ToHTMLType(nodes[0].Type))
return values.NewInt(common.FromHTMLType(nodes[0].Type))
}
func (el *HTMLElement) Close() error {
return nil
}
func (el *HTMLElement) NodeName() values.String {
func (el *HTMLElement) GetNodeName() values.String {
return values.NewString(goquery.NodeName(el.selection))
}
@@ -122,11 +126,11 @@ func (el *HTMLElement) SetValue(_ context.Context, value core.Value) error {
return nil
}
func (el *HTMLElement) InnerText(_ context.Context) values.String {
func (el *HTMLElement) GetInnerText(_ context.Context) values.String {
return values.NewString(el.selection.Text())
}
func (el *HTMLElement) InnerHTML(_ context.Context) values.String {
func (el *HTMLElement) GetInnerHTML(_ context.Context) values.String {
h, err := el.selection.Html()
if err != nil {

16
pkg/drivers/http/element_test.go
View File

@@ -244,7 +244,7 @@ func TestElement(t *testing.T) {
<svg xmlns="http://www.w3.org/2000/svg" width="348" height="225" viewBox="0 0 348 225" preserveAspectRatio="none" style="display: none; visibility: hidden; position: absolute; top: -100%; left: -100%;"><defs><style type="text/css"></style></defs><text x="0" y="17" style="font-weight:bold;font-size:17pt;font-family:Arial, Helvetica, Open Sans, sans-serif">Thumbnail</text></svg></body></html>
`
Convey(".NodeType", t, func() {
Convey(".GetNodeType", t, func() {
buff := bytes.NewBuffer([]byte(doc))
buff.Write([]byte(doc))
@@ -257,10 +257,10 @@ func TestElement(t *testing.T) {
So(err, ShouldBeNil)
So(el.NodeType(), ShouldEqual, 1)
So(el.GetNodeType(), ShouldEqual, 1)
})
Convey(".NodeName", t, func() {
Convey(".GetNodeName", t, func() {
buff := bytes.NewBuffer([]byte(doc))
buff.Write([]byte(doc))
@@ -273,7 +273,7 @@ func TestElement(t *testing.T) {
So(err, ShouldBeNil)
So(el.NodeName(), ShouldEqual, "body")
So(el.GetNodeName(), ShouldEqual, "body")
})
Convey(".Length", t, func() {
@@ -327,7 +327,7 @@ func TestElement(t *testing.T) {
So(v, ShouldEqual, "find")
})
Convey(".InnerText", t, func() {
Convey(".GetInnerText", t, func() {
buff := bytes.NewBuffer([]byte(`
<html>
<head></head>
@@ -349,7 +349,7 @@ func TestElement(t *testing.T) {
So(err, ShouldBeNil)
v := el.InnerText(context.Background())
v := el.GetInnerText(context.Background())
So(v, ShouldEqual, "Ferret")
})
@@ -376,7 +376,7 @@ func TestElement(t *testing.T) {
So(err, ShouldBeNil)
v := el.InnerHTML(context.Background())
v := el.GetInnerHTML(context.Background())
So(v, ShouldEqual, "<h2>Ferret</h2>")
})
@@ -396,7 +396,7 @@ func TestElement(t *testing.T) {
So(found, ShouldNotEqual, values.None)
v := found.(drivers.HTMLNode).NodeName()
v := found.(drivers.HTMLNode).GetNodeName()
So(err, ShouldBeNil)

199
pkg/drivers/http/page.go Normal file
View File

@@ -0,0 +1,199 @@
package http
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/PuerkitoBio/goquery"
"hash/fnv"
)
type HTMLPage struct {
document *HTMLDocument
cookies []drivers.HTTPCookie
frames *values.Array
}
func NewHTMLPage(
qdoc *goquery.Document,
url string,
cookies []drivers.HTTPCookie,
) (*HTMLPage, error) {
doc, err := NewRootHTMLDocument(qdoc, url)
if err != nil {
return nil, err
}
p := new(HTMLPage)
p.document = doc
p.cookies = cookies
p.frames = nil
return p, nil
}
func (p *HTMLPage) MarshalJSON() ([]byte, error) {
return p.document.MarshalJSON()
}
func (p *HTMLPage) Type() core.Type {
return drivers.HTMLPageType
}
func (p *HTMLPage) String() string {
return p.document.GetURL().String()
}
func (p *HTMLPage) Compare(other core.Value) int64 {
tc := drivers.Compare(p.Type(), other.Type())
if tc != 0 {
return tc
}
httpPage, ok := other.(*HTMLPage)
if !ok {
return 1
}
return p.document.GetURL().Compare(httpPage.GetURL())
}
func (p *HTMLPage) Unwrap() interface{} {
return p.document
}
func (p *HTMLPage) Hash() uint64 {
h := fnv.New64a()
h.Write([]byte("HTTP"))
h.Write([]byte(p.Type().String()))
h.Write([]byte(":"))
h.Write([]byte(p.document.GetURL()))
return h.Sum64()
}
func (p *HTMLPage) Copy() core.Value {
page, err := NewHTMLPage(p.document.doc, p.document.GetURL().String(), p.cookies[:])
if err != nil {
return values.None
}
return page
}
func (p *HTMLPage) Iterate(ctx context.Context) (core.Iterator, error) {
return p.document.Iterate(ctx)
}
func (p *HTMLPage) GetIn(ctx context.Context, path []core.Value) (core.Value, error) {
return common.GetInPage(ctx, p, path)
}
func (p *HTMLPage) SetIn(ctx context.Context, path []core.Value, value core.Value) error {
return common.SetInPage(ctx, p, path, value)
}
func (p *HTMLPage) Length() values.Int {
return p.document.Length()
}
func (p *HTMLPage) Close() error {
return nil
}
func (p *HTMLPage) IsClosed() values.Boolean {
return values.True
}
func (p *HTMLPage) GetURL() values.String {
return p.document.GetURL()
}
func (p *HTMLPage) GetMainFrame() drivers.HTMLDocument {
return p.document
}
func (p *HTMLPage) GetFrames(ctx context.Context) (*values.Array, error) {
if p.frames == nil {
arr := values.NewArray(10)
err := common.CollectFrames(ctx, arr, p.document)
if err != nil {
return values.NewArray(0), err
}
p.frames = arr
}
return p.frames, nil
}
func (p *HTMLPage) GetFrame(ctx context.Context, idx values.Int) (core.Value, error) {
if p.frames == nil {
arr := values.NewArray(10)
err := common.CollectFrames(ctx, arr, p.document)
if err != nil {
return values.None, err
}
p.frames = arr
}
return p.frames.Get(idx), nil
}
func (p *HTMLPage) GetCookies(_ context.Context) (*values.Array, error) {
if p.cookies == nil {
return values.NewArray(0), nil
}
arr := values.NewArray(len(p.cookies))
for _, c := range p.cookies {
arr.Push(c)
}
return arr, nil
}
func (p *HTMLPage) SetCookies(_ context.Context, _ ...drivers.HTTPCookie) error {
return core.ErrNotSupported
}
func (p *HTMLPage) DeleteCookies(_ context.Context, _ ...drivers.HTTPCookie) error {
return core.ErrNotSupported
}
func (p *HTMLPage) PrintToPDF(_ context.Context, _ drivers.PDFParams) (values.Binary, error) {
return nil, core.ErrNotSupported
}
func (p *HTMLPage) CaptureScreenshot(_ context.Context, _ drivers.ScreenshotParams) (values.Binary, error) {
return nil, core.ErrNotSupported
}
func (p *HTMLPage) WaitForNavigation(_ context.Context) error {
return core.ErrNotSupported
}
func (p *HTMLPage) Navigate(_ context.Context, _ values.String) error {
return core.ErrNotSupported
}
func (p *HTMLPage) NavigateBack(_ context.Context, _ values.Int) (values.Boolean, error) {
return false, core.ErrNotSupported
}
func (p *HTMLPage) NavigateForward(_ context.Context, _ values.Int) (values.Boolean, error) {
return false, core.ErrNotSupported
}

2
pkg/drivers/type.go
View File

@@ -7,6 +7,7 @@ var (
HTTPCookieType = core.NewType("HTTPCookie")
HTMLElementType = core.NewType("HTMLElement")
HTMLDocumentType = core.NewType("HTMLDocument")
HTMLPageType = core.NewType("HTMLPageType")
)
// Comparison table of builtin types
@@ -15,6 +16,7 @@ var typeComparisonTable = map[core.Type]uint64{
HTTPCookieType: 1,
HTMLElementType: 2,
HTMLDocumentType: 3,
HTMLPageType: 4,
}
func Compare(first, second core.Type) int64 {

77
pkg/drivers/value.go
View File

@@ -24,9 +24,11 @@ type (
collections.Measurable
io.Closer
NodeType() values.Int
IsDetached() values.Boolean
NodeName() values.String
GetNodeType() values.Int
GetNodeName() values.String
GetChildNodes(ctx context.Context) core.Value
@@ -41,13 +43,13 @@ type (
ExistsBySelector(ctx context.Context, selector values.String) values.Boolean
}
// HTMLElement is the most general base interface which most objects in a Document implement.
// HTMLElement is the most general base interface which most objects in a GetMainFrame implement.
HTMLElement interface {
HTMLNode
InnerText(ctx context.Context) values.String
GetInnerText(ctx context.Context) values.String
InnerHTML(ctx context.Context) values.String
GetInnerHTML(ctx context.Context) values.String
GetValue(ctx context.Context) core.Value
@@ -98,28 +100,20 @@ type (
WaitForClass(ctx context.Context, class values.String, when WaitEvent) error
}
// The Document interface represents any web page loaded in the browser
// and serves as an entry point into the web page's content, which is the DOM tree.
HTMLDocument interface {
HTMLNode
DocumentElement() HTMLElement
GetTitle() values.String
GetURL() core.Value
GetElement() HTMLElement
SetURL(ctx context.Context, url values.String) error
GetURL() values.String
GetCookies(ctx context.Context) (*values.Array, error)
GetName() values.String
SetCookies(ctx context.Context, cookies ...HTTPCookie) error
GetParentDocument() HTMLDocument
DeleteCookies(ctx context.Context, cookies ...HTTPCookie) error
Navigate(ctx context.Context, url values.String) error
NavigateBack(ctx context.Context, skip values.Int) (values.Boolean, error)
NavigateForward(ctx context.Context, skip values.Int) (values.Boolean, error)
GetChildDocuments(ctx context.Context) (*values.Array, error)
ClickBySelector(ctx context.Context, selector values.String) (values.Boolean, error)
@@ -129,10 +123,6 @@ type (
SelectBySelector(ctx context.Context, selector values.String, value *values.Array) (*values.Array, error)
PrintToPDF(ctx context.Context, params PDFParams) (values.Binary, error)
CaptureScreenshot(ctx context.Context, params ScreenshotParams) (values.Binary, error)
ScrollTop(ctx context.Context) error
ScrollBottom(ctx context.Context) error
@@ -145,8 +135,6 @@ type (
MoveMouseBySelector(ctx context.Context, selector values.String) error
WaitForNavigation(ctx context.Context) error
WaitForElement(ctx context.Context, selector values.String, when WaitEvent) error
WaitForAttributeBySelector(ctx context.Context, selector, name values.String, value core.Value, when WaitEvent) error
@@ -161,6 +149,45 @@ type (
WaitForClassBySelectorAll(ctx context.Context, selector, class values.String, when WaitEvent) error
}
// HTMLPage interface represents any web page loaded in the browser
// and serves as an entry point into the web page's content
HTMLPage interface {
core.Value
core.Iterable
core.Getter
core.Setter
collections.Measurable
io.Closer
IsClosed() values.Boolean
GetURL() values.String
GetMainFrame() HTMLDocument
GetFrames(ctx context.Context) (*values.Array, error)
GetFrame(ctx context.Context, idx values.Int) (core.Value, error)
GetCookies(ctx context.Context) (*values.Array, error)
SetCookies(ctx context.Context, cookies ...HTTPCookie) error
DeleteCookies(ctx context.Context, cookies ...HTTPCookie) error
PrintToPDF(ctx context.Context, params PDFParams) (values.Binary, error)
CaptureScreenshot(ctx context.Context, params ScreenshotParams) (values.Binary, error)
WaitForNavigation(ctx context.Context) error
Navigate(ctx context.Context, url values.String) error
NavigateBack(ctx context.Context, skip values.Int) (values.Boolean, error)
NavigateForward(ctx context.Context, skip values.Int) (values.Boolean, error)
}
)
const (

1
pkg/runtime/collections/indexed_test.go
View File

@@ -120,6 +120,7 @@ func TestArrayIterator(t *testing.T) {
So(item, ShouldBeNil)
So(err, ShouldBeNil)
So(res, ShouldHaveLength, int(arr.Length()))
})
Convey("Should NOT iterate over an empty array", t, func() {

2
pkg/runtime/core/type.go
View File

@@ -6,7 +6,7 @@ import (
)
// Type represents runtime type with id for quick type check
// and Name for error messages
// and GetName for error messages
//revive:disable-next-line:redefines-builtin-id
type (

3
pkg/runtime/expressions/operators/operator_test.go
View File

@@ -246,6 +246,7 @@ func TestAdd(t *testing.T) {
}
for _, argN := range args {
argN := argN
Convey(argN.Type().String(), func() {
So(operators.Add(arg1, argN), ShouldEqual, values.NewInt(1))
})
@@ -743,6 +744,7 @@ func TestMultiply(t *testing.T) {
}
for _, argN := range args {
argN := argN
Convey(argN.Type().String(), func() {
So(operators.Multiply(arg1, argN), ShouldEqual, values.NewInt(0))
})
@@ -1011,6 +1013,7 @@ func TestDivide(t *testing.T) {
}
for _, argN := range args {
argN := argN
Convey(argN.Type().String(), func() {
So(func() {
operators.Divide(arg1, argN)

10
pkg/runtime/values/boolean.go
View File

@@ -34,15 +34,7 @@ func ParseBoolean(input interface{}) (Boolean, error) {
s, ok := input.(string)
if ok {
s := strings.ToLower(s)
if s == "true" {
return True, nil
}
if s == "false" {
return False, nil
}
return Boolean(strings.ToLower(s) == "true"), nil
}
return False, core.Error(core.ErrInvalidType, "expected 'bool'")

2
pkg/runtime/values/types/types_test.go
View File

@@ -40,7 +40,7 @@ func (v TestValue) Copy() core.Value {
}
func TestType(t *testing.T) {
Convey(".Name", t, func() {
Convey(".GetName", t, func() {
So(types.None.String(), ShouldEqual, "none")
So(types.Boolean.String(), ShouldEqual, "boolean")
So(types.Int.String(), ShouldEqual, "int")

1
pkg/stdlib/arrays/minus.go
View File

@@ -23,6 +23,7 @@ func Minus(_ context.Context, args ...core.Value) (core.Value, error) {
capacity := values.NewInt(0)
for idx, i := range args {
idx := idx
err := core.ValidateType(i, types.Array)
if err != nil {

3
pkg/stdlib/html/attr_get.go
View File

@@ -3,6 +3,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@@ -18,7 +19,7 @@ func AttributeGet(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
el, err := resolveElement(args[0])
el, err := drivers.ToElement(args[0])
if err != nil {
return values.None, err

3
pkg/stdlib/html/attr_remove.go
View File

@@ -3,6 +3,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
@@ -18,7 +19,7 @@ func AttributeRemove(ctx context.Context, args ...core.Value) (core.Value, error
return values.None, err
}
el, err := resolveElement(args[0])
el, err := drivers.ToElement(args[0])
if err != nil {
return values.None, err

3
pkg/stdlib/html/attr_set.go
View File

@@ -3,6 +3,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
@@ -19,7 +20,7 @@ func AttributeSet(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
el, err := resolveElement(args[0])
el, err := drivers.ToElement(args[0])
if err != nil {
return values.None, err

7
pkg/stdlib/html/click.go
View File

@@ -3,12 +3,13 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
// Click dispatches click event on a given element
// @param source (Document | Element) - Event source.
// @param source (Open | GetElement) - Event source.
// @param selector (String, optional) - Optional selector. Only used when a document instance is passed.
func Click(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 1, 2)
@@ -19,7 +20,7 @@ func Click(ctx context.Context, args ...core.Value) (core.Value, error) {
// CLICK(el)
if len(args) == 1 {
el, err := toElement(args[0])
el, err := drivers.ToElement(args[0])
if err != nil {
return values.False, err
@@ -29,7 +30,7 @@ func Click(ctx context.Context, args ...core.Value) (core.Value, error) {
}
// CLICK(doc, selector)
doc, err := toDocument(args[0])
doc, err := drivers.ToDocument(args[0])
if err != nil {
return values.False, err

21
pkg/stdlib/html/click_all.go
View File

@@ -9,7 +9,7 @@ import (
)
// ClickAll dispatches click event on all matched element
// @param source (Document) - Document.
// @param source (Open) - Open.
// @param selector (String) - Selector.
// @returns (Boolean) - Returns true if matched at least one element.
func ClickAll(ctx context.Context, args ...core.Value) (core.Value, error) {
@@ -19,20 +19,13 @@ func ClickAll(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.False, err
}
arg1 := args[0]
doc, err := drivers.ToDocument(args[0])
if err != nil {
return values.None, err
}
selector := args[1].String()
err = core.ValidateType(arg1, drivers.HTMLDocumentType)
if err != nil {
return values.None, err
}
doc, err := toDocument(args[0])
if err != nil {
return values.None, err
}
return doc.ClickBySelectorAll(ctx, values.NewString(selector))
}

12
pkg/stdlib/html/cookie_del.go
View File

@@ -2,14 +2,15 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// CookieSet gets a cookie from a given document by name.
// @param source (HTMLDocument) - Target HTMLDocument.
// CookieSet gets a cookie from a given page by name.
// @param page (HTMLPage) - Target page.
// @param cookie (...HTTPCookie|String) - Cookie or cookie name to delete.
func CookieDel(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 2, core.MaxArgs)
@@ -18,13 +19,12 @@ func CookieDel(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
err = core.ValidateType(args[0], drivers.HTMLDocumentType)
page, err := drivers.ToPage(args[0])
if err != nil {
return values.None, err
}
doc := args[0].(drivers.HTMLDocument)
inputs := args[1:]
var currentCookies *values.Array
cookies := make([]drivers.HTTPCookie, 0, len(inputs))
@@ -33,7 +33,7 @@ func CookieDel(ctx context.Context, args ...core.Value) (core.Value, error) {
switch cookie := c.(type) {
case values.String:
if currentCookies == nil {
current, err := doc.GetCookies(ctx)
current, err := page.GetCookies(ctx)
if err != nil {
return values.None, err
@@ -60,5 +60,5 @@ func CookieDel(ctx context.Context, args ...core.Value) (core.Value, error) {
}
}
return values.None, doc.DeleteCookies(ctx, cookies...)
return values.None, page.DeleteCookies(ctx, cookies...)
}

9
pkg/stdlib/html/cookie_get.go
View File

@@ -9,8 +9,8 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// CookieSet gets a cookie from a given document by name.
// @param doc (HTMLDocument) - Target HTMLDocument.
// CookieSet gets a cookie from a given page by name.
// @param page (HTMLPage) - Target page.
// @param name (String) - Cookie or cookie name to delete.
func CookieGet(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 2, 2)
@@ -19,7 +19,7 @@ func CookieGet(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
err = core.ValidateType(args[0], drivers.HTMLDocumentType)
page, err := drivers.ToPage(args[0])
if err != nil {
return values.None, err
@@ -31,10 +31,9 @@ func CookieGet(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc := args[0].(drivers.HTMLDocument)
name := args[1].(values.String)
cookies, err := doc.GetCookies(ctx)
cookies, err := page.GetCookies(ctx)
if err != nil {
return values.None, err

10
pkg/stdlib/html/cookie_set.go
View File

@@ -8,8 +8,8 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/values"
)
// CookieSet sets cookies to a given document
// @param doc (HTMLDocument) - Target document.
// CookieSet sets cookies to a given page
// @param page (HTMLPage) - Target page.
// @param cookie... (HTTPCookie) - Target cookies.
func CookieSet(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 2, core.MaxArgs)
@@ -18,14 +18,12 @@ func CookieSet(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
err = core.ValidateType(args[0], drivers.HTMLDocumentType)
page, err := drivers.ToPage(args[0])
if err != nil {
return values.None, err
}
doc := args[0].(drivers.HTMLDocument)
cookies := make([]drivers.HTTPCookie, 0, len(args)-1)
for _, c := range args[1:] {
@@ -38,5 +36,5 @@ func CookieSet(ctx context.Context, args ...core.Value) (core.Value, error) {
cookies = append(cookies, cookie)
}
return values.None, doc.SetCookies(ctx, cookies...)
return values.None, page.SetCookies(ctx, cookies...)
}

26
pkg/stdlib/html/document.go
View File

@@ -12,22 +12,22 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
type DocumentLoadParams struct {
drivers.LoadDocumentParams
type PageLoadParams struct {
drivers.OpenPageParams
Driver string
Timeout time.Duration
}
// Document loads a HTML document by a given url.
// Open opens an HTML page by a given url.
// By default, loads a document by http call - resulted document does not support any interactions.
// If passed "true" as a second argument, headless browser is used for loading the document which support interactions.
// @param url (String) - Target url string. If passed "about:blank" for dynamic document - it will open an empty page.
// @param isDynamicOrParams (Boolean|DocumentLoadParams) - Either a boolean value that indicates whether to use dynamic page
// @param isDynamicOrParams (Boolean|PageLoadParams) - Either a boolean value that indicates whether to use dynamic page
// or an object with the following properties :
// dynamic (Boolean) - Optional, indicates whether to use dynamic page.
// timeout (Int) - Optional, Document load timeout.
// timeout (Int) - Optional, Open load timeout.
// @returns (HTMLDocument) - Returns loaded HTML document.
func Document(ctx context.Context, args ...core.Value) (core.Value, error) {
func Open(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 1, 2)
if err != nil {
@@ -42,12 +42,12 @@ func Document(ctx context.Context, args ...core.Value) (core.Value, error) {
url := args[0].(values.String)
var params DocumentLoadParams
var params PageLoadParams
if len(args) == 1 {
params = newDefaultDocLoadParams(url)
} else {
p, err := newDocLoadParams(url, args[1])
p, err := newPageLoadParams(url, args[1])
if err != nil {
return values.None, err
@@ -65,19 +65,19 @@ func Document(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
return drv.LoadDocument(ctx, params.LoadDocumentParams)
return drv.Open(ctx, params.OpenPageParams)
}
func newDefaultDocLoadParams(url values.String) DocumentLoadParams {
return DocumentLoadParams{
LoadDocumentParams: drivers.LoadDocumentParams{
func newDefaultDocLoadParams(url values.String) PageLoadParams {
return PageLoadParams{
OpenPageParams: drivers.OpenPageParams{
URL: url.String(),
},
Timeout: time.Second * 30,
}
}
func newDocLoadParams(url values.String, arg core.Value) (DocumentLoadParams, error) {
func newPageLoadParams(url values.String, arg core.Value) (PageLoadParams, error) {
res := newDefaultDocLoadParams(url)
if err := core.ValidateType(arg, types.Boolean, types.String, types.Object); err != nil {

4
pkg/stdlib/html/download.go
View File

@@ -10,8 +10,8 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// Download a resource from the given URL.
// @param URL (String) - URL to download.
// Download a resource from the given GetURL.
// @param GetURL (String) - GetURL to download.
// @returns data (Binary) - Returns a base64 encoded string in binary format.
func Download(_ context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 1, 1)

8
pkg/stdlib/html/element.go
View File

@@ -9,7 +9,7 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// Element finds an element by a given CSS selector.
// GetElement finds an element by a given CSS selector.
// Returns NONE if element not found.
// @param docOrEl (HTMLDocument|HTMLElement) - Parent document or element.
// @param selector (String) - CSS selector.
@@ -24,14 +24,14 @@ func Element(ctx context.Context, args ...core.Value) (core.Value, error) {
return el.QuerySelector(ctx, selector), nil
}
func queryArgs(args []core.Value) (drivers.HTMLNode, values.String, error) {
func queryArgs(args []core.Value) (drivers.HTMLElement, values.String, error) {
err := core.ValidateArgs(args, 2, 2)
if err != nil {
return nil, values.EmptyString, err
}
err = core.ValidateType(args[0], drivers.HTMLDocumentType, drivers.HTMLElementType)
el, err := drivers.ToElement(args[0])
if err != nil {
return nil, values.EmptyString, err
@@ -43,5 +43,5 @@ func queryArgs(args []core.Value) (drivers.HTMLNode, values.String, error) {
return nil, values.EmptyString, err
}
return args[0].(drivers.HTMLNode), args[1].(values.String), nil
return el, args[1].(values.String), nil
}

10
pkg/stdlib/html/hover.go
View File

@@ -20,8 +20,8 @@ func Hover(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
// document or element
err = core.ValidateType(args[0], drivers.HTMLDocumentType, drivers.HTMLElementType)
// page or document or element
err = core.ValidateType(args[0], drivers.HTMLPageType, drivers.HTMLDocumentType, drivers.HTMLElementType)
if err != nil {
return values.None, err
@@ -40,6 +40,12 @@ func Hover(ctx context.Context, args ...core.Value) (core.Value, error) {
}
switch n := args[0].(type) {
case drivers.HTMLPage:
if selector == values.EmptyString {
return values.None, core.Error(core.ErrMissedArgument, "selector")
}
return values.None, n.GetMainFrame().MoveMouseBySelector(ctx, selector)
case drivers.HTMLDocument:
if selector == values.EmptyString {
return values.None, core.Error(core.ErrMissedArgument, "selector")

14
pkg/stdlib/html/inner_html.go
View File

@@ -9,8 +9,8 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// InnerHTML Returns inner HTML string of a given or matched by CSS selector element
// @param doc (Document|Element) - Parent document or element.
// GetInnerHTML Returns inner HTML string of a given or matched by CSS selector element
// @param doc (Open|GetElement) - Parent document or element.
// @param selector (String, optional) - String of CSS selector.
// @returns (String) - Inner HTML string if an element found, otherwise empty string.
func InnerHTML(ctx context.Context, args ...core.Value) (core.Value, error) {
@@ -20,20 +20,14 @@ func InnerHTML(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.EmptyString, err
}
err = core.ValidateType(args[0], drivers.HTMLDocumentType, drivers.HTMLElementType)
if err != nil {
return values.None, err
}
el, err := resolveElement(args[0])
el, err := drivers.ToElement(args[0])
if err != nil {
return values.None, err
}
if len(args) == 1 {
return el.InnerHTML(ctx), nil
return el.GetInnerHTML(ctx), nil
}
err = core.ValidateType(args[1], types.String)

8
pkg/stdlib/html/inner_html_all.go
View File

@@ -20,19 +20,13 @@ func InnerHTMLAll(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
err = core.ValidateType(args[0], drivers.HTMLDocumentType, drivers.HTMLElementType)
if err != nil {
return values.None, err
}
err = core.ValidateType(args[1], types.String)
if err != nil {
return values.None, err
}
el, err := resolveElement(args[0])
el, err := drivers.ToElement(args[0])
if err != nil {
return values.None, err

7
pkg/stdlib/html/inner_text.go
View File

@@ -3,12 +3,13 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// InnerText returns inner text string of a given or matched by CSS selector element
// GetInnerText returns inner text string of a given or matched by CSS selector element
// @param doc (HTMLDocument|HTMLElement) - Parent document or element.
// @param selector (String, optional) - String of CSS selector.
// @returns (String) - Inner text if an element found, otherwise empty string.
@@ -19,14 +20,14 @@ func InnerText(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.EmptyString, err
}
el, err := resolveElement(args[0])
el, err := drivers.ToElement(args[0])
if err != nil {
return values.None, err
}
if len(args) == 1 {
return el.InnerText(ctx), nil
return el.GetInnerText(ctx), nil
}
err = core.ValidateType(args[1], types.String)

8
pkg/stdlib/html/inner_text_all.go
View File

@@ -20,19 +20,13 @@ func InnerTextAll(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
err = core.ValidateType(args[0], drivers.HTMLDocumentType, drivers.HTMLElementType)
if err != nil {
return values.None, err
}
err = core.ValidateType(args[1], types.String)
if err != nil {
return values.None, err
}
el, err := resolveElement(args[0])
el, err := drivers.ToElement(args[0])
if err != nil {
return values.None, err

19
pkg/stdlib/html/input.go
View File

@@ -10,7 +10,7 @@ import (
)
// Input types a value to an underlying input element.
// @param source (Document | Element) - Event target.
// @param source (Open | GetElement) - Event target.
// @param valueOrSelector (String) - Selector or a value.
// @param value (String) - Target value.
// @param delay (Int, optional) - Waits delay milliseconds between keystrokes
@@ -23,14 +23,18 @@ func Input(ctx context.Context, args ...core.Value) (core.Value, error) {
}
arg1 := args[0]
err = core.ValidateType(arg1, drivers.HTMLDocumentType, drivers.HTMLElementType)
err = core.ValidateType(arg1, drivers.HTMLPageType, drivers.HTMLDocumentType, drivers.HTMLElementType)
if err != nil {
return values.False, err
}
if arg1.Type() == drivers.HTMLDocumentType {
doc := arg1.(drivers.HTMLDocument)
if arg1.Type() == drivers.HTMLPageType || arg1.Type() == drivers.HTMLDocumentType {
doc, err := drivers.ToDocument(arg1)
if err != nil {
return values.False, err
}
// selector
arg2 := args[1]
@@ -57,7 +61,12 @@ func Input(ctx context.Context, args ...core.Value) (core.Value, error) {
return doc.InputBySelector(ctx, arg2.(values.String), args[2], delay)
}
el := arg1.(drivers.HTMLElement)
el, err := drivers.ToElement(arg1)
if err != nil {
return values.None, err
}
delay := values.Int(0)
if len(args) == 3 {

54
pkg/stdlib/html/lib.go
View File

@@ -22,7 +22,7 @@ func NewLib() map[string]core.Function {
"COOKIE_SET": CookieSet,
"CLICK": Click,
"CLICK_ALL": ClickAll,
"DOCUMENT": Document,
"DOCUMENT": Open,
"DOWNLOAD": Download,
"ELEMENT": Element,
"ELEMENT_EXISTS": ElementExists,
@@ -67,27 +67,29 @@ func NewLib() map[string]core.Function {
}
}
func ValidateDocument(ctx context.Context, value core.Value) (core.Value, error) {
err := core.ValidateType(value, drivers.HTMLDocumentType, types.String)
func OpenOrCastPage(ctx context.Context, value core.Value) (drivers.HTMLPage, bool, error) {
err := core.ValidateType(value, drivers.HTMLPageType, types.String)
if err != nil {
return values.None, err
return nil, false, err
}
var doc drivers.HTMLDocument
var page drivers.HTMLPage
var closeAfter bool
if value.Type() == types.String {
buf, err := Document(ctx, value, values.NewBoolean(true))
buf, err := Open(ctx, value, values.NewBoolean(true))
if err != nil {
return values.None, err
return nil, false, err
}
doc = buf.(drivers.HTMLDocument)
page = buf.(drivers.HTMLPage)
closeAfter = true
} else {
doc = value.(drivers.HTMLDocument)
page = value.(drivers.HTMLPage)
}
return doc, nil
return page, closeAfter, nil
}
func waitTimeout(ctx context.Context, value values.Int) (context.Context, context.CancelFunc) {
@@ -96,35 +98,3 @@ func waitTimeout(ctx context.Context, value values.Int) (context.Context, contex
time.Duration(value)*time.Millisecond,
)
}
func resolveElement(value core.Value) (drivers.HTMLElement, error) {
vt := value.Type()
if vt == drivers.HTMLDocumentType {
return value.(drivers.HTMLDocument).DocumentElement(), nil
} else if vt == drivers.HTMLElementType {
return value.(drivers.HTMLElement), nil
}
return nil, core.TypeError(value.Type(), drivers.HTMLDocumentType, drivers.HTMLElementType)
}
func toDocument(value core.Value) (drivers.HTMLDocument, error) {
err := core.ValidateType(value, drivers.HTMLDocumentType)
if err != nil {
return nil, err
}
return value.(drivers.HTMLDocument), nil
}
func toElement(value core.Value) (drivers.HTMLElement, error) {
err := core.ValidateType(value, drivers.HTMLElementType)
if err != nil {
return nil, err
}
return value.(drivers.HTMLElement), nil
}

4
pkg/stdlib/html/mouse_xy.go
View File

@@ -20,7 +20,7 @@ func MouseMoveXY(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
err = core.ValidateType(args[0], drivers.HTMLDocumentType)
doc, err := drivers.ToDocument(args[0])
if err != nil {
return values.None, err
@@ -41,7 +41,5 @@ func MouseMoveXY(ctx context.Context, args ...core.Value) (core.Value, error) {
x := values.ToFloat(args[0])
y := values.ToFloat(args[1])
doc := args[0].(drivers.HTMLDocument)
return values.None, doc.MoveMouseByXY(ctx, x, y)
}

10
pkg/stdlib/html/navigate.go
View File

@@ -2,15 +2,17 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// Navigate navigates a document to a new resource.
// Navigate navigates a given page to a new resource.
// The operation blocks the execution until the page gets loaded.
// Which means there is no need in WAIT_NAVIGATION function.
// @param doc (Document) - Target document.
// @param page (HTMLPage) - Target page.
// @param url (String) - Target url to navigate.
// @param timeout (Int, optional) - Optional timeout. Default is 5000.
func Navigate(ctx context.Context, args ...core.Value) (core.Value, error) {
@@ -20,7 +22,7 @@ func Navigate(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc, err := toDocument(args[0])
page, err := drivers.ToPage(args[0])
if err != nil {
return values.None, err
@@ -47,5 +49,5 @@ func Navigate(ctx context.Context, args ...core.Value) (core.Value, error) {
ctx, fn := waitTimeout(ctx, timeout)
defer fn()
return values.None, doc.Navigate(ctx, args[1].(values.String))
return values.None, page.Navigate(ctx, args[1].(values.String))
}

10
pkg/stdlib/html/navigate_back.go
View File

@@ -2,15 +2,17 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// NavigateBack navigates a document back within its navigation history.
// NavigateBack navigates a given page back within its navigation history.
// The operation blocks the execution until the page gets loaded.
// If the history is empty, the function returns FALSE.
// @param doc (Document) - Target document.
// @param page (HTMLPage) - Target page.
// @param entry (Int, optional) - Optional value indicating how many pages to skip. Default 1.
// @param timeout (Int, optional) - Optional timeout. Default is 5000.
// @returns (Boolean) - Returns TRUE if history exists and the operation succeeded, otherwise FALSE.
@@ -21,7 +23,7 @@ func NavigateBack(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.False, err
}
doc, err := toDocument(args[0])
page, err := drivers.ToPage(args[0])
if err != nil {
return values.None, err
@@ -53,5 +55,5 @@ func NavigateBack(ctx context.Context, args ...core.Value) (core.Value, error) {
ctx, fn := waitTimeout(ctx, timeout)
defer fn()
return doc.NavigateBack(ctx, skip)
return page.NavigateBack(ctx, skip)
}

10
pkg/stdlib/html/navigate_forward.go
View File

@@ -2,15 +2,17 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// NavigateForward navigates a document forward within its navigation history.
// NavigateForward navigates a given page forward within its navigation history.
// The operation blocks the execution until the page gets loaded.
// If the history is empty, the function returns FALSE.
// @param doc (Document) - Target document.
// @param page (HTMLPage) - Target page.
// @param entry (Int, optional) - Optional value indicating how many pages to skip. Default 1.
// @param timeout (Int, optional) - Optional timeout. Default is 5000.
// @returns (Boolean) - Returns TRUE if history exists and the operation succeeded, otherwise FALSE.
@@ -21,7 +23,7 @@ func NavigateForward(ctx context.Context, args ...core.Value) (core.Value, error
return values.False, err
}
doc, err := toDocument(args[0])
page, err := drivers.ToPage(args[0])
if err != nil {
return values.None, err
@@ -53,5 +55,5 @@ func NavigateForward(ctx context.Context, args ...core.Value) (core.Value, error
ctx, fn := waitTimeout(ctx, timeout)
defer fn()
return doc.NavigateForward(ctx, skip)
return page.NavigateForward(ctx, skip)
}

4
pkg/stdlib/html/pagination.go
View File

@@ -12,7 +12,7 @@ import (
// Pagination creates an iterator that goes through pages using CSS selector.
// The iterator starts from the current page i.e. it does not change the page on 1st iteration.
// That allows you to keep scraping logic inside FOR loop.
// @param doc (Document) - Target document.
// @param doc (Open) - Target document.
// @param selector (String) - CSS selector for a pagination on the page.
func Pagination(_ context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 2, 2)
@@ -21,7 +21,7 @@ func Pagination(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc, err := toDocument(args[0])
doc, err := drivers.ToDocument(args[0])
if err != nil {
return values.None, err

13
pkg/stdlib/html/pdf.go
View File

@@ -22,7 +22,7 @@ func ValidatePageRanges(pageRanges string) (bool, error) {
}
// PDF print a PDF of the current page.
// @param source (Document) - Document.
// @param target (HTMLPage|String) - Target page or url.
// @param params (Object) - Optional, An object containing the following properties :
// Landscape (Bool) - Paper orientation. Defaults to false.
// DisplayHeaderFooter (Bool) - Display header and footer. Defaults to false.
@@ -48,14 +48,17 @@ func PDF(ctx context.Context, args ...core.Value) (core.Value, error) {
}
arg1 := args[0]
val, err := ValidateDocument(ctx, arg1)
page, closeAfter, err := OpenOrCastPage(ctx, arg1)
if err != nil {
return values.None, err
}
doc := val.(drivers.HTMLDocument)
defer doc.Close()
defer func() {
if closeAfter {
page.Close()
}
}()
pdfParams := drivers.PDFParams{}
@@ -292,7 +295,7 @@ func PDF(ctx context.Context, args ...core.Value) (core.Value, error) {
}
}
pdf, err := doc.PrintToPDF(ctx, pdfParams)
pdf, err := page.PrintToPDF(ctx, pdfParams)
if err != nil {
return values.None, err

16
pkg/stdlib/html/screenshot.go
View File

@@ -10,8 +10,8 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// Screenshot takes a screenshot of the current page.
// @param source (Document) - Document.
// Screenshot takes a screenshot of a given page.
// @param target (HTMLPage|String) - Target page or url.
// @param params (Object) - Optional, An object containing the following properties :
// x (Float|Int) - Optional, X position of the viewport.
// x (Float|Int) - Optional,Y position of the viewport.
@@ -35,15 +35,17 @@ func Screenshot(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
val, err := ValidateDocument(ctx, arg1)
page, closeAfter, err := OpenOrCastPage(ctx, arg1)
if err != nil {
return values.None, err
}
doc := val.(drivers.HTMLDocument)
defer doc.Close()
defer func() {
if closeAfter {
page.Close()
}
}()
screenshotParams := drivers.ScreenshotParams{
X: 0,
@@ -155,7 +157,7 @@ func Screenshot(ctx context.Context, args ...core.Value) (core.Value, error) {
}
}
scr, err := doc.CaptureScreenshot(ctx, screenshotParams)
scr, err := page.CaptureScreenshot(ctx, screenshotParams)
if err != nil {
return values.None, err

8
pkg/stdlib/html/scroll_bottom.go
View File

@@ -17,13 +17,7 @@ func ScrollBottom(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
err = core.ValidateType(args[0], drivers.HTMLDocumentType)
if err != nil {
return values.None, err
}
doc, err := toDocument(args[0])
doc, err := drivers.ToDocument(args[0])
if err != nil {
return values.None, err

12
pkg/stdlib/html/scroll_element.go
View File

@@ -20,7 +20,7 @@ func ScrollInto(ctx context.Context, args ...core.Value) (core.Value, error) {
}
if len(args) == 2 {
err = core.ValidateType(args[0], drivers.HTMLDocumentType)
doc, err := drivers.ToDocument(args[0])
if err != nil {
return values.None, err
@@ -32,8 +32,6 @@ func ScrollInto(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
// Document with a selector
doc := args[0].(drivers.HTMLDocument)
selector := args[1].(values.String)
return values.None, doc.ScrollBySelector(ctx, selector)
@@ -45,8 +43,12 @@ func ScrollInto(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
// Element
el := args[0].(drivers.HTMLElement)
// GetElement
el, err := drivers.ToElement(args[0])
if err != nil {
return values.None, err
}
return values.None, el.ScrollIntoView(ctx)
}

8
pkg/stdlib/html/scroll_top.go
View File

@@ -17,13 +17,7 @@ func ScrollTop(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
err = core.ValidateType(args[0], drivers.HTMLDocumentType)
if err != nil {
return values.None, err
}
doc, err := toDocument(args[0])
doc, err := drivers.ToDocument(args[0])
if err != nil {
return values.None, err

4
pkg/stdlib/html/scroll_xy.go
View File

@@ -20,7 +20,7 @@ func ScrollXY(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
err = core.ValidateType(args[0], drivers.HTMLDocumentType)
doc, err := drivers.ToDocument(args[0])
if err != nil {
return values.None, err
@@ -41,7 +41,5 @@ func ScrollXY(ctx context.Context, args ...core.Value) (core.Value, error) {
x := values.ToFloat(args[1])
y := values.ToFloat(args[2])
doc := args[0].(drivers.HTMLDocument)
return values.None, doc.ScrollByXY(ctx, x, y)
}

12
pkg/stdlib/html/select.go
View File

@@ -10,7 +10,7 @@ import (
)
// Select selects a value from an underlying select element.
// @param source (Document | Element) - Event target.
// @param source (Open | GetElement) - Event target.
// @param valueOrSelector (String | Array<String>) - Selector or a an array of strings as a value.
// @param value (Array<String) - Target value. Optional.
// @returns (Array<String>) - Returns an array of selected values.
@@ -22,14 +22,18 @@ func Select(ctx context.Context, args ...core.Value) (core.Value, error) {
}
arg1 := args[0]
err = core.ValidateType(arg1, drivers.HTMLDocumentType, drivers.HTMLElementType)
err = core.ValidateType(arg1, drivers.HTMLPageType, drivers.HTMLDocumentType, drivers.HTMLElementType)
if err != nil {
return values.False, err
}
if arg1.Type() == drivers.HTMLDocumentType {
doc := arg1.(drivers.HTMLDocument)
if arg1.Type() == drivers.HTMLPageType || arg1.Type() == drivers.HTMLDocumentType {
doc, err := drivers.ToDocument(arg1)
if err != nil {
return values.None, err
}
// selector
arg2 := args[1]

3
pkg/stdlib/html/style_get.go
View File

@@ -3,6 +3,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@@ -18,7 +19,7 @@ func StyleGet(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
el, err := resolveElement(args[0])
el, err := drivers.ToElement(args[0])
if err != nil {
return values.None, err

3
pkg/stdlib/html/style_remove.go
View File

@@ -3,6 +3,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
@@ -18,7 +19,7 @@ func StyleRemove(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
el, err := resolveElement(args[0])
el, err := drivers.ToElement(args[0])
if err != nil {
return values.None, err

3
pkg/stdlib/html/style_set.go
View File

@@ -3,6 +3,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
@@ -19,7 +20,7 @@ func StyleSet(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
el, err := resolveElement(args[0])
el, err := drivers.ToElement(args[0])
if err != nil {
return values.None, err

11
pkg/stdlib/html/wait_attr.go
View File

@@ -26,7 +26,7 @@ func waitAttributeWhen(ctx context.Context, args []core.Value, when drivers.Wait
// document or element
arg1 := args[0]
err = core.ValidateType(arg1, drivers.HTMLDocumentType, drivers.HTMLElementType)
err = core.ValidateType(arg1, drivers.HTMLPageType, drivers.HTMLDocumentType, drivers.HTMLElementType)
if err != nil {
return values.None, err
@@ -43,7 +43,7 @@ func waitAttributeWhen(ctx context.Context, args []core.Value, when drivers.Wait
// if a document is passed
// WAIT_ATTR(doc, selector, attrName, attrValue, timeout)
if arg1.Type() == drivers.HTMLDocumentType {
if arg1.Type() == drivers.HTMLPageType || arg1.Type() == drivers.HTMLDocumentType {
// revalidate args with more accurate amount
err := core.ValidateArgs(args, 4, 5)
@@ -58,7 +58,12 @@ func waitAttributeWhen(ctx context.Context, args []core.Value, when drivers.Wait
return values.None, err
}
doc := arg1.(drivers.HTMLDocument)
doc, err := drivers.ToDocument(arg1)
if err != nil {
return values.None, err
}
selector := args[1].(values.String)
name := args[2].(values.String)
value := args[3]

3
pkg/stdlib/html/wait_attr_all.go
View File

@@ -2,6 +2,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
@@ -35,7 +36,7 @@ func waitAttributeAllWhen(ctx context.Context, args []core.Value, when drivers.W
return values.None, err
}
doc, err := toDocument(args[0])
doc, err := drivers.ToDocument(args[0])
if err != nil {
return values.None, err

12
pkg/stdlib/html/wait_class.go
View File

@@ -2,6 +2,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
@@ -43,7 +44,7 @@ func waitClassWhen(ctx context.Context, args []core.Value, when drivers.WaitEven
// document or element
arg1 := args[0]
err = core.ValidateType(arg1, drivers.HTMLDocumentType, drivers.HTMLElementType)
err = core.ValidateType(arg1, drivers.HTMLPageType, drivers.HTMLDocumentType, drivers.HTMLElementType)
if err != nil {
return values.None, err
@@ -59,7 +60,7 @@ func waitClassWhen(ctx context.Context, args []core.Value, when drivers.WaitEven
timeout := values.NewInt(defaultTimeout)
// if a document is passed
if arg1.Type() == drivers.HTMLDocumentType {
if arg1.Type() == drivers.HTMLPageType || arg1.Type() == drivers.HTMLDocumentType {
// revalidate args with more accurate amount
err := core.ValidateArgs(args, 3, 4)
@@ -74,7 +75,12 @@ func waitClassWhen(ctx context.Context, args []core.Value, when drivers.WaitEven
return values.None, err
}
doc := arg1.(drivers.HTMLDocument)
doc, err := drivers.ToDocument(arg1)
if err != nil {
return values.None, err
}
selector := args[1].(values.String)
class := args[2].(values.String)

3
pkg/stdlib/html/wait_class_all.go
View File

@@ -2,6 +2,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
@@ -35,7 +36,7 @@ func waitClassAllWhen(ctx context.Context, args []core.Value, when drivers.WaitE
return values.None, err
}
doc, err := toDocument(args[0])
doc, err := drivers.ToDocument(args[0])
if err != nil {
return values.None, err

3
pkg/stdlib/html/wait_element.go
View File

@@ -2,6 +2,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
@@ -33,7 +34,7 @@ func waitElementWhen(ctx context.Context, args []core.Value, when drivers.WaitEv
return values.None, err
}
doc, err := toDocument(args[0])
doc, err := drivers.ToDocument(args[0])
if err != nil {
return values.None, err

8
pkg/stdlib/html/wait_navigation.go
View File

@@ -2,14 +2,16 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// WaitNavigation waits for document to navigate to a new url.
// WaitNavigation waits for a given page to navigate to a new url.
// Stops the execution until the navigation ends or operation times out.
// @param doc (HTMLDocument) - Driver HTMLDocument.
// @param page (HTMLPage) - Target page.
// @param timeout (Int, optional) - Optional timeout. Default 5000 ms.
func WaitNavigation(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 1, 2)
@@ -18,7 +20,7 @@ func WaitNavigation(ctx context.Context, args ...core.Value) (core.Value, error)
return values.None, err
}
doc, err := toDocument(args[0])
doc, err := drivers.ToPage(args[0])
if err != nil {
return values.None, err

Some files were not shown because too many files have changed in this diff Show More