go/ferret
go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2025-02-03 13:11:45 +02:00
Code Issues Releases Activity

Bugfix/e2e tests (#648)

Browse Source 
* Fixed logger level

* Fixed WAITFOR EVENT parser

* Added tracing to Network Manager

* Updated logging

* Swtitched to value type of logger

* Added tracing

* Increased websocket maximum buffer size

* Ignore unimportant error message

* Added support of new CDP API for layouts

* Switched to value type of logger

* Added log level

* Fixed early context cancellation

* Updated example of 'click' action

* Switched to val for elements lookup

* Fixed unit tests

* Refactored 'eval' module

* Fixed SetStyle eval expression

* Fixed style deletion

* Updated logic of setting multiple styles
This commit is contained in:
Tim Voronov 2021-09-02 11:09:48 -04:00 committed by GitHub
parent 25c97b86b8
commit e6dd5689b4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
59 changed files with 2772 additions and 1507 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
e2e/cli.go
View File

@ -6,16 +6,18 @@ import (
"encoding/json"
"flag"
"fmt"
"github.com/rs/zerolog"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/MontFerret/ferret"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp"
"github.com/MontFerret/ferret/pkg/drivers/http"
"github.com/MontFerret/ferret/pkg/runtime"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/logging"
)
type Params []string
@ -61,8 +63,16 @@ var (
"",
"set CDP address",
)
logLevel = flag.String(
"log-level",
logging.ErrorLevel.String(),
"log level",
)
)
var logger zerolog.Logger
func main() {
var params Params
@ -74,10 +84,21 @@ func main() {
flag.Parse()
var query string
console := zerolog.ConsoleWriter{
Out: os.Stderr,
TimeFormat: "15:04:05.999",
}
logger = zerolog.New(console).
Level(zerolog.Level(logging.MustParseLevel(*logLevel))).
With().
Timestamp().
Logger()
stat, _ := os.Stdin.Stat()
var query string
var files []string
if (stat.Mode() & os.ModeCharDevice) == 0 {
// check whether the app is getting a query via standard input
std := bufio.NewReader(os.Stdin)
@ -91,18 +112,10 @@ func main() {
query = string(b)
} else if flag.NArg() > 0 {
// backward compatibility
content, err := ioutil.ReadFile(flag.Arg(0))
if err != nil {
fmt.Println(err)
os.Exit(1)
}
query = string(content)
files = flag.Args()
} else {
fmt.Println(flag.NArg())
fmt.Println("Missed file")
fmt.Println("File or input stream are required")
os.Exit(1)
}
@ -113,26 +126,121 @@ func main() {
os.Exit(1)
}
if err := execFile(query, p); err != nil {
engine := ferret.New()
_ = engine.Drivers().Register(http.NewDriver())
_ = engine.Drivers().Register(cdp.NewDriver(cdp.WithAddress(*conn)))
opts := []runtime.Option{
runtime.WithParams(p),
runtime.WithLog(console),
runtime.WithLogLevel(logging.MustParseLevel(*logLevel)),
}
if query != "" {
err = execQuery(engine, opts, query)
} else {
err = execFiles(engine, opts, files)
}
if err != nil {
fmt.Println(err)
os.Exit(1)
}
}
func execFile(query string, params map[string]interface{}) error {
ctx := drivers.WithContext(
context.Background(),
http.NewDriver(),
drivers.AsDefault(),
)
func execFiles(engine *ferret.Instance, opts []runtime.Option, files []string) error {
errList := make([]error, 0, len(files))
ctx = drivers.WithContext(
ctx,
cdp.NewDriver(cdp.WithAddress(*conn)),
)
for _, path := range files {
log := logger.With().Str("path", path).Logger()
log.Debug().Msg("checking path...")
i := ferret.New()
out, err := i.Exec(ctx, query, runtime.WithParams(params))
info, err := os.Stat(path)
if err != nil {
log.Debug().Err(err).Msg("failed to get path info")
errList = append(errList, err)
continue
}
if info.IsDir() {
log.Debug().Msg("path points to a directory. retrieving list of files...")
fileInfos, err := ioutil.ReadDir(path)
if err != nil {
log.Debug().Err(err).Msg("failed to retrieve list of files")
errList = append(errList, err)
continue
}
log.Debug().Int("size", len(fileInfos)).Msg("retrieved list of files. starting to iterate...")
dirFiles := make([]string, 0, len(fileInfos))
for _, info := range fileInfos {
if filepath.Ext(info.Name()) == ".fql" {
dirFiles = append(dirFiles, filepath.Join(path, info.Name()))
}
}
if len(dirFiles) > 0 {
if err := execFiles(engine, opts, dirFiles); err != nil {
log.Debug().Err(err).Msg("failed to execute files")
errList = append(errList, err)
} else {
log.Debug().Int("size", len(fileInfos)).Err(err).Msg("successfully executed files")
}
} else {
log.Debug().Int("size", len(fileInfos)).Err(err).Msg("no FQL files found")
}
continue
}
log.Debug().Msg("path points to a file. starting to read content")
out, err := ioutil.ReadFile(path)
if err != nil {
log.Debug().Err(err).Msg("failed to read content")
errList = append(errList, err)
continue
}
log.Debug().Msg("successfully read file")
log.Debug().Msg("executing file...")
err = execQuery(engine, opts, string(out))
if err != nil {
log.Debug().Err(err).Msg("failed to execute file")
errList = append(errList, err)
continue
}
log.Debug().Msg("successfully executed file")
}
if len(errList) > 0 {
if len(errList) == len(files) {
logger.Debug().Errs("errors", errList).Msg("failed to execute file(s)")
} else {
logger.Debug().Errs("errors", errList).Msg("executed with errors")
}
return core.Errors(errList...)
}
return nil
}
func execQuery(engine *ferret.Instance, opts []runtime.Option, query string) error {
out, err := engine.Exec(context.Background(), query, opts...)
if err != nil {
return err

2
e2e/tests/dynamic/doc/click/click.fql
View File

@ -5,4 +5,4 @@ CLICK(page, "#wait-class-random-btn")
WAIT_CLASS(page, "#wait-class-random-content", "alert-success")
RETURN ""
RETURN TRUE

12
e2e/tests/dynamic/doc/wait/frame_navigation.fql → e2e/tests/dynamic/doc/wait/frame_navigation.fqlx
View File

@ -1,13 +1,17 @@
LET url = @lab.cdn.dynamic + "?redirect=/iframe&src=/iframe"
// LET url = "http://192.168.4.23:8080/?redirect=/iframe&src=/iframe"
LET page = DOCUMENT(url, { driver: 'cdp' })
LET original = FIRST(FRAMES(page, "name", "nested"))
LET original = FIRST(FRAMES(page, "url", "/\?redirect=/iframe$"))
INPUT(original, "#url_input", "https://getbootstrap.com/")
// WAIT(3000)
// LET btn = ELEMENT(original, "#submit")
// CLICK(btn)
CLICK(original, "#submit")
WAIT_NAVIGATION(page, {
frame: original
})
WAITFOR EVENT "navigation" IN page OPTIONS { frame: original } 10000
LET current = FIRST(FRAMES(page, "name", "nested"))

9
e2e/tests/dynamic/element/wait/style.fql
View File

@ -4,18 +4,19 @@ LET doc = DOCUMENT(url, true)
WAIT_ELEMENT(doc, "#page-events")
LET el = ELEMENT(doc, "#wait-class-content")
LET original = el.style.color
ATTR_SET(el, "style", "color: black")
WAIT_STYLE(el, "color", "black")
WAIT_STYLE(el, "color", "rgb(0, 0, 0)")
LET prev = el.style
ATTR_REMOVE(el, "style")
WAIT_STYLE(el, "color", NONE)
WAIT_STYLE(el, "color", original)
LET curr = el.style
T::EQ(prev.color, "black")
T::NONE(curr.color, "style should be removed")
T::EQ(prev.color, "rgb(0, 0, 0)")
T::EQ(curr.color, original, "style should be returned to original")
RETURN NONE

0
e2e/tests/examples/navigate_back.yamlx → e2e/tests/examples/navigate_back.yaml
View File

0
e2e/tests/examples/navigate_back_2.yamlx → e2e/tests/examples/navigate_back_2.yaml
View File

0
e2e/tests/examples/navigate_forward.yamlx → e2e/tests/examples/navigate_forward.yaml
View File

0
e2e/tests/examples/navigate_forward_2.yamlx → e2e/tests/examples/navigate_forward_2.yaml
View File

0
e2e/tests/examples/pagination.yamlx → e2e/tests/examples/pagination.yaml
View File

0
e2e/tests/examples/pagination_uncontrolled.yamlx → e2e/tests/examples/pagination_uncontrolled.yaml
View File

12
examples/click.fql
View File

@ -1,10 +1,8 @@
LET doc = DOCUMENT("https://github.com/", { driver: "cdp" })
LET doc = DOCUMENT("https://www.montferret.dev/", { driver: "cdp" })
HOVER(doc, ".HeaderMenu-details")
CLICK(doc, ".HeaderMenu a")
CLICK(doc, "#repl")
WAIT_NAVIGATION(doc)
WAIT_ELEMENT(doc, 'main nav')
WAITFOR EVENT "navigation" IN doc
WAIT_ELEMENT(doc, '.code-editor-text')
FOR el IN ELEMENTS(doc, 'main nav a')
RETURN TRIM(el.innerText)
RETURN doc.url

15
examples/crawler.fql
View File

@ -1,15 +1,26 @@
LET doc = DOCUMENT('https://www.theverge.com/tech', {
driver: "cdp"
driver: "cdp",
ignore: {
resources: [
{
url: "*",
type: "image"
}
]
}
})
WAIT_ELEMENT(doc, '.c-compact-river__entry', 5000)
LET articles = ELEMENTS(doc, '.c-entry-box--compact__image-wrapper')
LET links = (
FOR article IN articles
FILTER article.attributes?.href LIKE 'https://www.theverge.com/*'
RETURN article.attributes.href
)
FOR link IN links
// The Verge has pretty heavy pages, so let's increase the navigation wait time
NAVIGATE(doc, link, 20000)
WAIT_ELEMENT(doc, '.c-entry-content', 5000)
WAIT_ELEMENT(doc, '.c-entry-content', 15000)
LET texter = ELEMENT(doc, '.c-entry-content')
RETURN texter.innerText

2
examples/disable-images.fql
View File

@ -10,4 +10,4 @@ LET p = DOCUMENT("https://www.gettyimages.com/", {
}
})
RETURN NONE
RETURN TRUE

24
examples/google-search.fql
View File

@ -6,20 +6,22 @@ LET google = DOCUMENT("https://www.google.com/", {
HOVER(google, 'input[name="q"]')
WAIT(RAND(100))
INPUT(google, 'input[name="q"]', @criteria, 30)
WAIT(RAND(100))
WAIT_ELEMENT(google, '.UUbT9')
WAIT(RAND(100))
CLICK(google, 'input[name="btnK"]')
WAIT_NAVIGATION(google)
WAITFOR EVENT "navigation" IN google
WAIT_ELEMENT(google, "#res")
FOR el IN ELEMENTS(google, '#kp-wp-tab-overview > [jsdata]')
// filter out extra elements like media and 'People also ask'
FILTER ELEMENT_EXISTS(el, "#media_result_group") == FALSE
FILTER ELEMENT_EXISTS(el, '[role="heading"]') == FALSE
LET descr = (FOR i IN ELEMENTS(el, "span") FILTER LENGTH(i.attributes) == 0 RETURN i)
FOR result IN ELEMENTS(google, '.g')
// filter out extra elements like videos and 'People also ask'
FILTER TRIM(result.attributes.class) == 'g'
RETURN {
title: INNER_TEXT(result, 'h3'),
description: INNER_TEXT(result, '.rc > div:nth-child(2) span'),
url: INNER_TEXT(result, 'cite')
title: INNER_TEXT(el, 'h3'),
description: FIRST(descr),
url: ELEMENT(el, 'a')?.attributes.href
}

2
examples/history-api.fql
View File

@ -11,6 +11,6 @@ CLICK(song)
WAIT_ELEMENT(doc, ".l-listen-hero")
RETURN {
page: page.url,
current: page.url,
first: doc.url
}

12
examples/pagination.fql
View File

@ -2,15 +2,15 @@ LET baseURL = 'https://www.amazon.com/'
LET amazon = DOCUMENT(baseURL, { driver: "cdp" })
INPUT(amazon, '#twotabsearchtextbox', @criteria)
CLICK(amazon, '.nav-search-submit input[type="submit"]')
CLICK(amazon, '#nav-search-submit-button')
WAIT_NAVIGATION(amazon)
LET resultListSelector = 'div.s-result-list'
LET resultListSelector = '[data-component-type="s-search-results"]'
LET resultItemSelector = '[data-component-type="s-search-result"]'
LET nextBtnSelector = 'ul.a-pagination .a-last a'
LET nextBtnSelector = '.s-pagination-next:not(.s-pagination-disabled)'
LET priceWholeSelector = '.a-price-whole'
LET priceFracSelector = '.a-price-fraction'
LET pagers = ELEMENTS(amazon, 'ul.a-pagination li.a-disabled')
LET pagers = ELEMENTS(amazon, '.s-pagination-item.s-pagination-disabled')
LET pages = LENGTH(pagers) > 0 ? TO_INT(INNER_TEXT(LAST(pagers))) : 0
LET result = (
@ -18,8 +18,7 @@ LET result = (
LIMIT @pages
LET clicked = pageNum == 1 ? false : CLICK(amazon, nextBtnSelector)
LET wait = clicked ? WAIT_NAVIGATION(amazon, 10000) : false
LET waitSelector = wait ? WAIT_ELEMENT(amazon, resultListSelector) : false
LET waitSelector = clicked ? WAIT_ELEMENT(amazon, resultListSelector) : false
PRINT("page:", pageNum, "clicked", clicked)
@ -32,6 +31,7 @@ LET result = (
LET anchor = ELEMENT(el, "a")
RETURN {
page: pageNum,
url: baseURL + anchor.attributes.href,
title: INNER_TEXT(el, 'h2'),
price

12
examples/pagination_uncontrolled.fql
View File

@ -2,12 +2,12 @@ LET baseURL = 'https://www.amazon.com/'
LET amazon = DOCUMENT(baseURL, { driver: "cdp" })
INPUT(amazon, '#twotabsearchtextbox', @criteria)
CLICK(amazon, '.nav-search-submit input[type="submit"]')
CLICK(amazon, '#nav-search-submit-button')
WAIT_NAVIGATION(amazon)
LET resultListSelector = '#s-results-list-atf'
LET resultListSelector = '[data-component-type="s-search-results"]'
LET resultItemSelector = '[data-component-type="s-search-result"]'
LET nextBtnSelector = 'ul.a-pagination .a-last a'
LET nextBtnSelector = '.s-pagination-next:not(.s-pagination-disabled)'
LET priceWholeSelector = '.a-price-whole'
LET priceFracSelector = '.a-price-fraction'
@ -15,8 +15,7 @@ LET result = (
FOR pageNum IN PAGINATION(amazon, nextBtnSelector)
LIMIT @pages
LET wait = pageNum > 0 ? WAIT_NAVIGATION(amazon, 20000) : false
LET waitSelector = wait ? WAIT_ELEMENT(amazon, resultListSelector) : false
LET waitSelector = pageNum > 0 ? WAIT_ELEMENT(amazon, resultListSelector) : false
LET items = (
FOR el IN ELEMENTS(amazon, resultItemSelector)
@ -26,9 +25,8 @@ LET result = (
LET price = TO_FLOAT(priceWholeTxt + "." + priceFracTxt)
LET anchor = ELEMENT(el, "a")
PRINT(priceWholeTxt, priceFracTxt)
RETURN {
page: pageNum,
url: baseURL + anchor.attributes.href,
title: INNER_TEXT(el, 'h2'),
price

3
examples/query-all.fql Normal file
View File

@ -0,0 +1,3 @@
let doc = document("https://github.com/MontFerret/ferret", { driver: "cdp" })
return elements(doc, '[role="row"]')

12
examples/redirects.fql
View File

@ -1,15 +1,7 @@
LET doc = DOCUMENT(@url, {
driver: 'cdp',
viewport: {
width: 1920,
height: 1080
}
})
LET doc = DOCUMENT(@url, { driver: 'cdp' })
CLICK(doc, '.click')
WAIT_NAVIGATION(doc, {
target: @targetURL
})
WAITFOR EVENT "navigation" IN doc { target: @targetURL }
RETURN ELEMENT(doc, '.title')

10
examples/while.fql
View File

@ -1,17 +1,17 @@
LET doc = DOCUMENT("https://github.com/MontFerret/ferret/stargazers", { driver: "cdp" })
LET nextSelector = ".paginate-container .BtnGroup a:nth-child(2)"
LET elementsSelector = '.follow-list li'
LET nextSelector = '[data-test-selector="pagination"] .btn:nth-child(2):not([disabled])'
LET elementsSelector = '#repos ol li'
FOR i DO WHILE ELEMENT_EXISTS(doc, nextSelector)
LIMIT 3
LET wait = i > 0 ? CLICK(doc, nextSelector) : false
LET nav = wait ? WAIT_NAVIGATION(doc) : false
LET nav = wait ? (WAITFOR EVENT "navigation" IN doc) : false
FOR el IN ELEMENTS(doc, elementsSelector)
FILTER ELEMENT_EXISTS(el, ".octicon-organization")
RETURN {
name: INNER_TEXT(el, ".follow-list-name"),
company: INNER_TEXT(el, ".follow-list-info span")
name: INNER_TEXT(el, 'div > div:nth-child(2) [data-hovercard-type="user"]'),
company: INNER_TEXT(el, "div > div:nth-child(2) p")
}

7
go.mod
View File

@ -13,11 +13,12 @@ require (
github.com/jarcoal/httpmock v1.0.8
github.com/mafredri/cdp v0.32.0
github.com/pkg/errors v0.9.1
github.com/rs/zerolog v1.23.0
github.com/rs/zerolog v1.24.0
github.com/sethgrid/pester v1.1.0
github.com/smartystreets/goconvey v1.6.4
github.com/wI2L/jettison v0.7.1
github.com/stretchr/testify v1.5.1
github.com/wI2L/jettison v0.7.2
golang.org/x/net v0.0.0-20210614182718-04defd469f4e
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/text v0.3.6
golang.org/x/text v0.3.7
)

40
go.sum
View File

@ -17,7 +17,6 @@ github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkb
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -37,14 +36,13 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k=
github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/klauspost/cpuid/v2 v2.0.5/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/mafredri/cdp v0.32.0 h1:JzW2F+zVK2y9ZhbNWyjrwafZLL9oNnl9Tf6JQ149Og8=
github.com/mafredri/cdp v0.32.0/go.mod h1:YTCwLXkZSa18SGSIxCPMOGZcUJODZSNlAhiMqbyxWJg=
github.com/mafredri/go-lint v0.0.0-20180911205320-920981dfc79e/go.mod h1:k/zdyxI3q6dup24o8xpYjJKTCf2F7rfxLp6w/efTiWs=
@ -56,30 +54,30 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/zerolog v1.23.0 h1:UskrK+saS9P9Y789yNNulYKdARjPZuS35B8gJF2x60g=
github.com/rs/zerolog v1.23.0/go.mod h1:6c7hFfxPOy7TacJc4Fcdi24/J0NKYGzjG8FWRI916Qo=
github.com/segmentio/encoding v0.1.10 h1:0b8dva47cSuNQR5ZcU3d0pfi9EnPpSK6q7y5ZGEW36Q=
github.com/segmentio/encoding v0.1.10/go.mod h1:RWhr02uzMB9gQC1x+MfYxedtmBibb9cZ6Vv9VxRSSbw=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.24.0 h1:76ivFxmVSRs1u2wUwJVg5VZDYQgeH1JpoS6ndgr9Wy8=
github.com/rs/zerolog v1.24.0/go.mod h1:7KHcEGe0QZPOm2IE4Kpb5rTh6n1h2hIgS5OOnu1rUaI=
github.com/segmentio/encoding v0.2.19 h1:Kshkmoz080qvUtdtakR8Bjk2sIlLS8wSvijFMEHRGow=
github.com/segmentio/encoding v0.2.19/go.mod h1:7E68jTSWMnNoYhHi1JbLd7NBSB6XfE4vzqhR88hDBQc=
github.com/sethgrid/pester v1.1.0 h1:IyEAVvwSUPjs2ACFZkBe5N59BBUpSIkQ71Hr6cM5A+w=
github.com/sethgrid/pester v1.1.0/go.mod h1:Ad7IjTpvzZO8Fl0vh9AzQ+j/jYZfyp2diGwI8m5q+ns=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/wI2L/jettison v0.7.1 h1:XNq/WvSOAiJhFww9F5JZZcBZtKFL2Y/9WHHEHLDq9TE=
github.com/wI2L/jettison v0.7.1/go.mod h1:dj49nOP41M7x6Jql62BqqF/+nW+XJgBaWzJR0hd6M84=
github.com/wI2L/jettison v0.7.2 h1:vJ66luIiQkqpswDHgN2dFq70+22cVL+9omH+NS0YxIo=
github.com/wI2L/jettison v0.7.2/go.mod h1:W3PPso417OeZeWs9nV/olfapp0o4eSZcaeZk4HeSzfM=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@ -91,6 +89,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -104,24 +103,27 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210319071255-635bc2c9138d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

66
pkg/compiler/compiler_waitfor_event_test.go
View File

@ -110,8 +110,72 @@ func newCompilerWithObservable() *compiler.Compiler {
}
func TestWaitforEventExpression(t *testing.T) {
Convey("WAITFOR EVENT X IN Y", t, func() {
Convey("WAITFOR EVENT parser", t, func() {
Convey("Should parse", func() {
c := newCompilerWithObservable()
_, err := c.Compile(`
LET obj = X::CREATE()
X::EMIT(obj, "test", 100)
WAITFOR EVENT "test" IN obj
RETURN NONE
`)
So(err, ShouldBeNil)
})
Convey("Should parse 2", func() {
c := newCompilerWithObservable()
_, err := c.Compile(`
LET obj = X::CREATE()
X::EMIT(obj, "test", 100)
WAITFOR EVENT "test" IN obj 1000
RETURN NONE
`)
So(err, ShouldBeNil)
})
Convey("Should parse 3", func() {
c := newCompilerWithObservable()
_, err := c.Compile(`
LET obj = X::CREATE()
X::EMIT(obj, "test", 100)
LET timeout = 1000
WAITFOR EVENT "test" IN obj timeout
RETURN NONE
`)
So(err, ShouldBeNil)
})
Convey("Should parse 4", func() {
c := newCompilerWithObservable()
_, err := c.Compile(`
LET obj = X::CREATE()
X::EMIT(obj, "test", 100)
LET timeout = 1000
WAITFOR EVENT "test" IN obj timeout
X::EMIT(obj, "test", 100)
RETURN NONE
`)
So(err, ShouldBeNil)
})
})
Convey("WAITFOR EVENT X IN Y runtime", t, func() {
Convey("Should wait for a given event", func() {
c := newCompilerWithObservable()

14
pkg/compiler/visitor.go
View File

@ -807,13 +807,13 @@ func (v *visitor) doVisitWaitForTimeoutValueContext(ctx *fql.WaitForTimeoutConte
return v.doVisitVariable(variable.(*fql.VariableContext), s)
}
if member := ctx.MemberExpression(); member != nil {
return v.doVisitMemberExpression(member.(*fql.MemberExpressionContext), s)
}
if fnCall := ctx.FunctionCallExpression(); fnCall != nil {
return v.doVisitFunctionCallExpression(fnCall.(*fql.FunctionCallExpressionContext), s)
}
//if member := ctx.MemberExpression(); member != nil {
// return v.doVisitMemberExpression(member.(*fql.MemberExpressionContext), s)
//}
//
//if fnCall := ctx.FunctionCallExpression(); fnCall != nil {
// return v.doVisitFunctionCallExpression(fnCall.(*fql.FunctionCallExpressionContext), s)
//}
return nil, ErrNotImplemented
}

27
pkg/drivers/cdp/dom/document.go
View File

@ -3,12 +3,12 @@ package dom
import (
"context"
"fmt"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"hash/fnv"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/pkg/errors"
"github.com/rs/zerolog"
@ -23,18 +23,18 @@ import (
)
type HTMLDocument struct {
logger *zerolog.Logger
logger zerolog.Logger
client *cdp.Client
dom *Manager
input *input.Manager
exec *eval.ExecutionContext
exec *eval.Runtime
frameTree page.FrameTree
element *HTMLElement
}
func LoadRootHTMLDocument(
ctx context.Context,
logger *zerolog.Logger,
logger zerolog.Logger,
client *cdp.Client,
domManager *Manager,
mouse *input.Mouse,
@ -52,7 +52,7 @@ func LoadRootHTMLDocument(
return nil, err
}
worldRepl, err := client.Page.CreateIsolatedWorld(ctx, page.NewCreateIsolatedWorldArgs(ftRepl.FrameTree.Frame.ID))
exec, err := eval.New(ctx, client, ftRepl.FrameTree.Frame.ID)
if err != nil {
return nil, err
@ -67,23 +67,22 @@ func LoadRootHTMLDocument(
keyboard,
gdRepl.Root,
ftRepl.FrameTree,
worldRepl.ExecutionContextID,
exec,
)
}
func LoadHTMLDocument(
ctx context.Context,
logger *zerolog.Logger,
logger zerolog.Logger,
client *cdp.Client,
domManager *Manager,
mouse *input.Mouse,
keyboard *input.Keyboard,
node dom.Node,
frameTree page.FrameTree,
execID runtime.ExecutionContextID,
exec *eval.Runtime,
) (*HTMLDocument, error) {
exec := eval.NewExecutionContext(client, frameTree.Frame, execID)
inputManager := input.NewManager(client, exec, keyboard, mouse)
inputManager := input.NewManager(logger, client, exec, keyboard, mouse)
rootElement, err := LoadHTMLElement(
ctx,
@ -111,16 +110,16 @@ func LoadHTMLDocument(
}
func NewHTMLDocument(
logger *zerolog.Logger,
logger zerolog.Logger,
client *cdp.Client,
domManager *Manager,
input *input.Manager,
exec *eval.ExecutionContext,
exec *eval.Runtime,
rootElement *HTMLElement,
frames page.FrameTree,
) *HTMLDocument {
doc := new(HTMLDocument)
doc.logger = logger
doc.logger = logging.WithName(logger.With(), "html_document").Logger()
doc.client = client
doc.dom = domManager
doc.input = input
@ -464,7 +463,7 @@ func (doc *HTMLDocument) ScrollByXY(ctx context.Context, x, y values.Float, opti
}
func (doc *HTMLDocument) Eval(ctx context.Context, expression string) (core.Value, error) {
return doc.exec.EvalWithReturnValue(ctx, expression)
return doc.exec.EvalValue(ctx, expression)
}
func (doc *HTMLDocument) logError(err error) *zerolog.Event {

348
pkg/drivers/cdp/dom/element.go
View File

@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"hash/fnv"
"strconv"
"strings"
@ -37,11 +38,11 @@ type (
}
HTMLElement struct {
logger *zerolog.Logger
logger zerolog.Logger
client *cdp.Client
dom *Manager
input *input.Manager
exec *eval.ExecutionContext
exec *eval.Runtime
id HTMLElementIdentity
nodeType html.NodeType
nodeName values.String
@ -50,11 +51,11 @@ type (
func LoadHTMLElement(
ctx context.Context,
logger *zerolog.Logger,
logger zerolog.Logger,
client *cdp.Client,
domManager *Manager,
input *input.Manager,
exec *eval.ExecutionContext,
exec *eval.Runtime,
nodeID dom.NodeID,
) (*HTMLElement, error) {
if client == nil {
@ -62,7 +63,7 @@ func LoadHTMLElement(
}
// getting a remote object that represents the current DOM Node
args := dom.NewResolveNodeArgs().SetNodeID(nodeID).SetExecutionContextID(exec.ID())
args := dom.NewResolveNodeArgs().SetNodeID(nodeID).SetExecutionContextID(exec.ContextID())
obj, err := client.DOM.ResolveNode(ctx, args)
@ -90,11 +91,11 @@ func LoadHTMLElement(
func LoadHTMLElementWithID(
ctx context.Context,
logger *zerolog.Logger,
logger zerolog.Logger,
client *cdp.Client,
domManager *Manager,
input *input.Manager,
exec *eval.ExecutionContext,
exec *eval.Runtime,
id HTMLElementIdentity,
) (*HTMLElement, error) {
node, err := client.DOM.DescribeNode(
@ -122,17 +123,22 @@ func LoadHTMLElementWithID(
}
func NewHTMLElement(
logger *zerolog.Logger,
logger zerolog.Logger,
client *cdp.Client,
domManager *Manager,
input *input.Manager,
exec *eval.ExecutionContext,
exec *eval.Runtime,
id HTMLElementIdentity,
nodeType int,
nodeName string,
) *HTMLElement {
el := new(HTMLElement)
el.logger = logger
el.logger = logging.
WithName(logger.With(), "dom_element").
Int("node_id", int(id.NodeID)).
Str("object_id", string(id.ObjectID)).
Str("node_name", nodeName).
Logger()
el.client = client
el.dom = domManager
el.input = input
@ -229,9 +235,11 @@ func (el *HTMLElement) GetNodeName() values.String {
}
func (el *HTMLElement) Length() values.Int {
value, err := el.exec.EvalWithArgumentsAndReturnValue(context.Background(), templates.GetChildrenCount(), runtime.CallArgument{
ObjectID: &el.id.ObjectID,
})
value, err := el.exec.EvalValue(
context.Background(),
templates.GetChildrenCount(),
eval.WithArgRef(el.id.ObjectID),
)
if err != nil {
el.logError(err)
@ -243,9 +251,11 @@ func (el *HTMLElement) Length() values.Int {
}
func (el *HTMLElement) GetStyles(ctx context.Context) (*values.Object, error) {
value, err := el.exec.EvalWithArgumentsAndReturnValue(ctx, templates.GetStyles(), runtime.CallArgument{
ObjectID: &el.id.ObjectID,
})
value, err := el.exec.EvalValue(
ctx,
templates.GetStyles(),
eval.WithArgRef(el.id.ObjectID),
)
if err != nil {
return values.NewObject(), err
@ -259,70 +269,27 @@ func (el *HTMLElement) GetStyles(ctx context.Context) (*values.Object, error) {
}
func (el *HTMLElement) GetStyle(ctx context.Context, name values.String) (core.Value, error) {
styles, err := el.GetStyles(ctx)
if err != nil {
return values.None, err
}
val, found := styles.Get(name)
if !found {
return values.None, nil
}
return val, nil
return el.exec.EvalValue(
ctx,
templates.GetStyle(name.String()),
eval.WithArgRef(el.id.ObjectID),
)
}
func (el *HTMLElement) SetStyles(ctx context.Context, styles *values.Object) error {
if styles == nil {
return nil
}
currentStyles, err := el.GetStyles(ctx)
if err != nil {
return err
}
styles.ForEach(func(value core.Value, key string) bool {
currentStyles.Set(values.NewString(key), value)
return true
})
str := common.SerializeStyles(ctx, currentStyles)
return el.SetAttribute(ctx, common.AttrNameStyle, str)
return el.exec.Eval(
ctx,
templates.SetStyles(styles),
eval.WithArgRef(el.id.ObjectID),
)
}
func (el *HTMLElement) SetStyle(ctx context.Context, name, value values.String) error {
// we manually set only those that are defined in attribute only
attrValue, err := el.GetAttribute(ctx, common.AttrNameStyle)
if err != nil {
return err
}
var styles *values.Object
if attrValue == values.None {
styles = values.NewObject()
} else {
styleAttr, ok := attrValue.(*values.Object)
if !ok {
return core.TypeError(attrValue.Type(), types.Object)
}
styles = styleAttr
}
styles.Set(name, value)
str := common.SerializeStyles(ctx, styles)
return el.SetAttribute(ctx, common.AttrNameStyle, str)
return el.exec.Eval(
ctx,
templates.SetStyle(name.String(), value.String()),
eval.WithArgRef(el.id.ObjectID),
)
}
func (el *HTMLElement) RemoveStyle(ctx context.Context, names ...values.String) error {
@ -330,30 +297,11 @@ func (el *HTMLElement) RemoveStyle(ctx context.Context, names ...values.String)
return nil
}
value, err := el.GetAttribute(ctx, common.AttrNameStyle)
if err != nil {
return err
}
// no attribute
if value == values.None {
return nil
}
styles, ok := value.(*values.Object)
if !ok {
return core.TypeError(styles.Type(), types.Object)
}
for _, name := range names {
styles.Remove(name)
}
str := common.SerializeStyles(ctx, styles)
return el.SetAttribute(ctx, common.AttrNameStyle, str)
return el.exec.Eval(
ctx,
templates.RemoveStyles(names),
eval.WithArgRef(el.id.ObjectID),
)
}
func (el *HTMLElement) GetAttributes(ctx context.Context) (*values.Object, error) {
@ -457,10 +405,10 @@ func (el *HTMLElement) RemoveAttribute(ctx context.Context, names ...values.Stri
}
func (el *HTMLElement) GetChildNodes(ctx context.Context) (*values.Array, error) {
out, err := el.exec.EvalWithArgumentsAndReturnReference(ctx, templates.GetChildren(),
runtime.CallArgument{
ObjectID: &el.id.ObjectID,
},
out, err := el.exec.EvalRef(
ctx,
templates.GetChildren(),
eval.WithArgRef(el.id.ObjectID),
)
if err != nil {
@ -485,10 +433,10 @@ func (el *HTMLElement) GetChildNodes(ctx context.Context) (*values.Array, error)
}
func (el *HTMLElement) GetChildNode(ctx context.Context, idx values.Int) (core.Value, error) {
out, err := el.exec.EvalWithArgumentsAndReturnReference(ctx, templates.GetChildByIndex(int64(idx)),
runtime.CallArgument{
ObjectID: &el.id.ObjectID,
},
out, err := el.exec.EvalRef(
ctx,
templates.GetChildByIndex(int64(idx)),
eval.WithArgRef(el.id.ObjectID),
)
if err != nil {
@ -511,9 +459,11 @@ func (el *HTMLElement) GetNextElementSibling(ctx context.Context) (core.Value, e
}
func (el *HTMLElement) evalAndGetElement(ctx context.Context, expr string) (core.Value, error) {
obj, err := el.exec.EvalWithArgumentsAndReturnReference(ctx, expr, runtime.CallArgument{
ObjectID: &el.id.ObjectID,
})
obj, err := el.exec.EvalRef(
ctx,
expr,
eval.WithArgRef(el.id.ObjectID),
)
if err != nil {
return values.None, err
@ -544,32 +494,17 @@ func (el *HTMLElement) evalAndGetElement(ctx context.Context, expr string) (core
}
func (el *HTMLElement) QuerySelector(ctx context.Context, selector values.String) (core.Value, error) {
selectorArgs := dom.NewQuerySelectorArgs(el.id.NodeID, selector.String())
found, err := el.client.DOM.QuerySelector(ctx, selectorArgs)
obj, err := el.exec.EvalRef(
ctx,
templates.QuerySelector(selector.String()),
eval.WithArgRef(el.id.ObjectID),
)
if err != nil {
return values.None, err
}
if found.NodeID == emptyNodeID {
return values.None, nil
}
res, err := LoadHTMLElement(
ctx,
el.logger,
el.client,
el.dom,
el.input,
el.exec,
found.NodeID,
)
if err != nil {
return values.None, nil
}
return res, nil
return el.convertEvalResult(ctx, obj)
}
func (el *HTMLElement) QuerySelectorAll(ctx context.Context, selector values.String) (*values.Array, error) {
@ -612,19 +547,11 @@ func (el *HTMLElement) QuerySelectorAll(ctx context.Context, selector values.Str
}
func (el *HTMLElement) XPath(ctx context.Context, expression values.String) (result core.Value, err error) {
exp, err := expression.MarshalJSON()
if err != nil {
return values.None, err
}
out, err := el.exec.EvalWithArgumentsAndReturnReference(ctx, templates.XPath(),
runtime.CallArgument{
ObjectID: &el.id.ObjectID,
},
runtime.CallArgument{
Value: exp,
},
out, err := el.exec.EvalRef(
ctx,
templates.XPath(),
eval.WithArgRef(el.id.ObjectID),
eval.WithArgValue(expression),
)
if err != nil {
@ -643,21 +570,11 @@ func (el *HTMLElement) SetInnerText(ctx context.Context, innerText values.String
}
func (el *HTMLElement) GetInnerTextBySelector(ctx context.Context, selector values.String) (values.String, error) {
sel, err := selector.MarshalJSON()
if err != nil {
return values.EmptyString, err
}
out, err := el.exec.EvalWithArgumentsAndReturnValue(
out, err := el.exec.EvalValue(
ctx,
templates.GetInnerTextBySelector(),
runtime.CallArgument{
ObjectID: &el.id.ObjectID,
},
runtime.CallArgument{
Value: sel,
},
eval.WithArgRef(el.id.ObjectID),
eval.WithArgValue(selector),
)
if err != nil {
@ -668,49 +585,21 @@ func (el *HTMLElement) GetInnerTextBySelector(ctx context.Context, selector valu
}
func (el *HTMLElement) SetInnerTextBySelector(ctx context.Context, selector, innerText values.String) error {
sel, err := selector.MarshalJSON()
if err != nil {
return err
}
val, err := innerText.MarshalJSON()
if err != nil {
return err
}
return el.exec.EvalWithArguments(
return el.exec.Eval(
ctx,
templates.SetInnerTextBySelector(),
runtime.CallArgument{
ObjectID: &el.id.ObjectID,
},
runtime.CallArgument{
Value: sel,
},
runtime.CallArgument{
Value: val,
},
eval.WithArgRef(el.id.ObjectID),
eval.WithArgValue(selector),
eval.WithArgValue(innerText),
)
}
func (el *HTMLElement) GetInnerTextBySelectorAll(ctx context.Context, selector values.String) (*values.Array, error) {
sel, err := selector.MarshalJSON()
if err != nil {
return nil, err
}
out, err := el.exec.EvalWithArgumentsAndReturnValue(
out, err := el.exec.EvalValue(
ctx,
templates.GetInnerTextBySelectorAll(),
runtime.CallArgument{
ObjectID: &el.id.ObjectID,
},
runtime.CallArgument{
Value: sel,
},
eval.WithArgRef(el.id.ObjectID),
eval.WithArgValue(selector),
)
if err != nil {
@ -735,21 +624,11 @@ func (el *HTMLElement) SetInnerHTML(ctx context.Context, innerHTML values.String
}
func (el *HTMLElement) GetInnerHTMLBySelector(ctx context.Context, selector values.String) (values.String, error) {
sel, err := selector.MarshalJSON()
if err != nil {
return values.EmptyString, err
}
out, err := el.exec.EvalWithArgumentsAndReturnValue(
out, err := el.exec.EvalValue(
ctx,
templates.GetInnerHTMLBySelector(),
runtime.CallArgument{
ObjectID: &el.id.ObjectID,
},
runtime.CallArgument{
Value: sel,
},
eval.WithArgRef(el.id.ObjectID),
eval.WithArgValue(selector),
)
if err != nil {
@ -760,49 +639,21 @@ func (el *HTMLElement) GetInnerHTMLBySelector(ctx context.Context, selector valu
}
func (el *HTMLElement) SetInnerHTMLBySelector(ctx context.Context, selector, innerHTML values.String) error {
sel, err := selector.MarshalJSON()
if err != nil {
return err
}
val, err := innerHTML.MarshalJSON()
if err != nil {
return err
}
return el.exec.EvalWithArguments(
return el.exec.Eval(
ctx,
templates.SetInnerHTMLBySelector(),
runtime.CallArgument{
ObjectID: &el.id.ObjectID,
},
runtime.CallArgument{
Value: sel,
},
runtime.CallArgument{
Value: val,
},
eval.WithArgRef(el.id.ObjectID),
eval.WithArgValue(selector),
eval.WithArgValue(innerHTML),
)
}
func (el *HTMLElement) GetInnerHTMLBySelectorAll(ctx context.Context, selector values.String) (*values.Array, error) {
sel, err := selector.MarshalJSON()
if err != nil {
return values.EmptyArray(), err
}
out, err := el.exec.EvalWithArgumentsAndReturnValue(
out, err := el.exec.EvalValue(
ctx,
templates.GetInnerHTMLBySelectorAll(),
runtime.CallArgument{
ObjectID: &el.id.ObjectID,
},
runtime.CallArgument{
Value: sel,
},
eval.WithArgRef(el.id.ObjectID),
eval.WithArgValue(selector),
)
if err != nil {
@ -912,9 +763,16 @@ func (el *HTMLElement) WaitForAttribute(
}
func (el *HTMLElement) WaitForStyle(ctx context.Context, name values.String, value core.Value, when drivers.WaitEvent) error {
task := events.NewValueWaitTask(when, value, func(ctx context.Context) (core.Value, error) {
return el.GetStyle(ctx, name)
}, events.DefaultPolling)
task := events.NewEvalWaitTask(
el.exec,
templates.WaitForStyle(
name.String(),
value.String(),
when,
),
events.DefaultPolling,
eval.WithArgRef(el.id.ObjectID),
)
_, err := task.Run(ctx)
@ -1023,9 +881,7 @@ func (el *HTMLElement) convertEvalResult(ctx context.Context, out runtime.Remote
}
switch typeName {
case "string", "number", "boolean":
return eval.Unmarshal(&out)
case "array":
case "array", "HTMLCollection":
if out.ObjectID == nil {
return values.None, nil
}
@ -1122,6 +978,8 @@ func (el *HTMLElement) convertEvalResult(ctx context.Context, out runtime.Remote
ObjectID: *out.ObjectID,
},
)
case "string", "number", "boolean":
return eval.Unmarshal(&out)
default:
return values.None, nil
}

5
pkg/drivers/cdp/dom/frame.go
View File

@ -8,9 +8,8 @@ import (
type (
Frame struct {
tree page.FrameTree
node *HTMLDocument
ready bool
tree page.FrameTree
node *HTMLDocument
}
AtomicFrameID struct {

156
pkg/drivers/cdp/dom/helpers.go
View File

@ -3,14 +3,14 @@ package dom
import (
"bytes"
"context"
"errors"
"time"
"regexp"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/pkg/errors"
"golang.org/x/net/html"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
@ -18,7 +18,7 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/values"
)
var emptyExpires = time.Time{}
var camelMatcher = regexp.MustCompile("[A-Za-z0-9]+")
// traverseAttrs is a helper function that parses a given interleaved array of node attribute names and values,
// and calls a given attribute on each key-value pair
@ -34,11 +34,11 @@ func traverseAttrs(attrs []string, predicate func(name, value string) bool) {
}
}
func setInnerHTML(ctx context.Context, client *cdp.Client, exec *eval.ExecutionContext, id HTMLElementIdentity, innerHTML values.String) error {
var objID *runtime.RemoteObjectID
func setInnerHTML(ctx context.Context, client *cdp.Client, exec *eval.Runtime, id HTMLElementIdentity, innerHTML values.String) error {
var objID runtime.RemoteObjectID
if id.ObjectID != "" {
objID = &id.ObjectID
objID = id.ObjectID
} else {
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.NodeID))
@ -50,28 +50,19 @@ func setInnerHTML(ctx context.Context, client *cdp.Client, exec *eval.ExecutionC
return errors.New("unable to resolve node")
}
objID = repl.Object.ObjectID
objID = *repl.Object.ObjectID
}
b, err := innerHTML.MarshalJSON()
if err != nil {
return err
}
err = exec.EvalWithArguments(ctx, templates.SetInnerHTML(),
runtime.CallArgument{
ObjectID: objID,
},
runtime.CallArgument{
Value: b,
},
return exec.Eval(
ctx,
templates.SetInnerHTML(),
eval.WithArgRef(objID),
eval.WithArgValue(innerHTML),
)
return err
}
func getInnerHTML(ctx context.Context, client *cdp.Client, exec *eval.ExecutionContext, id HTMLElementIdentity, nodeType html.NodeType) (values.String, error) {
func getInnerHTML(ctx context.Context, client *cdp.Client, exec *eval.Runtime, id HTMLElementIdentity, nodeType html.NodeType) (values.String, error) {
// not a document
if nodeType != html.DocumentNode {
var objID runtime.RemoteObjectID
@ -101,7 +92,7 @@ func getInnerHTML(ctx context.Context, client *cdp.Client, exec *eval.ExecutionC
return values.NewString(res.String()), nil
}
repl, err := exec.EvalWithReturnValue(ctx, "return document.documentElement.innerHTML")
repl, err := exec.EvalValue(ctx, "return document.documentElement.innerHTML")
if err != nil {
return "", err
@ -110,11 +101,11 @@ func getInnerHTML(ctx context.Context, client *cdp.Client, exec *eval.ExecutionC
return values.NewString(repl.String()), nil
}
func setInnerText(ctx context.Context, client *cdp.Client, exec *eval.ExecutionContext, id HTMLElementIdentity, innerText values.String) error {
var objID *runtime.RemoteObjectID
func setInnerText(ctx context.Context, client *cdp.Client, exec *eval.Runtime, id HTMLElementIdentity, innerText values.String) error {
var objID runtime.RemoteObjectID
if id.ObjectID != "" {
objID = &id.ObjectID
objID = id.ObjectID
} else {
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.NodeID))
@ -126,28 +117,18 @@ func setInnerText(ctx context.Context, client *cdp.Client, exec *eval.ExecutionC
return errors.New("unable to resolve node")
}
objID = repl.Object.ObjectID
objID = *repl.Object.ObjectID
}
b, err := innerText.MarshalJSON()
if err != nil {
return err
}
err = exec.EvalWithArguments(ctx, templates.SetInnerText(),
runtime.CallArgument{
ObjectID: objID,
},
runtime.CallArgument{
Value: b,
},
return exec.Eval(
ctx,
templates.SetInnerText(),
eval.WithArgRef(objID),
eval.WithArgValue(innerText),
)
return err
}
func getInnerText(ctx context.Context, client *cdp.Client, exec *eval.ExecutionContext, id HTMLElementIdentity, nodeType html.NodeType) (values.String, error) {
func getInnerText(ctx context.Context, client *cdp.Client, exec *eval.Runtime, id HTMLElementIdentity, nodeType html.NodeType) (values.String, error) {
// not a document
if nodeType != html.DocumentNode {
var objID runtime.RemoteObjectID
@ -177,7 +158,7 @@ func getInnerText(ctx context.Context, client *cdp.Client, exec *eval.ExecutionC
return values.NewString(res.String()), err
}
repl, err := exec.EvalWithReturnValue(ctx, "return document.documentElement.innerText")
repl, err := exec.EvalValue(ctx, "return document.documentElement.innerText")
if err != nil {
return "", err
@ -186,67 +167,34 @@ func getInnerText(ctx context.Context, client *cdp.Client, exec *eval.ExecutionC
return values.NewString(repl.String()), nil
}
func parseInnerText(innerHTML string) (values.String, error) {
buff := bytes.NewBuffer([]byte(innerHTML))
parsed, err := goquery.NewDocumentFromReader(buff)
func resolveFrame(ctx context.Context, client *cdp.Client, frameID page.FrameID) (dom.Node, *eval.Runtime, error) {
exec, err := eval.New(ctx, client, frameID)
if err != nil {
return values.EmptyString, err
return dom.Node{}, nil, errors.Wrap(err, "create JS executor")
}
return values.NewString(parsed.Text()), nil
}
func createChildrenArray(nodes []dom.Node) []HTMLElementIdentity {
children := make([]HTMLElementIdentity, len(nodes))
for idx, child := range nodes {
child := child
children[idx] = HTMLElementIdentity{
NodeID: child.NodeID,
}
}
return children
}
func resolveFrame(ctx context.Context, client *cdp.Client, frameID page.FrameID) (dom.Node, runtime.ExecutionContextID, error) {
worldRepl, err := client.Page.CreateIsolatedWorld(ctx, page.NewCreateIsolatedWorldArgs(frameID))
if err != nil {
return dom.Node{}, -1, err
}
evalRes, err := client.Runtime.Evaluate(
evalRes, err := exec.EvalRef(
ctx,
runtime.NewEvaluateArgs(eval.PrepareEval("return document")).
SetContextID(worldRepl.ExecutionContextID),
"return document",
)
if err != nil {
return dom.Node{}, -1, err
return dom.Node{}, nil, err
}
if evalRes.ExceptionDetails != nil {
exception := *evalRes.ExceptionDetails
return dom.Node{}, -1, errors.New(exception.Text)
if evalRes.ObjectID == nil {
return dom.Node{}, nil, errors.New("failed to resolve frame document")
}
if evalRes.Result.ObjectID == nil {
return dom.Node{}, -1, errors.New("failed to resolve frame document")
}
req, err := client.DOM.RequestNode(ctx, dom.NewRequestNodeArgs(*evalRes.Result.ObjectID))
req, err := client.DOM.RequestNode(ctx, dom.NewRequestNodeArgs(*evalRes.ObjectID))
if err != nil {
return dom.Node{}, -1, err
return dom.Node{}, nil, err
}
if req.NodeID == 0 {
return dom.Node{}, -1, errors.New("framed document is resolved with empty node id")
return dom.Node{}, nil, errors.New("framed document is resolved with empty node id")
}
desc, err := client.DOM.DescribeNode(
@ -258,12 +206,38 @@ func resolveFrame(ctx context.Context, client *cdp.Client, frameID page.FrameID)
)
if err != nil {
return dom.Node{}, -1, err
return dom.Node{}, nil, err
}
// Returned node, by some reason, does not contain the NodeID
// So, we have to set it manually
desc.Node.NodeID = req.NodeID
return desc.Node, worldRepl.ExecutionContextID, nil
return desc.Node, exec, nil
}
func toCamelCase(input string) string {
var buf bytes.Buffer
matched := camelMatcher.FindAllString(input, -1)
if matched == nil {
return ""
}
for i, match := range matched {
res := match
if i > 0 {
if len(match) > 1 {
res = strings.ToUpper(match[0:1]) + match[1:]
} else {
res = strings.ToUpper(match)
}
}
buf.WriteString(res)
}
return buf.String()
}

55
pkg/drivers/cdp/dom/helpers_test.go Normal file
View File

@ -0,0 +1,55 @@
package dom
import (
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func Test_toCamelCase(t *testing.T) {
Convey("toCamelCase", t, func() {
Convey("should format string into camel case", func() {
inputs := []struct {
actual string
expected string
}{
{
actual: "foo-bar",
expected: "fooBar",
},
{
actual: "foo-1-bar",
expected: "foo1Bar",
},
{
actual: "overscroll-behavior-x",
expected: "overscrollBehaviorX",
},
{
actual: "x",
expected: "x",
},
{
actual: "foo-x",
expected: "fooX",
},
{
actual: "foo-$",
expected: "foo",
},
{
actual: "color",
expected: "color",
},
{
actual: "textDecorationSkipInk",
expected: "textDecorationSkipInk",
},
}
for _, input := range inputs {
So(toCamelCase(input.actual), ShouldEqual, input.expected)
}
})
})
}

11
pkg/drivers/cdp/dom/manager.go
View File

@ -11,13 +11,14 @@ import (
"github.com/MontFerret/ferret/pkg/drivers/cdp/input"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type (
Manager struct {
mu sync.RWMutex
logger *zerolog.Logger
logger zerolog.Logger
client *cdp.Client
mouse *input.Mouse
keyboard *input.Keyboard
@ -27,14 +28,14 @@ type (
)
func New(
logger *zerolog.Logger,
logger zerolog.Logger,
client *cdp.Client,
mouse *input.Mouse,
keyboard *input.Keyboard,
) (manager *Manager, err error) {
manager = new(Manager)
manager.logger = logger
manager.logger = logging.WithName(logger.With(), "dom_manager").Logger()
manager.client = client
manager.mouse = mouse
manager.keyboard = keyboard
@ -212,7 +213,7 @@ func (m *Manager) getFrameInternal(ctx context.Context, frameID page.FrameID) (*
}
// the frames is not loaded yet
node, execID, err := resolveFrame(ctx, m.client, frameID)
node, exec, err := resolveFrame(ctx, m.client, frameID)
if err != nil {
return nil, errors.Wrapf(err, "failed to resolve frame node: %s", frameID)
@ -227,7 +228,7 @@ func (m *Manager) getFrameInternal(ctx context.Context, frameID page.FrameID) (*
m.keyboard,
node,
frame.tree,
execID,
exec,
)
if err != nil {

10
pkg/drivers/cdp/driver.go
View File

@ -52,9 +52,7 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
conn, err := drv.createConnection(ctx, params.KeepCookies)
if err != nil {
logger.
Error().
Timestamp().
logger.Error().
Err(err).
Str("driver", drv.options.Name).
Msg("failed to create a new connection")
@ -71,9 +69,7 @@ func (drv *Driver) Parse(ctx context.Context, params drivers.ParseParams) (drive
conn, err := drv.createConnection(ctx, true)
if err != nil {
logger.
Error().
Timestamp().
logger.Error().
Err(err).
Str("driver", drv.options.Name).
Msg("failed to create a new connection")
@ -158,7 +154,7 @@ func (drv *Driver) init(ctx context.Context) error {
bconn, err := rpcc.DialContext(
ctx,
ver.WebSocketDebuggerURL,
rpcc.WithWriteBufferSize(1048562),
rpcc.WithWriteBufferSize(104857586),
rpcc.WithCompression(),
)

366
pkg/drivers/cdp/eval/context.go
View File

@ -1,366 +0,0 @@
package eval
import (
"context"
"fmt"
"strings"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/pkg/errors"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
const EmptyExecutionContextID = runtime.ExecutionContextID(-1)
type ExecutionContext struct {
client *cdp.Client
frame page.Frame
contextID runtime.ExecutionContextID
}
func NewExecutionContextFrom(ctx context.Context, client *cdp.Client, frame page.Frame) (*ExecutionContext, error) {
world, err := client.Page.CreateIsolatedWorld(ctx, page.NewCreateIsolatedWorldArgs(frame.ID))
if err != nil {
return nil, err
}
return NewExecutionContext(client, frame, world.ExecutionContextID), nil
}
func NewExecutionContext(client *cdp.Client, frame page.Frame, contextID runtime.ExecutionContextID) *ExecutionContext {
ec := new(ExecutionContext)
ec.client = client
ec.frame = frame
ec.contextID = contextID
return ec
}
func (ec *ExecutionContext) ID() runtime.ExecutionContextID {
return ec.contextID
}
func (ec *ExecutionContext) Eval(ctx context.Context, exp string) error {
_, err := ec.evalWithValueInternal(
ctx,
runtime.
NewEvaluateArgs(PrepareEval(exp)),
)
return err
}
func (ec *ExecutionContext) EvalWithArguments(ctx context.Context, exp string, args ...runtime.CallArgument) error {
_, err := ec.evalWithArgumentsInternal(ctx, exp, args, false)
return err
}
func (ec *ExecutionContext) EvalWithArgumentsAndReturnValue(ctx context.Context, exp string, args ...runtime.CallArgument) (core.Value, error) {
out, err := ec.evalWithArgumentsInternal(ctx, exp, args, true)
if err != nil {
return values.None, err
}
return Unmarshal(&out)
}
func (ec *ExecutionContext) EvalWithArgumentsAndReturnReference(ctx context.Context, exp string, args ...runtime.CallArgument) (runtime.RemoteObject, error) {
return ec.evalWithArgumentsInternal(ctx, exp, args, false)
}
func (ec *ExecutionContext) EvalWithReturnValue(ctx context.Context, exp string) (core.Value, error) {
return ec.evalWithValueInternal(
ctx,
runtime.
NewEvaluateArgs(PrepareEval(exp)).
SetReturnByValue(true),
)
}
func (ec *ExecutionContext) EvalWithReturnReference(ctx context.Context, exp string) (runtime.RemoteObject, error) {
return ec.evalInternal(
ctx,
runtime.
NewEvaluateArgs(PrepareEval(exp)).
SetReturnByValue(false),
)
}
func (ec *ExecutionContext) EvalAsync(ctx context.Context, exp string) (core.Value, error) {
return ec.evalWithValueInternal(
ctx,
runtime.
NewEvaluateArgs(PrepareEval(exp)).
SetReturnByValue(true).
SetAwaitPromise(true),
)
}
func (ec *ExecutionContext) CallMethod(
ctx context.Context,
objectID runtime.RemoteObjectID,
methodName string,
args []runtime.CallArgument,
) (*runtime.RemoteObject, error) {
callArgs := runtime.NewCallFunctionOnArgs(methodName).
SetObjectID(objectID).
SetArguments(args)
if ec.contextID != EmptyExecutionContextID {
callArgs.SetExecutionContextID(ec.contextID)
}
found, err := ec.client.Runtime.CallFunctionOn(
ctx,
callArgs,
)
if err != nil {
return nil, err
}
err = ec.handleException(found.ExceptionDetails)
if err != nil {
return nil, err
}
if found.Result.ObjectID == nil {
return nil, nil
}
return &found.Result, nil
}
func (ec *ExecutionContext) ReadPropertyByNodeID(
ctx context.Context,
nodeID dom.NodeID,
propName string,
) (core.Value, error) {
obj, err := ec.client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(nodeID))
if err != nil {
return values.None, err
}
if obj.Object.ObjectID == nil {
return values.None, nil
}
return ec.ReadProperty(ctx, *obj.Object.ObjectID, propName)
}
func (ec *ExecutionContext) ReadProperty(
ctx context.Context,
objectID runtime.RemoteObjectID,
propName string,
) (core.Value, error) {
res, err := ec.client.Runtime.GetProperties(
ctx,
runtime.NewGetPropertiesArgs(objectID),
)
if err != nil {
return values.None, err
}
if res.ExceptionDetails != nil {
return values.None, res.ExceptionDetails
}
// all props
if propName == "" {
arr := values.NewArray(len(res.Result))
for _, prop := range res.Result {
val, err := Unmarshal(prop.Value)
if err != nil {
return values.None, err
}
arr.Push(val)
}
return arr, nil
}
for _, prop := range res.Result {
if prop.Name == propName {
return Unmarshal(prop.Value)
}
}
return values.None, nil
}
func (ec *ExecutionContext) DispatchEvent(
ctx context.Context,
objectID runtime.RemoteObjectID,
eventName string,
) (values.Boolean, error) {
args := runtime.NewEvaluateArgs(PrepareEval(fmt.Sprintf(`
return new window.MouseEvent('%s', { bubbles: true, cancelable: true })
`, eventName)))
if ec.contextID != EmptyExecutionContextID {
args.SetContextID(ec.contextID)
}
evt, err := ec.client.Runtime.Evaluate(ctx, args)
if err != nil {
return values.False, nil
}
err = ec.handleException(evt.ExceptionDetails)
if err != nil {
return values.False, nil
}
if evt.Result.ObjectID == nil {
return values.False, nil
}
evtID := evt.Result.ObjectID
// release the event object
defer ec.client.Runtime.ReleaseObject(ctx, runtime.NewReleaseObjectArgs(*evtID))
_, err = ec.CallMethod(
ctx,
objectID,
"dispatchEvent",
[]runtime.CallArgument{
{
ObjectID: evt.Result.ObjectID,
},
},
)
if err != nil {
return values.False, err
}
return values.True, nil
}
func (ec *ExecutionContext) ResolveRemoteObject(ctx context.Context, exp string) (runtime.RemoteObject, error) {
res, err := ec.evalInternal(ctx, runtime.NewEvaluateArgs(PrepareEval(exp)))
if err != nil {
return runtime.RemoteObject{}, err
}
if res.ObjectID == nil {
return runtime.RemoteObject{}, errors.Wrap(core.ErrUnexpected, "unable to resolve remote object")
}
return res, nil
}
func (ec *ExecutionContext) ResolveNode(ctx context.Context, nodeID dom.NodeID) (runtime.RemoteObject, error) {
args := dom.NewResolveNodeArgs().SetNodeID(nodeID)
if ec.contextID != EmptyExecutionContextID {
args.SetExecutionContextID(ec.contextID)
}
repl, err := ec.client.DOM.ResolveNode(ctx, args)
if err != nil {
return runtime.RemoteObject{}, err
}
if repl.Object.ObjectID == nil {
return runtime.RemoteObject{}, errors.Wrap(core.ErrUnexpected, "unable to resolve remote object")
}
return repl.Object, nil
}
func (ec *ExecutionContext) evalWithArgumentsInternal(ctx context.Context, exp string, args []runtime.CallArgument, ret bool) (runtime.RemoteObject, error) {
cfArgs := runtime.
NewCallFunctionOnArgs(exp).
SetArguments(args).
SetReturnByValue(ret)
if ec.contextID != EmptyExecutionContextID {
cfArgs.SetExecutionContextID(ec.contextID)
}
repl, err := ec.client.Runtime.CallFunctionOn(ctx, cfArgs)
if err != nil {
return runtime.RemoteObject{}, err
}
err = ec.handleException(repl.ExceptionDetails)
if err != nil {
return runtime.RemoteObject{}, err
}
return repl.Result, nil
}
func (ec *ExecutionContext) evalWithValueInternal(ctx context.Context, args *runtime.EvaluateArgs) (core.Value, error) {
obj, err := ec.evalInternal(ctx, args)
if err != nil {
return values.None, err
}
if obj.Type != "undefined" && obj.Type != "null" {
return values.Unmarshal(obj.Value)
}
return Unmarshal(&obj)
}
func (ec *ExecutionContext) evalInternal(ctx context.Context, args *runtime.EvaluateArgs) (runtime.RemoteObject, error) {
if ec.contextID != EmptyExecutionContextID {
args.SetContextID(ec.contextID)
}
out, err := ec.client.Runtime.Evaluate(ctx, args)
if err != nil {
return runtime.RemoteObject{}, err
}
err = ec.handleException(out.ExceptionDetails)
if err != nil {
return runtime.RemoteObject{}, err
}
return out.Result, nil
}
func (ec *ExecutionContext) handleException(details *runtime.ExceptionDetails) error {
if details == nil {
return nil
}
desc := *details.Exception.Description
if strings.Contains(desc, drivers.ErrNotFound.Error()) {
return drivers.ErrNotFound
}
return core.Error(
core.ErrUnexpected,
fmt.Sprintf("%s: %s", details.Text, desc),
)
}

127
pkg/drivers/cdp/eval/function.go Normal file
View File

@ -0,0 +1,127 @@
package eval
import (
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/mafredri/cdp/protocol/runtime"
"strings"
)
type (
FunctionReturnType int
Function struct {
exp string
ownerID *runtime.RemoteObjectID
args []runtime.CallArgument
returnType FunctionReturnType
async bool
}
FunctionOption func(op *Function)
)
const (
ReturnNothing FunctionReturnType = iota
ReturnValue
ReturnRef
)
func newFunction(exp string, opts []FunctionOption) *Function {
op := new(Function)
op.exp = exp
op.returnType = ReturnNothing
for _, opt := range opts {
opt(op)
}
return op
}
func (fn *Function) Use(opt FunctionOption) {
opt(fn)
}
func (fn *Function) toArgs(ctx runtime.ExecutionContextID) *runtime.CallFunctionOnArgs {
exp := strings.TrimSpace(fn.exp)
if !strings.HasPrefix(exp, "(") && !strings.HasPrefix(exp, "function") {
exp = wrapExp(exp)
}
call := runtime.NewCallFunctionOnArgs(exp).
SetAwaitPromise(fn.async)
if fn.returnType == ReturnValue {
call.SetReturnByValue(true)
}
if ctx != EmptyExecutionContextID {
call.SetExecutionContextID(ctx)
}
if fn.ownerID != nil {
call.SetObjectID(*fn.ownerID)
}
if len(fn.args) > 0 {
call.SetArguments(fn.args)
}
return call
}
func withReturnRef() FunctionOption {
return func(op *Function) {
op.returnType = ReturnRef
}
}
func withReturnValue() FunctionOption {
return func(op *Function) {
op.returnType = ReturnValue
}
}
func WithArgs(args ...runtime.CallArgument) FunctionOption {
return func(op *Function) {
if op.args == nil {
op.args = args
} else {
op.args = append(op.args, args...)
}
}
}
func WithArgValue(value core.Value) FunctionOption {
raw, err := value.MarshalJSON()
if err != nil {
// we defer error
return WithArgs(runtime.CallArgument{
Value: []byte(err.Error()),
})
}
return WithArgs(runtime.CallArgument{
Value: raw,
})
}
func WithArgRef(id runtime.RemoteObjectID) FunctionOption {
return WithArgs(runtime.CallArgument{
ObjectID: &id,
})
}
func WithOwner(ctx *runtime.RemoteObjectID) FunctionOption {
return func(op *Function) {
op.ownerID = ctx
}
}
func WithAsync() FunctionOption {
return func(op *Function) {
op.async = true
}
}

44
pkg/drivers/cdp/eval/helpers.go
View File

@ -1,17 +1,38 @@
package eval
import (
"fmt"
"strconv"
"strings"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
func PrepareEval(exp string) string {
return fmt.Sprintf("((function () {%s})())", exp)
func CastToValue(input interface{}) (core.Value, error) {
value, ok := input.(core.Value)
if !ok {
return values.None, core.Error(core.ErrInvalidType, "eval return type")
}
return value, nil
}
func CastToReference(input interface{}) (runtime.RemoteObject, error) {
value, ok := input.(runtime.RemoteObject)
if !ok {
return runtime.RemoteObject{}, core.Error(core.ErrInvalidType, "eval return type")
}
return value, nil
}
func wrapExp(exp string) string {
return "function () {" + exp + "}"
}
func Unmarshal(obj *runtime.RemoteObject) (core.Value, error) {
@ -34,3 +55,20 @@ func Unmarshal(obj *runtime.RemoteObject) (core.Value, error) {
return values.Unmarshal(obj.Value)
}
}
func parseRuntimeException(details *runtime.ExceptionDetails) error {
if details == nil || details.Exception == nil {
return nil
}
desc := *details.Exception.Description
if strings.Contains(desc, drivers.ErrNotFound.Error()) {
return drivers.ErrNotFound
}
return core.Error(
core.ErrUnexpected,
desc,
)
}

31
pkg/drivers/cdp/eval/param.go
View File

@ -1,6 +1,7 @@
package eval
import (
"bytes"
"strconv"
"github.com/MontFerret/ferret/pkg/runtime/core"
@ -24,6 +25,36 @@ func Param(input core.Value) string {
}
}
func ParamList(inputs []core.Value) string {
var buf bytes.Buffer
lastIndex := len(inputs) - 1
for i, input := range inputs {
buf.WriteString(Param(input))
if i != lastIndex {
buf.WriteString(",")
}
}
return buf.String()
}
func ParamStringList(inputs []values.String) string {
var buf bytes.Buffer
lastIndex := len(inputs) - 1
for i, input := range inputs {
buf.WriteString(ParamString(input.String()))
if i != lastIndex {
buf.WriteString(",")
}
}
return buf.String()
}
func ParamString(param string) string {
return "`" + param + "`"
}

146
pkg/drivers/cdp/eval/runtime.go Normal file
View File

@ -0,0 +1,146 @@
package eval
import (
"context"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/pkg/errors"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
const EmptyExecutionContextID = runtime.ExecutionContextID(-1)
type Runtime struct {
client *cdp.Client
frame page.Frame
contextID runtime.ExecutionContextID
}
func New(ctx context.Context, client *cdp.Client, frameID page.FrameID) (*Runtime, error) {
world, err := client.Page.CreateIsolatedWorld(ctx, page.NewCreateIsolatedWorldArgs(frameID))
if err != nil {
return nil, err
}
return Create(client, world.ExecutionContextID), nil
}
func Create(client *cdp.Client, contextID runtime.ExecutionContextID) *Runtime {
ec := new(Runtime)
ec.client = client
ec.contextID = contextID
return ec
}
func (ex *Runtime) ContextID() runtime.ExecutionContextID {
return ex.contextID
}
func (ex *Runtime) Eval(ctx context.Context, exp string, opts ...FunctionOption) error {
_, err := ex.call(ctx, newFunction(exp, opts))
return err
}
func (ex *Runtime) EvalValue(ctx context.Context, exp string, opts ...FunctionOption) (core.Value, error) {
fn := newFunction(exp, opts)
fn.Use(withReturnValue())
out, err := ex.call(ctx, fn)
if err != nil {
return values.None, err
}
return CastToValue(out)
}
func (ex *Runtime) EvalRef(ctx context.Context, exp string, opts ...FunctionOption) (runtime.RemoteObject, error) {
fn := newFunction(exp, opts)
fn.Use(withReturnRef())
out, err := ex.call(ctx, fn)
if err != nil {
return runtime.RemoteObject{}, err
}
return CastToReference(out)
}
func (ex *Runtime) ReadProperty(
ctx context.Context,
objectID runtime.RemoteObjectID,
propName string,
) (core.Value, error) {
res, err := ex.client.Runtime.GetProperties(
ctx,
runtime.NewGetPropertiesArgs(objectID),
)
if err != nil {
return values.None, err
}
if res.ExceptionDetails != nil {
return values.None, res.ExceptionDetails
}
// all props
if propName == "" {
arr := values.NewArray(len(res.Result))
for _, prop := range res.Result {
val, err := Unmarshal(prop.Value)
if err != nil {
return values.None, err
}
arr.Push(val)
}
return arr, nil
}
for _, prop := range res.Result {
if prop.Name == propName {
return Unmarshal(prop.Value)
}
}
return values.None, nil
}
func (ex *Runtime) call(ctx context.Context, fn *Function) (interface{}, error) {
repl, err := ex.client.Runtime.CallFunctionOn(ctx, fn.toArgs(ex.contextID))
if err != nil {
return nil, errors.Wrap(err, "runtime call")
}
if err := parseRuntimeException(repl.ExceptionDetails); err != nil {
return nil, err
}
switch fn.returnType {
case ReturnValue:
out := repl.Result
if out.Type != "undefined" && out.Type != "null" {
return values.Unmarshal(out.Value)
}
return Unmarshal(&out)
case ReturnRef:
return repl.Result, nil
default:
return nil, nil
}
}

53
pkg/drivers/cdp/events/loop.go
View File

@ -2,6 +2,7 @@ package events
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
"math/rand"
"sync"
)
@ -10,6 +11,7 @@ type Loop struct {
mu sync.RWMutex
sources *SourceCollection
listeners *ListenerCollection
cancel context.CancelFunc
}
func NewLoop() *Loop {
@ -20,8 +22,33 @@ func NewLoop() *Loop {
return loop
}
func (loop *Loop) Run(ctx context.Context) {
go loop.run(ctx)
func (loop *Loop) Run(ctx context.Context) error {
loop.mu.Lock()
defer loop.mu.Unlock()
if loop.cancel != nil {
return core.Error(core.ErrInvalidOperation, "loop is already running")
}
childCtx, cancel := context.WithCancel(ctx)
loop.cancel = cancel
go loop.run(childCtx)
return nil
}
func (loop *Loop) Close() error {
loop.mu.Lock()
defer loop.mu.Unlock()
if loop.cancel != nil {
loop.cancel()
loop.cancel = nil
}
return loop.sources.Close()
}
func (loop *Loop) AddSource(source Source) {
@ -38,6 +65,13 @@ func (loop *Loop) RemoveSource(source Source) {
loop.sources.Remove(source)
}
func (loop *Loop) Listeners(eventID ID) int {
loop.mu.RLock()
defer loop.mu.RUnlock()
return loop.listeners.Size(eventID)
}
func (loop *Loop) AddListener(eventID ID, handler Handler) ListenerID {
loop.mu.RLock()
defer loop.mu.RUnlock()
@ -67,11 +101,11 @@ func (loop *Loop) run(ctx context.Context) {
size := sources.Size()
counter := -1
// in case event array is empty
// we use this mock noop event source to simplify the logic
noop := newNoopSource()
for {
if isCtxDone(ctx) {
break
}
counter++
if counter >= size {
@ -88,13 +122,12 @@ func (loop *Loop) run(ctx context.Context) {
if err == nil {
source = found
} else {
// might be removed
source = noop
// force to reset counter
counter = size
continue
}
} else {
source = noop
continue
}
select {
@ -129,7 +162,7 @@ func (loop *Loop) emit(ctx context.Context, eventID ID, message interface{}, err
case <-ctx.Done():
return
default:
// if returned false, it means the loops should call the handler anymore
// if returned false, it means the loops should not call the handler anymore
if !listener.Handler(ctx, message) {
loop.mu.RLock()
loop.listeners.Remove(eventID, listener.ID)

51
pkg/drivers/cdp/events/loop_test.go
View File

@ -50,9 +50,13 @@ type (
var TestEvent = events.New("test_event")
func NewTestEventStream() *TestEventStream {
return NewBufferedTestEventStream(0)
}
func NewBufferedTestEventStream(buffer int) *TestEventStream {
es := new(TestEventStream)
es.ready = make(chan struct{})
es.message = make(chan interface{})
es.ready = make(chan struct{}, buffer)
es.message = make(chan interface{}, buffer)
return es
}
@ -290,7 +294,7 @@ func TestLoop(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
loop.Run(ctx)
So(loop.Run(ctx), ShouldBeNil)
defer cancel()
loop.AddListener(TestEvent, events.Always(func(ctx context.Context, message interface{}) {
@ -352,7 +356,7 @@ func TestLoop(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
loop.Run(ctx)
So(loop.Run(ctx), ShouldBeNil)
defer cancel()
time.Sleep(time.Duration(100) * time.Millisecond)
@ -363,6 +367,45 @@ func TestLoop(t *testing.T) {
So(counter.Value(), ShouldEqual, 1)
})
Convey("Should stop on Context.Done", t, func() {
loop := events.NewLoop()
eventsToFire := 5
counter := NewCounter()
onLoad := &TestLoadEventFiredClient{NewBufferedTestEventStream(10)}
loop.AddSource(events.NewSource(TestEvent, onLoad, func(_ rpcc.Stream) (i interface{}, e error) {
return onLoad.Recv()
}))
loop.AddListener(TestEvent, events.Always(func(ctx context.Context, message interface{}) {
counter.Increase()
}))
ctx, cancel := context.WithCancel(context.Background())
So(loop.Run(ctx), ShouldBeNil)
for i := 0; i <= eventsToFire; i++ {
time.Sleep(time.Duration(100) * time.Millisecond)
onLoad.Emit(&page.LoadEventFiredReply{})
}
// Stop the loop
cancel()
time.Sleep(time.Duration(100) * time.Millisecond)
onLoad.Emit(&page.LoadEventFiredReply{})
for i := 0; i <= eventsToFire; i++ {
time.Sleep(time.Duration(100) * time.Millisecond)
onLoad.Emit(&page.LoadEventFiredReply{})
}
So(counter.Value(), ShouldEqual, eventsToFire)
})
}
func BenchmarkLoop_AddListenerSync(b *testing.B) {

31
pkg/drivers/cdp/events/noop.go
View File

@ -1,31 +0,0 @@
package events
import (
"github.com/MontFerret/ferret/pkg/runtime/core"
)
type noopEvent struct {
c chan struct{}
}
func newNoopSource() Source {
return noopEvent{
c: make(chan struct{}),
}
}
func (n noopEvent) Ready() <-chan struct{} {
return n.c
}
func (n noopEvent) RecvMsg(_ interface{}) error {
return core.ErrNotSupported
}
func (n noopEvent) Close() error {
return nil
}
func (n noopEvent) Recv() (Event, error) {
return Event{}, core.ErrNotSupported
}

6
pkg/drivers/cdp/events/wait.go
View File

@ -57,15 +57,17 @@ func (task *WaitTask) Run(ctx context.Context) (core.Value, error) {
}
func NewEvalWaitTask(
ec *eval.ExecutionContext,
ec *eval.Runtime,
predicate string,
polling time.Duration,
opts ...eval.FunctionOption,
) *WaitTask {
return NewWaitTask(
func(ctx context.Context) (core.Value, error) {
return ec.EvalWithReturnValue(
return ec.EvalValue(
ctx,
predicate,
opts...,
)
},
polling,

521
pkg/drivers/cdp/input/manager.go
View File

@ -2,6 +2,8 @@ package input
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/rs/zerolog"
"time"
"github.com/mafredri/cdp"
@ -23,20 +25,25 @@ type (
}
Manager struct {
logger zerolog.Logger
client *cdp.Client
exec *eval.ExecutionContext
exec *eval.Runtime
keyboard *Keyboard
mouse *Mouse
}
)
func NewManager(
logger zerolog.Logger,
client *cdp.Client,
exec *eval.ExecutionContext,
exec *eval.Runtime,
keyboard *Keyboard,
mouse *Mouse,
) *Manager {
logger = logging.WithName(logger.With(), "input_manager").Logger()
return &Manager{
logger,
client,
exec,
keyboard,
@ -53,35 +60,111 @@ func (m *Manager) Mouse() *Mouse {
}
func (m *Manager) ScrollTop(ctx context.Context, options drivers.ScrollOptions) error {
return m.exec.Eval(ctx, templates.ScrollTop(options))
m.logger.Trace().
Str("behavior", options.Behavior.String()).
Str("block", options.Block.String()).
Str("inline", options.Inline.String()).
Msg("scrolling to the top")
if err := m.exec.Eval(ctx, templates.ScrollTop(options)); err != nil {
m.logger.Trace().Err(err).Msg("failed to scroll to the top")
return err
}
m.logger.Trace().Msg("scrolled to the top")
return nil
}
func (m *Manager) ScrollBottom(ctx context.Context, options drivers.ScrollOptions) error {
return m.exec.Eval(ctx, templates.ScrollBottom(options))
m.logger.Trace().
Str("behavior", options.Behavior.String()).
Str("block", options.Block.String()).
Str("inline", options.Inline.String()).
Msg("scrolling to the bottom")
if err := m.exec.Eval(ctx, templates.ScrollBottom(options)); err != nil {
m.logger.Trace().Err(err).Msg("failed to scroll to the bottom")
return err
}
m.logger.Trace().Msg("scrolled to the bottom")
return nil
}
func (m *Manager) ScrollIntoView(ctx context.Context, objectID runtime.RemoteObjectID, options drivers.ScrollOptions) error {
return m.exec.EvalWithArguments(
m.logger.Trace().
Str("object_id", string(objectID)).
Str("behavior", options.Behavior.String()).
Str("block", options.Block.String()).
Str("inline", options.Inline.String()).
Msg("scrolling to an element")
if err := m.exec.Eval(
ctx,
templates.ScrollIntoView(options),
runtime.CallArgument{
ObjectID: &objectID,
},
)
eval.WithArgRef(objectID),
); err != nil {
m.logger.Trace().Err(err).Msg("failed to scroll to an element")
return err
}
m.logger.Trace().Msg("scrolled to an element")
return nil
}
func (m *Manager) ScrollIntoViewBySelector(ctx context.Context, selector string, options drivers.ScrollOptions) error {
return m.exec.Eval(ctx, templates.ScrollIntoViewBySelector(selector, options))
m.logger.Trace().
Str("selector", selector).
Str("behavior", options.Behavior.String()).
Str("block", options.Block.String()).
Str("inline", options.Inline.String()).
Msg("scrolling to an element by selector")
if err := m.exec.Eval(ctx, templates.ScrollIntoViewBySelector(selector, options)); err != nil {
m.logger.Trace().Err(err).Msg("failed to scroll to an element by selector")
return err
}
m.logger.Trace().Msg("scrolled to an element by selector")
return nil
}
func (m *Manager) ScrollByXY(ctx context.Context, x, y float64, options drivers.ScrollOptions) error {
return m.exec.Eval(
m.logger.Trace().
Float64("x", x).
Float64("y", y).
Str("behavior", options.Behavior.String()).
Str("block", options.Block.String()).
Str("inline", options.Inline.String()).
Msg("scrolling to an element by given coordinates")
if err := m.exec.Eval(
ctx,
templates.Scroll(eval.ParamFloat(x), eval.ParamFloat(y), options),
)
); err != nil {
m.logger.Trace().Err(err).Msg("failed to scroll to an element by coordinates")
return err
}
m.logger.Trace().Msg("scrolled to an element by given coordinates")
return nil
}
func (m *Manager) Focus(ctx context.Context, objectID runtime.RemoteObjectID) error {
m.logger.Trace().
Str("object_id", string(objectID)).
Msg("focusing on an element")
err := m.ScrollIntoView(ctx, objectID, drivers.ScrollOptions{
Behavior: drivers.ScrollBehaviorAuto,
Block: drivers.ScrollVerticalAlignmentCenter,
@ -92,10 +175,23 @@ func (m *Manager) Focus(ctx context.Context, objectID runtime.RemoteObjectID) er
return err
}
return m.client.DOM.Focus(ctx, dom.NewFocusArgs().SetObjectID(objectID))
if err := m.client.DOM.Focus(ctx, dom.NewFocusArgs().SetObjectID(objectID)); err != nil {
m.logger.Trace().Err(err).Msg("failed focusing on an element")
return err
}
m.logger.Trace().Msg("focused on an element")
return nil
}
func (m *Manager) FocusBySelector(ctx context.Context, parentNodeID dom.NodeID, selector string) error {
m.logger.Trace().
Int("parent_node_id", int(parentNodeID)).
Str("selector", selector).
Msg("focusing on an element by selector")
err := m.ScrollIntoViewBySelector(ctx, selector, drivers.ScrollOptions{
Behavior: drivers.ScrollBehaviorAuto,
Block: drivers.ScrollVerticalAlignmentCenter,
@ -106,70 +202,171 @@ func (m *Manager) FocusBySelector(ctx context.Context, parentNodeID dom.NodeID,
return err
}
m.logger.Trace().Msg("resolving an element by selector")
found, err := m.client.DOM.QuerySelector(ctx, dom.NewQuerySelectorArgs(parentNodeID, selector))
if err != nil {
return nil
m.logger.Trace().
Err(err).
Msg("failed resolving an element by selector")
return err
}
return m.client.DOM.Focus(ctx, dom.NewFocusArgs().SetNodeID(found.NodeID))
if err := m.client.DOM.Focus(ctx, dom.NewFocusArgs().SetNodeID(found.NodeID)); err != nil {
m.logger.Trace().
Err(err).
Msg("failed focusing on an element by selector")
return err
}
m.logger.Trace().Msg("focused on an element")
return nil
}
func (m *Manager) Blur(ctx context.Context, objectID runtime.RemoteObjectID) error {
return m.exec.EvalWithArguments(ctx, templates.Blur(), runtime.CallArgument{
ObjectID: &objectID,
})
m.logger.Trace().
Str("object_id", string(objectID)).
Msg("removing focus from an element")
if err := m.exec.Eval(ctx, templates.Blur(), eval.WithArgRef(objectID)); err != nil {
m.logger.Trace().
Err(err).
Msg("failed removing focus from an element")
return err
}
m.logger.Trace().Msg("removed focus from an element")
return nil
}
func (m *Manager) BlurBySelector(ctx context.Context, parentObjectID runtime.RemoteObjectID, selector string) error {
return m.exec.EvalWithArguments(ctx, templates.BlurBySelector(selector), runtime.CallArgument{
ObjectID: &parentObjectID,
})
m.logger.Trace().
Str("parent_object_id", string(parentObjectID)).
Str("selector", selector).
Msg("removing focus from an element by selector")
if err := m.exec.Eval(ctx, templates.BlurBySelector(selector), eval.WithArgRef(parentObjectID)); err != nil {
m.logger.Trace().
Err(err).
Msg("failed removing focus from an element by selector")
return err
}
m.logger.Trace().Msg("removed focus from an element by selector")
return nil
}
func (m *Manager) MoveMouse(ctx context.Context, objectID runtime.RemoteObjectID) error {
m.logger.Trace().
Str("object_id", string(objectID)).
Msg("starting to move the mouse towards an element")
if err := m.ScrollIntoView(ctx, objectID, drivers.ScrollOptions{}); err != nil {
m.logger.Trace().Err(err).Msg("could not scroll into the object. failed to move the mouse")
return err
}
m.logger.Trace().Msg("calculating clickable element points")
q, err := GetClickablePointByObjectID(ctx, m.client, objectID)
if err != nil {
m.logger.Trace().Err(err).Msg("failed calculating clickable element points")
return err
}
return m.mouse.Move(ctx, q.X, q.Y)
m.logger.Trace().Float64("x", q.X).Float64("y", q.Y).Msg("calculated clickable element points")
if err := m.mouse.Move(ctx, q.X, q.Y); err != nil {
m.logger.Trace().Err(err).Msg("failed to move the mouse")
return err
}
m.logger.Trace().Msg("moved the mouse")
return nil
}
func (m *Manager) MoveMouseBySelector(ctx context.Context, parentNodeID dom.NodeID, selector string) error {
m.logger.Trace().
Int("parent_node_id", int(parentNodeID)).
Str("selector", selector).
Msg("starting to move the mouse towards an element by selector")
if err := m.ScrollIntoViewBySelector(ctx, selector, drivers.ScrollOptions{}); err != nil {
return err
}
m.logger.Trace().Msg("looking up for an element by selector")
found, err := m.client.DOM.QuerySelector(ctx, dom.NewQuerySelectorArgs(parentNodeID, selector))
if err != nil {
m.logger.Trace().Err(err).Msg("failed to find an element by selector")
return err
}
q, err := GetClickablePointByNodeID(ctx, m.client, found.NodeID)
m.logger.Trace().Int("node_id", int(found.NodeID)).Msg("calculating clickable element points")
points, err := GetClickablePointByNodeID(ctx, m.client, found.NodeID)
if err != nil {
m.logger.Trace().Err(err).Msg("failed calculating clickable element points")
return err
}
return m.mouse.Move(ctx, q.X, q.Y)
m.logger.Trace().Float64("x", points.X).Float64("y", points.Y).Msg("calculated clickable element points")
if err := m.mouse.Move(ctx, points.X, points.Y); err != nil {
m.logger.Trace().Err(err).Msg("failed to move the mouse")
return err
}
m.logger.Trace().Msg("moved the mouse")
return nil
}
func (m *Manager) MoveMouseByXY(ctx context.Context, x, y float64) error {
m.logger.Trace().
Float64("x", x).
Float64("y", y).
Msg("starting to move the mouse towards an element by given coordinates")
if err := m.ScrollByXY(ctx, x, y, drivers.ScrollOptions{}); err != nil {
return err
}
return m.mouse.Move(ctx, x, y)
if err := m.mouse.Move(ctx, x, y); err != nil {
m.logger.Trace().Err(err).Msg("failed to move the mouse towards an element by given coordinates")
return err
}
m.logger.Trace().Msg("moved the mouse")
return nil
}
func (m *Manager) Click(ctx context.Context, objectID runtime.RemoteObjectID, count int) error {
m.logger.Trace().
Str("object_id", string(objectID)).
Msg("starting to click on an element")
if err := m.ScrollIntoView(ctx, objectID, drivers.ScrollOptions{
Behavior: drivers.ScrollBehaviorAuto,
Block: drivers.ScrollVerticalAlignmentCenter,
@ -178,22 +375,42 @@ func (m *Manager) Click(ctx context.Context, objectID runtime.RemoteObjectID, co
return err
}
m.logger.Trace().Msg("calculating clickable element points")
points, err := GetClickablePointByObjectID(ctx, m.client, objectID)
if err != nil {
m.logger.Trace().Err(err).Msg("failed calculating clickable element points")
return err
}
m.logger.Trace().Float64("x", points.X).Float64("y", points.Y).Msg("calculated clickable element points")
delay := time.Duration(drivers.DefaultMouseDelay) * time.Millisecond
if err := m.mouse.ClickWithCount(ctx, points.X, points.Y, delay, count); err != nil {
return nil
m.logger.Trace().
Err(err).
Msg("failed to click on an element")
return err
}
m.logger.Trace().
Err(err).
Msg("clicked on an element")
return nil
}
func (m *Manager) ClickBySelector(ctx context.Context, parentNodeID dom.NodeID, selector string, count int) error {
m.logger.Trace().
Int("parent_node_id", int(parentNodeID)).
Str("selector", selector).
Int("count", count).
Msg("starting to click on an element by selector")
if err := m.ScrollIntoViewBySelector(ctx, selector, drivers.ScrollOptions{
Behavior: drivers.ScrollBehaviorAuto,
Block: drivers.ScrollVerticalAlignmentCenter,
@ -202,28 +419,47 @@ func (m *Manager) ClickBySelector(ctx context.Context, parentNodeID dom.NodeID,
return err
}
m.logger.Trace().Msg("looking up for an element by selector")
found, err := m.client.DOM.QuerySelector(ctx, dom.NewQuerySelectorArgs(parentNodeID, selector))
if err != nil {
m.logger.Trace().Err(err).Msg("failed to find an element by selector")
return err
}
m.logger.Trace().Int("node_id", int(found.NodeID)).Msg("calculating clickable element points")
points, err := GetClickablePointByNodeID(ctx, m.client, found.NodeID)
if err != nil {
m.logger.Trace().Err(err).Msg("failed calculating clickable element points")
return err
}
m.logger.Trace().Float64("x", points.X).Float64("y", points.Y).Msg("calculated clickable element points")
delay := time.Duration(drivers.DefaultMouseDelay) * time.Millisecond
if err := m.mouse.ClickWithCount(ctx, points.X, points.Y, delay, count); err != nil {
m.logger.Trace().Err(err).Msg("failed to click on an element")
return nil
}
m.logger.Trace().Msg("clicked on an element")
return nil
}
func (m *Manager) ClickBySelectorAll(ctx context.Context, parentNodeID dom.NodeID, selector string, count int) error {
m.logger.Trace().
Int("parent_node_id", int(parentNodeID)).
Str("selector", selector).
Int("count", count).
Msg("starting to click on elements by selector")
if err := m.ScrollIntoViewBySelector(ctx, selector, drivers.ScrollOptions{
Behavior: drivers.ScrollBehaviorAuto,
Block: drivers.ScrollVerticalAlignmentCenter,
@ -232,34 +468,56 @@ func (m *Manager) ClickBySelectorAll(ctx context.Context, parentNodeID dom.NodeI
return err
}
m.logger.Trace().Msg("looking up for elements by selector")
found, err := m.client.DOM.QuerySelectorAll(ctx, dom.NewQuerySelectorAllArgs(parentNodeID, selector))
if err != nil {
m.logger.Trace().Err(err).Msg("failed to find elements by selector")
return err
}
for _, nodeID := range found.NodeIDs {
beforeTypeDelay := time.Duration(core.NumberLowerBoundary(drivers.DefaultMouseDelay*10)) * time.Millisecond
for idx, nodeID := range found.NodeIDs {
if idx > 0 {
m.logger.Trace().Msg("pausing")
beforeClickDelay := time.Duration(core.NumberLowerBoundary(drivers.DefaultMouseDelay*10)) * time.Millisecond
time.Sleep(beforeTypeDelay)
time.Sleep(beforeClickDelay)
}
m.logger.Trace().Int("node_id", int(nodeID)).Msg("calculating clickable element points")
points, err := GetClickablePointByNodeID(ctx, m.client, nodeID)
if err != nil {
m.logger.Trace().Err(err).Msg("failed calculating clickable element points")
return err
}
m.logger.Trace().Float64("x", points.X).Float64("y", points.Y).Msg("calculated clickable element points")
delay := time.Duration(drivers.DefaultMouseDelay) * time.Millisecond
if err := m.mouse.ClickWithCount(ctx, points.X, points.Y, delay, count); err != nil {
m.logger.Trace().Err(err).Msg("failed to click on an element")
return nil
}
m.logger.Trace().Msg("clicked on an element")
}
m.logger.Trace().Msg("clicked on all elements")
return nil
}
func (m *Manager) Type(ctx context.Context, objectID runtime.RemoteObjectID, params TypeParams) error {
m.logger.Trace().
Str("object_id", string(objectID)).
Msg("starting to type text")
err := m.ScrollIntoView(ctx, objectID, drivers.ScrollOptions{
Behavior: drivers.ScrollBehaviorAuto,
Block: drivers.ScrollVerticalAlignmentCenter,
@ -270,19 +528,29 @@ func (m *Manager) Type(ctx context.Context, objectID runtime.RemoteObjectID, par
return err
}
err = m.client.DOM.Focus(ctx, dom.NewFocusArgs().SetObjectID(objectID))
m.logger.Trace().Msg("focusing on an element")
if err := m.client.DOM.Focus(ctx, dom.NewFocusArgs().SetObjectID(objectID)); err != nil {
m.logger.Trace().Msg("failed to focus on an element")
if err != nil {
return err
}
m.logger.Trace().Bool("clear", params.Clear).Msg("is clearing text required?")
if params.Clear {
m.logger.Trace().Msg("calculating clickable element points")
points, err := GetClickablePointByObjectID(ctx, m.client, objectID)
if err != nil {
m.logger.Trace().Err(err).Msg("failed calculating clickable element points")
return err
}
m.logger.Trace().Float64("x", points.X).Float64("y", points.Y).Msg("calculated clickable element points")
if err := m.ClearByXY(ctx, points); err != nil {
return err
}
@ -291,12 +559,29 @@ func (m *Manager) Type(ctx context.Context, objectID runtime.RemoteObjectID, par
d := core.NumberLowerBoundary(float64(params.Delay))
beforeTypeDelay := time.Duration(d)
m.logger.Trace().Float64("delay", d).Msg("calculated pause delay")
time.Sleep(beforeTypeDelay)
return m.keyboard.Type(ctx, params.Text, params.Delay)
m.logger.Trace().Msg("starting to type text")
if err := m.keyboard.Type(ctx, params.Text, params.Delay); err != nil {
m.logger.Trace().Err(err).Msg("failed to type text")
return err
}
m.logger.Trace().Msg("typed text")
return nil
}
func (m *Manager) TypeBySelector(ctx context.Context, parentNodeID dom.NodeID, selector string, params TypeParams) error {
m.logger.Trace().
Int("parent_node_id", int(parentNodeID)).
Str("selector", selector).
Msg("starting to type text by selector")
err := m.ScrollIntoViewBySelector(ctx, selector, drivers.ScrollOptions{
Behavior: drivers.ScrollBehaviorAuto,
Block: drivers.ScrollVerticalAlignmentCenter,
@ -307,25 +592,41 @@ func (m *Manager) TypeBySelector(ctx context.Context, parentNodeID dom.NodeID, s
return err
}
m.logger.Trace().Msg("looking up for an element by selector")
found, err := m.client.DOM.QuerySelector(ctx, dom.NewQuerySelectorArgs(parentNodeID, selector))
if err != nil {
m.logger.Trace().Err(err).Msg("failed to find an element by selector")
return err
}
m.logger.Trace().Int("node_id", int(found.NodeID)).Msg("focusing on an element")
err = m.client.DOM.Focus(ctx, dom.NewFocusArgs().SetNodeID(found.NodeID))
if err != nil {
m.logger.Trace().Err(err).Msg("failed to focus on an element")
return err
}
m.logger.Trace().Bool("clear", params.Clear).Msg("is clearing text required?")
if params.Clear {
m.logger.Trace().Msg("calculating clickable element points")
points, err := GetClickablePointByNodeID(ctx, m.client, found.NodeID)
if err != nil {
m.logger.Trace().Err(err).Msg("failed calculating clickable element points")
return err
}
m.logger.Trace().Float64("x", points.X).Float64("y", points.Y).Msg("calculated clickable element points")
if err := m.ClearByXY(ctx, points); err != nil {
return err
}
@ -334,12 +635,28 @@ func (m *Manager) TypeBySelector(ctx context.Context, parentNodeID dom.NodeID, s
d := core.NumberLowerBoundary(float64(params.Delay))
beforeTypeDelay := time.Duration(d)
m.logger.Trace().Float64("delay", d).Msg("calculated pause delay")
time.Sleep(beforeTypeDelay)
return m.keyboard.Type(ctx, params.Text, params.Delay)
m.logger.Trace().Msg("starting to type text")
if err := m.keyboard.Type(ctx, params.Text, params.Delay); err != nil {
m.logger.Trace().Err(err).Msg("failed to type text")
return err
}
m.logger.Trace().Msg("typed text")
return nil
}
func (m *Manager) Clear(ctx context.Context, objectID runtime.RemoteObjectID) error {
m.logger.Trace().
Str("object_id", string(objectID)).
Msg("starting to clear element")
err := m.ScrollIntoView(ctx, objectID, drivers.ScrollOptions{
Behavior: drivers.ScrollBehaviorAuto,
Block: drivers.ScrollVerticalAlignmentCenter,
@ -350,22 +667,46 @@ func (m *Manager) Clear(ctx context.Context, objectID runtime.RemoteObjectID) er
return err
}
m.logger.Trace().Msg("calculating clickable element points")
points, err := GetClickablePointByObjectID(ctx, m.client, objectID)
if err != nil {
m.logger.Trace().Err(err).Msg("failed calculating clickable element points")
return err
}
m.logger.Trace().Float64("x", points.X).Float64("y", points.Y).Msg("calculated clickable element points")
m.logger.Trace().Msg("focusing on an element")
err = m.client.DOM.Focus(ctx, dom.NewFocusArgs().SetObjectID(objectID))
if err != nil {
m.logger.Trace().Err(err).Msg("failed to focus on an element")
return err
}
return m.ClearByXY(ctx, points)
m.logger.Trace().Msg("clearing element")
if err := m.ClearByXY(ctx, points); err != nil {
m.logger.Trace().Err(err).Msg("failed to clear element")
return err
}
m.logger.Trace().Msg("cleared element")
return nil
}
func (m *Manager) ClearBySelector(ctx context.Context, parentNodeID dom.NodeID, selector string) error {
m.logger.Trace().
Int("parent_node_id", int(parentNodeID)).
Str("selector", selector).
Msg("starting to clear element by selector")
err := m.ScrollIntoViewBySelector(ctx, selector, drivers.ScrollOptions{
Behavior: drivers.ScrollBehaviorAuto,
Block: drivers.ScrollVerticalAlignmentCenter,
@ -376,43 +717,108 @@ func (m *Manager) ClearBySelector(ctx context.Context, parentNodeID dom.NodeID,
return err
}
m.logger.Trace().Msg("looking up for an element by selector")
found, err := m.client.DOM.QuerySelector(ctx, dom.NewQuerySelectorArgs(parentNodeID, selector))
if err != nil {
m.logger.Trace().Err(err).Msg("failed to find an element by selector")
return err
}
m.logger.Trace().Int("node_id", int(found.NodeID)).Msg("calculating clickable element points")
points, err := GetClickablePointByNodeID(ctx, m.client, found.NodeID)
if err != nil {
m.logger.Trace().Err(err).Msg("failed calculating clickable element points")
return err
}
m.logger.Trace().Float64("x", points.X).Float64("y", points.Y).Msg("calculated clickable element points")
m.logger.Trace().Msg("focusing on an element")
err = m.client.DOM.Focus(ctx, dom.NewFocusArgs().SetNodeID(found.NodeID))
if err != nil {
m.logger.Trace().Err(err).Msg("failed to focus on an element")
return err
}
return m.ClearByXY(ctx, points)
m.logger.Trace().Msg("clearing element")
if err := m.ClearByXY(ctx, points); err != nil {
m.logger.Trace().Err(err).Msg("failed to clear element")
return err
}
m.logger.Trace().Msg("cleared element")
return nil
}
func (m *Manager) ClearByXY(ctx context.Context, points Quad) error {
m.logger.Trace().
Float64("x", points.X).
Float64("y", points.Y).
Msg("starting to clear element by coordinates")
delay := time.Duration(drivers.DefaultMouseDelay) * time.Millisecond
m.logger.Trace().Dur("delay", delay).Msg("clicking mouse button to select text")
err := m.mouse.ClickWithCount(ctx, points.X, points.Y, delay, 3)
if err != nil {
m.logger.Trace().Err(err).Msg("failed to click mouse button")
return err
}
return m.keyboard.Press(ctx, []string{"Backspace"}, 1, time.Duration(drivers.DefaultKeyboardDelay)*time.Millisecond)
delay = time.Duration(drivers.DefaultKeyboardDelay) * time.Millisecond
m.logger.Trace().Dur("delay", delay).Msg("pressing 'Backspace'")
if err := m.keyboard.Press(ctx, []string{"Backspace"}, 1, delay); err != nil {
m.logger.Trace().Err(err).Msg("failed to press 'Backspace'")
return err
}
return err
}
func (m *Manager) Press(ctx context.Context, keys []string, count int) error {
return m.keyboard.Press(ctx, keys, count, time.Duration(drivers.DefaultKeyboardDelay)*time.Millisecond)
delay := time.Duration(drivers.DefaultKeyboardDelay) * time.Millisecond
m.logger.Trace().
Strs("keys", keys).
Int("count", count).
Dur("delay", delay).
Msg("pressing keyboard keys")
if err := m.keyboard.Press(ctx, keys, count, delay); err != nil {
m.logger.Trace().Err(err).Msg("failed to press keyboard keys")
return err
}
return nil
}
func (m *Manager) PressBySelector(ctx context.Context, parentNodeID dom.NodeID, selector string, keys []string, count int) error {
m.logger.Trace().
Int("parent_node_id", int(parentNodeID)).
Str("selector", selector).
Strs("keys", keys).
Int("count", count).
Msg("starting to press keyboard keys by selector")
if err := m.FocusBySelector(ctx, parentNodeID, selector); err != nil {
return err
}
@ -421,43 +827,76 @@ func (m *Manager) PressBySelector(ctx context.Context, parentNodeID dom.NodeID,
}
func (m *Manager) Select(ctx context.Context, objectID runtime.RemoteObjectID, value *values.Array) (*values.Array, error) {
m.logger.Trace().
Str("object_id", string(objectID)).
Msg("starting to select values")
if err := m.Focus(ctx, objectID); err != nil {
return values.NewArray(0), err
}
val, err := m.exec.EvalWithArgumentsAndReturnValue(ctx, templates.Select(value.String()), runtime.CallArgument{
ObjectID: &objectID,
})
m.logger.Trace().Msg("selecting values")
m.logger.Trace().Msg("evaluating a JS function")
val, err := m.exec.EvalValue(
ctx,
templates.Select(value.String()),
eval.WithArgRef(objectID),
)
if err != nil {
m.logger.Trace().Err(err).Msg("failed to evaluate a JS function")
return nil, err
}
m.logger.Trace().Msg("validating JS result")
arr, ok := val.(*values.Array)
if !ok {
m.logger.Trace().Err(err).Msg("JS result validation failed")
return values.NewArray(0), core.ErrUnexpected
}
m.logger.Trace().Msg("selected values")
return arr, nil
}
func (m *Manager) SelectBySelector(ctx context.Context, parentNodeID dom.NodeID, selector string, value *values.Array) (*values.Array, error) {
m.logger.Trace().
Int("parent_node_id", int(parentNodeID)).
Str("selector", selector).
Msg("starting to select values by selector")
if err := m.FocusBySelector(ctx, parentNodeID, selector); err != nil {
return values.NewArray(0), err
}
res, err := m.exec.EvalWithReturnValue(ctx, templates.SelectBySelector(selector, value.String()))
m.logger.Trace().Msg("selecting values")
m.logger.Trace().Msg("evaluating a JS function")
res, err := m.exec.EvalValue(ctx, templates.SelectBySelector(selector, value.String()))
if err != nil {
m.logger.Trace().Err(err).Msg("failed to evaluate a JS function")
return values.NewArray(0), err
}
m.logger.Trace().Msg("validating JS result")
arr, ok := res.(*values.Array)
if !ok {
m.logger.Trace().Err(err).Msg("JS result validation failed")
return values.NewArray(0), core.ErrUnexpected
}
m.logger.Trace().Msg("selected values")
return arr, nil
}

16
pkg/drivers/cdp/input/mouse.go
View File

@ -27,37 +27,37 @@ func (m *Mouse) ClickWithCount(ctx context.Context, x, y float64, delay time.Dur
return err
}
if err := m.DownWithCount(ctx, "left", count); err != nil {
if err := m.DownWithCount(ctx, input.MouseButtonLeft, count); err != nil {
return err
}
time.Sleep(randomDuration(int(delay)))
return m.UpWithCount(ctx, "left", count)
return m.UpWithCount(ctx, input.MouseButtonLeft, count)
}
func (m *Mouse) Down(ctx context.Context, button string) error {
func (m *Mouse) Down(ctx context.Context, button input.MouseButton) error {
return m.DownWithCount(ctx, button, 1)
}
func (m *Mouse) DownWithCount(ctx context.Context, button string, count int) error {
func (m *Mouse) DownWithCount(ctx context.Context, button input.MouseButton, count int) error {
return m.client.Input.DispatchMouseEvent(
ctx,
input.NewDispatchMouseEventArgs("mousePressed", m.x, m.y).
SetButton(input.MouseButton(button)).
SetButton(button).
SetClickCount(count),
)
}
func (m *Mouse) Up(ctx context.Context, button string) error {
func (m *Mouse) Up(ctx context.Context, button input.MouseButton) error {
return m.UpWithCount(ctx, button, 1)
}
func (m *Mouse) UpWithCount(ctx context.Context, button string, count int) error {
func (m *Mouse) UpWithCount(ctx context.Context, button input.MouseButton, count int) error {
return m.client.Input.DispatchMouseEvent(
ctx,
input.NewDispatchMouseEventArgs("mouseReleased", m.x, m.y).
SetButton(input.MouseButton(button)).
SetButton(button).
SetClickCount(count),
)
}

39
pkg/drivers/cdp/input/quad.go
View File

@ -8,6 +8,8 @@ import (
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/pkg/errors"
"github.com/MontFerret/ferret/pkg/drivers/cdp/utils"
)
type Quad struct {
@ -78,9 +80,7 @@ func getClickablePoint(ctx context.Context, client *cdp.Client, qargs *dom.GetCo
return Quad{}, err
}
clientWidth := layoutMetricsReply.CSSLayoutViewport.ClientWidth
clientHeight := layoutMetricsReply.CSSLayoutViewport.ClientHeight
clientWidth, clientHeight := utils.GetLayoutViewportWH(layoutMetricsReply)
quads := make([][]Quad, 0, len(contentQuadsReply.Quads))
for _, q := range contentQuadsReply.Quads {
@ -111,6 +111,39 @@ func getClickablePoint(ctx context.Context, client *cdp.Client, qargs *dom.GetCo
}, nil
}
func getClickablePoint2(ctx context.Context, client *cdp.Client, qargs *dom.GetContentQuadsArgs) (Quad, error) {
contentQuadsReply, err := client.DOM.GetContentQuads(ctx, qargs)
if err != nil {
return Quad{}, err
}
if contentQuadsReply.Quads == nil || len(contentQuadsReply.Quads) == 0 {
return Quad{}, errors.New("node is either not visible or not an HTMLElement")
}
content := contentQuadsReply.Quads[0]
c := len(content)
if c%2 != 0 || c < 1 {
return Quad{}, errors.New("node is either not visible or not an HTMLElement")
}
var x, y float64
for i := 0; i < c; i += 2 {
x += content[i]
y += content[i+1]
}
x /= float64(c / 2)
y /= float64(c / 2)
return Quad{
X: x,
Y: y,
}, nil
}
func GetClickablePointByNodeID(ctx context.Context, client *cdp.Client, nodeID dom.NodeID) (Quad, error) {
return getClickablePoint(ctx, client, dom.NewGetContentQuadsArgs().SetNodeID(nodeID))
}

367
pkg/drivers/cdp/network/manager.go
View File

@ -3,7 +3,7 @@ package network
import (
"context"
"encoding/json"
"io"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"regexp"
"sync"
@ -20,7 +20,6 @@ import (
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
"github.com/MontFerret/ferret/pkg/drivers/cdp/templates"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -31,11 +30,12 @@ type (
FrameLoadedListener = func(ctx context.Context, frame page.Frame)
Manager struct {
mu sync.Mutex
logger *zerolog.Logger
mu sync.RWMutex
logger zerolog.Logger
client *cdp.Client
headers *drivers.HTTPHeaders
eventLoop *events.Loop
foregroundLoop *events.Loop
backgroundLoop *events.Loop
cancel context.CancelFunc
responseListenerID events.ListenerID
filterListenerID events.ListenerID
@ -44,42 +44,37 @@ type (
)
func New(
ctx context.Context,
logger *zerolog.Logger,
logger zerolog.Logger,
client *cdp.Client,
options Options,
) (*Manager, error) {
ctx, cancel := context.WithCancel(ctx)
ctx, cancel := context.WithCancel(context.Background())
m := new(Manager)
m.logger = logger
m.logger = logging.WithName(logger.With(), "network_manager").Logger()
m.client = client
m.headers = drivers.NewHTTPHeaders()
m.eventLoop = events.NewLoop()
m.foregroundLoop = events.NewLoop()
m.cancel = cancel
m.response = new(sync.Map)
if options.Cookies != nil && len(options.Cookies) > 0 {
for url, cookies := range options.Cookies {
if err := m.setCookiesInternal(ctx, url, cookies); err != nil {
return nil, err
}
}
}
if options.Headers != nil && options.Headers.Length() > 0 {
if err := m.setHeadersInternal(ctx, options.Headers); err != nil {
return nil, err
}
}
var err error
closers := make([]io.Closer, 0, 10)
defer func() {
if err != nil {
common.CloseAll(logger, closers, "failed to close a DOM event stream")
cancel()
if m.foregroundLoop != nil {
if err := m.foregroundLoop.Close(); err != nil {
m.logger.Trace().Err(err).Msg("failed to close the foreground loop during cleanup")
}
}
if m.backgroundLoop != nil {
if err := m.backgroundLoop.Close(); err != nil {
m.logger.Trace().Err(err).Msg("failed to close the background loop during cleanup")
}
}
}
}()
@ -89,24 +84,24 @@ func New(
return nil, err
}
m.foregroundLoop.AddSource(events.NewSource(eventFrameLoad, frameNavigatedStream, func(stream rpcc.Stream) (interface{}, error) {
return stream.(page.FrameNavigatedClient).Recv()
}))
responseReceivedStream, err := m.client.Network.ResponseReceived(ctx)
if err != nil {
return nil, err
}
m.eventLoop.AddSource(events.NewSource(eventFrameLoad, frameNavigatedStream, func(stream rpcc.Stream) (interface{}, error) {
return stream.(page.FrameNavigatedClient).Recv()
}))
m.eventLoop.AddSource(events.NewSource(responseReceived, responseReceivedStream, func(stream rpcc.Stream) (interface{}, error) {
m.foregroundLoop.AddSource(events.NewSource(responseReceived, responseReceivedStream, func(stream rpcc.Stream) (interface{}, error) {
return stream.(network.ResponseReceivedClient).Recv()
}))
m.responseListenerID = m.eventLoop.AddListener(responseReceived, m.onResponse)
m.responseListenerID = m.foregroundLoop.AddListener(responseReceived, m.onResponse)
if options.Filter != nil && len(options.Filter.Patterns) > 0 {
el2 := events.NewLoop()
m.backgroundLoop = events.NewLoop()
err = m.client.Fetch.Enable(ctx, toFetchArgs(options.Filter.Patterns))
@ -120,18 +115,46 @@ func New(
return nil, err
}
el2.AddSource(events.NewSource(requestPaused, requestPausedStream, func(stream rpcc.Stream) (interface{}, error) {
m.backgroundLoop.AddSource(events.NewSource(requestPaused, requestPausedStream, func(stream rpcc.Stream) (interface{}, error) {
return stream.(fetch.RequestPausedClient).Recv()
}))
m.filterListenerID = el2.AddListener(requestPaused, m.onRequestPaused)
// run in a separate loop in order to get higher priority
// TODO: Consider adding support of event priorities to EventLoop
el2.Run(ctx)
m.filterListenerID = m.backgroundLoop.AddListener(requestPaused, m.onRequestPaused)
}
m.eventLoop.Run(ctx)
if options.Cookies != nil && len(options.Cookies) > 0 {
for url, cookies := range options.Cookies {
err = m.setCookiesInternal(ctx, url, cookies)
if err != nil {
return nil, err
}
}
}
if options.Headers != nil && options.Headers.Length() > 0 {
err = m.setHeadersInternal(ctx, options.Headers)
if err != nil {
return nil, err
}
}
err = m.foregroundLoop.Run(ctx)
if err != nil {
return nil, err
}
if m.backgroundLoop != nil {
// run in a separate loop in order to get higher priority
// TODO: Consider adding support of event priorities to EventLoop
err = m.backgroundLoop.Run(ctx)
if err != nil {
return nil, err
}
}
return m, nil
}
@ -140,24 +163,38 @@ func (m *Manager) Close() error {
m.mu.Lock()
defer m.mu.Unlock()
m.logger.Trace().Msg("closing")
if m.cancel != nil {
m.cancel()
m.cancel = nil
}
_ = m.foregroundLoop.Close()
if m.backgroundLoop != nil {
_ = m.backgroundLoop.Close()
}
return nil
}
func (m *Manager) GetCookies(ctx context.Context) (*drivers.HTTPCookies, error) {
m.logger.Trace().Msg("starting to get cookies")
repl, err := m.client.Network.GetAllCookies(ctx)
if err != nil {
m.logger.Trace().Err(err).Msg("failed to get cookies")
return nil, errors.Wrap(err, "failed to get cookies")
}
cookies := drivers.NewHTTPCookies()
if repl.Cookies == nil {
m.logger.Trace().Msg("no cookies found")
return cookies, nil
}
@ -165,6 +202,8 @@ func (m *Manager) GetCookies(ctx context.Context) (*drivers.HTTPCookies, error)
cookies.Set(toDriverCookie(c))
}
m.logger.Trace().Err(err).Msg("succeeded to get cookies")
return cookies, nil
}
@ -176,11 +215,17 @@ func (m *Manager) SetCookies(ctx context.Context, url string, cookies *drivers.H
}
func (m *Manager) setCookiesInternal(ctx context.Context, url string, cookies *drivers.HTTPCookies) error {
m.logger.Trace().Str("url", url).Msg("starting to set cookies")
if cookies == nil {
m.logger.Trace().Msg("nil cookies passed")
return errors.Wrap(core.ErrMissedArgument, "cookies")
}
if cookies.Length() == 0 {
m.logger.Trace().Msg("no cookies passed")
return nil
}
@ -192,30 +237,52 @@ func (m *Manager) setCookiesInternal(ctx context.Context, url string, cookies *d
return true
})
return m.client.Network.SetCookies(ctx, network.NewSetCookiesArgs(params))
err := m.client.Network.SetCookies(ctx, network.NewSetCookiesArgs(params))
if err != nil {
m.logger.Trace().Err(err).Msg("failed to set cookies")
return err
}
m.logger.Trace().Msg("succeeded to set cookies")
return nil
}
func (m *Manager) DeleteCookies(ctx context.Context, url string, cookies *drivers.HTTPCookies) error {
m.mu.Lock()
defer m.mu.Unlock()
m.logger.Trace().Str("url", url).Msg("starting to delete cookies")
if cookies == nil {
m.logger.Trace().Msg("nil cookies passed")
return errors.Wrap(core.ErrMissedArgument, "cookies")
}
if cookies.Length() == 0 {
m.logger.Trace().Msg("no cookies passed")
return nil
}
var err error
cookies.ForEach(func(value drivers.HTTPCookie, _ values.String) bool {
m.logger.Trace().Str("name", value.Name).Msg("deleting a cookie")
err = m.client.Network.DeleteCookies(ctx, fromDriverCookieDelete(url, value))
if err != nil {
m.logger.Trace().Err(err).Str("name", value.Name).Msg("failed to delete a cookie")
return false
}
m.logger.Trace().Str("name", value.Name).Msg("succeeded to delete a cookie")
return true
})
@ -241,33 +308,52 @@ func (m *Manager) SetHeaders(ctx context.Context, headers *drivers.HTTPHeaders)
}
func (m *Manager) setHeadersInternal(ctx context.Context, headers *drivers.HTTPHeaders) error {
m.logger.Trace().Msg("starting to set headers")
if headers.Length() == 0 {
m.logger.Trace().Msg("no headers passed")
return nil
}
m.headers = headers
m.logger.Trace().Msg("marshaling headers")
j, err := jettison.MarshalOpts(headers, jettison.NoHTMLEscaping())
if err != nil {
m.logger.Trace().Err(err).Msg("failed to marshal headers")
return errors.Wrap(err, "failed to marshal headers")
}
m.logger.Trace().Msg("sending headers to browser")
err = m.client.Network.SetExtraHTTPHeaders(
ctx,
network.NewSetExtraHTTPHeadersArgs(j),
)
if err != nil {
m.logger.Trace().Err(err).Msg("failed to set headers")
return errors.Wrap(err, "failed to set headers")
}
m.logger.Trace().Msg("succeeded to set headers")
return nil
}
func (m *Manager) GetResponse(_ context.Context, frameID page.FrameID) (drivers.HTTPResponse, error) {
value, found := m.response.Load(frameID)
m.logger.Trace().
Str("frame_id", string(frameID)).
Bool("found", found).
Msg("getting frame response")
if !found {
return drivers.HTTPResponse{}, core.ErrNotFound
}
@ -284,16 +370,21 @@ func (m *Manager) Navigate(ctx context.Context, url values.String) error {
}
urlStr := url.String()
m.logger.Trace().Str("url", urlStr).Msg("starting navigation")
repl, err := m.client.Page.Navigate(ctx, page.NewNavigateArgs(urlStr))
if err == nil && repl.ErrorText != nil {
err = errors.New(*repl.ErrorText)
}
if err != nil {
m.logger.Trace().Err(err).Msg("failed starting navigation")
return err
}
if repl.ErrorText != nil {
return errors.New(*repl.ErrorText)
}
m.logger.Trace().Msg("succeeded starting navigation")
return m.WaitForNavigation(ctx, nil)
}
@ -302,9 +393,17 @@ func (m *Manager) NavigateForward(ctx context.Context, skip values.Int) (values.
m.mu.Lock()
defer m.mu.Unlock()
m.logger.Trace().
Int64("skip", int64(skip)).
Msg("starting forward navigation")
history, err := m.client.Page.GetNavigationHistory(ctx)
if err != nil {
m.logger.Trace().
Err(err).
Msg("failed to get navigation history")
return values.False, err
}
@ -313,6 +412,12 @@ func (m *Manager) NavigateForward(ctx context.Context, skip values.Int) (values.
// nowhere to go forward
if history.CurrentIndex == lastIndex {
m.logger.Trace().
Int("history_entries", length).
Int("history_current_index", history.CurrentIndex).
Int("history_last_index", lastIndex).
Msg("no forward history. nowhere to navigate. done.")
return values.False, nil
}
@ -323,23 +428,52 @@ func (m *Manager) NavigateForward(ctx context.Context, skip values.Int) (values.
to := int(skip) + history.CurrentIndex
if to > lastIndex {
// TODO: Return error?
return values.False, nil
m.logger.Trace().
Int("history_entries", length).
Int("history_current_index", history.CurrentIndex).
Int("history_last_index", lastIndex).
Int("history_target_index", to).
Msg("not enough history items. using the edge index")
to = lastIndex
}
entry := history.Entries[to]
err = m.client.Page.NavigateToHistoryEntry(ctx, page.NewNavigateToHistoryEntryArgs(entry.ID))
if err != nil {
m.logger.Trace().
Int("history_entries", length).
Int("history_current_index", history.CurrentIndex).
Int("history_last_index", lastIndex).
Int("history_target_index", to).
Err(err).
Msg("failed to get navigation history entry")
return values.False, err
}
err = m.WaitForNavigation(ctx, nil)
if err != nil {
m.logger.Trace().
Int("history_entries", length).
Int("history_current_index", history.CurrentIndex).
Int("history_last_index", lastIndex).
Int("history_target_index", to).
Err(err).
Msg("failed to wait for navigation completion")
return values.False, err
}
m.logger.Trace().
Int("history_entries", length).
Int("history_current_index", history.CurrentIndex).
Int("history_last_index", lastIndex).
Int("history_target_index", to).
Msg("succeeded to wait for navigation completion")
return values.True, nil
}
@ -347,14 +481,27 @@ func (m *Manager) NavigateBack(ctx context.Context, skip values.Int) (values.Boo
m.mu.Lock()
defer m.mu.Unlock()
m.logger.Trace().
Int64("skip", int64(skip)).
Msg("starting backward navigation")
history, err := m.client.Page.GetNavigationHistory(ctx)
if err != nil {
m.logger.Trace().Err(err).Msg("failed to get navigation history")
return values.False, err
}
length := len(history.Entries)
// we are in the beginning
if history.CurrentIndex == 0 {
m.logger.Trace().
Int("history_entries", length).
Int("history_current_index", history.CurrentIndex).
Msg("no backward history. nowhere to navigate. done.")
return values.False, nil
}
@ -365,23 +512,48 @@ func (m *Manager) NavigateBack(ctx context.Context, skip values.Int) (values.Boo
to := history.CurrentIndex - int(skip)
if to < 0 {
// TODO: Return error?
return values.False, nil
m.logger.Trace().
Int("history_entries", length).
Int("history_current_index", history.CurrentIndex).
Int("history_target_index", to).
Msg("not enough history items. using 0 index")
to = 0
}
entry := history.Entries[to]
err = m.client.Page.NavigateToHistoryEntry(ctx, page.NewNavigateToHistoryEntryArgs(entry.ID))
if err != nil {
m.logger.Trace().
Int("history_entries", length).
Int("history_current_index", history.CurrentIndex).
Int("history_target_index", to).
Err(err).
Msg("failed to get navigation history entry")
return values.False, err
}
err = m.WaitForNavigation(ctx, nil)
if err != nil {
m.logger.Trace().
Int("history_entries", length).
Int("history_current_index", history.CurrentIndex).
Int("history_target_index", to).
Err(err).
Msg("failed to wait for navigation completion")
return values.False, err
}
m.logger.Trace().
Int("history_entries", length).
Int("history_current_index", history.CurrentIndex).
Int("history_target_index", to).
Msg("succeeded to wait for navigation completion")
return values.True, nil
}
@ -392,8 +564,27 @@ func (m *Manager) WaitForNavigation(ctx context.Context, pattern *regexp.Regexp)
func (m *Manager) WaitForFrameNavigation(ctx context.Context, frameID page.FrameID, urlPattern *regexp.Regexp) error {
onEvent := make(chan struct{})
m.eventLoop.AddListener(eventFrameLoad, func(_ context.Context, message interface{}) bool {
var urlPatternStr string
if urlPattern != nil {
urlPatternStr = urlPattern.String()
}
m.logger.Trace().
Str("fame_id", string(frameID)).
Str("url_pattern", urlPatternStr).
Msg("starting to wait for frame navigation event")
m.foregroundLoop.AddListener(eventFrameLoad, func(_ context.Context, message interface{}) bool {
repl := message.(*page.FrameNavigatedReply)
log := m.logger.With().
Str("fame_id", string(frameID)).
Str("event_fame_id", string(repl.Frame.ID)).
Str("event_fame_url", repl.Frame.URL).
Str("url_pattern", urlPatternStr).
Logger()
log.Trace().Msg("received framed navigation event")
var matched bool
@ -409,14 +600,23 @@ func (m *Manager) WaitForFrameNavigation(ctx context.Context, frameID page.Frame
}
if matched {
log.Trace().Msg("frame navigation url is matched with url pattern")
if ctx.Err() == nil {
ec, err := eval.NewExecutionContextFrom(ctx, m.client, repl.Frame)
log.Trace().Msg("creating frame execution context")
ec, err := eval.New(ctx, m.client, repl.Frame.ID)
if err != nil {
log.Trace().Err(err).Msg("failed to create frame execution context")
close(onEvent)
return false
}
log.Trace().Err(err).Msg("starting polling DOM ready event")
_, err = events.NewEvalWaitTask(
ec,
templates.DOMReady(),
@ -424,10 +624,15 @@ func (m *Manager) WaitForFrameNavigation(ctx context.Context, frameID page.Frame
).Run(ctx)
if err != nil {
log.Trace().Err(err).Msg("failed to poll DOM ready event")
close(onEvent)
return false
}
log.Trace().Msg("DOM is ready")
onEvent <- struct{}{}
close(onEvent)
}
@ -439,14 +644,25 @@ func (m *Manager) WaitForFrameNavigation(ctx context.Context, frameID page.Frame
select {
case <-onEvent:
m.logger.Trace().
Str("fame_id", string(frameID)).
Str("url_pattern", urlPatternStr).
Msg("navigation has completed")
return nil
case <-ctx.Done():
m.logger.Trace().
Err(core.ErrTimeout).
Str("fame_id", string(frameID)).
Str("url_pattern", urlPatternStr).
Msg("navigation has failed")
return core.ErrTimeout
}
}
func (m *Manager) AddFrameLoadedListener(listener FrameLoadedListener) events.ListenerID {
return m.eventLoop.AddListener(eventFrameLoad, func(ctx context.Context, message interface{}) bool {
return m.foregroundLoop.AddListener(eventFrameLoad, func(ctx context.Context, message interface{}) bool {
repl := message.(*page.FrameNavigatedReply)
listener(ctx, repl.Frame)
@ -456,7 +672,7 @@ func (m *Manager) AddFrameLoadedListener(listener FrameLoadedListener) events.Li
}
func (m *Manager) RemoveFrameLoadedListener(id events.ListenerID) {
m.eventLoop.RemoveListener(eventFrameLoad, id)
m.foregroundLoop.RemoveListener(eventFrameLoad, id)
}
func (m *Manager) onResponse(_ context.Context, message interface{}) (out bool) {
@ -472,10 +688,28 @@ func (m *Manager) onResponse(_ context.Context, message interface{}) (out bool)
return
}
if msg.FrameID == nil {
return
}
log := m.logger.With().
Str("frame_id", string(*msg.FrameID)).
Str("request_id", string(msg.RequestID)).
Str("loader_id", string(msg.LoaderID)).
Float64("timestamp", float64(msg.Timestamp)).
Str("url", msg.Response.URL).
Int("status_code", msg.Response.Status).
Str("status_text", msg.Response.StatusText).
Logger()
log.Trace().Msg("received browser response")
response := drivers.HTTPResponse{
StatusCode: msg.Response.Status,
Status: msg.Response.StatusText,
Headers: drivers.NewHTTPHeaders(),
URL: msg.Response.URL,
StatusCode: msg.Response.Status,
Status: msg.Response.StatusText,
Headers: drivers.NewHTTPHeaders(),
ResponseTime: float64(msg.Response.ResponseTime),
}
deserialized := make(map[string]string)
@ -484,7 +718,7 @@ func (m *Manager) onResponse(_ context.Context, message interface{}) (out bool)
err := json.Unmarshal(msg.Response.Headers, &deserialized)
if err != nil {
m.logger.Error().Err(err).Msg("failed to deserialize response headers")
log.Trace().Err(err).Msg("failed to deserialize response headers")
}
}
@ -494,6 +728,8 @@ func (m *Manager) onResponse(_ context.Context, message interface{}) (out bool)
m.response.Store(*msg.FrameID, response)
log.Trace().Msg("updated frame response information")
return
}
@ -505,18 +741,25 @@ func (m *Manager) onRequestPaused(ctx context.Context, message interface{}) (out
return
}
log := m.logger.With().
Str("request_id", string(msg.RequestID)).
Str("frame_id", string(msg.FrameID)).
Str("resource_type", string(msg.ResourceType)).
Str("url", msg.Request.URL).
Logger()
log.Trace().Msg("trying to block resource loading")
err := m.client.Fetch.FailRequest(ctx, &fetch.FailRequestArgs{
RequestID: msg.RequestID,
ErrorReason: network.ErrorReasonBlockedByClient,
})
if err != nil {
m.logger.
Err(err).
Str("resourceType", msg.ResourceType.String()).
Str("url", msg.Request.URL).
Msg("failed to terminate a request")
log.Trace().Err(err).Msg("failed to block resource loading")
}
log.Trace().Msg("succeeded to block resource loading")
return
}

295
pkg/drivers/cdp/network/manager_test.go Normal file
View File

@ -0,0 +1,295 @@
package network_test
import (
"context"
"os"
"testing"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/fetch"
network2 "github.com/mafredri/cdp/protocol/network"
"github.com/mafredri/cdp/protocol/page"
"github.com/pkg/errors"
"github.com/rs/zerolog"
. "github.com/smartystreets/goconvey/convey"
"github.com/stretchr/testify/mock"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/network"
)
type (
PageAPI struct {
mock.Mock
cdp.Page
frameNavigated func(ctx context.Context) (page.FrameNavigatedClient, error)
}
NetworkAPI struct {
mock.Mock
cdp.Network
responseReceived func(ctx context.Context) (network2.ResponseReceivedClient, error)
setExtraHTTPHeaders func(ctx context.Context, args *network2.SetExtraHTTPHeadersArgs) error
}
FetchAPI struct {
mock.Mock
cdp.Fetch
enable func(context.Context, *fetch.EnableArgs) error
requestPaused func(context.Context) (fetch.RequestPausedClient, error)
}
TestEventStream struct {
mock.Mock
ready chan struct{}
message chan interface{}
}
FrameNavigatedClient struct {
*TestEventStream
}
ResponseReceivedClient struct {
*TestEventStream
}
RequestPausedClient struct {
*TestEventStream
}
)
func (api *PageAPI) FrameNavigated(ctx context.Context) (page.FrameNavigatedClient, error) {
return api.frameNavigated(ctx)
}
func (api *NetworkAPI) ResponseReceived(ctx context.Context) (network2.ResponseReceivedClient, error) {
return api.responseReceived(ctx)
}
func (api *NetworkAPI) SetExtraHTTPHeaders(ctx context.Context, args *network2.SetExtraHTTPHeadersArgs) error {
return api.setExtraHTTPHeaders(ctx, args)
}
func (api *FetchAPI) Enable(ctx context.Context, args *fetch.EnableArgs) error {
return api.enable(ctx, args)
}
func (api *FetchAPI) RequestPaused(ctx context.Context) (fetch.RequestPausedClient, error) {
return api.requestPaused(ctx)
}
func NewTestEventStream() *TestEventStream {
return NewBufferedTestEventStream(0)
}
func NewBufferedTestEventStream(buffer int) *TestEventStream {
es := new(TestEventStream)
es.ready = make(chan struct{}, buffer)
es.message = make(chan interface{}, buffer)
return es
}
func (stream *TestEventStream) Ready() <-chan struct{} {
return stream.ready
}
func (stream *TestEventStream) RecvMsg(i interface{}) error {
return nil
}
func (stream *TestEventStream) Message() interface{} {
return <-stream.message
}
func (stream *TestEventStream) Close() error {
stream.Called()
close(stream.message)
close(stream.ready)
return nil
}
func (stream *TestEventStream) Emit(msg interface{}) {
stream.ready <- struct{}{}
stream.message <- msg
}
func NewFrameNavigatedClient() *FrameNavigatedClient {
return &FrameNavigatedClient{
TestEventStream: NewTestEventStream(),
}
}
func (stream *FrameNavigatedClient) Recv() (*page.FrameNavigatedReply, error) {
<-stream.Ready()
msg := stream.Message()
repl, ok := msg.(*page.FrameNavigatedReply)
if !ok {
panic("Invalid message type")
}
return repl, nil
}
func NewResponseReceivedClient() *ResponseReceivedClient {
return &ResponseReceivedClient{
TestEventStream: NewTestEventStream(),
}
}
func (stream *ResponseReceivedClient) Recv() (*network2.ResponseReceivedReply, error) {
<-stream.Ready()
msg := stream.Message()
repl, ok := msg.(*network2.ResponseReceivedReply)
if !ok {
panic("Invalid message type")
}
return repl, nil
}
func NewRequestPausedClient() *RequestPausedClient {
return &RequestPausedClient{
TestEventStream: NewTestEventStream(),
}
}
func (stream *RequestPausedClient) Recv() (*fetch.RequestPausedReply, error) {
<-stream.Ready()
msg := stream.Message()
repl, ok := msg.(*fetch.RequestPausedReply)
if !ok {
panic("Invalid message type")
}
return repl, nil
}
func TestManager(t *testing.T) {
Convey("Network manager", t, func() {
Convey("New", func() {
Convey("Should close all resources on error", func() {
frameNavigatedClient := NewFrameNavigatedClient()
frameNavigatedClient.On("Close", mock.Anything).Once().Return(nil)
pageAPI := new(PageAPI)
pageAPI.frameNavigated = func(ctx context.Context) (page.FrameNavigatedClient, error) {
return frameNavigatedClient, nil
}
responseReceivedClient := NewResponseReceivedClient()
responseReceivedClient.On("Close", mock.Anything).Once().Return(nil)
setExtraHTTPHeadersErr := errors.New("test error")
networkAPI := new(NetworkAPI)
networkAPI.responseReceived = func(ctx context.Context) (network2.ResponseReceivedClient, error) {
return responseReceivedClient, nil
}
networkAPI.setExtraHTTPHeaders = func(ctx context.Context, args *network2.SetExtraHTTPHeadersArgs) error {
return setExtraHTTPHeadersErr
}
requestPausedClient := NewRequestPausedClient()
requestPausedClient.On("Close", mock.Anything).Once().Return(nil)
fetchAPI := new(FetchAPI)
fetchAPI.enable = func(ctx context.Context, args *fetch.EnableArgs) error {
return nil
}
fetchAPI.requestPaused = func(ctx context.Context) (fetch.RequestPausedClient, error) {
return requestPausedClient, nil
}
client := &cdp.Client{
Page: pageAPI,
Network: networkAPI,
Fetch: fetchAPI,
}
_, err := network.New(
zerolog.New(os.Stdout).Level(zerolog.Disabled),
client,
network.Options{
Headers: drivers.NewHTTPHeadersWith(map[string][]string{"x-correlation-id": {"foo"}}),
Filter: &network.Filter{
Patterns: []drivers.ResourceFilter{
{
URL: "http://google.com",
Type: "img",
},
},
},
},
)
So(err, ShouldNotBeNil)
frameNavigatedClient.AssertExpectations(t)
responseReceivedClient.AssertExpectations(t)
requestPausedClient.AssertExpectations(t)
})
Convey("Should close all resources on Close", func() {
frameNavigatedClient := NewFrameNavigatedClient()
frameNavigatedClient.On("Close", mock.Anything).Once().Return(nil)
pageAPI := new(PageAPI)
pageAPI.frameNavigated = func(ctx context.Context) (page.FrameNavigatedClient, error) {
return frameNavigatedClient, nil
}
responseReceivedClient := NewResponseReceivedClient()
responseReceivedClient.On("Close", mock.Anything).Once().Return(nil)
networkAPI := new(NetworkAPI)
networkAPI.responseReceived = func(ctx context.Context) (network2.ResponseReceivedClient, error) {
return responseReceivedClient, nil
}
networkAPI.setExtraHTTPHeaders = func(ctx context.Context, args *network2.SetExtraHTTPHeadersArgs) error {
return nil
}
requestPausedClient := NewRequestPausedClient()
requestPausedClient.On("Close", mock.Anything).Once().Return(nil)
fetchAPI := new(FetchAPI)
fetchAPI.enable = func(ctx context.Context, args *fetch.EnableArgs) error {
return nil
}
fetchAPI.requestPaused = func(ctx context.Context) (fetch.RequestPausedClient, error) {
return requestPausedClient, nil
}
client := &cdp.Client{
Page: pageAPI,
Network: networkAPI,
Fetch: fetchAPI,
}
mgr, err := network.New(
zerolog.New(os.Stdout).Level(zerolog.Disabled),
client,
network.Options{
Headers: drivers.NewHTTPHeadersWith(map[string][]string{"x-correlation-id": {"foo"}}),
Filter: &network.Filter{
Patterns: []drivers.ResourceFilter{
{
URL: "http://google.com",
Type: "img",
},
},
},
},
)
So(err, ShouldBeNil)
So(mgr.Close(), ShouldBeNil)
frameNavigatedClient.AssertExpectations(t)
responseReceivedClient.AssertExpectations(t)
requestPausedClient.AssertExpectations(t)
})
})
})
}

41
pkg/drivers/cdp/page.go
View File

@ -18,6 +18,7 @@ import (
"github.com/MontFerret/ferret/pkg/drivers/cdp/input"
net "github.com/MontFerret/ferret/pkg/drivers/cdp/network"
"github.com/MontFerret/ferret/pkg/drivers/cdp/templates"
"github.com/MontFerret/ferret/pkg/drivers/cdp/utils"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/events"
@ -32,7 +33,7 @@ type (
HTMLPage struct {
mu sync.Mutex
closed values.Boolean
logger *zerolog.Logger
logger zerolog.Logger
conn *rpcc.Conn
client *cdp.Client
network *net.Manager
@ -93,7 +94,12 @@ func LoadHTMLPage(
}
}
netManager, err := net.New(ctx, logger, client, netOpts)
netManager, err := net.New(
logger,
client,
netOpts,
)
if err != nil {
return nil, err
}
@ -180,7 +186,7 @@ func LoadHTMLPageWithContent(
}
func NewHTMLPage(
logger *zerolog.Logger,
logger zerolog.Logger,
conn *rpcc.Conn,
client *cdp.Client,
netManager *net.Manager,
@ -190,7 +196,7 @@ func NewHTMLPage(
) *HTMLPage {
p := new(HTMLPage)
p.closed = values.False
p.logger = logger
p.logger = logging.WithName(logger.With(), "cdp_page").Logger()
p.conn = conn
p.client = client
p.network = netManager
@ -271,14 +277,13 @@ func (p *HTMLPage) Close() error {
p.mu.Lock()
defer p.mu.Unlock()
url := p.GetURL().String()
url := p.dom.GetMainFrame().GetURL().String()
p.closed = values.True
err := p.dom.Close()
if err != nil {
p.logger.Warn().
Timestamp().
Str("url", url).
Err(err).
Msg("failed to close dom manager")
@ -288,7 +293,6 @@ func (p *HTMLPage) Close() error {
if err != nil {
p.logger.Warn().
Timestamp().
Str("url", url).
Err(err).
Msg("failed to close network manager")
@ -298,23 +302,15 @@ func (p *HTMLPage) Close() error {
if err != nil {
p.logger.Warn().
Timestamp().
Str("url", url).
Err(err).
Msg("failed to close browser page")
}
err = p.conn.Close()
// Ignore errors from the connection object
p.conn.Close()
if err != nil {
p.logger.Warn().
Timestamp().
Str("url", url).
Err(err).
Msg("failed to close connection")
}
return err
return nil
}
func (p *HTMLPage) IsClosed() values.Boolean {
@ -332,7 +328,6 @@ func (p *HTMLPage) GetURL() values.String {
}
p.logger.Warn().
Timestamp().
Err(err).
Msg("failed to retrieve URL")
@ -477,12 +472,14 @@ func (p *HTMLPage) CaptureScreenshot(ctx context.Context, params drivers.Screens
params.Y = 0
}
clientWidth, clientHeight := utils.GetLayoutViewportWH(metrics)
if params.Width <= 0 {
params.Width = values.Float(metrics.CSSLayoutViewport.ClientWidth) - params.X
params.Width = values.Float(clientWidth) - params.X
}
if params.Height <= 0 {
params.Height = values.Float(metrics.CSSLayoutViewport.ClientHeight) - params.Y
params.Height = values.Float(clientHeight) - params.Y
}
clip := page.Viewport{
@ -615,6 +612,8 @@ func (p *HTMLPage) Subscribe(ctx context.Context, eventName string, options *val
Data: data,
Err: err,
}
close(ch)
}()
return ch

22
pkg/drivers/cdp/templates/get_styles.go
View File

@ -1,22 +0,0 @@
package templates
var getStylesTemplate = `
(el) => {
const out = {};
const styles = window.getComputedStyle(el);
Object.keys(styles).forEach((key) => {
if (!isNaN(parseFloat(key))) {
const name = styles[key];
const value = styles.getPropertyValue(name);
out[name] = value;
}
});
return out;
}
`
func GetStyles() string {
return getStylesTemplate
}

41
pkg/drivers/cdp/templates/query.go Normal file
View File

@ -0,0 +1,41 @@
package templates
import (
"fmt"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
)
func QuerySelector(selector string) string {
return fmt.Sprintf(`
(el) => {
const found = el.querySelector(%s);
if (found == null) {
throw new Error(%s);
}
return found;
}
`,
eval.ParamString(selector),
eval.ParamString(drivers.ErrNotFound.Error()),
)
}
func QuerySelectorAll(selector string) string {
return fmt.Sprintf(`
(el) => {
const found = el.querySelectorAll(%s);
if (found == null) {
throw new Error(%s);
}
return found;
}
`,
eval.ParamString(selector),
eval.ParamString(drivers.ErrNotFound.Error()),
)
}

77
pkg/drivers/cdp/templates/styles.go
View File

@ -2,11 +2,86 @@ package templates
import (
"fmt"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
var getStylesTemplate = `
(el) => {
const out = {};
const styles = window.getComputedStyle(el);
Object.keys(styles).forEach((key) => {
if (!isNaN(parseFloat(key))) {
const name = styles[key];
const value = styles.getPropertyValue(name);
out[name] = value;
}
});
return out;
}
`
func GetStyles() string {
return getStylesTemplate
}
func GetStyle(name string) string {
return fmt.Sprintf(`
(el) => {
const out = {};
const styles = window.getComputedStyle(el);
return styles[%s];
}
`, eval.ParamString(name))
}
func SetStyle(name, value string) string {
return fmt.Sprintf(`
(el) => {
el.style[%s] = %s;
}
`, eval.ParamString(name), eval.ParamString(value))
}
func SetStyles(pairs *values.Object) string {
return fmt.Sprintf(`
(el) => {
const values = %s;
Object.keys(values).forEach((key) => {
el.style[key] = values[key]
});
}
`, eval.Param(pairs))
}
func RemoveStyles(names []values.String) string {
return fmt.Sprintf(`
(el) => {
const style = el.style;
[%s].forEach((name) => { style[name] = "" })
}
`,
eval.ParamStringList(names),
)
}
func WaitForStyle(name, value string, when drivers.WaitEvent) string {
return fmt.Sprintf(`
(el) => {
const styles = window.getComputedStyle(el);
const actual = styles[%s];
const expected = %s;
// null means we need to repeat
return actual %s expected ? true : null ;
}
`, eval.ParamString(name), eval.ParamString(value), WaitEventToEqOperator(when))
}
func StyleRead(name values.String) string {
n := name.String()
return fmt.Sprintf(`

21
pkg/drivers/cdp/utils/layout.go Normal file
View File

@ -0,0 +1,21 @@
package utils
import "github.com/mafredri/cdp/protocol/page"
func GetLayoutViewportWH(metrics *page.GetLayoutMetricsReply) (width int, height int) {
if metrics.CSSLayoutViewport.ClientWidth > 0 {
width = metrics.CSSLayoutViewport.ClientWidth
} else {
// Chrome version <=89
width = metrics.LayoutViewport.ClientWidth
}
if metrics.CSSLayoutViewport.ClientHeight > 0 {
height = metrics.CSSLayoutViewport.ClientHeight
} else {
// Chrome version <=89
height = metrics.LayoutViewport.ClientHeight
}
return
}

2
pkg/drivers/common/errors.go
View File

@ -12,7 +12,7 @@ var (
ErrInvalidPath = core.Error(core.ErrInvalidOperation, "invalid path")
)
func CloseAll(logger *zerolog.Logger, closers []io.Closer, msg string) {
func CloseAll(logger zerolog.Logger, closers []io.Closer, msg string) {
for _, closer := range closers {
if err := closer.Close(); err != nil {
logger.Error().Err(err).Msg(msg)

25
pkg/drivers/response.go
View File

@ -3,18 +3,20 @@ package drivers
import (
"context"
"github.com/wI2L/jettison"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
"github.com/wI2L/jettison"
)
// HTTPResponse HTTP response object.
type HTTPResponse struct {
StatusCode int
Status string
Headers *HTTPHeaders
URL string
StatusCode int
Status string
Headers *HTTPHeaders
ResponseTime float64
}
func (resp *HTTPResponse) Type() core.Type {
@ -60,9 +62,11 @@ func (resp *HTTPResponse) Hash() uint64 {
// responseMarshal is a structure that repeats HTTPResponse. It allows
// easily Marshal the HTTPResponse object.
type responseMarshal struct {
StatusCode int `json:"status_code"`
Status string `json:"status"`
Headers *HTTPHeaders `json:"headers"`
URL string `json:"url"`
StatusCode int `json:"status_code"`
Status string `json:"status"`
Headers *HTTPHeaders `json:"headers"`
ResponseTime float64 `json:"response_time"`
}
func (resp *HTTPResponse) MarshalJSON() ([]byte, error) {
@ -85,6 +89,8 @@ func (resp *HTTPResponse) GetIn(ctx context.Context, path []core.Value) (core.Va
field := path[0].(values.String).String()
switch field {
case "url", "URL":
return values.NewString(resp.URL), nil
case "status":
return values.NewString(resp.Status), nil
case "statusCode":
@ -95,6 +101,9 @@ func (resp *HTTPResponse) GetIn(ctx context.Context, path []core.Value) (core.Va
}
return resp.Headers.GetIn(ctx, path[1:])
case "responseTime":
return values.NewFloat(resp.ResponseTime), nil
}
return values.None, nil

3
pkg/parser/antlr/FqlParser.g4
View File

@ -157,8 +157,7 @@ waitForExpression
waitForTimeout
: integerLiteral
| variable
| functionCallExpression
| memberExpression
| param
;
waitForEventName

2
pkg/parser/fql/FqlParser.interp
View File

File diff suppressed because one or more lines are too long

830
pkg/parser/fql/fql_parser.go
View File

File diff suppressed because it is too large Load Diff

45
pkg/runtime/logging/logger.go
View File

@ -8,7 +8,7 @@ import (
)
type (
Level uint8
Level int8
Options struct {
Writer io.Writer
@ -26,8 +26,34 @@ const (
PanicLevel
NoLevel
Disabled
TraceLevel Level = -1
)
func ParseLevel(input string) (Level, error) {
lvl, err := zerolog.ParseLevel(input)
if err != nil {
return NoLevel, err
}
return Level(lvl), nil
}
func MustParseLevel(input string) Level {
lvl, err := zerolog.ParseLevel(input)
if err != nil {
panic(err)
}
return Level(lvl)
}
func (l Level) String() string {
return zerolog.Level(l).String()
}
func WithContext(ctx context.Context, opts Options) context.Context {
c := zerolog.New(opts.Writer).With().Timestamp()
@ -35,12 +61,21 @@ func WithContext(ctx context.Context, opts Options) context.Context {
c = c.Interface(k, v)
}
logger := c.Logger()
logger.Level(zerolog.Level(opts.Level))
logger := c.Logger().Level(zerolog.Level(opts.Level))
return logger.WithContext(ctx)
}
func FromContext(ctx context.Context) *zerolog.Logger {
return zerolog.Ctx(ctx)
func FromContext(ctx context.Context) zerolog.Logger {
found := zerolog.Ctx(ctx)
if found == nil {
panic("logger is not set")
}
return *found
}
func WithName(ctx zerolog.Context, name string) zerolog.Context {
return ctx.Str("component", name)
}

12
pkg/runtime/program.go
View File

@ -2,7 +2,7 @@ package runtime
import (
"context"
"runtime"
"fmt"
"strings"
"github.com/pkg/errors"
@ -58,24 +58,20 @@ func (p *Program) Run(ctx context.Context, setters ...Option) (result []byte, er
defer func() {
if r := recover(); r != nil {
// find out exactly what the error was and set err
switch x := r.(type) {
case string:
err = errors.New(x)
case error:
err = x
err = errors.WithStack(err)
default:
err = errors.New("unknown panic")
}
b := make([]byte, 0, 20)
runtime.Stack(b, true)
logger.Error().
Timestamp().
Err(err).
Str("stack", string(b)).
Msg("Panic")
Str("stack", fmt.Sprintf("%+v", err)).
Msg("panic")
result = nil
}

11
pkg/stdlib/html/find_frames.go
View File

@ -5,13 +5,14 @@ import (
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"regexp"
)
// FRAMES finds HTML frames by a given property selector.
// Returns an empty array if frames not found.
// @param {HTMLPage} page - HTML page.
// @param {String} property - Property selector.
// @param {Any} value - Property value.
// @param {String} exp - Regular expression to match property value.
// @return {HTMLDocument[]} - Returns an array of found HTML frames.
func Frames(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 3, 3)
@ -33,7 +34,11 @@ func Frames(ctx context.Context, args ...core.Value) (core.Value, error) {
}
propName := values.ToString(args[1])
propValue := args[2]
matcher, err := regexp.Compile(values.ToString(args[2]).String())
if err != nil {
return values.None, err
}
result, _ := frames.Find(func(value core.Value, idx int) bool {
doc, e := drivers.ToDocument(value)
@ -51,7 +56,7 @@ func Frames(ctx context.Context, args ...core.Value) (core.Value, error) {
return false
}
return currentPropValue.Compare(propValue) == 0
return matcher.MatchString(currentPropValue.String())
})
return result, err

30
pkg/stdlib/html/pagination.go
View File

@ -2,11 +2,12 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
"github.com/rs/zerolog"
)
// PAGINATION creates an iterator that goes through pages using CSS selector.
@ -14,7 +15,7 @@ import (
// That allows you to keep scraping logic inside FOR loop.
// @param {HTMLPage | HTMLDocument | HTMLElement} node - Target html node.
// @param {String} selector - CSS selector for a pagination on the page.
func Pagination(_ context.Context, args ...core.Value) (core.Value, error) {
func Pagination(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 2, 2)
if err != nil {
@ -35,18 +36,25 @@ func Pagination(_ context.Context, args ...core.Value) (core.Value, error) {
selector := args[1].(values.String)
return &Paging{page, selector}, nil
logger := logging.
WithName(logging.FromContext(ctx).With(), "stdlib_html_pagination").
Str("selector", selector.String()).
Logger()
return &Paging{logger, page, selector}, nil
}
var PagingType = core.NewType("paging")
type (
Paging struct {
logger zerolog.Logger
page drivers.HTMLPage
selector values.String
}
PagingIterator struct {
logger zerolog.Logger
page drivers.HTMLPage
selector values.String
pos values.Int
@ -82,32 +90,46 @@ func (p *Paging) Copy() core.Value {
}
func (p *Paging) Iterate(_ context.Context) (core.Iterator, error) {
return &PagingIterator{p.page, p.selector, -1}, nil
return &PagingIterator{p.logger, p.page, p.selector, -1}, nil
}
func (i *PagingIterator) Next(ctx context.Context) (core.Value, core.Value, error) {
i.pos++
i.logger.Trace().Int("position", int(i.pos)).Msg("starting to advance iteration")
if i.pos == 0 {
i.logger.Trace().Msg("starting point of pagination. nothing to do. exit")
return values.ZeroInt, values.ZeroInt, nil
}
i.logger.Trace().Msg("checking if an element exists...")
exists, err := i.page.GetMainFrame().ExistsBySelector(ctx, i.selector)
if err != nil {
i.logger.Trace().Err(err).Msg("failed to check")
return values.None, values.None, err
}
if !exists {
i.logger.Trace().Bool("exists", bool(exists)).Msg("element does not exist. exit")
return values.None, values.None, core.ErrNoMoreData
}
i.logger.Trace().Bool("exists", bool(exists)).Msg("element exists. clicking...")
err = i.page.GetMainFrame().GetElement().ClickBySelector(ctx, i.selector, 1)
if err != nil {
i.logger.Trace().Err(err).Msg("failed to click. exit")
return values.None, values.None, err
}
i.logger.Trace().Msg("successfully clicked on element. iteration has succeeded")
// terminate
return i.pos, i.pos, nil
}