mirror of
https://github.com/MontFerret/ferret.git
synced 2025-08-15 20:02:56 +02:00
Added possibility to dispatch events on node
This commit is contained in:
28
Gopkg.lock
generated
28
Gopkg.lock
generated
@@ -66,7 +66,7 @@
|
||||
version = "v4.2.1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:383ac09778833e583aa5f74cf52a71367c72ba43a7de1460e8ff95182ad93b3b"
|
||||
digest = "1:7ffdd69928c5153fc351132aa80bbcc18a8f8122de1ba592cf42dccb65732361"
|
||||
name = "github.com/mafredri/cdp"
|
||||
packages = [
|
||||
".",
|
||||
@@ -115,6 +115,7 @@
|
||||
"protocol/tethering",
|
||||
"protocol/tracing",
|
||||
"rpcc",
|
||||
"session",
|
||||
]
|
||||
pruneopts = "UT"
|
||||
revision = "75b0ecc5efcff27ac756a33ec71f0db75dc3d21c"
|
||||
@@ -180,25 +181,6 @@
|
||||
pruneopts = "UT"
|
||||
revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:45ed1634373de798161249f031804bac6f5ec7707bbae94047ecd8e69875936c"
|
||||
name = "golang.org/x/text"
|
||||
packages = [
|
||||
"internal/colltab",
|
||||
"internal/gen",
|
||||
"internal/tag",
|
||||
"internal/triegen",
|
||||
"internal/ucd",
|
||||
"language",
|
||||
"search",
|
||||
"transform",
|
||||
"unicode/cldr",
|
||||
"unicode/norm",
|
||||
]
|
||||
pruneopts = "UT"
|
||||
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
|
||||
version = "v0.3.0"
|
||||
|
||||
[solve-meta]
|
||||
analyzer-name = "dep"
|
||||
analyzer-version = 1
|
||||
@@ -210,13 +192,17 @@
|
||||
"github.com/mafredri/cdp",
|
||||
"github.com/mafredri/cdp/devtool",
|
||||
"github.com/mafredri/cdp/protocol/dom",
|
||||
"github.com/mafredri/cdp/protocol/emulation",
|
||||
"github.com/mafredri/cdp/protocol/input",
|
||||
"github.com/mafredri/cdp/protocol/page",
|
||||
"github.com/mafredri/cdp/protocol/runtime",
|
||||
"github.com/mafredri/cdp/rpcc",
|
||||
"github.com/mafredri/cdp/session",
|
||||
"github.com/pkg/errors",
|
||||
"github.com/sethgrid/pester",
|
||||
"github.com/smartystreets/goconvey/convey",
|
||||
"golang.org/x/net/html",
|
||||
"golang.org/x/sync/errgroup",
|
||||
"golang.org/x/text/search",
|
||||
]
|
||||
solver-name = "gps-cdcl"
|
||||
solver-version = 1
|
||||
|
9
docs/examples/click.fql
Normal file
9
docs/examples/click.fql
Normal file
@@ -0,0 +1,9 @@
|
||||
LET doc = DOCUMENT("https://github.com/", true)
|
||||
LET btn = ELEMENT(doc, ".HeaderMenu a")
|
||||
LET clicked = CLICK(btn)
|
||||
|
||||
WAIT_ELEMENT(doc, '.IconNav', 5000)
|
||||
WAIT(5000)
|
||||
|
||||
FOR el IN ELEMENTS(doc, '.IconNav a')
|
||||
RETURN el.innerText
|
@@ -4,8 +4,6 @@ WAIT_ELEMENT(doc, '.chartTrack__details', 5000)
|
||||
|
||||
LET tracks = ELEMENTS(doc, '.chartTrack__details')
|
||||
|
||||
// LOG("found", LENGTH(tracks), "tracks")
|
||||
|
||||
FOR track IN tracks
|
||||
LET username = ELEMENT(track, '.chartTrack__username')
|
||||
LET title = ELEMENT(track, '.chartTrack__title')
|
@@ -1,8 +0,0 @@
|
||||
LET reddit = DOCUMENT("https://www.reddit.com/")
|
||||
LET urls = ELEMENTS(reddit, 'a[data-click-id="body"]')
|
||||
|
||||
FOR url IN urls
|
||||
LET subreddit = DOCUMENT("https://www.reddit.com" + TRIM(url.attributes.href))
|
||||
LET post = ELEMENT(subreddit, 'div[data-test-id="post-content"] > div:nth-child(2) > div')
|
||||
|
||||
RETURN { title: post.children[0].innerText }
|
56
pkg/stdlib/html/actions.go
Normal file
56
pkg/stdlib/html/actions.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package html
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
"github.com/MontFerret/ferret/pkg/stdlib/html/driver/browser"
|
||||
)
|
||||
|
||||
/*
|
||||
*
|
||||
*/
|
||||
func Click(_ context.Context, args ...core.Value) (core.Value, error) {
|
||||
err := core.ValidateArgs(args, 1, 2)
|
||||
|
||||
if err != nil {
|
||||
return values.False, err
|
||||
}
|
||||
|
||||
// CLICK(el)
|
||||
if len(args) == 1 {
|
||||
arg1 := args[0]
|
||||
|
||||
err := core.ValidateType(arg1, core.HtmlElementType)
|
||||
|
||||
if err != nil {
|
||||
return values.False, err
|
||||
}
|
||||
|
||||
el, ok := arg1.(*browser.HtmlElement)
|
||||
|
||||
if !ok {
|
||||
return values.False, core.Error(core.ErrInvalidType, "expected dynamic element")
|
||||
}
|
||||
|
||||
return el.Click()
|
||||
} else {
|
||||
// CLICK(doc, selector)
|
||||
arg1 := args[0]
|
||||
selector := args[1].String()
|
||||
|
||||
err = core.ValidateType(arg1, core.HtmlDocumentType)
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
doc, ok := arg1.(*browser.HtmlDocument)
|
||||
|
||||
if !ok {
|
||||
return values.False, core.Error(core.ErrInvalidType, "expected dynamic document")
|
||||
}
|
||||
|
||||
return doc.ClickBySelector(values.NewString(selector))
|
||||
}
|
||||
}
|
103
pkg/stdlib/html/driver/browser/broker.go
Normal file
103
pkg/stdlib/html/driver/browser/broker.go
Normal file
@@ -0,0 +1,103 @@
|
||||
package browser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
"github.com/mafredri/cdp/protocol/page"
|
||||
)
|
||||
|
||||
type (
|
||||
EventHandler func(event, message string)
|
||||
|
||||
EventBroker struct {
|
||||
client page.LifecycleEventClient
|
||||
handlers map[string][]EventHandler
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
)
|
||||
|
||||
func NewEventBroker(client page.LifecycleEventClient) *EventBroker {
|
||||
return &EventBroker{
|
||||
client,
|
||||
make(map[string][]EventHandler),
|
||||
nil,
|
||||
}
|
||||
}
|
||||
|
||||
func (broker *EventBroker) Start() error {
|
||||
if broker.cancel != nil {
|
||||
return core.Error(core.ErrInvalidOperation, "broker is already started")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
broker.cancel = cancel
|
||||
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-broker.client.Ready():
|
||||
reply, err := broker.client.Recv()
|
||||
|
||||
if err != nil {
|
||||
fmt.Println("FAILED TO GET EVENT", err)
|
||||
broker.Emit("error", err.Error())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println("EVENT", reply.Name)
|
||||
|
||||
broker.Emit(reply.Name, "")
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (broker *EventBroker) Stop() error {
|
||||
if broker.cancel == nil {
|
||||
return core.Error(core.ErrInvalidOperation, "broker is already stopped")
|
||||
}
|
||||
|
||||
broker.cancel()
|
||||
broker.client = nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (broker *EventBroker) Close() error {
|
||||
if broker.cancel != nil {
|
||||
broker.Stop()
|
||||
}
|
||||
|
||||
return broker.client.Close()
|
||||
}
|
||||
|
||||
func (broker *EventBroker) AddListener(event string, handler EventHandler) {
|
||||
handlers, ok := broker.handlers[event]
|
||||
|
||||
if !ok {
|
||||
handlers = make([]EventHandler, 0, 5)
|
||||
|
||||
broker.handlers[event] = handlers
|
||||
}
|
||||
|
||||
handlers = append(handlers, handler)
|
||||
}
|
||||
|
||||
func (broker *EventBroker) Emit(name, message string) {
|
||||
handlers, ok := broker.handlers[name]
|
||||
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
for _, handler := range handlers {
|
||||
handler(name, message)
|
||||
}
|
||||
}
|
@@ -5,8 +5,11 @@ import (
|
||||
"fmt"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
"github.com/corpix/uarand"
|
||||
"github.com/mafredri/cdp"
|
||||
"github.com/mafredri/cdp/protocol/dom"
|
||||
"github.com/mafredri/cdp/protocol/emulation"
|
||||
"github.com/mafredri/cdp/protocol/page"
|
||||
"github.com/mafredri/cdp/rpcc"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -16,6 +19,7 @@ type HtmlDocument struct {
|
||||
*HtmlElement
|
||||
conn *rpcc.Conn
|
||||
client *cdp.Client
|
||||
events *EventBroker
|
||||
url string
|
||||
}
|
||||
|
||||
@@ -39,6 +43,13 @@ func NewHtmlDocument(
|
||||
return client.Page.Enable(ctx)
|
||||
},
|
||||
|
||||
func() error {
|
||||
return client.Page.SetLifecycleEventsEnabled(
|
||||
ctx,
|
||||
page.NewSetLifecycleEventsEnabledArgs(true),
|
||||
)
|
||||
},
|
||||
|
||||
func() error {
|
||||
return client.DOM.Enable(ctx)
|
||||
},
|
||||
@@ -46,44 +57,93 @@ func NewHtmlDocument(
|
||||
func() error {
|
||||
return client.Runtime.Enable(ctx)
|
||||
},
|
||||
|
||||
func() error {
|
||||
return client.Emulation.SetUserAgentOverride(
|
||||
ctx,
|
||||
emulation.NewSetUserAgentOverrideArgs(uarand.GetRandom()),
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
loadEventFired, err := client.Page.LoadEventFired(ctx)
|
||||
err = waitForLoadEvent(ctx, client)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
root, err := getRootElement(ctx, client)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
events, err := createEventBroker(ctx, client)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
doc := &HtmlDocument{
|
||||
NewHtmlElement(client, root.NodeID, root),
|
||||
conn,
|
||||
client,
|
||||
events,
|
||||
url,
|
||||
}
|
||||
|
||||
doc.init()
|
||||
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
func waitForLoadEvent(ctx context.Context, client *cdp.Client) error {
|
||||
loadEventFired, err := client.Page.LoadEventFired(ctx)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = loadEventFired.Recv()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return err
|
||||
}
|
||||
|
||||
loadEventFired.Close()
|
||||
return loadEventFired.Close()
|
||||
}
|
||||
|
||||
func getRootElement(ctx context.Context, client *cdp.Client) (dom.Node, error) {
|
||||
args := dom.NewGetDocumentArgs()
|
||||
args.Depth = PointerInt(-1) // lets load the entire document
|
||||
|
||||
d, err := client.DOM.GetDocument(ctx, args)
|
||||
|
||||
if err != nil {
|
||||
return dom.Node{}, err
|
||||
}
|
||||
|
||||
return d.Root, nil
|
||||
}
|
||||
|
||||
func createEventBroker(ctx context.Context, client *cdp.Client) (*EventBroker, error) {
|
||||
lfc, err := client.Page.LifecycleEvent(ctx)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &HtmlDocument{
|
||||
&HtmlElement{client, d.Root.NodeID, d.Root, nil},
|
||||
conn,
|
||||
client,
|
||||
url,
|
||||
}, nil
|
||||
return NewEventBroker(lfc), nil
|
||||
}
|
||||
|
||||
func (doc *HtmlDocument) Close() error {
|
||||
doc.events.Stop()
|
||||
doc.events.Close()
|
||||
|
||||
doc.client.Page.Close(context.Background())
|
||||
|
||||
return doc.conn.Close()
|
||||
@@ -112,6 +172,36 @@ func (doc *HtmlDocument) Compare(other core.Value) int {
|
||||
}
|
||||
}
|
||||
|
||||
func (doc *HtmlDocument) ClickBySelector(selector values.String) (values.Boolean, error) {
|
||||
res, err := Eval(
|
||||
doc.client,
|
||||
fmt.Sprintf(`
|
||||
var el = document.querySelector("%s");
|
||||
|
||||
if (el == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
var evt = new window.MouseEvent('click', { bubbles: true });
|
||||
el.dispatchEvent(evt);
|
||||
|
||||
return true;
|
||||
`, selector),
|
||||
true,
|
||||
false,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return values.False, err
|
||||
}
|
||||
|
||||
if res.Type() == core.BooleanType {
|
||||
return res.(values.Boolean), nil
|
||||
}
|
||||
|
||||
return values.False, nil
|
||||
}
|
||||
|
||||
func (doc *HtmlDocument) WaitForSelector(selector values.String, timeout values.Int) error {
|
||||
task := NewWaitTask(
|
||||
doc.client,
|
||||
@@ -125,9 +215,27 @@ func (doc *HtmlDocument) WaitForSelector(selector values.String, timeout values.
|
||||
return null;
|
||||
`, selector),
|
||||
time.Millisecond*time.Duration(timeout),
|
||||
DefaultPolling,
|
||||
)
|
||||
|
||||
_, err := task.Run()
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (doc *HtmlDocument) init() {
|
||||
// doc.events.AddListener("")
|
||||
}
|
||||
|
||||
func (doc *HtmlDocument) reload() error {
|
||||
root, err := getRootElement(context.Background(), doc.client)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
doc.url = *root.BaseURL
|
||||
doc.id = root.NodeID
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/mafredri/cdp"
|
||||
"github.com/mafredri/cdp/protocol/dom"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
@@ -19,13 +18,17 @@ import (
|
||||
const DefaultTimeout = time.Second * 30
|
||||
|
||||
type HtmlElement struct {
|
||||
client *cdp.Client
|
||||
id dom.NodeID
|
||||
node dom.Node
|
||||
attributes *values.Object
|
||||
client *cdp.Client
|
||||
id dom.NodeID
|
||||
nodeType values.Int
|
||||
nodeName values.String
|
||||
value string
|
||||
attributes *values.Object
|
||||
children []dom.NodeID
|
||||
loadedChildren *values.Array
|
||||
}
|
||||
|
||||
func NewHtmlElement(
|
||||
func LoadElement(
|
||||
client *cdp.Client,
|
||||
id dom.NodeID,
|
||||
) (*HtmlElement, error) {
|
||||
@@ -42,15 +45,46 @@ func NewHtmlElement(
|
||||
dom.
|
||||
NewDescribeNodeArgs().
|
||||
SetNodeID(id).
|
||||
SetDepth(-1),
|
||||
SetDepth(1),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Println("ERROR:", err)
|
||||
return nil, core.Error(err, strconv.Itoa(int(id)))
|
||||
}
|
||||
|
||||
return &HtmlElement{client, id, node.Node, nil}, nil
|
||||
return NewHtmlElement(client, id, node.Node), nil
|
||||
}
|
||||
|
||||
func NewHtmlElement(
|
||||
client *cdp.Client,
|
||||
id dom.NodeID,
|
||||
node dom.Node,
|
||||
) *HtmlElement {
|
||||
el := new(HtmlElement)
|
||||
el.client = client
|
||||
el.id = id
|
||||
el.nodeType = values.NewInt(node.NodeType)
|
||||
el.nodeName = values.NewString(node.NodeName)
|
||||
el.value = ""
|
||||
el.attributes = parseAttrs(node.Attributes)
|
||||
|
||||
var childCount int
|
||||
|
||||
if node.ChildNodeCount != nil {
|
||||
childCount = *node.ChildNodeCount
|
||||
}
|
||||
|
||||
if node.Value != nil {
|
||||
el.value = *node.Value
|
||||
}
|
||||
|
||||
el.children = make([]dom.NodeID, childCount)
|
||||
|
||||
for idx, child := range node.Children {
|
||||
el.children[idx] = child.NodeID
|
||||
}
|
||||
|
||||
return el
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Close() error {
|
||||
@@ -69,19 +103,19 @@ func (el *HtmlElement) MarshalJSON() ([]byte, error) {
|
||||
defer cancelFn()
|
||||
|
||||
args := dom.NewGetOuterHTMLArgs()
|
||||
args.NodeID = &el.node.NodeID
|
||||
args.NodeID = &el.id
|
||||
|
||||
reply, err := el.client.DOM.GetOuterHTML(ctx, args)
|
||||
|
||||
if err != nil {
|
||||
return nil, core.Error(err, strconv.Itoa(int(el.node.NodeID)))
|
||||
return nil, core.Error(err, strconv.Itoa(int(el.id)))
|
||||
}
|
||||
|
||||
return json.Marshal(reply.OuterHTML)
|
||||
}
|
||||
|
||||
func (el *HtmlElement) String() string {
|
||||
return *el.node.Value
|
||||
return el.value
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Compare(other core.Value) int {
|
||||
@@ -89,8 +123,8 @@ func (el *HtmlElement) Compare(other core.Value) int {
|
||||
case core.HtmlDocumentType:
|
||||
other := other.(*HtmlElement)
|
||||
|
||||
id := int(el.node.NodeID)
|
||||
otherId := int(other.node.NodeID)
|
||||
id := int(el.id)
|
||||
otherId := int(other.id)
|
||||
|
||||
if id == otherId {
|
||||
return 0
|
||||
@@ -111,13 +145,13 @@ func (el *HtmlElement) Compare(other core.Value) int {
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Unwrap() interface{} {
|
||||
return el.node
|
||||
return el
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Hash() int {
|
||||
h := sha512.New()
|
||||
|
||||
out, err := h.Write([]byte(*el.node.Value))
|
||||
out, err := h.Write([]byte(el.value))
|
||||
|
||||
if err != nil {
|
||||
return 0
|
||||
@@ -131,34 +165,22 @@ func (el *HtmlElement) Value() core.Value {
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Length() values.Int {
|
||||
if el.node.ChildNodeCount == nil {
|
||||
return values.ZeroInt
|
||||
}
|
||||
|
||||
return values.NewInt(*el.node.ChildNodeCount)
|
||||
return values.NewInt(len(el.children))
|
||||
}
|
||||
|
||||
func (el *HtmlElement) NodeType() values.Int {
|
||||
return values.NewInt(el.node.NodeType)
|
||||
return el.nodeType
|
||||
}
|
||||
|
||||
func (el *HtmlElement) NodeName() values.String {
|
||||
return values.NewString(el.node.NodeName)
|
||||
return el.nodeName
|
||||
}
|
||||
|
||||
func (el *HtmlElement) GetAttributes() core.Value {
|
||||
if el.attributes == nil {
|
||||
el.attributes = el.parseAttrs()
|
||||
}
|
||||
|
||||
return el.attributes
|
||||
}
|
||||
|
||||
func (el *HtmlElement) GetAttribute(name values.String) core.Value {
|
||||
if el.attributes == nil {
|
||||
el.attributes = el.parseAttrs()
|
||||
}
|
||||
|
||||
val, found := el.attributes.Get(name)
|
||||
|
||||
if !found {
|
||||
@@ -169,27 +191,19 @@ func (el *HtmlElement) GetAttribute(name values.String) core.Value {
|
||||
}
|
||||
|
||||
func (el *HtmlElement) GetChildNodes() core.Value {
|
||||
arr := values.NewArray(len(el.node.Children))
|
||||
|
||||
for idx := range el.node.Children {
|
||||
el := el.GetChildNode(values.NewInt(idx))
|
||||
|
||||
if el != values.None {
|
||||
arr.Push(el)
|
||||
}
|
||||
if el.loadedChildren == nil {
|
||||
el.loadedChildren = loadNodes(el.client, el.children)
|
||||
}
|
||||
|
||||
return arr
|
||||
return el.loadedChildren
|
||||
}
|
||||
|
||||
func (el *HtmlElement) GetChildNode(idx values.Int) core.Value {
|
||||
if el.Length() < idx {
|
||||
return values.None
|
||||
if el.loadedChildren == nil {
|
||||
el.loadedChildren = loadNodes(el.client, el.children)
|
||||
}
|
||||
|
||||
childNode := el.node.Children[idx]
|
||||
|
||||
return &HtmlElement{el.client, childNode.NodeID, childNode, nil}
|
||||
return el.loadedChildren.Get(idx)
|
||||
}
|
||||
|
||||
func (el *HtmlElement) QuerySelector(selector values.String) core.Value {
|
||||
@@ -199,11 +213,10 @@ func (el *HtmlElement) QuerySelector(selector values.String) core.Value {
|
||||
found, err := el.client.DOM.QuerySelector(ctx, selectorArgs)
|
||||
|
||||
if err != nil {
|
||||
el.logErr(err, selector.String())
|
||||
return values.None
|
||||
}
|
||||
|
||||
res, err := NewHtmlElement(el.client, found.NodeID)
|
||||
res, err := LoadElement(el.client, found.NodeID)
|
||||
|
||||
if err != nil {
|
||||
return values.None
|
||||
@@ -219,14 +232,13 @@ func (el *HtmlElement) QuerySelectorAll(selector values.String) core.Value {
|
||||
res, err := el.client.DOM.QuerySelectorAll(ctx, selectorArgs)
|
||||
|
||||
if err != nil {
|
||||
el.logErr(err, selector.String())
|
||||
return values.None
|
||||
}
|
||||
|
||||
arr := values.NewArray(len(res.NodeIDs))
|
||||
|
||||
for _, id := range res.NodeIDs {
|
||||
childEl, err := NewHtmlElement(el.client, id)
|
||||
childEl, err := LoadElement(el.client, id)
|
||||
|
||||
if err != nil {
|
||||
return values.None
|
||||
@@ -257,31 +269,37 @@ func (el *HtmlElement) InnerText() values.String {
|
||||
}
|
||||
|
||||
func (el *HtmlElement) InnerHtml() values.String {
|
||||
ctx, cancelFn := el.createCtx()
|
||||
ctx, cancelFn := createCtx()
|
||||
|
||||
defer cancelFn()
|
||||
|
||||
res, err := el.client.DOM.GetOuterHTML(ctx, dom.NewGetOuterHTMLArgs().SetNodeID(el.id))
|
||||
|
||||
if err != nil {
|
||||
el.logErr(err)
|
||||
|
||||
return values.EmptyString
|
||||
}
|
||||
|
||||
return values.NewString(res.OuterHTML)
|
||||
}
|
||||
|
||||
func (el *HtmlElement) createCtx() (context.Context, context.CancelFunc) {
|
||||
func (el *HtmlElement) Click() (values.Boolean, error) {
|
||||
ctx, cancel := createCtx()
|
||||
|
||||
defer cancel()
|
||||
|
||||
return DispatchEvent(ctx, el.client, el.id, "click")
|
||||
}
|
||||
|
||||
func createCtx() (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.Background(), DefaultTimeout)
|
||||
}
|
||||
|
||||
func (el *HtmlElement) parseAttrs() *values.Object {
|
||||
func parseAttrs(attrs []string) *values.Object {
|
||||
var attr values.String
|
||||
|
||||
res := values.NewObject()
|
||||
|
||||
for _, el := range el.node.Attributes {
|
||||
for _, el := range attrs {
|
||||
str := values.NewString(el)
|
||||
|
||||
if common.IsAttribute(el) {
|
||||
@@ -299,11 +317,18 @@ func (el *HtmlElement) parseAttrs() *values.Object {
|
||||
return res
|
||||
}
|
||||
|
||||
func (el *HtmlElement) logErr(values ...interface{}) {
|
||||
args := make([]interface{}, 0, len(values)+1)
|
||||
args = append(args, "ERROR:")
|
||||
args = append(args, values...)
|
||||
args = append(args, "id:", el.node.NodeID)
|
||||
func loadNodes(client *cdp.Client, nodes []dom.NodeID) *values.Array {
|
||||
arr := values.NewArray(len(nodes))
|
||||
|
||||
log.Println(args...)
|
||||
for _, id := range nodes {
|
||||
child, err := LoadElement(client, id)
|
||||
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
|
||||
arr.Push(child)
|
||||
}
|
||||
|
||||
return arr
|
||||
}
|
||||
|
@@ -1,6 +1,16 @@
|
||||
package browser
|
||||
|
||||
import "golang.org/x/sync/errgroup"
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
"github.com/mafredri/cdp"
|
||||
"github.com/mafredri/cdp/protocol/dom"
|
||||
"github.com/mafredri/cdp/protocol/runtime"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
func PointerInt(input int) *int {
|
||||
return &input
|
||||
@@ -17,3 +27,107 @@ func RunBatch(funcs ...BatchFunc) error {
|
||||
|
||||
return eg.Wait()
|
||||
}
|
||||
|
||||
func PrepareEval(exp string) string {
|
||||
return fmt.Sprintf("((function () {%s})())", exp)
|
||||
}
|
||||
|
||||
func Eval(client *cdp.Client, exp string, ret bool, async bool) (core.Value, error) {
|
||||
args := runtime.
|
||||
NewEvaluateArgs(PrepareEval(exp)).
|
||||
SetReturnByValue(ret).
|
||||
SetAwaitPromise(async)
|
||||
|
||||
out, err := client.Runtime.Evaluate(context.Background(), args)
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
if out.ExceptionDetails != nil {
|
||||
ex := out.ExceptionDetails
|
||||
|
||||
return values.None, core.Error(
|
||||
core.ErrUnexpected,
|
||||
fmt.Sprintf("%s: %s", ex.Text, *ex.Exception.Description),
|
||||
)
|
||||
}
|
||||
|
||||
if out.Result.Type != "undefined" {
|
||||
var o interface{}
|
||||
|
||||
err := json.Unmarshal(out.Result.Value, &o)
|
||||
|
||||
if err != nil {
|
||||
return values.None, core.Error(core.ErrUnexpected, err.Error())
|
||||
}
|
||||
|
||||
return values.Parse(o), nil
|
||||
}
|
||||
|
||||
return values.None, nil
|
||||
}
|
||||
|
||||
func DispatchEvent(
|
||||
ctx context.Context,
|
||||
client *cdp.Client,
|
||||
id dom.NodeID,
|
||||
eventName string,
|
||||
) (values.Boolean, error) {
|
||||
// get a ref to remote object representing the node
|
||||
obj, err := client.DOM.ResolveNode(
|
||||
ctx,
|
||||
dom.NewResolveNodeArgs().
|
||||
SetNodeID(id),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return values.False, err
|
||||
}
|
||||
|
||||
if obj.Object.ObjectID == nil {
|
||||
return values.False, nil
|
||||
}
|
||||
|
||||
evt, err := client.Runtime.Evaluate(ctx, runtime.NewEvaluateArgs(PrepareEval(fmt.Sprintf(`
|
||||
return new window.MouseEvent('%s', { bubbles: true })
|
||||
`, eventName))))
|
||||
|
||||
if err != nil {
|
||||
return values.False, nil
|
||||
}
|
||||
|
||||
if evt.ExceptionDetails != nil {
|
||||
return values.False, evt.ExceptionDetails
|
||||
}
|
||||
|
||||
if evt.Result.ObjectID == nil {
|
||||
return values.False, nil
|
||||
}
|
||||
|
||||
evtId := evt.Result.ObjectID
|
||||
|
||||
// release the event object
|
||||
defer client.Runtime.ReleaseObject(ctx, runtime.NewReleaseObjectArgs(*evtId))
|
||||
|
||||
res, err := client.Runtime.CallFunctionOn(
|
||||
ctx,
|
||||
runtime.NewCallFunctionOnArgs("dispatchEvent").
|
||||
SetObjectID(*obj.Object.ObjectID).
|
||||
SetArguments([]runtime.CallArgument{
|
||||
{
|
||||
ObjectID: evt.Result.ObjectID,
|
||||
},
|
||||
}),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return values.False, err
|
||||
}
|
||||
|
||||
if res.ExceptionDetails != nil {
|
||||
return values.False, res.ExceptionDetails
|
||||
}
|
||||
|
||||
return values.True, nil
|
||||
}
|
||||
|
@@ -1,13 +1,9 @@
|
||||
package browser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
"github.com/mafredri/cdp"
|
||||
"github.com/mafredri/cdp/protocol/runtime"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -15,82 +11,65 @@ type WaitTask struct {
|
||||
client *cdp.Client
|
||||
predicate string
|
||||
timeout time.Duration
|
||||
polling time.Duration
|
||||
}
|
||||
|
||||
const DefaultPolling = time.Millisecond * time.Duration(200)
|
||||
|
||||
func NewWaitTask(
|
||||
client *cdp.Client,
|
||||
predicate string,
|
||||
timeout time.Duration,
|
||||
polling time.Duration,
|
||||
) *WaitTask {
|
||||
return &WaitTask{
|
||||
client,
|
||||
fmt.Sprintf("((function () {%s})())", predicate),
|
||||
predicate,
|
||||
timeout,
|
||||
polling,
|
||||
}
|
||||
}
|
||||
|
||||
func (task *WaitTask) Run() (core.Value, error) {
|
||||
var result core.Value = values.None
|
||||
var err error
|
||||
var done bool
|
||||
timer := time.NewTimer(task.timeout)
|
||||
|
||||
for !done {
|
||||
for {
|
||||
select {
|
||||
case <-timer.C:
|
||||
err = core.ErrTimeout
|
||||
done = true
|
||||
return values.None, core.ErrTimeout
|
||||
default:
|
||||
out, e := task.exec()
|
||||
out, err := task.eval()
|
||||
|
||||
if e != nil {
|
||||
done = true
|
||||
// JS expression failed
|
||||
// terminating
|
||||
if err != nil {
|
||||
timer.Stop()
|
||||
err = e
|
||||
|
||||
break
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
// JS output is not empty
|
||||
// terminating
|
||||
if out != values.None {
|
||||
timer.Stop()
|
||||
|
||||
result = out
|
||||
done = true
|
||||
break
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Nothing yet, let's wait before the next try
|
||||
time.Sleep(task.polling)
|
||||
}
|
||||
}
|
||||
|
||||
return result, err
|
||||
// TODO: Do we need this code?
|
||||
return values.None, core.ErrTimeout
|
||||
}
|
||||
|
||||
func (task *WaitTask) exec() (core.Value, error) {
|
||||
args := runtime.NewEvaluateArgs(task.predicate).SetReturnByValue(true)
|
||||
out, err := task.client.Runtime.Evaluate(context.Background(), args)
|
||||
|
||||
if err != nil {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
if out.ExceptionDetails != nil {
|
||||
ex := out.ExceptionDetails
|
||||
return values.None, core.Error(
|
||||
core.ErrUnexpected,
|
||||
fmt.Sprintf("%s %s", ex.Text, *ex.Exception.Description),
|
||||
)
|
||||
}
|
||||
|
||||
if out.Result.Type != "undefined" {
|
||||
var o interface{}
|
||||
|
||||
err := json.Unmarshal(out.Result.Value, &o)
|
||||
|
||||
if err != nil {
|
||||
return values.None, core.Error(core.ErrUnexpected, err.Error())
|
||||
}
|
||||
|
||||
return values.Parse(o), nil
|
||||
}
|
||||
|
||||
return values.None, nil
|
||||
func (task *WaitTask) eval() (core.Value, error) {
|
||||
return Eval(
|
||||
task.client,
|
||||
task.predicate,
|
||||
true,
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
@@ -16,7 +16,7 @@ func WaitElement(_ context.Context, args ...core.Value) (core.Value, error) {
|
||||
|
||||
arg := args[0]
|
||||
selector := args[1].String()
|
||||
timeout := values.NewInt(1000)
|
||||
timeout := values.NewInt(5000)
|
||||
|
||||
if len(args) > 2 {
|
||||
if args[2].Type() == core.IntType {
|
||||
@@ -30,7 +30,11 @@ func WaitElement(_ context.Context, args ...core.Value) (core.Value, error) {
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
doc := arg.(*browser.HtmlDocument)
|
||||
doc, ok := arg.(*browser.HtmlDocument)
|
||||
|
||||
if !ok {
|
||||
return values.False, core.Error(core.ErrInvalidType, "expected dynamic document")
|
||||
}
|
||||
|
||||
return values.None, doc.WaitForSelector(values.NewString(selector), timeout)
|
||||
}
|
||||
|
@@ -9,5 +9,6 @@ func NewLib() map[string]core.Function {
|
||||
"ELEMENT": Element,
|
||||
"ELEMENTS": Elements,
|
||||
"WAIT_ELEMENT": WaitElement,
|
||||
"CLICK": Click,
|
||||
}
|
||||
}
|
||||
|
@@ -4,7 +4,7 @@ import "github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
|
||||
func NewLib() map[string]core.Function {
|
||||
return map[string]core.Function{
|
||||
"SLEEP": Sleep,
|
||||
"LOG": Log,
|
||||
"WAIT": Wait,
|
||||
"LOG": Log,
|
||||
}
|
||||
}
|
||||
|
@@ -8,7 +8,7 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func Sleep(_ context.Context, inputs ...core.Value) (core.Value, error) {
|
||||
func Wait(_ context.Context, inputs ...core.Value) (core.Value, error) {
|
||||
err := core.ValidateArgs(inputs, 1, 1)
|
||||
|
||||
if err != nil {
|
||||
|
Reference in New Issue
Block a user