#22 Added WAIT_NAVIGATION

This commit is contained in:
Tim Voronov
2018-09-25 17:58:57 -04:00
parent 64d0f585b4
commit ea6007b99e
13 changed files with 602 additions and 287 deletions
-103
View File
@@ -1,103 +0,0 @@
package browser
import (
"context"
"fmt"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/mafredri/cdp/protocol/page"
)
type (
EventHandler func(event, message string)
EventBroker struct {
client page.LifecycleEventClient
handlers map[string][]EventHandler
cancel context.CancelFunc
}
)
func NewEventBroker(client page.LifecycleEventClient) *EventBroker {
return &EventBroker{
client,
make(map[string][]EventHandler),
nil,
}
}
func (broker *EventBroker) Start() error {
if broker.cancel != nil {
return core.Error(core.ErrInvalidOperation, "broker is already started")
}
ctx, cancel := context.WithCancel(context.Background())
broker.cancel = cancel
go func() {
for {
select {
case <-ctx.Done():
return
case <-broker.client.Ready():
reply, err := broker.client.Recv()
if err != nil {
fmt.Println("FAILED TO GET EVENT", err)
broker.Emit("error", err.Error())
return
}
fmt.Println("EVENT", reply.Name)
broker.Emit(reply.Name, "")
}
}
}()
return nil
}
func (broker *EventBroker) Stop() error {
if broker.cancel == nil {
return core.Error(core.ErrInvalidOperation, "broker is already stopped")
}
broker.cancel()
broker.client = nil
return nil
}
func (broker *EventBroker) Close() error {
if broker.cancel != nil {
broker.Stop()
}
return broker.client.Close()
}
func (broker *EventBroker) AddListener(event string, handler EventHandler) {
handlers, ok := broker.handlers[event]
if !ok {
handlers = make([]EventHandler, 0, 5)
broker.handlers[event] = handlers
}
handlers = append(handlers, handler)
}
func (broker *EventBroker) Emit(name, message string) {
handlers, ok := broker.handlers[name]
if !ok {
return
}
for _, handler := range handlers {
handler(name, message)
}
}
+1 -1
View File
@@ -54,7 +54,7 @@ func (drv *CdpDriver) GetDocument(ctx context.Context, url string) (values.HtmlN
return nil, err
}
return NewHtmlDocument(ctx, conn, url)
return LoadHtmlDocument(ctx, conn, url)
}
func (drv *CdpDriver) Close() error {
+227 -45
View File
@@ -2,9 +2,12 @@ package browser
import (
"context"
"crypto/sha512"
"fmt"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/stdlib/html/driver/browser/eval"
"github.com/MontFerret/ferret/pkg/stdlib/html/driver/browser/events"
"github.com/corpix/uarand"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
@@ -12,18 +15,20 @@ import (
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/rpcc"
"strings"
"sync"
"time"
)
type HtmlDocument struct {
*HtmlElement
conn *rpcc.Conn
client *cdp.Client
events *EventBroker
url string
sync.Mutex
conn *rpcc.Conn
client *cdp.Client
events *events.EventBroker
url string
element *HtmlElement
}
func NewHtmlDocument(
func LoadHtmlDocument(
ctx context.Context,
conn *rpcc.Conn,
url string,
@@ -76,29 +81,19 @@ func NewHtmlDocument(
return nil, err
}
root, err := getRootElement(ctx, client)
root, err := getRootElement(client)
if err != nil {
return nil, err
}
events, err := createEventBroker(ctx, client)
broker, err := createEventBroker(client)
if err != nil {
return nil, err
}
doc := &HtmlDocument{
NewHtmlElement(client, root.NodeID, root),
conn,
client,
events,
url,
}
doc.init()
return doc, nil
return NewHtmlDocument(conn, client, root, broker), nil
}
func waitForLoadEvent(ctx context.Context, client *cdp.Client) error {
@@ -117,11 +112,11 @@ func waitForLoadEvent(ctx context.Context, client *cdp.Client) error {
return loadEventFired.Close()
}
func getRootElement(ctx context.Context, client *cdp.Client) (dom.Node, error) {
func getRootElement(client *cdp.Client) (dom.Node, error) {
args := dom.NewGetDocumentArgs()
args.Depth = PointerInt(-1) // lets load the entire document
args.Depth = PointerInt(1) // lets load the entire document
d, err := client.DOM.GetDocument(ctx, args)
d, err := client.DOM.GetDocument(context.Background(), args)
if err != nil {
return dom.Node{}, err
@@ -130,23 +125,79 @@ func getRootElement(ctx context.Context, client *cdp.Client) (dom.Node, error) {
return d.Root, nil
}
func createEventBroker(ctx context.Context, client *cdp.Client) (*EventBroker, error) {
lfc, err := client.Page.LifecycleEvent(ctx)
func createEventBroker(client *cdp.Client) (*events.EventBroker, error) {
load, err := client.Page.LoadEventFired(context.Background())
if err != nil {
return nil, err
}
return NewEventBroker(lfc), nil
broker := events.NewEventBroker()
broker.AddEventStream("load", load, func() interface{} {
return new(page.LoadEventFiredReply)
})
err = broker.Start()
if err != nil {
broker.Close()
return nil, err
}
return broker, nil
}
func (doc *HtmlDocument) Close() error {
doc.events.Stop()
doc.events.Close()
func NewHtmlDocument(
conn *rpcc.Conn,
client *cdp.Client,
root dom.Node,
broker *events.EventBroker,
) *HtmlDocument {
doc := new(HtmlDocument)
doc.conn = conn
doc.client = client
doc.events = broker
doc.element = NewHtmlElement(client, root.NodeID, root)
doc.url = ""
doc.client.Page.Close(context.Background())
if root.BaseURL != nil {
doc.url = *root.BaseURL
}
return doc.conn.Close()
broker.AddEventListener("load", func(_ interface{}) {
doc.Lock()
defer doc.Unlock()
fmt.Println("NAVIGATED")
updated, err := getRootElement(client)
if err != nil {
// TODO: We need somehow log all errors outside of stdout
return
}
// close an old root element
doc.element.Close()
// create a new root element wrapper
doc.element = NewHtmlElement(client, updated.NodeID, updated)
doc.url = ""
if updated.BaseURL != nil {
doc.url = *updated.BaseURL
}
})
return doc
}
func (doc *HtmlDocument) MarshalJSON() ([]byte, error) {
doc.Lock()
defer doc.Unlock()
return doc.element.MarshalJSON()
}
func (doc *HtmlDocument) Type() core.Type {
@@ -154,10 +205,38 @@ func (doc *HtmlDocument) Type() core.Type {
}
func (doc *HtmlDocument) String() string {
doc.Lock()
defer doc.Unlock()
return doc.url
}
func (doc *HtmlDocument) Unwrap() interface{} {
doc.Lock()
defer doc.Unlock()
return doc.element
}
func (doc *HtmlDocument) Hash() int {
doc.Lock()
defer doc.Unlock()
h := sha512.New()
out, err := h.Write([]byte(doc.url))
if err != nil {
return 0
}
return out
}
func (doc *HtmlDocument) Compare(other core.Value) int {
doc.Lock()
defer doc.Unlock()
switch other.Type() {
case core.HtmlDocumentType:
other := other.(*HtmlDocument)
@@ -172,8 +251,104 @@ func (doc *HtmlDocument) Compare(other core.Value) int {
}
}
func (doc *HtmlDocument) Close() error {
doc.Lock()
defer doc.Unlock()
doc.events.Stop()
doc.events.Close()
doc.client.Page.Close(context.Background())
return doc.conn.Close()
}
func (doc *HtmlDocument) NodeType() values.Int {
doc.Lock()
defer doc.Unlock()
return doc.element.NodeType()
}
func (doc *HtmlDocument) NodeName() values.String {
doc.Lock()
defer doc.Unlock()
return doc.element.NodeName()
}
func (doc *HtmlDocument) Length() values.Int {
doc.Lock()
defer doc.Unlock()
return doc.element.Length()
}
func (doc *HtmlDocument) InnerText() values.String {
doc.Lock()
defer doc.Unlock()
return doc.element.InnerText()
}
func (doc *HtmlDocument) InnerHtml() values.String {
doc.Lock()
defer doc.Unlock()
return doc.element.InnerHtml()
}
func (doc *HtmlDocument) Value() core.Value {
doc.Lock()
defer doc.Unlock()
return doc.element.Value()
}
func (doc *HtmlDocument) GetAttributes() core.Value {
doc.Lock()
defer doc.Unlock()
return doc.element.GetAttributes()
}
func (doc *HtmlDocument) GetAttribute(name values.String) core.Value {
doc.Lock()
defer doc.Unlock()
return doc.element.GetAttribute(name)
}
func (doc *HtmlDocument) GetChildNodes() core.Value {
doc.Lock()
defer doc.Unlock()
return doc.element.GetChildNodes()
}
func (doc *HtmlDocument) GetChildNode(idx values.Int) core.Value {
doc.Lock()
defer doc.Unlock()
return doc.element.GetChildNode(idx)
}
func (doc *HtmlDocument) QuerySelector(selector values.String) core.Value {
doc.Lock()
defer doc.Unlock()
return doc.element.QuerySelector(selector)
}
func (doc *HtmlDocument) QuerySelectorAll(selector values.String) core.Value {
doc.Lock()
defer doc.Unlock()
return doc.element.QuerySelectorAll(selector)
}
func (doc *HtmlDocument) ClickBySelector(selector values.String) (values.Boolean, error) {
res, err := Eval(
res, err := eval.Eval(
doc.client,
fmt.Sprintf(`
var el = document.querySelector("%s");
@@ -203,7 +378,7 @@ func (doc *HtmlDocument) ClickBySelector(selector values.String) (values.Boolean
}
func (doc *HtmlDocument) WaitForSelector(selector values.String, timeout values.Int) error {
task := NewWaitTask(
task := events.NewWaitTask(
doc.client,
fmt.Sprintf(`
el = document.querySelector("%s");
@@ -215,7 +390,7 @@ func (doc *HtmlDocument) WaitForSelector(selector values.String, timeout values.
return null;
`, selector),
time.Millisecond*time.Duration(timeout),
DefaultPolling,
events.DefaultPolling,
)
_, err := task.Run()
@@ -223,19 +398,26 @@ func (doc *HtmlDocument) WaitForSelector(selector values.String, timeout values.
return err
}
func (doc *HtmlDocument) init() {
// doc.events.AddListener("")
}
func (doc *HtmlDocument) reload() error {
root, err := getRootElement(context.Background(), doc.client)
if err != nil {
return err
func (doc *HtmlDocument) WaitForNavigation(timeout values.Int) error {
timer := time.NewTimer(time.Millisecond * time.Duration(timeout))
onEvent := make(chan bool)
listener := func(_ interface{}) {
onEvent <- true
}
doc.url = *root.BaseURL
doc.id = root.NodeID
defer doc.events.RemoveEventListener("load", listener)
defer close(onEvent)
return nil
doc.events.AddEventListener("load", listener)
for {
select {
case <-onEvent:
timer.Stop()
return nil
case <-timer.C:
return core.ErrTimeout
}
}
}
+2 -1
View File
@@ -7,6 +7,7 @@ import (
"encoding/json"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/stdlib/html/driver/browser/events"
"github.com/MontFerret/ferret/pkg/stdlib/html/driver/common"
"github.com/PuerkitoBio/goquery"
"github.com/mafredri/cdp"
@@ -287,7 +288,7 @@ func (el *HtmlElement) Click() (values.Boolean, error) {
defer cancel()
return DispatchEvent(ctx, el.client, el.id, "click")
return events.DispatchEvent(ctx, el.client, el.id, "click")
}
func createCtx() (context.Context, context.CancelFunc) {
@@ -0,0 +1,51 @@
package eval
import (
"context"
"encoding/json"
"fmt"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/runtime"
)
func PrepareEval(exp string) string {
return fmt.Sprintf("((function () {%s})())", exp)
}
func Eval(client *cdp.Client, exp string, ret bool, async bool) (core.Value, error) {
args := runtime.
NewEvaluateArgs(PrepareEval(exp)).
SetReturnByValue(ret).
SetAwaitPromise(async)
out, err := client.Runtime.Evaluate(context.Background(), args)
if err != nil {
return values.None, err
}
if out.ExceptionDetails != nil {
ex := out.ExceptionDetails
return values.None, core.Error(
core.ErrUnexpected,
fmt.Sprintf("%s: %s", ex.Text, *ex.Exception.Description),
)
}
if out.Result.Type != "undefined" {
var o interface{}
err := json.Unmarshal(out.Result.Value, &o)
if err != nil {
return values.None, core.Error(core.ErrUnexpected, err.Error())
}
return values.Parse(o), nil
}
return values.None, nil
}
@@ -0,0 +1,190 @@
package events
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/mafredri/cdp/rpcc"
"reflect"
"sync"
"time"
)
type (
MessageFactory func() interface{}
EventStream struct {
stream rpcc.Stream
message MessageFactory
}
EventListener func(message interface{})
EventBroker struct {
sync.Mutex
events map[string]*EventStream
listeners map[string][]EventListener
cancel context.CancelFunc
}
)
func NewEventBroker() *EventBroker {
broker := new(EventBroker)
broker.events = make(map[string]*EventStream)
broker.listeners = make(map[string][]EventListener)
return broker
}
func (broker *EventBroker) AddEventStream(name string, stream rpcc.Stream, msg MessageFactory) error {
broker.Lock()
defer broker.Unlock()
_, exists := broker.events[name]
if exists {
return core.Error(core.ErrNotUnique, name)
}
broker.events[name] = &EventStream{stream, msg}
return nil
}
func (broker *EventBroker) AddEventListener(event string, listener EventListener) {
broker.Lock()
defer broker.Unlock()
listeners, ok := broker.listeners[event]
if !ok {
listeners = make([]EventListener, 0, 5)
}
broker.listeners[event] = append(listeners, listener)
}
func (broker *EventBroker) RemoveEventListener(event string, listener EventListener) {
broker.Lock()
defer broker.Unlock()
idx := -1
listeners, ok := broker.listeners[event]
if !ok {
return
}
listenerPointer := reflect.ValueOf(listener).Pointer()
for i, l := range listeners {
itemPointer := reflect.ValueOf(l).Pointer()
if itemPointer == listenerPointer {
idx = i
break
}
}
if idx < 0 {
return
}
var modifiedListeners []EventListener = nil
if len(listeners) > 1 {
modifiedListeners = append(listeners[:idx], listeners[idx+1:]...)
}
broker.listeners[event] = modifiedListeners
}
func (broker *EventBroker) Start() error {
broker.Lock()
defer broker.Unlock()
if broker.cancel != nil {
return core.Error(core.ErrInvalidOperation, "broker is already started")
}
ctx, cancel := context.WithCancel(context.Background())
broker.cancel = cancel
go func() {
counter := 0
eventsCount := len(broker.events)
for {
for name, event := range broker.events {
counter += 1
select {
case <-ctx.Done():
return
case <-event.stream.Ready():
msg := event.message()
err := event.stream.RecvMsg(msg)
if err != nil {
broker.emit("error", err)
return
}
broker.emit(name, msg)
default:
// we have iterated over all events
// lets pause
if counter == eventsCount {
counter = 0
time.Sleep(DefaultPolling)
}
continue
}
}
}
}()
return nil
}
func (broker *EventBroker) Stop() error {
broker.Lock()
defer broker.Unlock()
if broker.cancel == nil {
return core.Error(core.ErrInvalidOperation, "broker is already stopped")
}
broker.cancel()
return nil
}
func (broker *EventBroker) Close() error {
broker.Lock()
defer broker.Unlock()
if broker.cancel != nil {
broker.Stop()
}
for _, event := range broker.events {
event.stream.Close()
}
return nil
}
func (broker *EventBroker) emit(name string, message interface{}) {
broker.Lock()
defer broker.Unlock()
listeners, ok := broker.listeners[name]
if !ok {
return
}
for _, listener := range listeners {
listener(message)
}
}
@@ -0,0 +1,75 @@
package events
import (
"context"
"fmt"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/stdlib/html/driver/browser/eval"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/runtime"
)
func DispatchEvent(
ctx context.Context,
client *cdp.Client,
id dom.NodeID,
eventName string,
) (values.Boolean, error) {
// get a ref to remote object representing the node
obj, err := client.DOM.ResolveNode(
ctx,
dom.NewResolveNodeArgs().
SetNodeID(id),
)
if err != nil {
return values.False, err
}
if obj.Object.ObjectID == nil {
return values.False, nil
}
evt, err := client.Runtime.Evaluate(ctx, runtime.NewEvaluateArgs(eval.PrepareEval(fmt.Sprintf(`
return new window.MouseEvent('%s', { bubbles: true })
`, eventName))))
if err != nil {
return values.False, nil
}
if evt.ExceptionDetails != nil {
return values.False, evt.ExceptionDetails
}
if evt.Result.ObjectID == nil {
return values.False, nil
}
evtId := evt.Result.ObjectID
// release the event object
defer client.Runtime.ReleaseObject(ctx, runtime.NewReleaseObjectArgs(*evtId))
res, err := client.Runtime.CallFunctionOn(
ctx,
runtime.NewCallFunctionOnArgs("dispatchEvent").
SetObjectID(*obj.Object.ObjectID).
SetArguments([]runtime.CallArgument{
{
ObjectID: evt.Result.ObjectID,
},
}),
)
if err != nil {
return values.False, err
}
if res.ExceptionDetails != nil {
return values.False, res.ExceptionDetails
}
return values.True, nil
}
@@ -1,8 +1,9 @@
package browser
package events
import (
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/stdlib/html/driver/browser/eval"
"github.com/mafredri/cdp"
"time"
)
@@ -38,7 +39,12 @@ func (task *WaitTask) Run() (core.Value, error) {
case <-timer.C:
return values.None, core.ErrTimeout
default:
out, err := task.eval()
out, err := eval.Eval(
task.client,
task.predicate,
true,
false,
)
// JS expression failed
// terminating
@@ -60,16 +66,4 @@ func (task *WaitTask) Run() (core.Value, error) {
time.Sleep(task.polling)
}
}
// TODO: Do we need this code?
return values.None, core.ErrTimeout
}
func (task *WaitTask) eval() (core.Value, error) {
return Eval(
task.client,
task.predicate,
true,
false,
)
}
-112
View File
@@ -1,14 +1,6 @@
package browser
import (
"context"
"encoding/json"
"fmt"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/runtime"
"golang.org/x/sync/errgroup"
)
@@ -27,107 +19,3 @@ func RunBatch(funcs ...BatchFunc) error {
return eg.Wait()
}
func PrepareEval(exp string) string {
return fmt.Sprintf("((function () {%s})())", exp)
}
func Eval(client *cdp.Client, exp string, ret bool, async bool) (core.Value, error) {
args := runtime.
NewEvaluateArgs(PrepareEval(exp)).
SetReturnByValue(ret).
SetAwaitPromise(async)
out, err := client.Runtime.Evaluate(context.Background(), args)
if err != nil {
return values.None, err
}
if out.ExceptionDetails != nil {
ex := out.ExceptionDetails
return values.None, core.Error(
core.ErrUnexpected,
fmt.Sprintf("%s: %s", ex.Text, *ex.Exception.Description),
)
}
if out.Result.Type != "undefined" {
var o interface{}
err := json.Unmarshal(out.Result.Value, &o)
if err != nil {
return values.None, core.Error(core.ErrUnexpected, err.Error())
}
return values.Parse(o), nil
}
return values.None, nil
}
func DispatchEvent(
ctx context.Context,
client *cdp.Client,
id dom.NodeID,
eventName string,
) (values.Boolean, error) {
// get a ref to remote object representing the node
obj, err := client.DOM.ResolveNode(
ctx,
dom.NewResolveNodeArgs().
SetNodeID(id),
)
if err != nil {
return values.False, err
}
if obj.Object.ObjectID == nil {
return values.False, nil
}
evt, err := client.Runtime.Evaluate(ctx, runtime.NewEvaluateArgs(PrepareEval(fmt.Sprintf(`
return new window.MouseEvent('%s', { bubbles: true })
`, eventName))))
if err != nil {
return values.False, nil
}
if evt.ExceptionDetails != nil {
return values.False, evt.ExceptionDetails
}
if evt.Result.ObjectID == nil {
return values.False, nil
}
evtId := evt.Result.ObjectID
// release the event object
defer client.Runtime.ReleaseObject(ctx, runtime.NewReleaseObjectArgs(*evtId))
res, err := client.Runtime.CallFunctionOn(
ctx,
runtime.NewCallFunctionOnArgs("dispatchEvent").
SetObjectID(*obj.Object.ObjectID).
SetArguments([]runtime.CallArgument{
{
ObjectID: evt.Result.ObjectID,
},
}),
)
if err != nil {
return values.False, err
}
if res.ExceptionDetails != nil {
return values.False, res.ExceptionDetails
}
return values.True, nil
}
+34
View File
@@ -38,3 +38,37 @@ func WaitElement(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, doc.WaitForSelector(values.NewString(selector), timeout)
}
func WaitNavigation(_ context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 1, 2)
if err != nil {
return values.None, err
}
err = core.ValidateType(args[0], core.HtmlDocumentType)
if err != nil {
return values.None, err
}
doc, ok := args[0].(*browser.HtmlDocument)
if !ok {
return values.None, core.Error(core.ErrInvalidType, "expected dynamic document")
}
timeout := values.NewInt(5000)
if len(args) > 1 {
err = core.ValidateType(args[1], core.IntType)
if err != nil {
return values.None, err
}
timeout = args[1].(values.Int)
}
return values.None, doc.WaitForNavigation(timeout)
}
+7 -6
View File
@@ -4,11 +4,12 @@ import "github.com/MontFerret/ferret/pkg/runtime/core"
func NewLib() map[string]core.Function {
return map[string]core.Function{
"DOCUMENT": Document,
"DOCUMENT_PARSE": DocumentParse,
"ELEMENT": Element,
"ELEMENTS": Elements,
"WAIT_ELEMENT": WaitElement,
"CLICK": Click,
"DOCUMENT": Document,
"DOCUMENT_PARSE": DocumentParse,
"ELEMENT": Element,
"ELEMENTS": Elements,
"WAIT_ELEMENT": WaitElement,
"WAIT_NAVIGATION": WaitNavigation,
"CLICK": Click,
}
}