go/ferret
go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2025-03-05 15:16:07 +02:00
Code Issues Releases Activity

Refactoring/input manager (#316)

Browse Source 
* Normalized and externalized input logic

* Fixed linting issue

* Removed redundant mutex

* Added missed locks in Page

* Fixed deadlock
This commit is contained in:
Tim Voronov 2019-06-20 13:21:48 -04:00 committed by GitHub
parent d7b923e4c3
commit eee801fb5b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 677 additions and 576 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

307
pkg/drivers/cdp/document.go
View File

@ -3,35 +3,31 @@ package cdp
import (
"context"
"fmt"
"hash/fnv"
"sync"
"time"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/input"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"hash/fnv"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
"github.com/MontFerret/ferret/pkg/drivers/cdp/input"
"github.com/MontFerret/ferret/pkg/drivers/cdp/templates"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
const BlankPageURL = "about:blank"
type HTMLDocument struct {
mu sync.Mutex
logger *zerolog.Logger
client *cdp.Client
events *events.EventBroker
input *input.Manager
exec *eval.ExecutionContext
frames page.FrameTree
element *HTMLElement
@ -43,7 +39,9 @@ func LoadRootHTMLDocument(
ctx context.Context,
logger *zerolog.Logger,
client *cdp.Client,
broker *events.EventBroker,
events *events.EventBroker,
mouse *input.Mouse,
keyboard *input.Keyboard,
) (*HTMLDocument, error) {
gdRepl, err := client.DOM.GetDocument(ctx, dom.NewGetDocumentArgs().SetDepth(1))
@ -61,7 +59,9 @@ func LoadRootHTMLDocument(
ctx,
logger,
client,
broker,
events,
mouse,
keyboard,
gdRepl.Root,
ftRepl.FrameTree,
eval.EmptyExecutionContextID,
@ -73,19 +73,23 @@ func LoadHTMLDocument(
ctx context.Context,
logger *zerolog.Logger,
client *cdp.Client,
broker *events.EventBroker,
events *events.EventBroker,
mouse *input.Mouse,
keyboard *input.Keyboard,
node dom.Node,
tree page.FrameTree,
execID runtime.ExecutionContextID,
parent *HTMLDocument,
) (*HTMLDocument, error) {
exec := eval.NewExecutionContext(client, tree.Frame, execID)
inputManager := input.NewManager(client, exec, keyboard, mouse)
rootElement, err := LoadHTMLElement(
ctx,
logger,
client,
broker,
events,
inputManager,
exec,
node.NodeID,
node.BackendNodeID,
@ -98,7 +102,8 @@ func LoadHTMLDocument(
return NewHTMLDocument(
logger,
client,
broker,
events,
inputManager,
exec,
rootElement,
tree,
@ -109,7 +114,8 @@ func LoadHTMLDocument(
func NewHTMLDocument(
logger *zerolog.Logger,
client *cdp.Client,
broker *events.EventBroker,
events *events.EventBroker,
input *input.Manager,
exec *eval.ExecutionContext,
rootElement *HTMLElement,
frames page.FrameTree,
@ -118,7 +124,8 @@ func NewHTMLDocument(
doc := new(HTMLDocument)
doc.logger = logger
doc.client = client
doc.events = broker
doc.events = events
doc.input = input
doc.exec = exec
doc.element = rootElement
doc.frames = frames
@ -129,9 +136,6 @@ func NewHTMLDocument(
}
func (doc *HTMLDocument) MarshalJSON() ([]byte, error) {
doc.mu.Lock()
defer doc.mu.Unlock()
return doc.element.MarshalJSON()
}
@ -140,23 +144,14 @@ func (doc *HTMLDocument) Type() core.Type {
}
func (doc *HTMLDocument) String() string {
doc.mu.Lock()
defer doc.mu.Unlock()
return doc.frames.Frame.URL
}
func (doc *HTMLDocument) Unwrap() interface{} {
doc.mu.Lock()
defer doc.mu.Unlock()
return doc.element
}
func (doc *HTMLDocument) Hash() uint64 {
doc.mu.Lock()
defer doc.mu.Unlock()
h := fnv.New64a()
h.Write([]byte(doc.Type().String()))
@ -172,9 +167,6 @@ func (doc *HTMLDocument) Copy() core.Value {
}
func (doc *HTMLDocument) Compare(other core.Value) int64 {
doc.mu.Lock()
defer doc.mu.Unlock()
switch other.Type() {
case drivers.HTMLDocumentType:
other := other.(drivers.HTMLDocument)
@ -198,9 +190,6 @@ func (doc *HTMLDocument) SetIn(ctx context.Context, path []core.Value, value cor
}
func (doc *HTMLDocument) Close() error {
doc.mu.Lock()
defer doc.mu.Unlock()
errs := make([]error, 0, 5)
if doc.children.Ready() {
@ -239,9 +228,6 @@ func (doc *HTMLDocument) Close() error {
}
func (doc *HTMLDocument) IsDetached() values.Boolean {
doc.mu.Lock()
defer doc.mu.Unlock()
return doc.element.IsDetached()
}
@ -254,51 +240,30 @@ func (doc *HTMLDocument) GetNodeName() values.String {
}
func (doc *HTMLDocument) GetChildNodes(ctx context.Context) core.Value {
doc.mu.Lock()
defer doc.mu.Unlock()
return doc.element.GetChildNodes(ctx)
}
func (doc *HTMLDocument) GetChildNode(ctx context.Context, idx values.Int) core.Value {
doc.mu.Lock()
defer doc.mu.Unlock()
return doc.element.GetChildNode(ctx, idx)
}
func (doc *HTMLDocument) QuerySelector(ctx context.Context, selector values.String) core.Value {
doc.mu.Lock()
defer doc.mu.Unlock()
return doc.element.QuerySelector(ctx, selector)
}
func (doc *HTMLDocument) QuerySelectorAll(ctx context.Context, selector values.String) core.Value {
doc.mu.Lock()
defer doc.mu.Unlock()
return doc.element.QuerySelectorAll(ctx, selector)
}
func (doc *HTMLDocument) CountBySelector(ctx context.Context, selector values.String) values.Int {
doc.mu.Lock()
defer doc.mu.Unlock()
return doc.element.CountBySelector(ctx, selector)
}
func (doc *HTMLDocument) ExistsBySelector(ctx context.Context, selector values.String) values.Boolean {
doc.mu.Lock()
defer doc.mu.Unlock()
return doc.element.ExistsBySelector(ctx, selector)
}
func (doc *HTMLDocument) GetTitle() values.String {
doc.mu.Lock()
defer doc.mu.Unlock()
value, err := doc.exec.ReadProperty(context.Background(), doc.element.id.objectID, "title")
if err != nil {
@ -311,9 +276,6 @@ func (doc *HTMLDocument) GetTitle() values.String {
}
func (doc *HTMLDocument) GetName() values.String {
doc.mu.Lock()
defer doc.mu.Unlock()
if doc.frames.Frame.Name != nil {
return values.NewString(*doc.frames.Frame.Name)
}
@ -322,16 +284,10 @@ func (doc *HTMLDocument) GetName() values.String {
}
func (doc *HTMLDocument) GetParentDocument() drivers.HTMLDocument {
doc.mu.Lock()
defer doc.mu.Unlock()
return doc.parent
}
func (doc *HTMLDocument) GetChildDocuments(ctx context.Context) (*values.Array, error) {
doc.mu.Lock()
defer doc.mu.Unlock()
children, err := doc.children.Read(ctx)
if err != nil {
@ -342,204 +298,59 @@ func (doc *HTMLDocument) GetChildDocuments(ctx context.Context) (*values.Array,
}
func (doc *HTMLDocument) Length() values.Int {
doc.mu.Lock()
defer doc.mu.Unlock()
return doc.element.Length()
}
func (doc *HTMLDocument) GetElement() drivers.HTMLElement {
doc.mu.Lock()
defer doc.mu.Unlock()
return doc.element
}
func (doc *HTMLDocument) GetURL() values.String {
doc.mu.Lock()
defer doc.mu.Unlock()
return values.NewString(doc.frames.Frame.URL)
}
func (doc *HTMLDocument) ClickBySelector(ctx context.Context, selector values.String) (values.Boolean, error) {
res, err := doc.exec.EvalWithReturn(
ctx,
fmt.Sprintf(`
var el = document.querySelector(%s);
if (el == null) {
return false;
}
var evt = new window.MouseEvent('click', { bubbles: true, cancelable: true });
el.dispatchEvent(evt);
return true;
`, eval.ParamString(selector.String())),
)
if err != nil {
if err := doc.input.ClickBySelector(ctx, doc.element.id.nodeID, selector); err != nil {
return values.False, err
}
if res.Type() == types.Boolean {
return res.(values.Boolean), nil
}
return values.False, nil
return values.True, nil
}
func (doc *HTMLDocument) ClickBySelectorAll(ctx context.Context, selector values.String) (values.Boolean, error) {
res, err := doc.exec.EvalWithReturn(
ctx,
fmt.Sprintf(`
var elements = document.querySelectorAll(%s);
if (elements == null) {
return false;
}
elements.forEach((el) => {
var evt = new window.MouseEvent('click', { bubbles: true, cancelable: true });
el.dispatchEvent(evt);
});
return true;
`, eval.ParamString(selector.String())),
)
found, err := doc.client.DOM.QuerySelectorAll(ctx, dom.NewQuerySelectorAllArgs(doc.element.id.nodeID, selector.String()))
if err != nil {
return values.False, err
}
if res.Type() == types.Boolean {
return res.(values.Boolean), nil
for _, nodeID := range found.NodeIDs {
if err := doc.input.ClickByNodeID(ctx, nodeID); err != nil {
return values.False, err
}
}
return values.False, nil
return values.True, nil
}
func (doc *HTMLDocument) InputBySelector(ctx context.Context, selector values.String, value core.Value, delay values.Int) (values.Boolean, error) {
valStr := value.String()
res, err := doc.exec.EvalWithReturn(
ctx,
fmt.Sprintf(`
var el = document.querySelector(%s);
if (el == null) {
return false;
}
el.focus();
return true;
`, eval.ParamString(selector.String())),
)
if err != nil {
if err := doc.input.TypeBySelector(ctx, doc.element.id.nodeID, selector, value, delay); err != nil {
return values.False, err
}
if res.Type() == types.Boolean && res.(values.Boolean) == values.False {
return values.False, nil
}
// Initial delay after focusing but before typing
time.Sleep(time.Duration(delay) * time.Millisecond)
for _, ch := range valStr {
for _, ev := range []string{"keyDown", "keyUp"} {
ke := input.NewDispatchKeyEventArgs(ev).SetText(string(ch))
if err := doc.client.Input.DispatchKeyEvent(ctx, ke); err != nil {
return values.False, err
}
}
time.Sleep(randomDuration(delay) * time.Millisecond)
}
return values.True, nil
}
func (doc *HTMLDocument) SelectBySelector(ctx context.Context, selector values.String, value *values.Array) (*values.Array, error) {
res, err := doc.exec.EvalWithReturn(
ctx,
fmt.Sprintf(`
var element = document.querySelector(%s);
if (element == null) {
return [];
}
var values = %s;
if (element.nodeName.toLowerCase() !== 'select') {
throw new Error('GetElement is not a <select> element.');
}
var options = Array.from(element.options);
element.value = undefined;
for (var option of options) {
option.selected = values.includes(option.value);
if (option.selected && !element.multiple) {
break;
}
}
element.dispatchEvent(new Event('input', { 'bubbles': true, cancelable: true }));
element.dispatchEvent(new Event('change', { 'bubbles': true, cancelable: true }));
return options.filter(option => option.selected).map(option => option.value);
`,
eval.ParamString(selector.String()),
value.String(),
),
)
if err != nil {
return nil, err
}
arr, ok := res.(*values.Array)
if ok {
return arr, nil
}
return nil, core.TypeError(types.Array, res.Type())
return doc.input.SelectBySelector(ctx, doc.element.id.nodeID, selector, value)
}
func (doc *HTMLDocument) MoveMouseBySelector(ctx context.Context, selector values.String) error {
err := doc.ScrollBySelector(ctx, selector)
if err != nil {
return err
}
selectorArgs := dom.NewQuerySelectorArgs(doc.element.id.nodeID, selector.String())
found, err := doc.client.DOM.QuerySelector(ctx, selectorArgs)
if err != nil {
doc.element.logError(err).
Str("selector", selector.String()).
Msg("failed to retrieve a node by selector")
return err
}
if found.NodeID <= 0 {
return errors.New("element not found")
}
q, err := getClickablePoint(ctx, doc.client, HTMLElementIdentity{
nodeID: found.NodeID,
})
if err != nil {
return err
}
return doc.client.Input.DispatchMouseEvent(
ctx,
input.NewDispatchMouseEventArgs("mouseMoved", q.X, q.Y),
)
return doc.input.MoveMouseBySelector(ctx, doc.element.id.nodeID, selector)
}
func (doc *HTMLDocument) MoveMouseByXY(ctx context.Context, x, y values.Float) error {
return doc.client.Input.DispatchMouseEvent(
ctx,
input.NewDispatchMouseEventArgs("mouseMoved", float64(x), float64(y)),
)
return doc.input.MoveMouse(ctx, x, y)
}
func (doc *HTMLDocument) WaitForElement(ctx context.Context, selector values.String, when drivers.WaitEvent) error {
@ -690,50 +501,19 @@ func (doc *HTMLDocument) WaitForStyleBySelectorAll(ctx context.Context, selector
}
func (doc *HTMLDocument) ScrollTop(ctx context.Context) error {
return doc.exec.Eval(ctx, `
window.scrollTo({
left: 0,
top: 0,
behavior: 'instant'
});
`)
return doc.input.ScrollTop(ctx)
}
func (doc *HTMLDocument) ScrollBottom(ctx context.Context) error {
return doc.exec.Eval(ctx, `
window.scrollTo({
left: 0,
top: window.document.body.scrollHeight,
behavior: 'instant'
});
`)
return doc.input.ScrollBottom(ctx)
}
func (doc *HTMLDocument) ScrollBySelector(ctx context.Context, selector values.String) error {
return doc.exec.Eval(ctx, fmt.Sprintf(`
var el = document.querySelector(%s);
if (el == null) {
throw new Error("element not found");
}
el.scrollIntoView({
behavior: 'instant'
});
return true;
`, eval.ParamString(selector.String()),
))
return doc.input.ScrollIntoViewBySelector(ctx, selector)
}
func (doc *HTMLDocument) ScrollByXY(ctx context.Context, x, y values.Float) error {
return doc.exec.Eval(ctx, fmt.Sprintf(`
window.scrollBy({
top: %s,
left: %s,
behavior: 'instant'
});
`,
eval.ParamFloat(float64(x)),
eval.ParamFloat(float64(y)),
))
return doc.input.Scroll(ctx, x, y)
}
func (doc *HTMLDocument) loadChildren(ctx context.Context) (value core.Value, e error) {
@ -747,7 +527,18 @@ func (doc *HTMLDocument) loadChildren(ctx context.Context) (value core.Value, e
return nil, errors.Wrap(err, "failed to resolve frame node")
}
cfDocument, err := LoadHTMLDocument(ctx, doc.logger, doc.client, doc.events, cfNode, cf, cfExecID, doc)
cfDocument, err := LoadHTMLDocument(
ctx,
doc.logger,
doc.client,
doc.events,
doc.input.Mouse(),
doc.input.Keyboard(),
cfNode,
cf,
cfExecID,
doc,
)
if err != nil {
return nil, errors.Wrap(err, "failed to load frame document")

210
pkg/drivers/cdp/element.go
View File

@ -9,19 +9,18 @@ import (
"strconv"
"strings"
"sync"
"time"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
"github.com/MontFerret/ferret/pkg/drivers/cdp/input"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
"github.com/gofrs/uuid"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/input"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/rs/zerolog"
)
@ -41,6 +40,7 @@ type (
logger *zerolog.Logger
client *cdp.Client
events *events.EventBroker
input *input.Manager
exec *eval.ExecutionContext
connected values.Boolean
id HTMLElementIdentity
@ -61,6 +61,7 @@ func LoadHTMLElement(
logger *zerolog.Logger,
client *cdp.Client,
broker *events.EventBroker,
input *input.Manager,
exec *eval.ExecutionContext,
nodeID dom.NodeID,
backendID dom.BackendNodeID,
@ -128,6 +129,7 @@ func LoadHTMLElement(
logger,
client,
broker,
input,
exec,
id,
node.Node.NodeType,
@ -142,6 +144,7 @@ func NewHTMLElement(
logger *zerolog.Logger,
client *cdp.Client,
broker *events.EventBroker,
input *input.Manager,
exec *eval.ExecutionContext,
id HTMLElementIdentity,
nodeType int,
@ -154,6 +157,7 @@ func NewHTMLElement(
el.logger = logger
el.client = client
el.events = broker
el.input = input
el.exec = exec
el.connected = values.True
el.id = id
@ -502,7 +506,7 @@ func (el *HTMLElement) QuerySelector(ctx context.Context, selector values.String
return values.None
}
res, err := LoadHTMLElement(ctx, el.logger, el.client, el.events, el.exec, found.NodeID, emptyBackendID)
res, err := LoadHTMLElement(ctx, el.logger, el.client, el.events, el.input, el.exec, found.NodeID, emptyBackendID)
if err != nil {
el.logError(err).
@ -543,7 +547,7 @@ func (el *HTMLElement) QuerySelectorAll(ctx context.Context, selector values.Str
continue
}
childEl, err := LoadHTMLElement(ctx, el.logger, el.client, el.events, el.exec, id, emptyBackendID)
childEl, err := LoadHTMLElement(ctx, el.logger, el.client, el.events, el.input, el.exec, id, emptyBackendID)
if err != nil {
el.logError(err).
@ -891,46 +895,7 @@ func (el *HTMLElement) WaitForStyle(ctx context.Context, name values.String, val
}
func (el *HTMLElement) Click(ctx context.Context) (values.Boolean, error) {
if err := el.ScrollIntoView(ctx); err != nil {
return values.False, err
}
points, err := getClickablePoint(ctx, el.client, el.id)
if err != nil {
return values.False, err
}
moveArgs := input.NewDispatchMouseEventArgs("mouseMoved", points.X, points.Y)
if err := el.client.Input.DispatchMouseEvent(ctx, moveArgs); err != nil {
return values.False, err
}
beforePressDelay := time.Duration(core.Random(100, 50))
time.Sleep(beforePressDelay)
btn := "left"
clickCount := 1
downArgs := input.NewDispatchMouseEventArgs("mousePressed", points.X, points.Y)
downArgs.ClickCount = &clickCount
downArgs.Button = &btn
if err := el.client.Input.DispatchMouseEvent(ctx, downArgs); err != nil {
return values.False, err
}
beforeReleaseDelay := time.Duration(core.Random(50, 25))
time.Sleep(beforeReleaseDelay * time.Millisecond)
upArgs := input.NewDispatchMouseEventArgs("mouseReleased", points.X, points.Y)
upArgs.ClickCount = &clickCount
upArgs.Button = &btn
if err := el.client.Input.DispatchMouseEvent(ctx, upArgs); err != nil {
if err := el.input.ClickByNodeID(ctx, el.id.nodeID); err != nil {
return values.False, err
}
@ -938,163 +903,23 @@ func (el *HTMLElement) Click(ctx context.Context) (values.Boolean, error) {
}
func (el *HTMLElement) Input(ctx context.Context, value core.Value, delay values.Int) error {
if err := el.client.DOM.Focus(ctx, dom.NewFocusArgs().SetObjectID(el.id.objectID)); err != nil {
el.logError(err).Msg("failed to focus")
return err
if el.GetNodeName() != "INPUT" {
return core.Error(core.ErrInvalidOperation, "element is not an <input> element.")
}
delayMs := time.Duration(delay)
time.Sleep(delayMs * time.Millisecond)
valStr := value.String()
for _, ch := range valStr {
for _, ev := range []string{"keyDown", "keyUp"} {
ke := input.NewDispatchKeyEventArgs(ev).SetText(string(ch))
if err := el.client.Input.DispatchKeyEvent(ctx, ke); err != nil {
el.logError(err).Str("value", value.String()).Msg("failed to input a value")
return err
}
time.Sleep(delayMs * time.Millisecond)
}
}
return nil
return el.input.TypeByNodeID(ctx, el.id.nodeID, value, delay)
}
func (el *HTMLElement) Select(ctx context.Context, value *values.Array) (*values.Array, error) {
var attrID = "data-ferret-select"
if el.GetNodeName() != "SELECT" {
return nil, core.Error(core.ErrInvalidOperation, "element is not a <select> element.")
}
id, err := uuid.NewV4()
if err != nil {
return nil, err
}
err = el.client.DOM.SetAttributeValue(ctx, dom.NewSetAttributeValueArgs(el.id.nodeID, attrID, id.String()))
if err != nil {
return nil, err
}
res, err := el.exec.EvalWithReturn(
ctx,
fmt.Sprintf(`
var el = document.querySelector('[%s="%s"]');
if (el == null) {
return [];
}
var values = %s;
if (el.nodeName.toLowerCase() !== 'select') {
throw new Error('element is not a <select> element.');
}
var options = Array.from(el.options);
el.value = undefined;
for (var option of options) {
option.selected = values.includes(option.value);
if (option.selected && !el.multiple) {
break;
}
}
el.dispatchEvent(new Event('input', { 'bubbles': true }));
el.dispatchEvent(new Event('change', { 'bubbles': true }));
return options.filter(option => option.selected).map(option => option.value);
`,
attrID,
id.String(),
value.String(),
),
)
if err != nil {
return nil, err
}
err = el.client.DOM.RemoveAttribute(ctx, dom.NewRemoveAttributeArgs(el.id.nodeID, attrID))
if err != nil {
return nil, err
}
arr, ok := res.(*values.Array)
if ok {
return arr, nil
}
return nil, core.TypeError(types.Array, res.Type())
return el.input.SelectByNodeID(ctx, el.id.nodeID, value)
}
func (el *HTMLElement) ScrollIntoView(ctx context.Context) error {
var attrID = "data-ferret-scroll"
id, err := uuid.NewV4()
if err != nil {
return err
}
err = el.client.DOM.SetAttributeValue(ctx, dom.NewSetAttributeValueArgs(el.id.nodeID, attrID, id.String()))
if err != nil {
return err
}
err = el.exec.Eval(
ctx,
fmt.Sprintf(`
var el = document.querySelector('[%s="%s"]');
if (el == null) {
throw new Error('element not found');
}
el.scrollIntoView({
behavior: 'instant',
inline: 'center',
block: 'center'
});
`,
attrID,
id.String(),
))
if err != nil {
return err
}
err = el.client.DOM.RemoveAttribute(ctx, dom.NewRemoveAttributeArgs(el.id.nodeID, attrID))
return err
return el.input.ScrollIntoViewByNodeID(ctx, el.id.nodeID)
}
func (el *HTMLElement) Hover(ctx context.Context) error {
err := el.ScrollIntoView(ctx)
if err != nil {
return err
}
q, err := getClickablePoint(ctx, el.client, el.id)
if err != nil {
return err
}
return el.client.Input.DispatchMouseEvent(
ctx,
input.NewDispatchMouseEventArgs("mouseMoved", q.X, q.Y),
)
return el.input.MoveMouseByNodeID(ctx, el.id.nodeID)
}
func (el *HTMLElement) IsDetached() values.Boolean {
@ -1157,6 +982,7 @@ func (el *HTMLElement) loadChildren(ctx context.Context) (core.Value, error) {
el.logger,
el.client,
el.events,
el.input,
el.exec,
childID.nodeID,
childID.backendID,
@ -1343,7 +1169,7 @@ func (el *HTMLElement) handleChildInserted(ctx context.Context, message interfac
el.loadedChildren.Write(ctx, func(v core.Value, _ error) {
loadedArr := v.(*values.Array)
loadedEl, err := LoadHTMLElement(ctx, el.logger, el.client, el.events, el.exec, nextID, emptyBackendID)
loadedEl, err := LoadHTMLElement(ctx, el.logger, el.client, el.events, el.input, el.exec, nextID, emptyBackendID)
if err != nil {
el.logError(err).Msg("failed to load an inserted element")

122
pkg/drivers/cdp/helpers.go
View File

@ -5,15 +5,14 @@ import (
"context"
"errors"
"golang.org/x/net/html"
"math"
"strings"
"time"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/PuerkitoBio/goquery"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
@ -27,11 +26,6 @@ var emptyExpires = time.Time{}
type (
batchFunc = func() error
Quad struct {
X float64
Y float64
}
)
func runBatch(funcs ...batchFunc) error {
@ -44,113 +38,6 @@ func runBatch(funcs ...batchFunc) error {
return eg.Wait()
}
func fromProtocolQuad(quad dom.Quad) []Quad {
return []Quad{
{
X: quad[0],
Y: quad[1],
},
{
X: quad[2],
Y: quad[3],
},
{
X: quad[4],
Y: quad[5],
},
{
X: quad[6],
Y: quad[7],
},
}
}
func computeQuadArea(quads []Quad) float64 {
var area float64
for i := range quads {
p1 := quads[i]
p2 := quads[(i+1)%len(quads)]
area += (p1.X*p2.Y - p2.X*p1.Y) / 2
}
return math.Abs(area)
}
func intersectQuadWithViewport(quad []Quad, width, height float64) []Quad {
quads := make([]Quad, 0, len(quad))
for _, point := range quad {
quads = append(quads, Quad{
X: math.Min(math.Max(point.X, 0), width),
Y: math.Min(math.Max(point.Y, 0), height),
})
}
return quads
}
func getClickablePoint(ctx context.Context, client *cdp.Client, id HTMLElementIdentity) (Quad, error) {
qargs := dom.NewGetContentQuadsArgs()
switch {
case id.objectID != "":
qargs.SetObjectID(id.objectID)
case id.backendID != 0:
qargs.SetBackendNodeID(id.backendID)
default:
qargs.SetNodeID(id.nodeID)
}
contentQuadsReply, err := client.DOM.GetContentQuads(ctx, qargs)
if err != nil {
return Quad{}, err
}
if contentQuadsReply.Quads == nil || len(contentQuadsReply.Quads) == 0 {
return Quad{}, errors.New("node is either not visible or not an HTMLElement")
}
layoutMetricsReply, err := client.Page.GetLayoutMetrics(ctx)
if err != nil {
return Quad{}, err
}
clientWidth := layoutMetricsReply.LayoutViewport.ClientWidth
clientHeight := layoutMetricsReply.LayoutViewport.ClientHeight
quads := make([][]Quad, 0, len(contentQuadsReply.Quads))
for _, q := range contentQuadsReply.Quads {
quad := intersectQuadWithViewport(fromProtocolQuad(q), float64(clientWidth), float64(clientHeight))
if computeQuadArea(quad) > 1 {
quads = append(quads, quad)
}
}
if len(quads) == 0 {
return Quad{}, errors.New("node is either not visible or not an HTMLElement")
}
// Return the middle point of the first quad.
quad := quads[0]
var x float64
var y float64
for _, q := range quad {
x += q.X
y += q.Y
}
return Quad{
X: x / 4,
Y: y / 4,
}, nil
}
func parseAttrs(attrs []string) *values.Object {
var attr values.String
@ -406,13 +293,6 @@ func normalizeCookieURL(url string) string {
return httpPrefix + url
}
func randomDuration(delay values.Int) time.Duration {
max, min := core.NumberBoundaries(float64(int64(delay)))
value := core.Random(max, min)
return time.Duration(int64(value))
}
func resolveFrame(ctx context.Context, client *cdp.Client, frame page.Frame) (dom.Node, runtime.ExecutionContextID, error) {
worldRepl, err := client.Page.CreateIsolatedWorld(ctx, page.NewCreateIsolatedWorldArgs(frame.ID))

14
pkg/drivers/cdp/input/helpers.go Normal file
View File

@ -0,0 +1,14 @@
package input
import (
"time"
"github.com/MontFerret/ferret/pkg/runtime/core"
)
func randomDuration(delay int) time.Duration {
max, min := core.NumberBoundaries(float64(delay))
value := core.Random(max, min)
return time.Duration(int64(value))
}

52
pkg/drivers/cdp/input/keyboard.go Normal file
View File

@ -0,0 +1,52 @@
package input
import (
"context"
"time"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/input"
)
type Keyboard struct {
client *cdp.Client
}
func NewKeyboard(client *cdp.Client) *Keyboard {
return &Keyboard{client}
}
func (k *Keyboard) Down(ctx context.Context, char string) error {
return k.client.Input.DispatchKeyEvent(
ctx,
input.NewDispatchKeyEventArgs("keyDown").
SetText(char),
)
}
func (k *Keyboard) Up(ctx context.Context, char string) error {
return k.client.Input.DispatchKeyEvent(
ctx,
input.NewDispatchKeyEventArgs("keyUp").
SetText(char),
)
}
func (k *Keyboard) Type(ctx context.Context, text string, delay int) error {
for _, ch := range text {
ch := string(ch)
if err := k.Down(ctx, ch); err != nil {
return err
}
releaseDelay := randomDuration(delay)
time.Sleep(releaseDelay)
if err := k.Up(ctx, ch); err != nil {
return err
}
}
return nil
}

305
pkg/drivers/cdp/input/manager.go Normal file
View File

@ -0,0 +1,305 @@
package input
import (
"context"
"fmt"
"time"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
"github.com/gofrs/uuid"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
)
type Manager struct {
client *cdp.Client
exec *eval.ExecutionContext
keyboard *Keyboard
mouse *Mouse
}
func NewManager(
client *cdp.Client,
exec *eval.ExecutionContext,
keyboard *Keyboard,
mouse *Mouse,
) *Manager {
return &Manager{
client,
exec,
keyboard,
mouse,
}
}
func (m *Manager) Keyboard() *Keyboard {
return m.keyboard
}
func (m *Manager) Mouse() *Mouse {
return m.mouse
}
func (m *Manager) Scroll(ctx context.Context, x, y values.Float) error {
return m.exec.Eval(ctx, fmt.Sprintf(`
window.scrollBy({
top: %s,
left: %s,
behavior: 'instant'
});
`,
eval.ParamFloat(float64(x)),
eval.ParamFloat(float64(y)),
))
}
func (m *Manager) ScrollIntoViewBySelector(ctx context.Context, selector values.String) error {
return m.exec.Eval(ctx, fmt.Sprintf(`
var el = document.querySelector(%s);
if (el == null) {
throw new Error("element not found");
}
el.scrollIntoView({
behavior: 'instant'
});
return true;
`, eval.ParamString(selector.String()),
))
}
func (m *Manager) ScrollIntoViewByNodeID(ctx context.Context, nodeID dom.NodeID) error {
var attrID = "data-ferret-scroll"
id, err := uuid.NewV4()
if err != nil {
return err
}
err = m.client.DOM.SetAttributeValue(ctx, dom.NewSetAttributeValueArgs(nodeID, attrID, id.String()))
if err != nil {
return err
}
err = m.exec.Eval(
ctx,
fmt.Sprintf(`
var el = document.querySelector('[%s="%s"]');
if (el == null) {
throw new Error('element not found');
}
el.scrollIntoView({
behavior: 'instant',
inline: 'center',
block: 'center'
});
`,
attrID,
id.String(),
))
if err != nil {
return err
}
err = m.client.DOM.RemoveAttribute(ctx, dom.NewRemoveAttributeArgs(nodeID, attrID))
return err
}
func (m *Manager) ScrollTop(ctx context.Context) error {
return m.exec.Eval(ctx, `
window.scrollTo({
left: 0,
top: 0,
behavior: 'instant'
});
`)
}
func (m *Manager) ScrollBottom(ctx context.Context) error {
return m.exec.Eval(ctx, `
window.scrollTo({
left: 0,
top: window.document.body.scrollHeight,
behavior: 'instant'
});
`)
}
func (m *Manager) MoveMouseBySelector(ctx context.Context, parentNodeID dom.NodeID, selector values.String) error {
found, err := m.client.DOM.QuerySelector(ctx, dom.NewQuerySelectorArgs(parentNodeID, selector.String()))
if err != nil {
return err
}
return m.MoveMouseByNodeID(ctx, found.NodeID)
}
func (m *Manager) MoveMouseByNodeID(ctx context.Context, nodeID dom.NodeID) error {
err := m.ScrollIntoViewByNodeID(ctx, nodeID)
if err != nil {
return err
}
q, err := GetClickablePointByNodeID(ctx, m.client, nodeID)
if err != nil {
return err
}
return m.mouse.Move(ctx, q.X, q.Y)
}
func (m *Manager) MoveMouse(ctx context.Context, x, y values.Float) error {
return m.mouse.Move(ctx, float64(x), float64(y))
}
func (m *Manager) ClickBySelector(ctx context.Context, parentNodeID dom.NodeID, selector values.String) error {
found, err := m.client.DOM.QuerySelector(ctx, dom.NewQuerySelectorArgs(parentNodeID, selector.String()))
if err != nil {
return err
}
return m.ClickByNodeID(ctx, found.NodeID)
}
func (m *Manager) ClickByNodeID(ctx context.Context, nodeID dom.NodeID) error {
if err := m.ScrollIntoViewByNodeID(ctx, nodeID); err != nil {
return err
}
points, err := GetClickablePointByNodeID(ctx, m.client, nodeID)
if err != nil {
return err
}
if err := m.mouse.Click(ctx, points.X, points.Y, 50); err != nil {
return nil
}
return nil
}
func (m *Manager) TypeBySelector(ctx context.Context, parentNodeID dom.NodeID, selector values.String, text core.Value, delay values.Int) error {
found, err := m.client.DOM.QuerySelector(ctx, dom.NewQuerySelectorArgs(parentNodeID, selector.String()))
if err != nil {
return err
}
return m.TypeByNodeID(ctx, found.NodeID, text, delay)
}
func (m *Manager) TypeByNodeID(ctx context.Context, nodeID dom.NodeID, text core.Value, delay values.Int) error {
if err := m.ScrollIntoViewByNodeID(ctx, nodeID); err != nil {
return err
}
if err := m.client.DOM.Focus(ctx, dom.NewFocusArgs().SetNodeID(nodeID)); err != nil {
return err
}
_, min := core.NumberBoundaries(float64(delay))
beforeTypeDelay := time.Duration(min)
time.Sleep(beforeTypeDelay * time.Millisecond)
return m.keyboard.Type(ctx, text.String(), int(delay))
}
func (m *Manager) SelectBySelector(ctx context.Context, parentNodeID dom.NodeID, selector values.String, value *values.Array) (*values.Array, error) {
found, err := m.client.DOM.QuerySelector(ctx, dom.NewQuerySelectorArgs(parentNodeID, selector.String()))
if err != nil {
return nil, err
}
return m.SelectByNodeID(ctx, found.NodeID, value)
}
func (m *Manager) SelectByNodeID(ctx context.Context, nodeID dom.NodeID, value *values.Array) (*values.Array, error) {
if err := m.ScrollIntoViewByNodeID(ctx, nodeID); err != nil {
return nil, err
}
if err := m.client.DOM.Focus(ctx, dom.NewFocusArgs().SetNodeID(nodeID)); err != nil {
return nil, err
}
var attrID = "data-ferret-select"
id, err := uuid.NewV4()
if err != nil {
return nil, err
}
err = m.client.DOM.SetAttributeValue(ctx, dom.NewSetAttributeValueArgs(nodeID, attrID, id.String()))
if err != nil {
return nil, err
}
res, err := m.exec.EvalWithReturn(
ctx,
fmt.Sprintf(`
var el = document.querySelector('[%s="%s"]');
if (el == null) {
return [];
}
var values = %s;
if (el.nodeName.toLowerCase() !== 'select') {
throw new Error('element is not a <select> element.');
}
var options = Array.from(el.options);
el.value = undefined;
for (var option of options) {
option.selected = values.includes(option.value);
if (option.selected && !el.multiple) {
break;
}
}
el.dispatchEvent(new Event('input', { 'bubbles': true }));
el.dispatchEvent(new Event('change', { 'bubbles': true }));
return options.filter(option => option.selected).map(option => option.value);
`,
attrID,
id.String(),
value.String(),
),
)
if err != nil {
return nil, err
}
err = m.client.DOM.RemoveAttribute(ctx, dom.NewRemoveAttributeArgs(nodeID, attrID))
if err != nil {
return nil, err
}
arr, ok := res.(*values.Array)
if ok {
return arr, nil
}
return nil, core.TypeError(types.Array, res.Type())
}

82
pkg/drivers/cdp/input/mouse.go Normal file
View File

@ -0,0 +1,82 @@
package input
import (
"context"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/input"
"time"
)
type Mouse struct {
client *cdp.Client
x float64
y float64
}
func NewMouse(client *cdp.Client) *Mouse {
return &Mouse{client, 0, 0}
}
func (m *Mouse) Click(ctx context.Context, x, y float64, delay int) error {
if err := m.Move(ctx, x, y); err != nil {
return err
}
if err := m.Down(ctx, "left"); err != nil {
return err
}
releaseDelay := randomDuration(delay)
time.Sleep(releaseDelay * time.Millisecond)
return m.Up(ctx, "left")
}
func (m *Mouse) Down(ctx context.Context, button string) error {
return m.client.Input.DispatchMouseEvent(
ctx,
input.NewDispatchMouseEventArgs("mousePressed", m.x, m.y).
SetClickCount(1).
SetButton(button),
)
}
func (m *Mouse) Up(ctx context.Context, button string) error {
return m.client.Input.DispatchMouseEvent(
ctx,
input.NewDispatchMouseEventArgs("mouseReleased", m.x, m.y).
SetClickCount(1).
SetButton(button),
)
}
func (m *Mouse) Move(ctx context.Context, x, y float64) error {
return m.MoveBySteps(ctx, x, y, 1)
}
func (m *Mouse) MoveBySteps(ctx context.Context, x, y float64, steps int) error {
fromX := m.x
fromY := m.y
for i := 0; i <= steps; i++ {
iFloat := float64(i)
stepFloat := float64(steps)
toX := fromX + (x-fromX)*(iFloat/stepFloat)
toY := fromY + (y-fromY)*(iFloat/stepFloat)
err := m.client.Input.DispatchMouseEvent(
ctx,
input.NewDispatchMouseEventArgs("mouseMoved", toX, toY),
)
if err != nil {
return err
}
}
m.x = x
m.y = y
return nil
}

124
pkg/drivers/cdp/input/quad.go Normal file
View File

@ -0,0 +1,124 @@
package input
import (
"context"
"github.com/mafredri/cdp/protocol/runtime"
"math"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/pkg/errors"
)
type Quad struct {
X float64
Y float64
}
func fromProtocolQuad(quad dom.Quad) []Quad {
return []Quad{
{
X: quad[0],
Y: quad[1],
},
{
X: quad[2],
Y: quad[3],
},
{
X: quad[4],
Y: quad[5],
},
{
X: quad[6],
Y: quad[7],
},
}
}
func computeQuadArea(quads []Quad) float64 {
var area float64
for i := range quads {
p1 := quads[i]
p2 := quads[(i+1)%len(quads)]
area += (p1.X*p2.Y - p2.X*p1.Y) / 2
}
return math.Abs(area)
}
func intersectQuadWithViewport(quad []Quad, width, height float64) []Quad {
quads := make([]Quad, 0, len(quad))
for _, point := range quad {
quads = append(quads, Quad{
X: math.Min(math.Max(point.X, 0), width),
Y: math.Min(math.Max(point.Y, 0), height),
})
}
return quads
}
func getClickablePoint(ctx context.Context, client *cdp.Client, qargs *dom.GetContentQuadsArgs) (Quad, error) {
contentQuadsReply, err := client.DOM.GetContentQuads(ctx, qargs)
if err != nil {
return Quad{}, err
}
if contentQuadsReply.Quads == nil || len(contentQuadsReply.Quads) == 0 {
return Quad{}, errors.New("node is either not visible or not an HTMLElement")
}
layoutMetricsReply, err := client.Page.GetLayoutMetrics(ctx)
if err != nil {
return Quad{}, err
}
clientWidth := layoutMetricsReply.LayoutViewport.ClientWidth
clientHeight := layoutMetricsReply.LayoutViewport.ClientHeight
quads := make([][]Quad, 0, len(contentQuadsReply.Quads))
for _, q := range contentQuadsReply.Quads {
quad := intersectQuadWithViewport(fromProtocolQuad(q), float64(clientWidth), float64(clientHeight))
if computeQuadArea(quad) > 1 {
quads = append(quads, quad)
}
}
if len(quads) == 0 {
return Quad{}, errors.New("node is either not visible or not an HTMLElement")
}
// Return the middle point of the first quad.
quad := quads[0]
var x float64
var y float64
for _, q := range quad {
x += q.X
y += q.Y
}
return Quad{
X: x / 4,
Y: y / 4,
}, nil
}
func GetClickablePointByNodeID(ctx context.Context, client *cdp.Client, nodeID dom.NodeID) (Quad, error) {
return getClickablePoint(ctx, client, dom.NewGetContentQuadsArgs().SetNodeID(nodeID))
}
func GetClickablePointByObjectID(ctx context.Context, client *cdp.Client, objectID runtime.RemoteObjectID) (Quad, error) {
return getClickablePoint(ctx, client, dom.NewGetContentQuadsArgs().SetObjectID(objectID))
}
func GetClickablePointByBackendID(ctx context.Context, client *cdp.Client, backendID dom.BackendNodeID) (Quad, error) {
return getClickablePoint(ctx, client, dom.NewGetContentQuadsArgs().SetBackendNodeID(backendID))
}

37
pkg/drivers/cdp/page.go
View File

@ -3,20 +3,21 @@ package cdp
import (
"context"
"encoding/json"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/mafredri/cdp/protocol/emulation"
"github.com/mafredri/cdp/protocol/page"
"hash/fnv"
"sync"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/emulation"
"github.com/mafredri/cdp/protocol/network"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/rpcc"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
"github.com/MontFerret/ferret/pkg/drivers/cdp/input"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
@ -29,6 +30,8 @@ type HTMLPage struct {
conn *rpcc.Conn
client *cdp.Client
events *events.EventBroker
mouse *input.Mouse
keyboard *input.Keyboard
document *HTMLDocument
frames *common.LazyValue
}
@ -184,7 +187,10 @@ func LoadHTMLPage(
return nil, errors.Wrap(err, "failed to create event events")
}
doc, err := LoadRootHTMLDocument(ctx, logger, client, broker)
mouse := input.NewMouse(client)
keyboard := input.NewKeyboard(client)
doc, err := LoadRootHTMLDocument(ctx, logger, client, broker, mouse, keyboard)
if err != nil {
broker.StopAndClose()
@ -198,6 +204,8 @@ func LoadHTMLPage(
conn,
client,
broker,
mouse,
keyboard,
doc,
), nil
}
@ -207,6 +215,8 @@ func NewHTMLPage(
conn *rpcc.Conn,
client *cdp.Client,
broker *events.EventBroker,
mouse *input.Mouse,
keyboard *input.Keyboard,
document *HTMLDocument,
) *HTMLPage {
p := new(HTMLPage)
@ -215,6 +225,8 @@ func NewHTMLPage(
p.conn = conn
p.client = client
p.events = broker
p.mouse = mouse
p.keyboard = keyboard
p.document = document
p.frames = common.NewLazyValue(p.unfoldFrames)
@ -465,6 +477,9 @@ func (p *HTMLPage) DeleteCookies(ctx context.Context, cookies ...drivers.HTTPCoo
}
func (p *HTMLPage) PrintToPDF(ctx context.Context, params drivers.PDFParams) (values.Binary, error) {
p.mu.Lock()
defer p.mu.Unlock()
args := page.NewPrintToPDFArgs()
args.
SetLandscape(bool(params.Landscape)).
@ -523,6 +538,9 @@ func (p *HTMLPage) PrintToPDF(ctx context.Context, params drivers.PDFParams) (va
}
func (p *HTMLPage) CaptureScreenshot(ctx context.Context, params drivers.ScreenshotParams) (values.Binary, error) {
p.mu.Lock()
defer p.mu.Unlock()
metrics, err := p.client.Page.GetLayoutMetrics(ctx)
if err != nil {
@ -575,6 +593,9 @@ func (p *HTMLPage) CaptureScreenshot(ctx context.Context, params drivers.Screens
}
func (p *HTMLPage) Navigate(ctx context.Context, url values.String) error {
p.mu.Lock()
defer p.mu.Unlock()
if url == "" {
url = BlankPageURL
}
@ -593,6 +614,9 @@ func (p *HTMLPage) Navigate(ctx context.Context, url values.String) error {
}
func (p *HTMLPage) NavigateBack(ctx context.Context, skip values.Int) (values.Boolean, error) {
p.mu.Lock()
defer p.mu.Unlock()
history, err := p.client.Page.GetNavigationHistory(ctx)
if err != nil {
@ -632,6 +656,9 @@ func (p *HTMLPage) NavigateBack(ctx context.Context, skip values.Int) (values.Bo
}
func (p *HTMLPage) NavigateForward(ctx context.Context, skip values.Int) (values.Boolean, error) {
p.mu.Lock()
defer p.mu.Unlock()
history, err := p.client.Page.GetNavigationHistory(ctx)
if err != nil {
@ -698,7 +725,7 @@ func (p *HTMLPage) handlePageLoad(ctx context.Context, _ interface{}) {
p.mu.Lock()
defer p.mu.Unlock()
nextDoc, err := LoadRootHTMLDocument(ctx, p.logger, p.client, p.events)
nextDoc, err := LoadRootHTMLDocument(ctx, p.logger, p.client, p.events, p.mouse, p.keyboard)
if err != nil {
p.logger.Error().