go/ferret
go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2025-01-06 03:03:57 +02:00
Code Issues Releases Activity

Feature/pre compiled eval scripts (#658)

Browse Source 
* Added support of pre-compiled eval expressions

* Added unit tests for eval.Function

* Added RemoteType and RemoteObjectType enums

* Refactored function generation

* Refactored Document and Element loading logic

* Removed redundant fields from cdp.Page

* Exposed eval.Runtime to external callers

* Added new eval.RemoteValue interface
This commit is contained in:
Tim Voronov 2021-09-19 19:35:54 -04:00 committed by GitHub
parent 90427cd537
commit 847dda1f10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 1264 additions and 417 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
e2e/tests/dynamic/doc/iframes/element_exists.fql
View File

@ -10,6 +10,7 @@ LET frames = (
LET doc = FIRST(frames)
T::NOT::NONE(doc)
T::TRUE(ELEMENT_EXISTS(doc, '.text-center'))
T::FALSE(ELEMENT_EXISTS(doc, '.foo-bar'))

68
pkg/drivers/cdp/dom/document.go
View File

@ -2,7 +2,6 @@ package dom
import (
"context"
"github.com/mafredri/cdp/protocol/runtime"
"hash/fnv"
"github.com/mafredri/cdp"
@ -31,69 +30,6 @@ type HTMLDocument struct {
element *HTMLElement
}
func LoadRootHTMLDocument(
ctx context.Context,
logger zerolog.Logger,
client *cdp.Client,
domManager *Manager,
mouse *input.Mouse,
keyboard *input.Keyboard,
) (*HTMLDocument, error) {
ftRepl, err := client.Page.GetFrameTree(ctx)
if err != nil {
return nil, err
}
return LoadHTMLDocument(
ctx,
logger,
client,
domManager,
mouse,
keyboard,
ftRepl.FrameTree,
)
}
func LoadHTMLDocument(
ctx context.Context,
logger zerolog.Logger,
client *cdp.Client,
domManager *Manager,
mouse *input.Mouse,
keyboard *input.Keyboard,
frameTree page.FrameTree,
) (*HTMLDocument, error) {
exec, err := eval.Create(ctx, logger, client, frameTree.Frame.ID)
if err != nil {
return nil, err
}
inputManager := input.NewManager(logger, client, exec, keyboard, mouse)
exec.SetLoader(func(ctx context.Context, remoteType eval.RemoteType, id runtime.RemoteObjectID) (core.Value, error) {
return NewHTMLElement(logger, client, domManager, inputManager, exec, id), nil
})
rootElement, err := exec.EvalElement(ctx, templates.GetDocument())
if err != nil {
return nil, errors.Wrap(err, "failed to load root element")
}
return NewHTMLDocument(
logger,
client,
domManager,
inputManager,
exec,
rootElement.(*HTMLElement),
frameTree,
), nil
}
func NewHTMLDocument(
logger zerolog.Logger,
client *cdp.Client,
@ -396,8 +332,8 @@ func (doc *HTMLDocument) Scroll(ctx context.Context, options drivers.ScrollOptio
return doc.input.ScrollByXY(ctx, options)
}
func (doc *HTMLDocument) Eval(ctx context.Context, expression string) (core.Value, error) {
return doc.eval.EvalValue(ctx, eval.F(expression))
func (doc *HTMLDocument) Eval() *eval.Runtime {
return doc.eval
}
func (doc *HTMLDocument) logError(err error) *zerolog.Event {

1
pkg/drivers/cdp/dom/document_test.go
View File

@ -1 +0,0 @@
package dom

138
pkg/drivers/cdp/dom/element.go
View File

@ -2,13 +2,11 @@ package dom
import (
"context"
"fmt"
"hash/fnv"
"strings"
"time"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/pkg/errors"
"github.com/rs/zerolog"
@ -30,48 +28,12 @@ type HTMLElement struct {
client *cdp.Client
dom *Manager
input *input.Manager
exec *eval.Runtime
eval *eval.Runtime
id runtime.RemoteObjectID
nodeType *common.LazyValue
nodeName *common.LazyValue
}
func LoadHTMLElement(
ctx context.Context,
logger zerolog.Logger,
client *cdp.Client,
domManager *Manager,
input *input.Manager,
exec *eval.Runtime,
nodeID dom.NodeID,
) (*HTMLElement, error) {
if client == nil {
return nil, core.Error(core.ErrMissedArgument, "client")
}
// getting a remote object that represents the current DOM Node
args := dom.NewResolveNodeArgs().SetNodeID(nodeID).SetExecutionContextID(exec.ContextID())
ref, err := client.DOM.ResolveNode(ctx, args)
if err != nil {
return nil, err
}
if ref.Object.ObjectID == nil {
return nil, core.Error(core.ErrNotFound, fmt.Sprintf("element %s", ref.Object.Value))
}
return NewHTMLElement(
logger,
client,
domManager,
input,
exec,
*ref.Object.ObjectID,
), nil
}
func NewHTMLElement(
logger zerolog.Logger,
client *cdp.Client,
@ -88,18 +50,22 @@ func NewHTMLElement(
el.client = client
el.dom = domManager
el.input = input
el.exec = exec
el.eval = exec
el.id = id
el.nodeType = common.NewLazyValue(func(ctx context.Context) (core.Value, error) {
return el.exec.EvalValue(ctx, templates.GetNodeType(el.id))
return el.eval.EvalValue(ctx, templates.GetNodeType(el.id))
})
el.nodeName = common.NewLazyValue(func(ctx context.Context) (core.Value, error) {
return el.exec.EvalValue(ctx, templates.GetNodeName(el.id))
return el.eval.EvalValue(ctx, templates.GetNodeName(el.id))
})
return el
}
func (el *HTMLElement) RemoteID() runtime.RemoteObjectID {
return el.id
}
func (el *HTMLElement) Close() error {
return nil
}
@ -169,11 +135,11 @@ func (el *HTMLElement) SetIn(ctx context.Context, path []core.Value, value core.
}
func (el *HTMLElement) GetValue(ctx context.Context) (core.Value, error) {
return el.exec.EvalValue(ctx, templates.GetValue(el.id))
return el.eval.EvalValue(ctx, templates.GetValue(el.id))
}
func (el *HTMLElement) SetValue(ctx context.Context, value core.Value) error {
return el.exec.Eval(ctx, templates.SetValue(el.id, value))
return el.eval.Eval(ctx, templates.SetValue(el.id, value))
}
func (el *HTMLElement) GetNodeType(ctx context.Context) (values.Int, error) {
@ -197,7 +163,7 @@ func (el *HTMLElement) GetNodeName(ctx context.Context) (values.String, error) {
}
func (el *HTMLElement) Length() values.Int {
value, err := el.exec.EvalValue(context.Background(), templates.GetChildrenCount(el.id))
value, err := el.eval.EvalValue(context.Background(), templates.GetChildrenCount(el.id))
if err != nil {
el.logError(err)
@ -209,7 +175,7 @@ func (el *HTMLElement) Length() values.Int {
}
func (el *HTMLElement) GetStyles(ctx context.Context) (*values.Object, error) {
out, err := el.exec.EvalValue(ctx, templates.GetStyles(el.id))
out, err := el.eval.EvalValue(ctx, templates.GetStyles(el.id))
if err != nil {
return values.NewObject(), err
@ -219,23 +185,23 @@ func (el *HTMLElement) GetStyles(ctx context.Context) (*values.Object, error) {
}
func (el *HTMLElement) GetStyle(ctx context.Context, name values.String) (core.Value, error) {
return el.exec.EvalValue(ctx, templates.GetStyle(el.id, name))
return el.eval.EvalValue(ctx, templates.GetStyle(el.id, name))
}
func (el *HTMLElement) SetStyles(ctx context.Context, styles *values.Object) error {
return el.exec.Eval(ctx, templates.SetStyles(el.id, styles))
return el.eval.Eval(ctx, templates.SetStyles(el.id, styles))
}
func (el *HTMLElement) SetStyle(ctx context.Context, name, value values.String) error {
return el.exec.Eval(ctx, templates.SetStyle(el.id, name, value))
return el.eval.Eval(ctx, templates.SetStyle(el.id, name, value))
}
func (el *HTMLElement) RemoveStyle(ctx context.Context, names ...values.String) error {
return el.exec.Eval(ctx, templates.RemoveStyles(el.id, names))
return el.eval.Eval(ctx, templates.RemoveStyles(el.id, names))
}
func (el *HTMLElement) GetAttributes(ctx context.Context) (*values.Object, error) {
out, err := el.exec.EvalValue(ctx, templates.GetAttributes(el.id))
out, err := el.eval.EvalValue(ctx, templates.GetAttributes(el.id))
if err != nil {
return values.NewObject(), err
@ -245,55 +211,55 @@ func (el *HTMLElement) GetAttributes(ctx context.Context) (*values.Object, error
}
func (el *HTMLElement) GetAttribute(ctx context.Context, name values.String) (core.Value, error) {
return el.exec.EvalValue(ctx, templates.GetAttribute(el.id, name))
return el.eval.EvalValue(ctx, templates.GetAttribute(el.id, name))
}
func (el *HTMLElement) SetAttributes(ctx context.Context, attrs *values.Object) error {
return el.exec.Eval(ctx, templates.SetAttributes(el.id, attrs))
return el.eval.Eval(ctx, templates.SetAttributes(el.id, attrs))
}
func (el *HTMLElement) SetAttribute(ctx context.Context, name, value values.String) error {
return el.exec.Eval(ctx, templates.SetAttribute(el.id, name, value))
return el.eval.Eval(ctx, templates.SetAttribute(el.id, name, value))
}
func (el *HTMLElement) RemoveAttribute(ctx context.Context, names ...values.String) error {
return el.exec.Eval(ctx, templates.RemoveAttributes(el.id, names))
return el.eval.Eval(ctx, templates.RemoveAttributes(el.id, names))
}
func (el *HTMLElement) GetChildNodes(ctx context.Context) (*values.Array, error) {
return el.exec.EvalElements(ctx, templates.GetChildren(el.id))
return el.eval.EvalElements(ctx, templates.GetChildren(el.id))
}
func (el *HTMLElement) GetChildNode(ctx context.Context, idx values.Int) (core.Value, error) {
return el.exec.EvalElement(ctx, templates.GetChildByIndex(el.id, idx))
return el.eval.EvalElement(ctx, templates.GetChildByIndex(el.id, idx))
}
func (el *HTMLElement) GetParentElement(ctx context.Context) (core.Value, error) {
return el.exec.EvalElement(ctx, templates.GetParent(el.id))
return el.eval.EvalElement(ctx, templates.GetParent(el.id))
}
func (el *HTMLElement) GetPreviousElementSibling(ctx context.Context) (core.Value, error) {
return el.exec.EvalElement(ctx, templates.GetPreviousElementSibling(el.id))
return el.eval.EvalElement(ctx, templates.GetPreviousElementSibling(el.id))
}
func (el *HTMLElement) GetNextElementSibling(ctx context.Context) (core.Value, error) {
return el.exec.EvalElement(ctx, templates.GetNextElementSibling(el.id))
return el.eval.EvalElement(ctx, templates.GetNextElementSibling(el.id))
}
func (el *HTMLElement) QuerySelector(ctx context.Context, selector drivers.QuerySelector) (core.Value, error) {
return el.exec.EvalElement(ctx, templates.QuerySelector(el.id, selector))
return el.eval.EvalElement(ctx, templates.QuerySelector(el.id, selector))
}
func (el *HTMLElement) QuerySelectorAll(ctx context.Context, selector drivers.QuerySelector) (*values.Array, error) {
return el.exec.EvalElements(ctx, templates.QuerySelectorAll(el.id, selector))
return el.eval.EvalElements(ctx, templates.QuerySelectorAll(el.id, selector))
}
func (el *HTMLElement) XPath(ctx context.Context, expression values.String) (result core.Value, err error) {
return el.exec.EvalValue(ctx, templates.XPath(el.id, expression))
return el.eval.EvalValue(ctx, templates.XPath(el.id, expression))
}
func (el *HTMLElement) GetInnerText(ctx context.Context) (values.String, error) {
out, err := el.exec.EvalValue(ctx, templates.GetInnerText(el.id))
out, err := el.eval.EvalValue(ctx, templates.GetInnerText(el.id))
if err != nil {
return values.EmptyString, err
@ -303,14 +269,14 @@ func (el *HTMLElement) GetInnerText(ctx context.Context) (values.String, error)
}
func (el *HTMLElement) SetInnerText(ctx context.Context, innerText values.String) error {
return el.exec.Eval(
return el.eval.Eval(
ctx,
templates.SetInnerText(el.id, innerText),
)
}
func (el *HTMLElement) GetInnerTextBySelector(ctx context.Context, selector drivers.QuerySelector) (values.String, error) {
out, err := el.exec.EvalValue(ctx, templates.GetInnerTextBySelector(el.id, selector))
out, err := el.eval.EvalValue(ctx, templates.GetInnerTextBySelector(el.id, selector))
if err != nil {
return values.EmptyString, err
@ -320,14 +286,14 @@ func (el *HTMLElement) GetInnerTextBySelector(ctx context.Context, selector driv
}
func (el *HTMLElement) SetInnerTextBySelector(ctx context.Context, selector drivers.QuerySelector, innerText values.String) error {
return el.exec.Eval(
return el.eval.Eval(
ctx,
templates.SetInnerTextBySelector(el.id, selector, innerText),
)
}
func (el *HTMLElement) GetInnerTextBySelectorAll(ctx context.Context, selector drivers.QuerySelector) (*values.Array, error) {
out, err := el.exec.EvalValue(ctx, templates.GetInnerTextBySelectorAll(el.id, selector))
out, err := el.eval.EvalValue(ctx, templates.GetInnerTextBySelectorAll(el.id, selector))
if err != nil {
return values.EmptyArray(), err
@ -337,7 +303,7 @@ func (el *HTMLElement) GetInnerTextBySelectorAll(ctx context.Context, selector d
}
func (el *HTMLElement) GetInnerHTML(ctx context.Context) (values.String, error) {
out, err := el.exec.EvalValue(ctx, templates.GetInnerHTML(el.id))
out, err := el.eval.EvalValue(ctx, templates.GetInnerHTML(el.id))
if err != nil {
return values.EmptyString, err
@ -347,11 +313,11 @@ func (el *HTMLElement) GetInnerHTML(ctx context.Context) (values.String, error)
}
func (el *HTMLElement) SetInnerHTML(ctx context.Context, innerHTML values.String) error {
return el.exec.Eval(ctx, templates.SetInnerHTML(el.id, innerHTML))
return el.eval.Eval(ctx, templates.SetInnerHTML(el.id, innerHTML))
}
func (el *HTMLElement) GetInnerHTMLBySelector(ctx context.Context, selector drivers.QuerySelector) (values.String, error) {
out, err := el.exec.EvalValue(ctx, templates.GetInnerHTMLBySelector(el.id, selector))
out, err := el.eval.EvalValue(ctx, templates.GetInnerHTMLBySelector(el.id, selector))
if err != nil {
return values.EmptyString, err
@ -361,11 +327,11 @@ func (el *HTMLElement) GetInnerHTMLBySelector(ctx context.Context, selector driv
}
func (el *HTMLElement) SetInnerHTMLBySelector(ctx context.Context, selector drivers.QuerySelector, innerHTML values.String) error {
return el.exec.Eval(ctx, templates.SetInnerHTMLBySelector(el.id, selector, innerHTML))
return el.eval.Eval(ctx, templates.SetInnerHTMLBySelector(el.id, selector, innerHTML))
}
func (el *HTMLElement) GetInnerHTMLBySelectorAll(ctx context.Context, selector drivers.QuerySelector) (*values.Array, error) {
out, err := el.exec.EvalValue(ctx, templates.GetInnerHTMLBySelectorAll(el.id, selector))
out, err := el.eval.EvalValue(ctx, templates.GetInnerHTMLBySelectorAll(el.id, selector))
if err != nil {
return values.EmptyArray(), err
@ -375,7 +341,7 @@ func (el *HTMLElement) GetInnerHTMLBySelectorAll(ctx context.Context, selector d
}
func (el *HTMLElement) CountBySelector(ctx context.Context, selector drivers.QuerySelector) (values.Int, error) {
out, err := el.exec.EvalValue(ctx, templates.CountBySelector(el.id, selector))
out, err := el.eval.EvalValue(ctx, templates.CountBySelector(el.id, selector))
if err != nil {
return values.ZeroInt, err
@ -385,7 +351,7 @@ func (el *HTMLElement) CountBySelector(ctx context.Context, selector drivers.Que
}
func (el *HTMLElement) ExistsBySelector(ctx context.Context, selector drivers.QuerySelector) (values.Boolean, error) {
out, err := el.exec.EvalValue(ctx, templates.ExistsBySelector(el.id, selector))
out, err := el.eval.EvalValue(ctx, templates.ExistsBySelector(el.id, selector))
if err != nil {
return values.False, err
@ -396,7 +362,7 @@ func (el *HTMLElement) ExistsBySelector(ctx context.Context, selector drivers.Qu
func (el *HTMLElement) WaitForElement(ctx context.Context, selector drivers.QuerySelector, when drivers.WaitEvent) error {
task := events.NewEvalWaitTask(
el.exec,
el.eval,
templates.WaitForElement(el.id, selector, when),
events.DefaultPolling,
)
@ -408,7 +374,7 @@ func (el *HTMLElement) WaitForElement(ctx context.Context, selector drivers.Quer
func (el *HTMLElement) WaitForElementAll(ctx context.Context, selector drivers.QuerySelector, when drivers.WaitEvent) error {
task := events.NewEvalWaitTask(
el.exec,
el.eval,
templates.WaitForElementAll(el.id, selector, when),
events.DefaultPolling,
)
@ -420,7 +386,7 @@ func (el *HTMLElement) WaitForElementAll(ctx context.Context, selector drivers.Q
func (el *HTMLElement) WaitForClass(ctx context.Context, class values.String, when drivers.WaitEvent) error {
task := events.NewEvalWaitTask(
el.exec,
el.eval,
templates.WaitForClass(el.id, class, when),
events.DefaultPolling,
)
@ -432,7 +398,7 @@ func (el *HTMLElement) WaitForClass(ctx context.Context, class values.String, wh
func (el *HTMLElement) WaitForClassBySelector(ctx context.Context, selector drivers.QuerySelector, class values.String, when drivers.WaitEvent) error {
task := events.NewEvalWaitTask(
el.exec,
el.eval,
templates.WaitForClassBySelector(el.id, selector, class, when),
events.DefaultPolling,
)
@ -444,7 +410,7 @@ func (el *HTMLElement) WaitForClassBySelector(ctx context.Context, selector driv
func (el *HTMLElement) WaitForClassBySelectorAll(ctx context.Context, selector drivers.QuerySelector, class values.String, when drivers.WaitEvent) error {
task := events.NewEvalWaitTask(
el.exec,
el.eval,
templates.WaitForClassBySelectorAll(el.id, selector, class, when),
events.DefaultPolling,
)
@ -461,7 +427,7 @@ func (el *HTMLElement) WaitForAttribute(
when drivers.WaitEvent,
) error {
task := events.NewEvalWaitTask(
el.exec,
el.eval,
templates.WaitForAttribute(el.id, name, value, when),
events.DefaultPolling,
)
@ -473,7 +439,7 @@ func (el *HTMLElement) WaitForAttribute(
func (el *HTMLElement) WaitForAttributeBySelector(ctx context.Context, selector drivers.QuerySelector, name values.String, value core.Value, when drivers.WaitEvent) error {
task := events.NewEvalWaitTask(
el.exec,
el.eval,
templates.WaitForAttributeBySelector(el.id, selector, name, value, when),
events.DefaultPolling,
)
@ -485,7 +451,7 @@ func (el *HTMLElement) WaitForAttributeBySelector(ctx context.Context, selector
func (el *HTMLElement) WaitForAttributeBySelectorAll(ctx context.Context, selector drivers.QuerySelector, name values.String, value core.Value, when drivers.WaitEvent) error {
task := events.NewEvalWaitTask(
el.exec,
el.eval,
templates.WaitForAttributeBySelectorAll(el.id, selector, name, value, when),
events.DefaultPolling,
)
@ -497,7 +463,7 @@ func (el *HTMLElement) WaitForAttributeBySelectorAll(ctx context.Context, select
func (el *HTMLElement) WaitForStyle(ctx context.Context, name values.String, value core.Value, when drivers.WaitEvent) error {
task := events.NewEvalWaitTask(
el.exec,
el.eval,
templates.WaitForStyle(el.id, name, value, when),
events.DefaultPolling,
)
@ -509,7 +475,7 @@ func (el *HTMLElement) WaitForStyle(ctx context.Context, name values.String, val
func (el *HTMLElement) WaitForStyleBySelector(ctx context.Context, selector drivers.QuerySelector, name values.String, value core.Value, when drivers.WaitEvent) error {
task := events.NewEvalWaitTask(
el.exec,
el.eval,
templates.WaitForStyleBySelector(el.id, selector, name, value, when),
events.DefaultPolling,
)
@ -521,7 +487,7 @@ func (el *HTMLElement) WaitForStyleBySelector(ctx context.Context, selector driv
func (el *HTMLElement) WaitForStyleBySelectorAll(ctx context.Context, selector drivers.QuerySelector, name values.String, value core.Value, when drivers.WaitEvent) error {
task := events.NewEvalWaitTask(
el.exec,
el.eval,
templates.WaitForStyleBySelectorAll(el.id, selector, name, value, when),
events.DefaultPolling,
)

22
pkg/drivers/cdp/dom/loader.go Normal file
View File

@ -0,0 +1,22 @@
package dom
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/protocol/runtime"
)
type NodeLoader struct {
dom *Manager
}
func NewNodeLoader(dom *Manager) eval.ValueLoader {
return &NodeLoader{dom}
}
func (n *NodeLoader) Load(ctx context.Context, frameID page.FrameID, _ eval.RemoteObjectType, _ eval.RemoteClassName, id runtime.RemoteObjectID) (core.Value, error) {
return n.dom.ResolveElement(ctx, frameID, id)
}

99
pkg/drivers/cdp/dom/manager.go
View File

@ -6,26 +6,27 @@ import (
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/drivers/cdp/input"
"github.com/MontFerret/ferret/pkg/drivers/cdp/templates"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type (
Manager struct {
mu sync.RWMutex
logger zerolog.Logger
client *cdp.Client
mouse *input.Mouse
keyboard *input.Keyboard
mainFrame *AtomicFrameID
frames *AtomicFrameCollection
}
)
type Manager struct {
mu sync.RWMutex
logger zerolog.Logger
client *cdp.Client
mouse *input.Mouse
keyboard *input.Keyboard
mainFrame *AtomicFrameID
frames *AtomicFrameCollection
}
func New(
logger zerolog.Logger,
@ -66,6 +67,70 @@ func (m *Manager) Close() error {
return nil
}
func (m *Manager) LoadRootDocument(ctx context.Context) (*HTMLDocument, error) {
ftRepl, err := m.client.Page.GetFrameTree(ctx)
if err != nil {
return nil, err
}
return m.LoadDocument(ctx, ftRepl.FrameTree)
}
func (m *Manager) LoadDocument(ctx context.Context, frame page.FrameTree) (*HTMLDocument, error) {
exec, err := eval.Create(ctx, m.logger, m.client, frame.Frame.ID)
if err != nil {
return nil, err
}
inputs := input.New(m.logger, m.client, exec, m.keyboard, m.mouse)
ref, err := exec.EvalRef(ctx, templates.GetDocument())
if err != nil {
return nil, errors.Wrap(err, "failed to load root element")
}
exec.SetLoader(NewNodeLoader(m))
rootElement := NewHTMLElement(
m.logger,
m.client,
m,
inputs,
exec,
*ref.ObjectID,
)
return NewHTMLDocument(
m.logger,
m.client,
m,
inputs,
exec,
rootElement,
frame,
), nil
}
func (m *Manager) ResolveElement(ctx context.Context, frameID page.FrameID, id runtime.RemoteObjectID) (*HTMLElement, error) {
doc, err := m.GetFrameNode(ctx, frameID)
if err != nil {
return nil, err
}
return NewHTMLElement(
m.logger,
m.client,
m,
doc.input,
doc.eval,
id,
), nil
}
func (m *Manager) GetMainFrame() *HTMLDocument {
m.mu.RLock()
defer m.mu.RUnlock()
@ -212,16 +277,8 @@ func (m *Manager) getFrameInternal(ctx context.Context, frameID page.FrameID) (*
return frame.node, nil
}
// the frames is not loaded yet
doc, err := LoadHTMLDocument(
ctx,
m.logger,
m.client,
m,
m.mouse,
m.keyboard,
frame.tree,
)
// the frame is not loaded yet
doc, err := m.LoadDocument(ctx, frame.tree)
if err != nil {
return nil, errors.Wrap(err, "failed to load frame document")

18
pkg/drivers/cdp/eval/arguments.go Normal file
View File

@ -0,0 +1,18 @@
package eval
import (
"github.com/mafredri/cdp/protocol/runtime"
"github.com/rs/zerolog"
)
type FunctionArguments []runtime.CallArgument
func (args FunctionArguments) MarshalZerologArray(a *zerolog.Array) {
for _, arg := range args {
if arg.ObjectID != nil {
a.Str(string(*arg.ObjectID))
} else {
a.RawJSON(arg.Value)
}
}
}

213
pkg/drivers/cdp/eval/function.go
View File

@ -4,31 +4,24 @@ import (
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/rs/zerolog"
"github.com/wI2L/jettison"
"strconv"
"strings"
)
type (
FunctionReturnType int
FunctionArguments []runtime.CallArgument
Function struct {
exp string
ownerID runtime.RemoteObjectID
args FunctionArguments
returnType FunctionReturnType
async bool
}
)
const defaultArgsCount = 5
type Function struct {
exp string
name string
ownerID runtime.RemoteObjectID
args FunctionArguments
returnType ReturnType
async bool
}
const (
ReturnNothing FunctionReturnType = iota
ReturnValue
ReturnRef
defaultArgsCount = 5
expName = "$exp"
compiledExpName = "$$exp"
)
func F(exp string) *Function {
@ -39,7 +32,7 @@ func F(exp string) *Function {
return op
}
func (fn *Function) AsPartOf(id runtime.RemoteObjectID) *Function {
func (fn *Function) CallOn(id runtime.RemoteObjectID) *Function {
fn.ownerID = id
return fn
@ -57,6 +50,24 @@ func (fn *Function) AsSync() *Function {
return fn
}
func (fn *Function) AsAnonymous() *Function {
fn.name = ""
return fn
}
func (fn *Function) AsNamed(name string) *Function {
if name != "" {
fn.name = name
}
return fn
}
func (fn *Function) WithArgRemoteValue(value RemoteValue) *Function {
return fn.WithArgRef(value.RemoteID())
}
func (fn *Function) WithArgRef(id runtime.RemoteObjectID) *Function {
return fn.withArg(runtime.CallArgument{
ObjectID: &id,
@ -95,6 +106,16 @@ func (fn *Function) String() string {
return fn.exp
}
func (fn *Function) Length() int {
return len(fn.args)
}
func (fn *Function) returnNothing() *Function {
fn.returnType = ReturnNothing
return fn
}
func (fn *Function) returnRef() *Function {
fn.returnType = ReturnRef
@ -117,26 +138,17 @@ func (fn *Function) withArg(arg runtime.CallArgument) *Function {
return fn
}
func (fn *Function) build(ctx runtime.ExecutionContextID) *runtime.CallFunctionOnArgs {
exp := strings.TrimSpace(fn.exp)
if !strings.HasPrefix(exp, "(") && !strings.HasPrefix(exp, "function") {
exp = wrapExp(exp, len(fn.args))
}
func (fn *Function) eval(ctx runtime.ExecutionContextID) *runtime.CallFunctionOnArgs {
exp := fn.prepExp()
call := runtime.NewCallFunctionOnArgs(exp).
SetAwaitPromise(fn.async)
SetAwaitPromise(fn.async).
SetReturnByValue(fn.returnType == ReturnValue)
if fn.returnType == ReturnValue {
call.SetReturnByValue(true)
}
if ctx != EmptyExecutionContextID {
call.SetExecutionContextID(ctx)
}
if fn.ownerID != "" {
if fn.ownerID != EmptyObjectID {
call.SetObjectID(fn.ownerID)
} else if ctx != EmptyExecutionContextID {
call.SetExecutionContextID(ctx)
}
if len(fn.args) > 0 {
@ -146,23 +158,124 @@ func (fn *Function) build(ctx runtime.ExecutionContextID) *runtime.CallFunctionO
return call
}
func (rt FunctionReturnType) String() string {
switch rt {
case ReturnValue:
return "value"
case ReturnRef:
return "reference"
default:
return "nothing"
func (fn *Function) compile(ctx runtime.ExecutionContextID) *runtime.CompileScriptArgs {
exp := fn.precompileExp()
call := runtime.NewCompileScriptArgs(exp, "", true)
if ctx != EmptyExecutionContextID {
call.SetExecutionContextID(ctx)
}
return call
}
func (args FunctionArguments) MarshalZerologArray(a *zerolog.Array) {
for _, arg := range args {
if arg.ObjectID != nil {
a.Str(string(*arg.ObjectID))
} else {
a.RawJSON(arg.Value)
func (fn *Function) prepExp() string {
var invoke bool
exp := strings.TrimSpace(fn.exp)
name := fn.name
// If the given expression is either an arrow or plain function
if strings.HasPrefix(exp, "(") || strings.HasPrefix(exp, "function") {
// And if this function must be an anonymous
// we just pass the expression as is without wrapping it.
if name == "" {
return exp
}
// But if the function must be identified (named)
// we need to wrap the given function with a named one/
// And then eval it with passing available arguments.
invoke = true
}
// Start building a wrapper
var buf strings.Builder
buf.WriteString("function")
// Name the function if the name is set
if name != "" {
buf.WriteString(" ")
buf.WriteString(name)
}
buf.WriteString("(")
// If the given expression is a function then we do not need to define wrapper's function arguments.
// Any available arguments will be passed down via 'arguments' runtime variable.
// Otherwise, we define a list of arguments as argN, so the given expression could access them by name.
if !invoke {
args := len(fn.args)
lastIndex := args - 1
for i := 0; i < args; i++ {
buf.WriteString("arg")
buf.WriteString(strconv.Itoa(i + 1))
if i != lastIndex {
buf.WriteString(",")
}
}
}
buf.WriteString(") {\n")
if !invoke {
buf.WriteString(exp)
} else {
buf.WriteString("const ")
buf.WriteString(expName)
buf.WriteString(" = ")
buf.WriteString(exp)
buf.WriteString(";\n")
buf.WriteString("return ")
buf.WriteString(expName)
buf.WriteString(".apply(this, arguments);")
}
buf.WriteString("\n}")
return buf.String()
}
func (fn *Function) precompileExp() string {
exp := fn.prepExp()
args := fn.args
var buf strings.Builder
var l = len(args)
buf.WriteString("const args = [")
if l > 0 {
for i := 0; i < l; i++ {
buf.WriteRune('\n')
arg := args[i]
if arg.Value != nil {
buf.Write(arg.Value)
} else if arg.ObjectID != nil {
buf.WriteString("(() => { throw new Error('Reference values cannot be used in pre-compiled scrips')})()")
}
buf.WriteString(",")
}
buf.WriteRune('\n')
}
buf.WriteString("];")
buf.WriteRune('\n')
buf.WriteString("const ")
buf.WriteString(compiledExpName)
buf.WriteString(" = ")
buf.WriteString(exp)
buf.WriteString(";\n")
buf.WriteString(compiledExpName)
buf.WriteString(".apply(this, args);")
buf.WriteString("\n")
return buf.String()
}

46
pkg/drivers/cdp/eval/function_compiled.go Normal file
View File

@ -0,0 +1,46 @@
package eval
import "github.com/mafredri/cdp/protocol/runtime"
type CompiledFunction struct {
id runtime.ScriptID
src *Function
}
func CF(id runtime.ScriptID, src *Function) *CompiledFunction {
op := new(CompiledFunction)
op.id = id
op.src = src
return op
}
func (fn *CompiledFunction) returnNothing() *CompiledFunction {
fn.src.returnNothing()
return fn
}
func (fn *CompiledFunction) returnRef() *CompiledFunction {
fn.src.returnRef()
return fn
}
func (fn *CompiledFunction) returnValue() *CompiledFunction {
fn.src.returnValue()
return fn
}
func (fn *CompiledFunction) call(ctx runtime.ExecutionContextID) *runtime.RunScriptArgs {
call := runtime.NewRunScriptArgs(fn.id).
SetAwaitPromise(fn.src.async).
SetReturnByValue(fn.src.returnType == ReturnValue)
if ctx != EmptyExecutionContextID {
call.SetExecutionContextID(ctx)
}
return call
}

464
pkg/drivers/cdp/eval/function_test.go Normal file
View File

@ -0,0 +1,464 @@
package eval
import (
"testing"
"github.com/mafredri/cdp/protocol/runtime"
. "github.com/smartystreets/goconvey/convey"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
func TestFunction(t *testing.T) {
Convey("Function", t, func() {
Convey(".AsAsync", func() {
Convey("Should set async=true", func() {
f := F("return 'foo'").AsAsync()
args := f.eval(EmptyExecutionContextID)
So(*args.AwaitPromise, ShouldBeTrue)
})
})
Convey(".AsSync", func() {
Convey("Should set async=false", func() {
f := F("return 'foo'").AsAsync()
args := f.eval(EmptyExecutionContextID)
So(*args.AwaitPromise, ShouldBeTrue)
args = f.AsSync().eval(EmptyExecutionContextID)
So(*args.AwaitPromise, ShouldBeFalse)
})
})
Convey(".AsNamed", func() {
Convey("When without args", func() {
Convey("Should generate a wrapper with a given function name", func() {
name := "getFoo"
exp := "return 'foo'"
f := F(exp).AsNamed(name)
So(f.name, ShouldEqual, name)
call := f.eval(EmptyExecutionContextID)
expected := "function " + name + "() {\n" + exp + "\n}"
So(call.FunctionDeclaration, ShouldEqual, expected)
})
})
Convey("When with args", func() {
Convey("When a declaration is an expression", func() {
Convey("Should generate a wrapper with a given function name", func() {
name := "getFoo"
exp := "return 'foo'"
f := F(exp).
AsNamed(name).
WithArg("bar").
WithArg(1)
So(f.name, ShouldEqual, name)
call := f.eval(EmptyExecutionContextID)
expected := "function " + name + "(arg1,arg2) {\n" + exp + "\n}"
So(call.FunctionDeclaration, ShouldEqual, expected)
})
})
Convey("When a declaration is an arrow function", func() {
Convey("Should generate a wrapper with a given function name", func() {
name := "getValue"
exp := "(el) => el.value"
f := F(exp).
AsNamed(name).
WithArgRef("my_element").
WithArg(1)
So(f.name, ShouldEqual, name)
call := f.eval(EmptyExecutionContextID)
expected := "function " + name + "() {\n" +
"const $exp = " + exp + ";\n" +
"return $exp.apply(this, arguments);\n" +
"}"
So(call.FunctionDeclaration, ShouldEqual, expected)
})
})
Convey("When a declaration is a plain function", func() {
Convey("Should generate a wrapper with a given function name", func() {
name := "getValue"
exp := "function getElementValue(el) => el.value"
f := F(exp).
AsNamed(name).
WithArgRef("my_element").
WithArg(1)
So(f.name, ShouldEqual, name)
call := f.eval(EmptyExecutionContextID)
expected := "function " + name + "() {\n" +
"const $exp = " + exp + ";\n" +
"return $exp.apply(this, arguments);\n" +
"}"
So(call.FunctionDeclaration, ShouldEqual, expected)
})
})
})
})
Convey(".AsAnonymous", func() {
Convey("When without args", func() {
Convey("Should generate an anonymous wrapper", func() {
name := ""
exp := "return 'foo'"
f := F(exp).AsNamed("getFoo").AsAnonymous()
So(f.name, ShouldEqual, name)
call := f.eval(EmptyExecutionContextID)
expected := "function() {\n" + exp + "\n}"
So(call.FunctionDeclaration, ShouldEqual, expected)
})
})
Convey("When with args", func() {
Convey("When a declaration is an expression", func() {
Convey("Should generate an anonymous wrapper", func() {
name := ""
exp := "return 'foo'"
f := F(exp).
AsNamed("getFoo").
AsAnonymous().
WithArg("bar").
WithArg(1)
So(f.name, ShouldEqual, name)
call := f.eval(EmptyExecutionContextID)
expected := "function(arg1,arg2) {\n" + exp + "\n}"
So(call.FunctionDeclaration, ShouldEqual, expected)
})
})
Convey("When a declaration is an arrow function", func() {
Convey("Should NOT generate a wrapper", func() {
name := ""
exp := "(el) => el.value"
f := F(exp).
AsNamed("getValue").
AsAnonymous().
WithArgRef("my_element").
WithArg(1)
So(f.name, ShouldEqual, name)
call := f.eval(EmptyExecutionContextID)
So(call.FunctionDeclaration, ShouldEqual, exp)
})
})
Convey("When a declaration is a plain function", func() {
Convey("Should NOT generate a wrapper", func() {
name := ""
exp := "function(el) => el.value"
f := F(exp).
AsNamed("getValue").
AsAnonymous().
WithArgRef("my_element").
WithArg(1)
So(f.name, ShouldEqual, name)
call := f.eval(EmptyExecutionContextID)
So(call.FunctionDeclaration, ShouldEqual, exp)
})
})
})
})
Convey(".CallOn", func() {
Convey("It should use a given ownerID over ContextID", func() {
ownerID := runtime.RemoteObjectID("foo")
contextID := runtime.ExecutionContextID(42)
f := F("return 'foo'").CallOn(ownerID)
call := f.eval(contextID)
So(call.ExecutionContextID, ShouldBeNil)
So(call.ObjectID, ShouldNotBeNil)
So(*call.ObjectID, ShouldEqual, ownerID)
})
Convey("It should use a given ContextID when ownerID is empty or nil", func() {
ownerID := runtime.RemoteObjectID("")
contextID := runtime.ExecutionContextID(42)
f := F("return 'foo'").CallOn(ownerID)
call := f.eval(contextID)
So(call.ExecutionContextID, ShouldNotBeNil)
So(call.ObjectID, ShouldBeNil)
So(*call.ExecutionContextID, ShouldEqual, contextID)
})
})
Convey(".WithArgRef", func() {
Convey("Should add argument with a given RemoteObjectID", func() {
f := F("return 'foo'")
id1 := runtime.RemoteObjectID("foo")
id2 := runtime.RemoteObjectID("bar")
id3 := runtime.RemoteObjectID("baz")
f.WithArgRef(id1).WithArgRef(id2).WithArgRef(id3)
So(f.Length(), ShouldEqual, 3)
arg1 := f.args[0]
arg2 := f.args[1]
arg3 := f.args[2]
So(*arg1.ObjectID, ShouldEqual, id1)
So(arg1.Value, ShouldBeNil)
So(arg1.UnserializableValue, ShouldBeNil)
So(*arg2.ObjectID, ShouldEqual, id2)
So(arg2.Value, ShouldBeNil)
So(arg2.UnserializableValue, ShouldBeNil)
So(*arg3.ObjectID, ShouldEqual, id3)
So(arg3.Value, ShouldBeNil)
So(arg3.UnserializableValue, ShouldBeNil)
})
})
Convey(".WithArgValue", func() {
Convey("Should add argument with a given Value", func() {
f := F("return 'foo'")
val1 := values.NewString("foo")
val2 := values.NewInt(1)
val3 := values.NewBoolean(true)
f.WithArgValue(val1).WithArgValue(val2).WithArgValue(val3)
So(f.Length(), ShouldEqual, 3)
arg1 := f.args[0]
arg2 := f.args[1]
arg3 := f.args[2]
So(arg1.ObjectID, ShouldBeNil)
So(arg1.Value, ShouldResemble, values.MustMarshal(val1))
So(arg1.UnserializableValue, ShouldBeNil)
So(arg2.ObjectID, ShouldBeNil)
So(arg2.Value, ShouldResemble, values.MustMarshal(val2))
So(arg2.UnserializableValue, ShouldBeNil)
So(arg3.ObjectID, ShouldBeNil)
So(arg3.Value, ShouldResemble, values.MustMarshal(val3))
So(arg3.UnserializableValue, ShouldBeNil)
})
})
Convey(".WithArg", func() {
Convey("Should add argument with a given any type", func() {
f := F("return 'foo'")
val1 := "foo"
val2 := 1
val3 := true
f.WithArg(val1).WithArg(val2).WithArg(val3)
So(f.Length(), ShouldEqual, 3)
arg1 := f.args[0]
arg2 := f.args[1]
arg3 := f.args[2]
So(arg1.ObjectID, ShouldBeNil)
So(arg1.Value, ShouldResemble, values.MustMarshalAny(val1))
So(arg1.UnserializableValue, ShouldBeNil)
So(arg2.ObjectID, ShouldBeNil)
So(arg2.Value, ShouldResemble, values.MustMarshalAny(val2))
So(arg2.UnserializableValue, ShouldBeNil)
So(arg3.ObjectID, ShouldBeNil)
So(arg3.Value, ShouldResemble, values.MustMarshalAny(val3))
So(arg3.UnserializableValue, ShouldBeNil)
})
})
Convey(".WithArgSelector", func() {
Convey("Should add argument with a given QuerySelector", func() {
f := F("return 'foo'")
val1 := drivers.NewCSSSelector(".foo-bar")
val2 := drivers.NewCSSSelector("#submit")
val3 := drivers.NewXPathSelector("//*[@id='q']")
f.WithArgSelector(val1).WithArgSelector(val2).WithArgSelector(val3)
So(f.Length(), ShouldEqual, 3)
arg1 := f.args[0]
arg2 := f.args[1]
arg3 := f.args[2]
So(arg1.ObjectID, ShouldBeNil)
So(arg1.Value, ShouldResemble, values.MustMarshalAny(val1.String()))
So(arg1.UnserializableValue, ShouldBeNil)
So(arg2.ObjectID, ShouldBeNil)
So(arg2.Value, ShouldResemble, values.MustMarshalAny(val2.String()))
So(arg2.UnserializableValue, ShouldBeNil)
So(arg3.ObjectID, ShouldBeNil)
So(arg3.Value, ShouldResemble, values.MustMarshalAny(val3.String()))
So(arg3.UnserializableValue, ShouldBeNil)
})
})
Convey(".String", func() {
Convey("It should return a function expression", func() {
exp := "return 'foo'"
f := F(exp)
So(f.String(), ShouldEqual, exp)
})
})
Convey(".returnNothing", func() {
Convey("It should set return by value to false", func() {
f := F("return 'foo'").returnNothing()
call := f.eval(EmptyExecutionContextID)
So(*call.ReturnByValue, ShouldBeFalse)
})
})
Convey(".returnValue", func() {
Convey("It should set return by value to true", func() {
f := F("return 'foo'").returnValue()
call := f.eval(EmptyExecutionContextID)
So(*call.ReturnByValue, ShouldBeTrue)
})
})
Convey(".returnRef", func() {
Convey("It should set return by value to false", func() {
f := F("return 'foo'").returnValue()
call := f.eval(EmptyExecutionContextID)
So(*call.ReturnByValue, ShouldBeTrue)
f.returnRef()
call = f.eval(EmptyExecutionContextID)
So(*call.ReturnByValue, ShouldBeFalse)
})
})
Convey(".compile", func() {
Convey("When Anonymous", func() {
Convey("When without args", func() {
Convey("Should generate an expression", func() {
name := ""
exp := "return 'foo'"
f := F(exp)
So(f.name, ShouldEqual, name)
call := f.compile(EmptyExecutionContextID)
expected := "const args = [];\n" +
"const " + compiledExpName + " = function() {\n" + exp + "\n};\n" +
compiledExpName + ".apply(this, args);\n"
So(call.Expression, ShouldEqual, expected)
})
Convey("When a function is given", func() {
Convey("Should generate an expression", func() {
name := ""
exp := "() => return 'foo'"
f := F(exp)
So(f.name, ShouldEqual, name)
call := f.compile(EmptyExecutionContextID)
expected := "const args = [];\n" +
"const " + compiledExpName + " = " + exp + ";\n" +
compiledExpName + ".apply(this, args);\n"
So(call.Expression, ShouldEqual, expected)
})
})
})
Convey("When with args", func() {
Convey("Should generate an expression", func() {
name := ""
exp := "return 'foo'"
f := F(exp).WithArg(1).WithArg("test").WithArg([]int{1, 2})
So(f.name, ShouldEqual, name)
call := f.compile(EmptyExecutionContextID)
expected := "const args = [\n" +
"1,\n" +
"\"test\",\n" +
"[1,2],\n" +
"];\n" +
"const " + compiledExpName + " = function(arg1,arg2,arg3) {\n" + exp + "\n};\n" +
compiledExpName + ".apply(this, args);\n"
So(call.Expression, ShouldEqual, expected)
})
Convey("When a function is given", func() {
Convey("Should generate an expression", func() {
name := ""
exp := "() => return 'foo'"
f := F(exp).WithArg(1).WithArg("test").WithArg([]int{1, 2})
So(f.name, ShouldEqual, name)
call := f.compile(EmptyExecutionContextID)
expected := "const args = [\n" +
"1,\n" +
"\"test\",\n" +
"[1,2],\n" +
"];\n" +
"const " + compiledExpName + " = " + exp + ";\n" +
compiledExpName + ".apply(this, args);\n"
So(call.Expression, ShouldEqual, expected)
})
})
})
})
})
})
}

67
pkg/drivers/cdp/eval/helpers.go
View File

@ -1,81 +1,14 @@
package eval
import (
"strconv"
"strings"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
func CastToValue(input interface{}) (core.Value, error) {
value, ok := input.(core.Value)
if !ok {
return values.None, core.Error(core.ErrInvalidType, "eval return type")
}
return value, nil
}
func CastToReference(input interface{}) (runtime.RemoteObject, error) {
value, ok := input.(runtime.RemoteObject)
if !ok {
return runtime.RemoteObject{}, core.Error(core.ErrInvalidType, "eval return type")
}
return value, nil
}
func wrapExp(exp string, args int) string {
if args == 0 {
return "() => {\n" + exp + "\n}"
}
var buf strings.Builder
lastIndex := args - 1
for i := 0; i < args; i++ {
buf.WriteString("arg")
buf.WriteString(strconv.Itoa(i + 1))
if i != lastIndex {
buf.WriteString(",")
}
}
return "(" + buf.String() + ") => {\n" + exp + "\n}"
}
func Unmarshal(obj runtime.RemoteObject) (core.Value, error) {
switch obj.Type {
case "string":
str, err := strconv.Unquote(string(obj.Value))
if err != nil {
return values.None, err
}
return values.NewString(str), nil
case "object":
if obj.Subtype != nil {
subtype := *obj.Subtype
if subtype == "null" || subtype == "undefined" {
return values.None, nil
}
}
return values.Unmarshal(obj.Value)
default:
return values.Unmarshal(obj.Value)
}
}
func parseRuntimeException(details *runtime.ExceptionDetails) error {
if details == nil || details.Exception == nil {
return nil

19
pkg/drivers/cdp/eval/helpers_test.go
View File

@ -1,20 +1 @@
package eval
import (
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func TestWrapExp(t *testing.T) {
Convey("wrapExp", t, func() {
Convey("When a plain expression is passed", func() {
exp := "return true"
So(wrapExp(exp, 0), ShouldEqual, "() => {\n"+exp+"\n}")
})
Convey("When a plain expression is passed with args > 0", func() {
exp := "return true"
So(wrapExp(exp, 3), ShouldEqual, "(arg1,arg2,arg3) => {\n"+exp+"\n}")
})
})
}

88
pkg/drivers/cdp/eval/resolver.go
View File

@ -2,25 +2,56 @@ package eval
import (
"context"
"strconv"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/pkg/errors"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/pkg/errors"
)
type (
ValueLoader func(ctx context.Context, remoteType RemoteType, id runtime.RemoteObjectID) (core.Value, error)
ValueLoader interface {
Load(
ctx context.Context,
frameID page.FrameID,
remoteType RemoteObjectType,
remoteClass RemoteClassName,
id runtime.RemoteObjectID,
) (core.Value, error)
}
ValueLoaderFn func(
ctx context.Context,
frameID page.FrameID,
remoteType RemoteObjectType,
remoteClass RemoteClassName,
id runtime.RemoteObjectID,
) (core.Value, error)
Resolver struct {
runtime cdp.Runtime
frameID page.FrameID
loader ValueLoader
}
)
func NewResolver(runtime cdp.Runtime) *Resolver {
return &Resolver{runtime, nil}
func (f ValueLoaderFn) Load(
ctx context.Context,
frameID page.FrameID,
remoteType RemoteObjectType,
remoteClass RemoteClassName,
id runtime.RemoteObjectID,
) (core.Value, error) {
return f(ctx, frameID, remoteType, remoteClass, id)
}
func NewResolver(runtime cdp.Runtime, frameID page.FrameID) *Resolver {
return &Resolver{runtime, frameID, nil}
}
func (r *Resolver) SetLoader(loader ValueLoader) *Resolver {
@ -32,13 +63,19 @@ func (r *Resolver) SetLoader(loader ValueLoader) *Resolver {
func (r *Resolver) ToValue(ctx context.Context, ref runtime.RemoteObject) (core.Value, error) {
// It's not an actual ref but rather a plain value
if ref.ObjectID == nil {
return values.Unmarshal(ref.Value)
if ref.Value != nil {
return values.Unmarshal(ref.Value)
}
return values.None, nil
}
switch ToRemoteType(ref) {
case NullType, UndefinedType:
subtype := ToRemoteObjectType(ref)
switch subtype {
case NullObjectType, UndefinedObjectType:
return values.None, nil
case ArrayType:
case ArrayObjectType:
props, err := r.runtime.GetProperties(ctx, runtime.NewGetPropertiesArgs(*ref.ObjectID).SetOwnProperties(true))
if err != nil {
@ -72,15 +109,32 @@ func (r *Resolver) ToValue(ctx context.Context, ref runtime.RemoteObject) (core.
}
return result, nil
case NodeType:
// could it be possible?
case NodeObjectType:
// is it even possible?
if ref.ObjectID == nil {
return values.Unmarshal(ref.Value)
}
return r.loadValue(ctx, NodeType, *ref.ObjectID)
return r.loadValue(ctx, NodeObjectType, ToRemoteClassName(ref), *ref.ObjectID)
default:
return Unmarshal(ref)
switch ToRemoteType(ref) {
case StringType:
str, err := strconv.Unquote(string(ref.Value))
if err != nil {
return values.None, err
}
return values.NewString(str), nil
case ObjectType:
if subtype == NullObjectType || subtype == UnknownObjectType {
return values.None, nil
}
return values.Unmarshal(ref.Value)
default:
return values.Unmarshal(ref.Value)
}
}
}
@ -89,7 +143,7 @@ func (r *Resolver) ToElement(ctx context.Context, ref runtime.RemoteObject) (dri
return nil, core.Error(core.ErrInvalidArgument, "ref id")
}
val, err := r.loadValue(ctx, ToRemoteType(ref), *ref.ObjectID)
val, err := r.loadValue(ctx, ToRemoteObjectType(ref), ToRemoteClassName(ref), *ref.ObjectID)
if err != nil {
return nil, err
@ -163,10 +217,10 @@ func (r *Resolver) ToProperties(
return arr, nil
}
func (r *Resolver) loadValue(ctx context.Context, remoteType RemoteType, id runtime.RemoteObjectID) (core.Value, error) {
func (r *Resolver) loadValue(ctx context.Context, remoteType RemoteObjectType, remoteClass RemoteClassName, id runtime.RemoteObjectID) (core.Value, error) {
if r.loader == nil {
return values.None, core.Error(core.ErrNotImplemented, "ValueLoader")
}
return r.loader(ctx, remoteType, id)
return r.loader.Load(ctx, r.frameID, remoteType, remoteClass, id)
}

20
pkg/drivers/cdp/eval/return.go Normal file
View File

@ -0,0 +1,20 @@
package eval
type ReturnType int
const (
ReturnNothing ReturnType = iota
ReturnValue
ReturnRef
)
func (rt ReturnType) String() string {
switch rt {
case ReturnValue:
return "value"
case ReturnRef:
return "reference"
default:
return "nothing"
}
}

173
pkg/drivers/cdp/eval/runtime.go
View File

@ -15,7 +15,10 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/values"
)
const EmptyExecutionContextID = runtime.ExecutionContextID(-1)
const (
EmptyExecutionContextID = runtime.ExecutionContextID(-1)
EmptyObjectID = runtime.RemoteObjectID("")
)
type Runtime struct {
logger zerolog.Logger
@ -37,19 +40,20 @@ func Create(
return nil, err
}
return New(logger, client, world.ExecutionContextID), nil
return New(logger, client, frameID, world.ExecutionContextID), nil
}
func New(
logger zerolog.Logger,
client *cdp.Client,
frameID page.FrameID,
contextID runtime.ExecutionContextID,
) *Runtime {
rt := new(Runtime)
rt.logger = logging.WithName(logger.With(), "js-eval").Logger()
rt.client = client
rt.contextID = contextID
rt.resolver = NewResolver(client.Runtime)
rt.resolver = NewResolver(client.Runtime, frameID)
return rt
}
@ -65,13 +69,13 @@ func (rt *Runtime) ContextID() runtime.ExecutionContextID {
}
func (rt *Runtime) Eval(ctx context.Context, fn *Function) error {
_, err := rt.call(ctx, fn)
_, err := rt.evalInternal(ctx, fn.returnNothing())
return err
}
func (rt *Runtime) EvalRef(ctx context.Context, fn *Function) (runtime.RemoteObject, error) {
out, err := rt.call(ctx, fn.returnRef())
out, err := rt.evalInternal(ctx, fn.returnRef())
if err != nil {
return runtime.RemoteObject{}, err
@ -81,7 +85,7 @@ func (rt *Runtime) EvalRef(ctx context.Context, fn *Function) (runtime.RemoteObj
}
func (rt *Runtime) EvalValue(ctx context.Context, fn *Function) (core.Value, error) {
out, err := rt.call(ctx, fn.returnValue())
out, err := rt.evalInternal(ctx, fn.returnValue())
if err != nil {
return values.None, err
@ -122,7 +126,104 @@ func (rt *Runtime) EvalElements(ctx context.Context, fn *Function) (*values.Arra
return values.NewArrayWith(val), nil
}
func (rt *Runtime) call(ctx context.Context, fn *Function) (runtime.RemoteObject, error) {
func (rt *Runtime) Compile(ctx context.Context, fn *Function) (*CompiledFunction, error) {
log := rt.logger.With().
Str("expression", fn.String()).
Array("arguments", fn.args).
Logger()
arg := fn.compile(rt.contextID)
log.Trace().Str("script", arg.Expression).Msg("compiling expression...")
repl, err := rt.client.Runtime.CompileScript(ctx, arg)
if err != nil {
log.Trace().Err(err).Msg("failed compiling expression")
return nil, err
}
if err := parseRuntimeException(repl.ExceptionDetails); err != nil {
log.Trace().Err(err).Msg("compilation has failed with runtime exception")
return nil, err
}
if repl.ScriptID == nil {
log.Trace().Err(core.ErrUnexpected).Msg("compilation did not return script id")
return nil, core.ErrUnexpected
}
id := *repl.ScriptID
log.Trace().
Str("script-id", string(id)).
Msg("succeeded compiling expression")
return CF(id, fn), nil
}
func (rt *Runtime) Call(ctx context.Context, fn *CompiledFunction) error {
_, err := rt.callInternal(ctx, fn.returnNothing())
return err
}
func (rt *Runtime) CallRef(ctx context.Context, fn *CompiledFunction) (runtime.RemoteObject, error) {
out, err := rt.callInternal(ctx, fn.returnRef())
if err != nil {
return runtime.RemoteObject{}, err
}
return out, nil
}
func (rt *Runtime) CallValue(ctx context.Context, fn *CompiledFunction) (core.Value, error) {
out, err := rt.callInternal(ctx, fn.returnValue())
if err != nil {
return values.None, err
}
return rt.resolver.ToValue(ctx, out)
}
func (rt *Runtime) CallElement(ctx context.Context, fn *CompiledFunction) (drivers.HTMLElement, error) {
ref, err := rt.CallRef(ctx, fn)
if err != nil {
return nil, err
}
return rt.resolver.ToElement(ctx, ref)
}
func (rt *Runtime) CallElements(ctx context.Context, fn *CompiledFunction) (*values.Array, error) {
ref, err := rt.CallRef(ctx, fn)
if err != nil {
return nil, err
}
val, err := rt.resolver.ToValue(ctx, ref)
if err != nil {
return nil, err
}
arr, ok := val.(*values.Array)
if ok {
return arr, nil
}
return values.NewArrayWith(val), nil
}
func (rt *Runtime) evalInternal(ctx context.Context, fn *Function) (runtime.RemoteObject, error) {
log := rt.logger.With().
Str("expression", fn.String()).
Str("returns", fn.returnType.String()).
@ -133,12 +234,12 @@ func (rt *Runtime) call(ctx context.Context, fn *Function) (runtime.RemoteObject
log.Trace().Msg("executing expression...")
repl, err := rt.client.Runtime.CallFunctionOn(ctx, fn.build(rt.contextID))
repl, err := rt.client.Runtime.CallFunctionOn(ctx, fn.eval(rt.contextID))
if err != nil {
log.Trace().Err(err).Msg("failed executing expression")
return runtime.RemoteObject{}, errors.Wrap(err, "runtime call")
return runtime.RemoteObject{}, errors.Wrap(err, "runtime evalInternal")
}
if err := parseRuntimeException(repl.ExceptionDetails); err != nil {
@ -160,11 +261,57 @@ func (rt *Runtime) call(ctx context.Context, fn *Function) (runtime.RemoteObject
}
log.Trace().
Str("return-type", repl.Result.Type).
Str("return-sub-type", subtype).
Str("return-class-name", className).
Str("return-value", string(repl.Result.Value)).
Str("returned-type", repl.Result.Type).
Str("returned-sub-type", subtype).
Str("returned-class-name", className).
Str("returned-value", string(repl.Result.Value)).
Msg("succeeded executing expression")
return repl.Result, nil
}
func (rt *Runtime) callInternal(ctx context.Context, fn *CompiledFunction) (runtime.RemoteObject, error) {
log := rt.logger.With().
Str("script-id", string(fn.id)).
Str("returns", fn.src.returnType.String()).
Bool("is-async", fn.src.async).
Array("arguments", fn.src.args).
Logger()
log.Trace().Msg("executing compiled script...")
repl, err := rt.client.Runtime.RunScript(ctx, fn.call(rt.contextID))
if err != nil {
log.Trace().Err(err).Msg("failed executing compiled script")
return runtime.RemoteObject{}, errors.Wrap(err, "runtime evalInternal")
}
if err := parseRuntimeException(repl.ExceptionDetails); err != nil {
log.Trace().Err(err).Msg("compiled script has failed with runtime exception")
return runtime.RemoteObject{}, err
}
var className string
if repl.Result.ClassName != nil {
className = *repl.Result.ClassName
}
var subtype string
if repl.Result.Subtype != nil {
subtype = *repl.Result.Subtype
}
log.Trace().
Str("returned-type", repl.Result.Type).
Str("returned-sub-type", subtype).
Str("returned-class-name", className).
Str("returned-value", string(repl.Result.Value)).
Msg("succeeded executing compiled script")
return repl.Result, nil
}

138
pkg/drivers/cdp/eval/types.go
View File

@ -4,37 +4,121 @@ import (
"github.com/mafredri/cdp/protocol/runtime"
)
type RemoteType string
type (
RemoteType string
// List of possible remote types
const (
UnknownType RemoteType = ""
NullType RemoteType = "null"
UndefinedType RemoteType = "undefined"
ArrayType RemoteType = "array"
NodeType RemoteType = "node"
RegexpType RemoteType = "regexp"
DateType RemoteType = "date"
MapType RemoteType = "map"
SetType RemoteType = "set"
WeakMapType RemoteType = "weakmap"
WeakSetType RemoteType = "weakset"
IteratorType RemoteType = "iterator"
GeneratorType RemoteType = "generator"
ErrorType RemoteType = "error"
ProxyType RemoteType = "proxy"
PromiseType RemoteType = "promise"
TypedArrayType RemoteType = "typedarray"
ArrayBufferType RemoteType = "arraybuffer"
DataViewType RemoteType = "dataview"
RemoteObjectType string
RemoteClassName string
)
func ToRemoteType(ref runtime.RemoteObject) RemoteType {
var subtype string
// List of possible remote types
// "object", "function", "undefined", "string", "number", "boolean", "symbol", "bigint"
const (
UnknownType RemoteType = ""
UndefinedType RemoteType = "undefined"
StringType RemoteType = "string"
NumberType RemoteType = "number"
BooleanType RemoteType = "boolean"
SymbolType RemoteType = "symbol"
BigintType RemoteType = "bigint"
ObjectType RemoteType = "object"
)
if ref.Subtype != nil {
subtype = *ref.Subtype
var remoteTypeMap = map[string]RemoteType{
string(UndefinedType): UndefinedType,
string(StringType): StringType,
string(NumberType): NumberType,
string(BooleanType): BooleanType,
string(SymbolType): SymbolType,
string(BigintType): BigintType,
string(ObjectType): ObjectType,
}
// List of possible remote object types
const (
UnknownObjectType RemoteObjectType = ""
NullObjectType RemoteObjectType = "null"
UndefinedObjectType RemoteObjectType = "undefined"
ArrayObjectType RemoteObjectType = "array"
NodeObjectType RemoteObjectType = "node"
RegexpObjectType RemoteObjectType = "regexp"
DateObjectType RemoteObjectType = "date"
MapObjectType RemoteObjectType = "map"
SetObjectType RemoteObjectType = "set"
WeakMapObjectType RemoteObjectType = "weakmap"
WeakSetObjectType RemoteObjectType = "weakset"
IteratorObjectType RemoteObjectType = "iterator"
GeneratorObjectType RemoteObjectType = "generator"
ErrorObjectType RemoteObjectType = "error"
ProxyObjectType RemoteObjectType = "proxy"
PromiseObjectType RemoteObjectType = "promise"
TypedArrayObjectType RemoteObjectType = "typedarray"
ArrayBufferObjectType RemoteObjectType = "arraybuffer"
DataViewObjectType RemoteObjectType = "dataview"
)
var remoteObjectTypeMap = map[string]RemoteObjectType{
string(NullObjectType): NullObjectType,
string(UndefinedObjectType): UndefinedObjectType,
string(ArrayObjectType): ArrayObjectType,
string(NodeObjectType): NodeObjectType,
string(RegexpObjectType): RegexpObjectType,
string(DateObjectType): DateObjectType,
string(MapObjectType): MapObjectType,
string(SetObjectType): SetObjectType,
string(WeakMapObjectType): WeakMapObjectType,
string(WeakSetObjectType): WeakSetObjectType,
string(IteratorObjectType): IteratorObjectType,
string(GeneratorObjectType): GeneratorObjectType,
string(ErrorObjectType): ErrorObjectType,
string(ProxyObjectType): ProxyObjectType,
string(PromiseObjectType): PromiseObjectType,
string(TypedArrayObjectType): TypedArrayObjectType,
string(ArrayBufferObjectType): ArrayBufferObjectType,
string(DataViewObjectType): DataViewObjectType,
}
// List of supported remote classses
const (
UnknownClassName RemoteClassName = ""
DocumentClassName RemoteClassName = "HTMLDocument"
)
var remoteClassNameMap = map[string]RemoteClassName{
string(DocumentClassName): DocumentClassName,
}
func ToRemoteType(ref runtime.RemoteObject) RemoteType {
remoteType, found := remoteTypeMap[ref.Type]
if found {
return remoteType
}
return RemoteType(subtype)
return UnknownType
}
func ToRemoteObjectType(ref runtime.RemoteObject) RemoteObjectType {
if ref.Subtype != nil {
remoteObjectType, found := remoteObjectTypeMap[*ref.Subtype]
if found {
return remoteObjectType
}
}
return UnknownObjectType
}
func ToRemoteClassName(ref runtime.RemoteObject) RemoteClassName {
if ref.ClassName != nil {
remoteClassName, found := remoteClassNameMap[*ref.ClassName]
if found {
return remoteClassName
}
}
return UnknownClassName
}

7
pkg/drivers/cdp/eval/value.go Normal file
View File

@ -0,0 +1,7 @@
package eval
import "github.com/mafredri/cdp/protocol/runtime"
type RemoteValue interface {
RemoteID() runtime.RemoteObjectID
}

13
pkg/drivers/cdp/events/wait.go
View File

@ -69,6 +69,19 @@ func NewEvalWaitTask(
)
}
func NewCallWaitTask(
ec *eval.Runtime,
fn *eval.CompiledFunction,
polling time.Duration,
) *WaitTask {
return NewWaitTask(
func(ctx context.Context) (core.Value, error) {
return ec.CallValue(ctx, fn)
},
polling,
)
}
func NewValueWaitTask(
when drivers.WaitEvent,
value core.Value,

2
pkg/drivers/cdp/input/manager.go
View File

@ -33,7 +33,7 @@ type (
}
)
func NewManager(
func New(
logger zerolog.Logger,
client *cdp.Client,
exec *eval.Runtime,

47
pkg/drivers/cdp/page.go
View File

@ -31,22 +31,16 @@ type (
HTMLPageEvent string
HTMLPage struct {
mu sync.Mutex
closed values.Boolean
logger zerolog.Logger
conn *rpcc.Conn
client *cdp.Client
network *net.Manager
dom *dom.Manager
mouse *input.Mouse
keyboard *input.Keyboard
mu sync.Mutex
closed values.Boolean
logger zerolog.Logger
conn *rpcc.Conn
client *cdp.Client
network *net.Manager
dom *dom.Manager
}
)
const (
HTMLPageEventNavigation HTMLPageEvent = "navigation"
)
func LoadHTMLPage(
ctx context.Context,
conn *rpcc.Conn,
@ -113,6 +107,7 @@ func LoadHTMLPage(
mouse,
keyboard,
)
if err != nil {
return nil, err
}
@ -123,8 +118,6 @@ func LoadHTMLPage(
client,
netManager,
domManager,
mouse,
keyboard,
)
if params.URL != BlankPageURL && params.URL != "" {
@ -191,8 +184,6 @@ func NewHTMLPage(
client *cdp.Client,
netManager *net.Manager,
domManager *dom.Manager,
mouse *input.Mouse,
keyboard *input.Keyboard,
) *HTMLPage {
p := new(HTMLPage)
p.closed = values.False
@ -201,8 +192,6 @@ func NewHTMLPage(
p.client = client
p.network = netManager
p.dom = domManager
p.mouse = mouse
p.keyboard = keyboard
return p
}
@ -327,7 +316,7 @@ func (p *HTMLPage) IsClosed() values.Boolean {
}
func (p *HTMLPage) GetURL() values.String {
res, err := p.getCurrentDocument().Eval(context.Background(), templates.GetURL().String())
res, err := p.getCurrentDocument().Eval().EvalValue(context.Background(), templates.GetURL())
if err == nil {
return values.ToString(res)
@ -693,14 +682,7 @@ func (p *HTMLPage) reloadMainFrame(ctx context.Context) error {
}
}
next, err := dom.LoadRootHTMLDocument(
ctx,
p.logger,
p.client,
p.dom,
p.mouse,
p.keyboard,
)
next, err := p.dom.LoadRootDocument(ctx)
if err != nil {
p.logger.Error().Err(err).Msg("failed to load a new root document")
@ -714,14 +696,7 @@ func (p *HTMLPage) reloadMainFrame(ctx context.Context) error {
}
func (p *HTMLPage) loadMainFrame(ctx context.Context) error {
next, err := dom.LoadRootHTMLDocument(
ctx,
p.logger,
p.client,
p.dom,
p.mouse,
p.keyboard,
)
next, err := p.dom.LoadRootDocument(ctx)
if err != nil {
return err

33
pkg/runtime/values/helpers.go
View File

@ -4,6 +4,7 @@ import (
"context"
"encoding/binary"
"encoding/json"
"github.com/wI2L/jettison"
"hash/fnv"
"reflect"
"sort"
@ -292,6 +293,26 @@ func Unmarshal(value json.RawMessage) (core.Value, error) {
return Parse(o), nil
}
func MustMarshal(value core.Value) json.RawMessage {
out, err := value.MarshalJSON()
if err != nil {
panic(err)
}
return out
}
func MustMarshalAny(input interface{}) json.RawMessage {
out, err := jettison.MarshalOpts(input, jettison.NoHTMLEscaping())
if err != nil {
panic(err)
}
return out
}
func ToBoolean(input core.Value) Boolean {
switch input.Type() {
case types.Boolean:
@ -515,17 +536,7 @@ func ToObject(ctx context.Context, input core.Value) *Object {
}
}
func ToStrings(input []core.Value) []String {
res := make([]String, len(input))
for i, v := range input {
res[i] = NewString(v.String())
}
return res
}
func ToStrings2(input *Array) []String {
func ToStrings(input *Array) []String {
res := make([]String, input.Length())
input.ForEach(func(v core.Value, i int) bool {

2
pkg/stdlib/html/press.go
View File

@ -41,7 +41,7 @@ func Press(ctx context.Context, args ...core.Value) (core.Value, error) {
case values.String:
return values.True, el.Press(ctx, []values.String{keys}, count)
case *values.Array:
return values.True, el.Press(ctx, values.ToStrings2(keys), count)
return values.True, el.Press(ctx, values.ToStrings(keys), count)
default:
return values.None, core.TypeError(keysArg.Type(), types.String, types.Array)
}

2
pkg/stdlib/html/press_selector.go
View File

@ -49,7 +49,7 @@ func PressSelector(ctx context.Context, args ...core.Value) (core.Value, error)
case values.String:
return values.True, el.PressBySelector(ctx, selector, []values.String{keys}, count)
case *values.Array:
return values.True, el.PressBySelector(ctx, selector, values.ToStrings2(keys), count)
return values.True, el.PressBySelector(ctx, selector, values.ToStrings(keys), count)
default:
return values.None, core.TypeError(keysArg.Type(), types.String, types.Array)
}