go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2025-08-13 19:52:52 +02:00
Code Issues Releases Activity

Removed redundant element value caching

Browse Source
This commit is contained in:
Tim Voronov
2019-09-07 13:12:19 -04:00
parent 926cc305df
commit 45bab19c0a
3 changed files with 77 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/drivers/cdp/document.go
View File

@@ -268,7 +268,7 @@ func (doc *HTMLDocument) ExistsBySelector(ctx context.Context, selector values.S
}
func (doc *HTMLDocument) GetTitle() values.String {
value, err := doc.exec.ReadProperty(context.Background(), doc.element.id.objectID, "title")
value, err := doc.exec.ReadProperty(context.Background(), doc.element.id.ObjectID, "title")
if err != nil {
doc.logError(errors.Wrap(err, "failed to read document title"))

146
pkg/drivers/cdp/element.go
View File

@@ -10,6 +10,13 @@ import (
"sync"
"time"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"golang.org/x/net/html"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
@@ -19,21 +26,14 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"golang.org/x/net/html"
)
var emptyNodeID = dom.NodeID(0)
type (
HTMLElementIdentity struct {
nodeID dom.NodeID
objectID runtime.RemoteObjectID
NodeID dom.NodeID
ObjectID runtime.RemoteObjectID
}
HTMLElement struct {
@@ -49,7 +49,6 @@ type (
nodeName values.String
innerHTML *common.LazyValue
innerText *common.LazyValue
value core.Value
attributes *common.LazyValue
style *common.LazyValue
children []HTMLElementIdentity
@@ -83,10 +82,6 @@ func LoadHTMLElement(
return nil, core.Error(core.ErrNotFound, fmt.Sprintf("element %d", nodeID))
}
id := HTMLElementIdentity{}
id.nodeID = nodeID
id.objectID = *obj.Object.ObjectID
return LoadHTMLElementWithID(
ctx,
logger,
@@ -94,7 +89,10 @@ func LoadHTMLElement(
broker,
input,
exec,
id,
HTMLElementIdentity{
NodeID: nodeID,
ObjectID: *obj.Object.ObjectID,
},
)
}
@@ -111,18 +109,12 @@ func LoadHTMLElementWithID(
ctx,
dom.
NewDescribeNodeArgs().
SetObjectID(id.objectID).
SetObjectID(id.ObjectID).
SetDepth(1),
)
if err != nil {
return nil, core.Error(err, strconv.Itoa(int(id.nodeID)))
}
var val string
if node.Node.Value != nil {
val = *node.Node.Value
return nil, core.Error(err, strconv.Itoa(int(id.NodeID)))
}
return NewHTMLElement(
@@ -134,7 +126,6 @@ func LoadHTMLElementWithID(
id,
node.Node.NodeType,
node.Node.NodeName,
val,
createChildrenArray(node.Node.Children),
), nil
}
@@ -148,7 +139,6 @@ func NewHTMLElement(
id HTMLElementIdentity,
nodeType int,
nodeName string,
value string,
children []HTMLElementIdentity,
) *HTMLElement {
el := new(HTMLElement)
@@ -165,9 +155,7 @@ func NewHTMLElement(
el.innerText = common.NewLazyValue(el.loadInnerText)
el.attributes = common.NewLazyValue(el.loadAttrs)
el.style = common.NewLazyValue(el.parseStyle)
el.value = values.EmptyString
el.loadedChildren = common.NewLazyValue(el.loadChildren)
el.value = values.NewString(value)
el.children = children
broker.AddEventListener(events.EventReload, el.handlePageReload)
@@ -243,7 +231,7 @@ func (el *HTMLElement) Hash() uint64 {
h.Write([]byte(el.Type().String()))
h.Write([]byte(":"))
h.Write([]byte(strconv.Itoa(int(el.id.nodeID))))
h.Write([]byte(strconv.Itoa(int(el.id.NodeID))))
return h.Sum64()
}
@@ -269,15 +257,7 @@ func (el *HTMLElement) GetValue(ctx context.Context) (core.Value, error) {
return values.None, drivers.ErrDetached
}
val, err := el.exec.ReadProperty(ctx, el.id.objectID, "value")
if err != nil {
return values.None, err
}
el.value = val
return val, nil
return el.exec.ReadProperty(ctx, el.id.ObjectID, "value")
}
func (el *HTMLElement) SetValue(ctx context.Context, value core.Value) error {
@@ -285,7 +265,7 @@ func (el *HTMLElement) SetValue(ctx context.Context, value core.Value) error {
return drivers.ErrDetached
}
return el.client.DOM.SetNodeValue(ctx, dom.NewSetNodeValueArgs(el.id.nodeID, value.String()))
return el.client.DOM.SetNodeValue(ctx, dom.NewSetNodeValueArgs(el.id.NodeID, value.String()))
}
func (el *HTMLElement) GetNodeType() values.Int {
@@ -437,7 +417,7 @@ func (el *HTMLElement) SetAttributes(ctx context.Context, attrs *values.Object)
func (el *HTMLElement) SetAttribute(ctx context.Context, name, value values.String) error {
return el.client.DOM.SetAttributeValue(
ctx,
dom.NewSetAttributeValueArgs(el.id.nodeID, string(name), string(value)),
dom.NewSetAttributeValueArgs(el.id.NodeID, string(name), string(value)),
)
}
@@ -445,7 +425,7 @@ func (el *HTMLElement) RemoveAttribute(ctx context.Context, names ...values.Stri
for _, name := range names {
err := el.client.DOM.RemoveAttribute(
ctx,
dom.NewRemoveAttributeArgs(el.id.nodeID, name.String()),
dom.NewRemoveAttributeArgs(el.id.NodeID, name.String()),
)
if err != nil {
@@ -481,7 +461,7 @@ func (el *HTMLElement) QuerySelector(ctx context.Context, selector values.String
return values.None, drivers.ErrDetached
}
selectorArgs := dom.NewQuerySelectorArgs(el.id.nodeID, selector.String())
selectorArgs := dom.NewQuerySelectorArgs(el.id.NodeID, selector.String())
found, err := el.client.DOM.QuerySelector(ctx, selectorArgs)
if err != nil {
@@ -506,7 +486,7 @@ func (el *HTMLElement) QuerySelectorAll(ctx context.Context, selector values.Str
return values.NewArray(0), drivers.ErrDetached
}
selectorArgs := dom.NewQuerySelectorAllArgs(el.id.nodeID, selector.String())
selectorArgs := dom.NewQuerySelectorAllArgs(el.id.NodeID, selector.String())
res, err := el.client.DOM.QuerySelectorAll(ctx, selectorArgs)
if err != nil {
@@ -554,7 +534,7 @@ func (el *HTMLElement) XPath(ctx context.Context, expression values.String) (res
out, err := el.exec.EvalWithArgumentsAndReturnReference(ctx, templates.XPath(),
runtime.CallArgument{
ObjectID: &el.id.objectID,
ObjectID: &el.id.ObjectID,
},
runtime.CallArgument{
Value: json.RawMessage(exp),
@@ -633,8 +613,8 @@ func (el *HTMLElement) XPath(ctx context.Context, expression values.String) (res
el.input,
el.exec,
HTMLElementIdentity{
nodeID: repl.NodeID,
objectID: *descr.Value.ObjectID,
NodeID: repl.NodeID,
ObjectID: *descr.Value.ObjectID,
},
)
@@ -665,8 +645,8 @@ func (el *HTMLElement) XPath(ctx context.Context, expression values.String) (res
el.input,
el.exec,
HTMLElementIdentity{
nodeID: repl.NodeID,
objectID: *out.ObjectID,
NodeID: repl.NodeID,
ObjectID: *out.ObjectID,
},
)
default:
@@ -713,7 +693,7 @@ func (el *HTMLElement) GetInnerTextBySelector(ctx context.Context, selector valu
ctx,
templates.GetInnerTextBySelector(),
runtime.CallArgument{
ObjectID: &el.id.objectID,
ObjectID: &el.id.ObjectID,
},
runtime.CallArgument{
Value: sel,
@@ -748,7 +728,7 @@ func (el *HTMLElement) SetInnerTextBySelector(ctx context.Context, selector, inn
ctx,
templates.SetInnerTextBySelector(),
runtime.CallArgument{
ObjectID: &el.id.objectID,
ObjectID: &el.id.ObjectID,
},
runtime.CallArgument{
Value: sel,
@@ -774,7 +754,7 @@ func (el *HTMLElement) GetInnerTextBySelectorAll(ctx context.Context, selector v
ctx,
templates.GetInnerTextBySelectorAll(),
runtime.CallArgument{
ObjectID: &el.id.objectID,
ObjectID: &el.id.ObjectID,
},
runtime.CallArgument{
Value: sel,
@@ -833,7 +813,7 @@ func (el *HTMLElement) GetInnerHTMLBySelector(ctx context.Context, selector valu
ctx,
templates.GetInnerHTMLBySelector(),
runtime.CallArgument{
ObjectID: &el.id.objectID,
ObjectID: &el.id.ObjectID,
},
runtime.CallArgument{
Value: sel,
@@ -868,7 +848,7 @@ func (el *HTMLElement) SetInnerHTMLBySelector(ctx context.Context, selector, inn
ctx,
templates.SetInnerHTMLBySelector(),
runtime.CallArgument{
ObjectID: &el.id.objectID,
ObjectID: &el.id.ObjectID,
},
runtime.CallArgument{
Value: sel,
@@ -894,7 +874,7 @@ func (el *HTMLElement) GetInnerHTMLBySelectorAll(ctx context.Context, selector v
ctx,
templates.GetInnerHTMLBySelectorAll(),
runtime.CallArgument{
ObjectID: &el.id.objectID,
ObjectID: &el.id.ObjectID,
},
runtime.CallArgument{
Value: sel,
@@ -919,7 +899,7 @@ func (el *HTMLElement) CountBySelector(ctx context.Context, selector values.Stri
return values.ZeroInt, drivers.ErrDetached
}
selectorArgs := dom.NewQuerySelectorAllArgs(el.id.nodeID, selector.String())
selectorArgs := dom.NewQuerySelectorAllArgs(el.id.NodeID, selector.String())
res, err := el.client.DOM.QuerySelectorAll(ctx, selectorArgs)
if err != nil {
@@ -935,7 +915,7 @@ func (el *HTMLElement) ExistsBySelector(ctx context.Context, selector values.Str
}
// TODO: Can we use RemoteObjectID or BackendID instead of NodeId?
selectorArgs := dom.NewQuerySelectorArgs(el.id.nodeID, selector.String())
selectorArgs := dom.NewQuerySelectorArgs(el.id.NodeID, selector.String())
res, err := el.client.DOM.QuerySelector(ctx, selectorArgs)
if err != nil {
@@ -1027,15 +1007,15 @@ func (el *HTMLElement) WaitForStyle(ctx context.Context, name values.String, val
}
func (el *HTMLElement) Click(ctx context.Context, count values.Int) error {
return el.input.Click(ctx, el.id.objectID, int(count))
return el.input.Click(ctx, el.id.ObjectID, int(count))
}
func (el *HTMLElement) ClickBySelector(ctx context.Context, selector values.String, count values.Int) error {
return el.input.ClickBySelector(ctx, el.id.nodeID, selector.String(), int(count))
return el.input.ClickBySelector(ctx, el.id.NodeID, selector.String(), int(count))
}
func (el *HTMLElement) ClickBySelectorAll(ctx context.Context, selector values.String, count values.Int) error {
return el.input.ClickBySelectorAll(ctx, el.id.nodeID, selector.String(), int(count))
return el.input.ClickBySelectorAll(ctx, el.id.NodeID, selector.String(), int(count))
}
func (el *HTMLElement) Input(ctx context.Context, value core.Value, delay values.Int) error {
@@ -1043,7 +1023,7 @@ func (el *HTMLElement) Input(ctx context.Context, value core.Value, delay values
return core.Error(core.ErrInvalidOperation, "element is not an <input> element.")
}
return el.input.Type(ctx, el.id.objectID, input.TypeParams{
return el.input.Type(ctx, el.id.ObjectID, input.TypeParams{
Text: value.String(),
Clear: false,
Delay: time.Duration(delay) * time.Millisecond,
@@ -1051,7 +1031,7 @@ func (el *HTMLElement) Input(ctx context.Context, value core.Value, delay values
}
func (el *HTMLElement) InputBySelector(ctx context.Context, selector values.String, value core.Value, delay values.Int) error {
return el.input.TypeBySelector(ctx, el.id.nodeID, selector.String(), input.TypeParams{
return el.input.TypeBySelector(ctx, el.id.NodeID, selector.String(), input.TypeParams{
Text: value.String(),
Clear: false,
Delay: time.Duration(delay) * time.Millisecond,
@@ -1059,39 +1039,39 @@ func (el *HTMLElement) InputBySelector(ctx context.Context, selector values.Stri
}
func (el *HTMLElement) Clear(ctx context.Context) error {
return el.input.Clear(ctx, el.id.objectID)
return el.input.Clear(ctx, el.id.ObjectID)
}
func (el *HTMLElement) ClearBySelector(ctx context.Context, selector values.String) error {
return el.input.ClearBySelector(ctx, el.id.nodeID, selector.String())
return el.input.ClearBySelector(ctx, el.id.NodeID, selector.String())
}
func (el *HTMLElement) Select(ctx context.Context, value *values.Array) (*values.Array, error) {
return el.input.Select(ctx, el.id.objectID, value)
return el.input.Select(ctx, el.id.ObjectID, value)
}
func (el *HTMLElement) SelectBySelector(ctx context.Context, selector values.String, value *values.Array) (*values.Array, error) {
return el.input.SelectBySelector(ctx, el.id.nodeID, selector.String(), value)
return el.input.SelectBySelector(ctx, el.id.NodeID, selector.String(), value)
}
func (el *HTMLElement) ScrollIntoView(ctx context.Context) error {
return el.input.ScrollIntoView(ctx, el.id.objectID)
return el.input.ScrollIntoView(ctx, el.id.ObjectID)
}
func (el *HTMLElement) Focus(ctx context.Context) error {
return el.input.Focus(ctx, el.id.objectID)
return el.input.Focus(ctx, el.id.ObjectID)
}
func (el *HTMLElement) FocusBySelector(ctx context.Context, selector values.String) error {
return el.input.FocusBySelector(ctx, el.id.nodeID, selector.String())
return el.input.FocusBySelector(ctx, el.id.NodeID, selector.String())
}
func (el *HTMLElement) Hover(ctx context.Context) error {
return el.input.MoveMouse(ctx, el.id.objectID)
return el.input.MoveMouse(ctx, el.id.ObjectID)
}
func (el *HTMLElement) HoverBySelector(ctx context.Context, selector values.String) error {
return el.input.MoveMouseBySelector(ctx, el.id.nodeID, selector.String())
return el.input.MoveMouseBySelector(ctx, el.id.NodeID, selector.String())
}
func (el *HTMLElement) IsDetached() values.Boolean {
@@ -1146,7 +1126,7 @@ func (el *HTMLElement) loadInnerText(ctx context.Context) (core.Value, error) {
}
func (el *HTMLElement) loadAttrs(ctx context.Context) (core.Value, error) {
repl, err := el.client.DOM.GetAttributes(ctx, dom.NewGetAttributesArgs(el.id.nodeID))
repl, err := el.client.DOM.GetAttributes(ctx, dom.NewGetAttributesArgs(el.id.NodeID))
if err != nil {
return values.None, err
@@ -1170,7 +1150,7 @@ func (el *HTMLElement) loadChildren(ctx context.Context) (core.Value, error) {
el.events,
el.input,
el.exec,
childID.nodeID,
childID.NodeID,
)
if err != nil {
@@ -1216,7 +1196,7 @@ func (el *HTMLElement) handleAttrModified(ctx context.Context, message interface
}
// it's not for this el
if reply.NodeID != el.id.nodeID {
if reply.NodeID != el.id.NodeID {
return
}
@@ -1260,7 +1240,7 @@ func (el *HTMLElement) handleAttrRemoved(ctx context.Context, message interface{
}
// it's not for this el
if reply.NodeID != el.id.nodeID {
if reply.NodeID != el.id.NodeID {
return
}
@@ -1302,7 +1282,7 @@ func (el *HTMLElement) handleChildrenCountChanged(ctx context.Context, message i
return
}
if reply.NodeID != el.id.nodeID {
if reply.NodeID != el.id.NodeID {
return
}
@@ -1315,7 +1295,7 @@ func (el *HTMLElement) handleChildrenCountChanged(ctx context.Context, message i
node, err := el.client.DOM.DescribeNode(
ctx,
dom.NewDescribeNodeArgs().SetObjectID(el.id.objectID),
dom.NewDescribeNodeArgs().SetObjectID(el.id.ObjectID),
)
if err != nil {
@@ -1334,7 +1314,7 @@ func (el *HTMLElement) handleChildInserted(ctx context.Context, message interfac
return
}
if reply.ParentNodeID != el.id.nodeID {
if reply.ParentNodeID != el.id.NodeID {
return
}
@@ -1350,7 +1330,7 @@ func (el *HTMLElement) handleChildInserted(ctx context.Context, message interfac
defer el.mu.Unlock()
for idx, id := range el.children {
if id.nodeID == prevID {
if id.NodeID == prevID {
targetIDx = idx
break
}
@@ -1361,7 +1341,7 @@ func (el *HTMLElement) handleChildInserted(ctx context.Context, message interfac
}
nextIdentity := HTMLElementIdentity{
nodeID: reply.Node.NodeID,
NodeID: reply.Node.NodeID,
}
arr := el.children
@@ -1395,7 +1375,7 @@ func (el *HTMLElement) handleChildRemoved(ctx context.Context, message interface
return
}
if reply.ParentNodeID != el.id.nodeID {
if reply.ParentNodeID != el.id.NodeID {
return
}
@@ -1410,7 +1390,7 @@ func (el *HTMLElement) handleChildRemoved(ctx context.Context, message interface
defer el.mu.Unlock()
for idx, id := range el.children {
if id.nodeID == targetID {
if id.NodeID == targetID {
targetIDx = idx
break
}
@@ -1432,7 +1412,7 @@ func (el *HTMLElement) handleChildRemoved(ctx context.Context, message interface
el.logger.Error().
Timestamp().
Err(err).
Int("nodeID", int(el.id.nodeID)).
Int("nodeID", int(el.id.NodeID)).
Msg("failed to update element")
return
@@ -1450,7 +1430,7 @@ func (el *HTMLElement) logError(err error) *zerolog.Event {
return el.logger.
Error().
Timestamp().
Int("nodeID", int(el.id.nodeID)).
Str("objectID", string(el.id.objectID)).
Int("nodeID", int(el.id.NodeID)).
Str("objectID", string(el.id.ObjectID)).
Err(err)
}

26
pkg/drivers/cdp/helpers.go
View File

@@ -71,10 +71,10 @@ func parseAttrs(attrs []string) *values.Object {
func setInnerHTML(ctx context.Context, client *cdp.Client, exec *eval.ExecutionContext, id HTMLElementIdentity, innerHTML values.String) error {
var objID *runtime.RemoteObjectID
if id.objectID != "" {
objID = &id.objectID
if id.ObjectID != "" {
objID = &id.ObjectID
} else {
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.nodeID))
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.NodeID))
if err != nil {
return err
@@ -110,10 +110,10 @@ func getInnerHTML(ctx context.Context, client *cdp.Client, exec *eval.ExecutionC
if nodeType != html.DocumentNode {
var objID runtime.RemoteObjectID
if id.objectID != "" {
objID = id.objectID
if id.ObjectID != "" {
objID = id.ObjectID
} else {
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.nodeID))
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.NodeID))
if err != nil {
return "", err
@@ -147,10 +147,10 @@ func getInnerHTML(ctx context.Context, client *cdp.Client, exec *eval.ExecutionC
func setInnerText(ctx context.Context, client *cdp.Client, exec *eval.ExecutionContext, id HTMLElementIdentity, innerText values.String) error {
var objID *runtime.RemoteObjectID
if id.objectID != "" {
objID = &id.objectID
if id.ObjectID != "" {
objID = &id.ObjectID
} else {
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.nodeID))
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.NodeID))
if err != nil {
return err
@@ -186,10 +186,10 @@ func getInnerText(ctx context.Context, client *cdp.Client, exec *eval.ExecutionC
if nodeType != html.DocumentNode {
var objID runtime.RemoteObjectID
if id.objectID != "" {
objID = id.objectID
if id.ObjectID != "" {
objID = id.ObjectID
} else {
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.nodeID))
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.NodeID))
if err != nil {
return "", err
@@ -238,7 +238,7 @@ func createChildrenArray(nodes []dom.Node) []HTMLElementIdentity {
for idx, child := range nodes {
child := child
children[idx] = HTMLElementIdentity{
nodeID: child.NodeID,
NodeID: child.NodeID,
}
}