go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2025-07-05 00:49:00 +02:00
Code Issues Releases Activity

Added support of computed styles (#570)

Browse Source 
* Added support of computed styles

* Updated style updates

* Fixed linter issues

* Updated styles manipulation in static driver

* Updated e2e tests

* Updated methods

* Updated e2e tests

* Updated README
This commit is contained in:
Tim Voronov
2020-11-20 20:09:21 -05:00
committed by GitHub
parent 01088247e2
commit 7eed93721c
26 changed files with 200 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
README.md
View File

@ -48,7 +48,14 @@ LET google = DOCUMENT("https://www.google.com/", {
userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36"
})
INPUT(google, 'input[name="q"]', "ferret")
HOVER(google, 'input[name="q"]')
WAIT(RAND(100))
INPUT(google, 'input[name="q"]', @criteria, 30)
WAIT(RAND(100))
WAIT_ELEMENT(google, '.UUbT9')
WAIT(RAND(100))
CLICK(google, 'input[name="btnK"]')
WAIT_NAVIGATION(google)
@ -58,7 +65,7 @@ FOR result IN ELEMENTS(google, '.g')
FILTER TRIM(result.attributes.class) == 'g'
RETURN {
title: INNER_TEXT(result, 'h3'),
description: INNER_TEXT(result, '.st'),
description: INNER_TEXT(result, '.rc > div:nth-child(2) span'),
url: INNER_TEXT(result, 'cite')
}
```
@ -76,13 +83,13 @@ After some time looking for a tool that would let me declare which data I needed
### Inspiration
FQL (Ferret Query Language) is meant to feel like writing a database query.
It is heavily inspired by [AQL](https://www.arangodb.com/) (ArangoDB Query Language).
But due to the domain specifics, there are some differences in syntax and how things work.
But, due to the domain specifics, there are some differences in syntax and how things work.
## Installation
### Binary
You can download latest binaries from [here](https://github.com/MontFerret/ferret/releases).
You can download the latest binaries from [here](https://github.com/MontFerret/ferret/releases).
### Source code
#### Production
@ -158,7 +165,7 @@ ferret < ./docs/examples/static-page.fql
### Browser mode
By default, ``ferret`` loads HTML pages directly via HTTP protocol, because it's faster.
But nowadays, more and more websites are rendered with JavaScript, and this 'old school' approach does not really work.
But, nowadays, more and more websites are rendered with JavaScript, and this 'old school' approach does not really work.
For these dynamic websites, you may fetch documents using Chrome or Chromium via Chrome DevTools protocol (aka CDP).
First, you need to make sure that you launched Chrome with ```remote-debugging-port=9222``` flag (see "Environment" in this README for instructions on setting this up).
Second, you need to pass the address to ```ferret``` CLI.
@ -176,8 +183,6 @@ Alternatively, you can tell CLI to launch Chrome for you.
ferret --cdp-launch
```
**NOTE:** Launch command is currently broken on MacOS.
Once ```ferret``` knows how to communicate with Chrome, you can use the function ```DOCUMENT(url, isDynamic)```, setting ```isDynamic``` to ```{driver: "cdp"}``` for dynamic pages:
```shell
@ -216,7 +221,7 @@ Please use `exit` or `Ctrl-D` to exit this program.
### Embedded mode
```ferret``` is a very modular system.
It can be be embedded into your Go application in only a few lines of code.
It can be embedded into your Go application in only a few lines of code.
Here is an example of a short Go application that defines an `fql` query, compiles it, executes it, then returns the results.
@ -583,29 +588,3 @@ FOR url IN urls
## References
Further documentation is available [at our website](https://www.montferret.dev/docs/introduction/)
## Contributors
Thanks to everyone who contributed.
<a href="https://github.com/MontFerret/ferret/graphs/contributors"><img src="https://opencollective.com/ferret/contributors.svg?width=890&button=false" /></a>
## Financial support
Support this project by becoming a sponsor. Your logo will show up here with a link to your website.
<p>
<a href="https://opencollective.com/ferret#section-contributors" target="_blank">
<img src="https://opencollective.com/ferret/sponsors.svg?width=890&button=false" />
</a>
</p>
<p>
<a href="https://opencollective.com/ferret#section-contributors" target="_blank">
<img src="https://opencollective.com/ferret/backers.svg?width=890&button=false" />
</a>
</p>
<p align="center">
<a href="https://opencollective.com/ferret/donate" target="_blank">
<img src="https://opencollective.com/ferret/donate/button@2x.png?color=blue" width="300" />
</a>
</p>

16
e2e/tests/dynamic/doc/wait/no_style.fql
View File

@ -6,17 +6,19 @@ WAIT_ELEMENT(doc, "#page-events")
LET el = ELEMENT(doc, selector)
ATTR_SET(el, "style", "width: 100%")
WAIT_STYLE(doc, selector, "width", "100%")
STYLE_SET(el, "color", "green")
WAIT(200)
WAIT_STYLE(doc, selector, "color", "rgb(0, 128, 0)")
LET prev = el.style
ATTR_SET(el, "style", "width: 50%")
WAIT_NO_STYLE(doc, selector, "width", "100%")
STYLE_SET(el, "color", "red")
WAIT_NO_STYLE(doc, selector, "color", "rgb(0, 128, 0)")
WAIT_STYLE(doc, selector, "color", "rgb(255, 0, 0)")
LET curr = el.style
T::EQ(prev.width, "100%")
T::EQ(curr.width, "50%")
T::EQ(prev.color, "rgb(0, 128, 0)")
T::EQ(curr.color, "rgb(255, 0, 0)")
RETURN NONE

6
e2e/tests/dynamic/doc/wait/no_style_all.fql
View File

@ -11,7 +11,7 @@ LET n = (
RETURN NONE
)
WAIT_STYLE_ALL(doc, selector, "color", "black", 10000)
WAIT_STYLE_ALL(doc, selector, "color", "rgb(0, 0, 0)", 10000)
LET n2 = (
FOR el IN ELEMENTS(doc, selector)
@ -20,13 +20,13 @@ LET n2 = (
RETURN NONE
)
WAIT_NO_STYLE_ALL(doc, selector, "color", "black", 10000)
WAIT_NO_STYLE_ALL(doc, selector, "color", "rgb(0, 0, 0)", 10000)
LET results = (
FOR el IN ELEMENTS(doc, selector)
RETURN el.style.color
)
T::EQ(CONCAT(results), "redred", "styles should be updated")
T::EQ(results, ["rgb(255, 0, 0)","rgb(255, 0, 0)"])
RETURN NONE

8
e2e/tests/dynamic/doc/wait/style.fql
View File

@ -7,12 +7,12 @@ WAIT_ELEMENT(doc, "#page-events")
LET el = ELEMENT(doc, selector)
LET prev = el.style
ATTR_SET(el, "style", "width: 100%")
WAIT_STYLE(doc, selector, "width", "100%")
ATTR_SET(el, "style", "width: 200px")
WAIT_STYLE(doc, selector, "width", "200px")
LET curr = el.style
T::NONE(prev.width)
T::EQ(curr.width, "100%", "style should be updated")
T::NOT::EQ(prev.width, "200px")
T::EQ(curr.width, "200px")
RETURN NONE

8
e2e/tests/dynamic/doc/wait/style_all.fql
View File

@ -6,18 +6,18 @@ WAIT_ELEMENT(doc, "#page-events")
LET n = (
FOR el IN ELEMENTS(doc, selector)
ATTR_SET(el, "style", "color: black")
ATTR_SET(el, "style", "width: 200px")
RETURN NONE
)
WAIT_STYLE_ALL(doc, selector, "color", "black", 10000)
WAIT_STYLE_ALL(doc, selector, "width", "200px", 10000)
LET results = (
FOR el IN ELEMENTS(doc, selector)
RETURN el.style.color
RETURN el.style.width
)
T::EQ(CONCAT(results), "blackblack", "styles should be updated")
T::EQ(results, ["200px","200px"])
RETURN NONE

4
e2e/tests/dynamic/element/attrs/get_2.fql
View File

@ -8,6 +8,8 @@ WAIT_ELEMENT(doc, pageSelector)
LET el = ELEMENT(doc, elemSelector)
LET attrs = ATTR_GET(el, "style")
T::EQ(attrs.style, "display: block;")
T::EQ(attrs.style, {
display: "block"
})
RETURN NONE

8
e2e/tests/dynamic/element/attrs/remove.fql
View File

@ -10,11 +10,11 @@ LET prev = el.attributes.style
ATTR_REMOVE(el, "style")
WAIT(1000)
LET curr = el.attributes.style
T::EQ(prev, "display: block;")
T::NONE(curr, "expected attribute to be removed")
T::EQ(prev, {
display: "block"
})
T::NONE(curr)
RETURN NONE

14
e2e/tests/dynamic/element/attrs/set.fql
View File

@ -6,14 +6,18 @@ LET elemSelector = "#wait-no-style-content"
WAIT_ELEMENT(doc, pageSelector)
LET el = ELEMENT(doc, elemSelector)
LET prev = el.style
LET prev = el.attributes.style
ATTR_SET(el, "style", "color: black;")
ATTR_SET(el, "style", {
color: "black"
})
WAIT(1000)
WAIT(200)
LET curr = el.style
LET curr = el.attributes.style
T::EQ(curr.color, "black", "styles should be updated")
PRINT(el.attributes.style)
T::EQ(curr.color, "black")
RETURN NONE

11
e2e/tests/dynamic/element/attrs/set_many.fql
View File

@ -7,11 +7,12 @@ WAIT_ELEMENT(doc, pageSelector)
LET el = ELEMENT(doc, elemSelector)
ATTR_SET(el, { style: "color: black;", "data-ferret-x": "test" })
ATTR_SET(el, {
style: "color: black;",
"data-ferret-x": "test"
})
WAIT(1000)
T::EQ(el.style.color, "black")
T::EQ(el.attributes["data-ferret-x"], "test", "styles should be updated")
T::EQ(el.attributes.style.color, "black")
T::EQ(el.attributes["data-ferret-x"], "test")
RETURN NONE

2
e2e/tests/dynamic/element/styles/get.fql
View File

@ -8,6 +8,6 @@ WAIT_ELEMENT(doc, pageSelector)
LET el = ELEMENT(doc, elemSelector)
LET val = STYLE_GET(el, "display")
T::EQ(val.display, "block", "could not get style values")
T::EQ(val, {display: "block"})
RETURN NONE

8
e2e/tests/dynamic/element/styles/remove.fql
View File

@ -1,20 +1,20 @@
LET url = @lab.cdn.dynamic + "?redirect=/events"
LET doc = DOCUMENT(url, true)
LET doc = DOCUMENT(url, { driver: "cdp" })
LET pageSelector = "#page-events"
LET elemSelector = "#wait-no-style-content"
WAIT_ELEMENT(doc, pageSelector)
LET el = ELEMENT(doc, elemSelector)
LET prev = el.style
LET prev = el.attributes.style
STYLE_REMOVE(el, "display")
WAIT(1000)
LET curr = el.style
LET curr = el.attributes.style
T::EQ(prev.display, "block")
T::NONE(curr.display, "expected style to be removed")
T::NONE(curr.display)
RETURN NONE

2
e2e/tests/dynamic/element/styles/set_2.fql
View File

@ -14,6 +14,6 @@ WAIT(1000)
LET curr = el.style
T::EQ(curr.color, "black", "styles should be updated")
T::EQ(curr.color, "rgb(0, 0, 0)")
RETURN NONE

7
e2e/tests/dynamic/element/styles/set_many.fql
View File

@ -8,14 +8,13 @@ WAIT_ELEMENT(doc, pageSelector)
LET el = ELEMENT(doc, elemSelector)
LET prev = el.style
STYLE_SET(el, { color: "black", "min-width": "100px", "background-color": "#11111" })
STYLE_SET(el, { color: "black", "font-size": "10px"})
WAIT(1000)
LET curr = el.style
T::EQ(curr.color, "black", "color should be updated")
T::EQ(curr["min-width"], "100px", "min width should be updated")
T::EQ(curr["background-color"], "#11111", "background color should be updated")
T::EQ(curr.color, "rgb(0, 0, 0)")
T::EQ(curr["font-size"], "10px")
RETURN NONE

6
e2e/tests/dynamic/element/wait/no_style.fql
View File

@ -6,7 +6,7 @@ WAIT_ELEMENT(doc, "#page-events")
LET el = ELEMENT(doc, "#wait-class-content")
ATTR_SET(el, "style", "color: black")
WAIT_STYLE(el, "color", "black")
WAIT_STYLE(el, "color", "rgb(0, 0, 0)")
LET prev = el.style
@ -15,7 +15,7 @@ WAIT_NO_STYLE(el, "color", "black")
LET curr = el.style
T::EQ(prev.color, "black")
T::EQ(curr.color, "red", "style should be changed")
T::EQ(prev.color, "rgb(0, 0, 0)")
T::EQ(curr.color, "rgb(255, 0, 0)")
RETURN NONE

8
e2e/tests/examples/download.yaml
View File

@ -1,8 +0,0 @@
query:
ref: ../../../examples/download.fql
assert:
text: |
LET result = @lab.data.query.result
T::EQ(result.type, "png")
T::NOT::EMPTY(result.data)
RETURN NONE

2
examples/google-search.fql
View File

@ -20,6 +20,6 @@ FOR result IN ELEMENTS(google, '.g')
FILTER TRIM(result.attributes.class) == 'g'
RETURN {
title: INNER_TEXT(result, 'h3'),
description: INNER_TEXT(result, '.st'),
description: INNER_TEXT(result, '.rc > div:nth-child(2) span'),
url: INNER_TEXT(result, 'cite')
}

84
pkg/drivers/cdp/dom/element.go
View File

@ -278,21 +278,19 @@ func (el *HTMLElement) GetStyles(ctx context.Context) (*values.Object, error) {
return values.NewObject(), drivers.ErrDetached
}
value, err := el.GetAttribute(ctx, "style")
value, err := el.exec.EvalWithArgumentsAndReturnValue(ctx, templates.GetStyles(), runtime.CallArgument{
ObjectID: &el.id.ObjectID,
})
if err != nil {
return values.NewObject(), err
}
if value == values.None {
return values.NewObject(), nil
if value.Type() == types.Object {
return value.(*values.Object), err
}
if value.Type() != types.String {
return values.NewObject(), nil
}
return common.DeserializeStyles(value.(values.String))
return values.NewObject(), core.TypeError(value.Type(), types.Object)
}
func (el *HTMLElement) GetStyle(ctx context.Context, name values.String) (core.Value, error) {
@ -338,25 +336,40 @@ func (el *HTMLElement) SetStyles(ctx context.Context, styles *values.Object) err
str := common.SerializeStyles(ctx, currentStyles)
return el.SetAttribute(ctx, "style", str)
return el.SetAttribute(ctx, common.AttrNameStyle, str)
}
func (el *HTMLElement) SetStyle(ctx context.Context, name values.String, value core.Value) error {
func (el *HTMLElement) SetStyle(ctx context.Context, name, value values.String) error {
if el.IsDetached() {
return drivers.ErrDetached
}
styles, err := el.GetStyles(ctx)
// we manually set only those that are defined in attribute only
attrValue, err := el.GetAttribute(ctx, common.AttrNameStyle)
if err != nil {
return err
}
var styles *values.Object
if attrValue == values.None {
styles = values.NewObject()
} else {
styleAttr, ok := attrValue.(*values.Object)
if !ok {
return core.TypeError(attrValue.Type(), types.Object)
}
styles = styleAttr
}
styles.Set(name, value)
str := common.SerializeStyles(ctx, styles)
return el.SetAttribute(ctx, "style", str)
return el.SetAttribute(ctx, common.AttrNameStyle, str)
}
func (el *HTMLElement) RemoveStyle(ctx context.Context, names ...values.String) error {
@ -368,19 +381,30 @@ func (el *HTMLElement) RemoveStyle(ctx context.Context, names ...values.String)
return nil
}
styles, err := el.GetStyles(ctx)
value, err := el.GetAttribute(ctx, common.AttrNameStyle)
if err != nil {
return err
}
// no attribute
if value == values.None {
return nil
}
styles, ok := value.(*values.Object)
if !ok {
return core.TypeError(styles.Type(), types.Object)
}
for _, name := range names {
styles.Remove(name)
}
str := common.SerializeStyles(ctx, styles)
return el.SetAttribute(ctx, "style", str)
return el.SetAttribute(ctx, common.AttrNameStyle, str)
}
func (el *HTMLElement) GetAttributes(ctx context.Context) (*values.Object, error) {
@ -397,7 +421,22 @@ func (el *HTMLElement) GetAttributes(ctx context.Context) (*values.Object, error
attrs := values.NewObject()
traverseAttrs(repl.Attributes, func(name, value string) bool {
attrs.Set(values.NewString(name), values.NewString(value))
key := values.NewString(name)
var val core.Value = values.None
if name != common.AttrNameStyle {
val = values.NewString(value)
} else {
parsed, err := common.DeserializeStyles(values.NewString(value))
if err == nil {
val = parsed
} else {
val = values.NewObject()
}
}
attrs.Set(key, val)
return true
})
@ -416,13 +455,22 @@ func (el *HTMLElement) GetAttribute(ctx context.Context, name values.String) (co
return values.None, err
}
var result core.Value
result = values.None
var result core.Value = values.None
targetName := strings.ToLower(name.String())
traverseAttrs(repl.Attributes, func(name, value string) bool {
if name == targetName {
if name != common.AttrNameStyle {
result = values.NewString(value)
} else {
parsed, err := common.DeserializeStyles(values.NewString(value))
if err == nil {
result = parsed
} else {
result = values.NewObject()
}
}
return false
}
@ -669,7 +717,7 @@ func (el *HTMLElement) XPath(ctx context.Context, expression values.String) (res
ObjectID: &el.id.ObjectID,
},
runtime.CallArgument{
Value: json.RawMessage(exp),
Value: exp,
},
)

7
pkg/drivers/cdp/driver.go
View File

@ -187,7 +187,12 @@ func (drv *Driver) init(ctx context.Context) error {
return errors.Wrap(err, "failed to initialize driver")
}
bconn, err := rpcc.DialContext(ctx, ver.WebSocketDebuggerURL)
bconn, err := rpcc.DialContext(
ctx,
ver.WebSocketDebuggerURL,
rpcc.WithWriteBufferSize(1048562),
rpcc.WithCompression(),
)
if err != nil {
return errors.Wrap(err, "failed to initialize driver")

22
pkg/drivers/cdp/templates/get_styles.go Normal file
View File

@ -0,0 +1,22 @@
package templates
var getStylesTemplate = `
(el) => {
const out = {};
const styles = window.getComputedStyle(el);
Object.keys(styles).forEach((key) => {
if (!isNaN(parseFloat(key))) {
const name = styles[key];
const value = styles.getPropertyValue(name);
out[name] = value;
}
});
return out;
}
`
func GetStyles() string {
return getStylesTemplate
}

13
pkg/drivers/cdp/templates/styles.go
View File

@ -9,9 +9,12 @@ import (
func StyleRead(name values.String) string {
n := name.String()
return fmt.Sprintf(
`el.style[%s] != "" ? el.style[%s] : null`,
eval.ParamString(n),
eval.ParamString(n),
)
return fmt.Sprintf(`
((function() {
const cs = window.getComputedStyle(el);
const currentValue = cs.getPropertyValue(%s);
return currentValue || null;
})())
`, eval.ParamString(n))
}

4
pkg/drivers/common/attrs.go
View File

@ -140,6 +140,10 @@ var Attributes = []string{
"wrap",
}
const (
AttrNameStyle = "style"
)
var attrMap = make(map[string]bool)
func init() {

4
pkg/drivers/common/setter.go
View File

@ -74,7 +74,7 @@ func SetInElement(ctx context.Context, el drivers.HTMLElement, path []core.Value
if len(path) > 1 {
attrName := path[1]
return el.SetStyle(ctx, values.NewString(attrName.String()), value)
return el.SetStyle(ctx, values.NewString(attrName.String()), values.NewString(value.String()))
}
err := core.ValidateType(value, types.Object)
@ -93,7 +93,7 @@ func SetInElement(ctx context.Context, el drivers.HTMLElement, path []core.Value
obj := value.(*values.Object)
obj.ForEach(func(value core.Value, key string) bool {
err = el.SetStyle(ctx, values.NewString(key), value)
err = el.SetStyle(ctx, values.NewString(key), values.NewString(value.String()))
return err == nil
})

12
pkg/drivers/http/element.go
View File

@ -174,7 +174,7 @@ func (el *HTMLElement) GetStyle(ctx context.Context, name values.String) (core.V
return el.styles.MustGet(name), nil
}
func (el *HTMLElement) SetStyle(ctx context.Context, name values.String, value core.Value) error {
func (el *HTMLElement) SetStyle(ctx context.Context, name, value values.String) error {
if err := el.ensureStyles(ctx); err != nil {
return err
}
@ -248,15 +248,23 @@ func (el *HTMLElement) GetAttributes(_ context.Context) (*values.Object, error)
return el.attrs.Copy().(*values.Object), nil
}
func (el *HTMLElement) GetAttribute(_ context.Context, name values.String) (core.Value, error) {
func (el *HTMLElement) GetAttribute(ctx context.Context, name values.String) (core.Value, error) {
el.ensureAttrs()
if name == common.AttrNameStyle {
return el.GetStyles(ctx)
}
return el.attrs.MustGet(name), nil
}
func (el *HTMLElement) SetAttribute(_ context.Context, name, value values.String) error {
el.ensureAttrs()
if name == common.AttrNameStyle {
el.styles = nil
}
el.attrs.Set(name, value)
el.selection.SetAttr(string(name), string(value))

2
pkg/drivers/value.go
View File

@ -67,7 +67,7 @@ type (
SetStyles(ctx context.Context, values *values.Object) error
SetStyle(ctx context.Context, name values.String, value core.Value) error
SetStyle(ctx context.Context, name, value values.String) error
RemoveStyle(ctx context.Context, name ...values.String) error

16
pkg/stdlib/html/attr_set.go
View File

@ -2,6 +2,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
@ -35,13 +36,18 @@ func AttributeSet(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, nil
}
arg2, ok := args[2].(values.String)
if !ok {
return values.None, core.TypeError(arg1.Type(), types.String, types.Object)
switch arg2 := args[2].(type) {
case values.String:
return values.None, el.SetAttribute(ctx, arg1, arg2)
case *values.Object:
if arg1 == common.AttrNameStyle {
return values.None, el.SetAttribute(ctx, arg1, common.SerializeStyles(ctx, arg2))
}
return values.None, el.SetAttribute(ctx, arg1, arg2)
return values.None, el.SetAttribute(ctx, arg1, values.NewString(arg2.String()))
default:
return values.None, core.TypeError(arg1.Type(), types.String, types.Object)
}
case *values.Object:
// ATTR_SET(el, values)
return values.None, el.SetAttributes(ctx, arg1)

4
pkg/stdlib/html/style_set.go
View File

@ -14,7 +14,7 @@ import (
// @param {String | Object} nameOrObj - Style name or an object representing a key-value pair of attributes.
// @param {String} value - If a second parameter is a string value, this parameter represent a style value.
func StyleSet(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 2, core.MaxArgs)
err := core.ValidateArgs(args, 2, 3)
if err != nil {
return values.None, err
@ -35,7 +35,7 @@ func StyleSet(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, nil
}
return values.None, el.SetStyle(ctx, arg1, args[2])
return values.None, el.SetStyle(ctx, arg1, values.NewString(args[2].String()))
case *values.Object:
// STYLE_SET(el, values)
return values.None, el.SetStyles(ctx, arg1)