go/ferret
go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2024-12-16 11:37:36 +02:00
Code Issues Releases Activity

Merge branch 'master' of github.com:MontFerret/ferret

Browse Source
This commit is contained in:
Tim Voronov 2018-11-21 22:45:22 -05:00
commit bf10628875
18 changed files with 285 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
e2e/pages/static/simple.html Normal file
View File

@ -0,0 +1,10 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Title</title>
</head>
<body>
<h1>Hello world</h1>
</body>
</html>

2
e2e/runner/lib.go
View File

@ -24,5 +24,5 @@ func expect(_ context.Context, args ...core.Value) (core.Value, error) {
return values.EmptyString, nil
}
return values.NewString(fmt.Sprintf(`expected "%s"", but got "%s"`, args[0], args[1])), nil
return values.NewString(fmt.Sprintf(`expected "%s", but got "%s"`, args[0], args[1])), nil
}

2
e2e/runner/runner.go
View File

@ -8,6 +8,7 @@ import (
"github.com/pkg/errors"
"github.com/rs/zerolog"
"io/ioutil"
"os"
"path/filepath"
"time"
)
@ -132,6 +133,7 @@ func (r *Runner) runQuery(c *compiler.FqlCompiler, name, script string) Result {
out, err := p.Run(
context.Background(),
runtime.WithLog(os.Stdout),
runtime.WithBrowser(r.settings.CDPAddress),
runtime.WithParam("static", r.settings.StaticServerAddress),
runtime.WithParam("dynamic", r.settings.DynamicServerAddress),

19
e2e/tests/doc_inner_html_1_arg.fql Normal file
View File

@ -0,0 +1,19 @@
LET url = @static + '/simple.html'
LET doc = DOCUMENT(url)
LET expected = `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8"/>
<title>Title</title>
</head>
<body>
<h1>Hello world</h1>
</body>
</html>`
LET actual = INNER_HTML(doc)
LET r1 = '(\s|\")'
LET r2 = '(\n|\s|\")'
RETURN EXPECT(REGEXP_REPLACE(expected, r1, ''), REGEXP_REPLACE(TRIM(actual), r2, ''))

29
e2e/tests/doc_inner_html_1_arg_d.fql Normal file
View File

@ -0,0 +1,29 @@
LET url = @dynamic
LET doc = DOCUMENT(url, true)
LET expected = `<!DOCTYPE html><html lang="en"><head>
<meta charset="utf-8">
<meta http-equiv="x-ua-compatible" content="ie=edge">
<title>Ferret E2E SPA</title>
<meta name="description" content="">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
<link rel="stylesheet" href="index.css">
</head>
<body class="text-center">
<div id="root"><div id="layout"><nav class="navbar navbar-expand-md navbar-dark bg-dark mb-4"><a class="navbar-brand active" aria-current="page" href="/">Ferret</a><button class="navbar-toggler" type="button"><span class="navbar-toggler-icon"></span></button><div class="collapse navbar-collapse"><ul class="navbar-nav mr-auto"><li class="nav-item"><a class="nav-link" href="/forms">Forms</a></li><li class="nav-item"><a class="nav-link" href="/navigation">Navigation</a></li><li class="nav-item"><a class="nav-link" href="/events">Events</a></li></ul></div></nav><main class="container"><div class="jumbotron"><div><h1>Welcome to Ferret E2E test page!</h1></div><div><p class="lead">It has several pages for testing different possibilities of the library</p></div></div></main></div></div>
<script src="https://unpkg.com/react@16.6.1/umd/react.production.min.js"></script>
<script src="https://unpkg.com/react-dom@16.6.1/umd/react-dom.production.min.js"></script>
<script src="https://unpkg.com/history@4.7.2/umd/history.min.js"></script>
<script src="https://unpkg.com/react-router@4.3.1/umd/react-router.js"></script>
<script src="https://unpkg.com/react-router-dom@4.3.1/umd/react-router-dom.js"></script>
<script src="index.js" type="module"></script>
</body></html>`
LET actual = INNER_HTML(doc)
LET r1 = '(\s|\")'
LET r2 = '(\n|\s|\")'
RETURN EXPECT(REGEXP_REPLACE(expected, r1, ''), REGEXP_REPLACE(TRIM(actual), r2, ''))

10
e2e/tests/doc_inner_text_1_arg.fql Normal file
View File

@ -0,0 +1,10 @@
LET url = @static + '/simple.html'
LET doc = DOCUMENT(url)
LET expected = `Title Hello world`
LET actual = INNER_TEXT(doc)
LET r1 = '(\s|\")'
LET r2 = '(\n|\s|\")'
RETURN EXPECT(REGEXP_REPLACE(expected, r1, ''), REGEXP_REPLACE(TRIM(actual), r2, ''))

17
e2e/tests/doc_inner_text_1_arg_d.fql Normal file
View File

@ -0,0 +1,17 @@
LET url = @dynamic
LET doc = DOCUMENT(url, true)
LET expected = `Ferret E2E SPA
Ferret
Forms
Navigation
Events
Welcome to Ferret E2E test page!
It has several pages for testing different possibilities of the library
`
LET actual = INNER_TEXT(doc)
LET r1 = '(\n|\s)'
LET r2 = '(\n|\s)'
RETURN EXPECT(REGEXP_REPLACE(expected, r1, ''), REGEXP_REPLACE(TRIM(actual), r2, ''))

11
e2e/tests/el_inner_html_1_arg.fql Normal file
View File

@ -0,0 +1,11 @@
LET url = @static + '/simple.html'
LET doc = DOCUMENT(url)
LET el = ELEMENT(doc, "body")
LET expected = `<h1>Hello world</h1>`
LET actual = INNER_HTML(el)
LET r1 = '(\s|\")'
LET r2 = '(\n|\s|\")'
RETURN EXPECT(REGEXP_REPLACE(expected, r1, ''), REGEXP_REPLACE(TRIM(actual), r2, ''))

11
e2e/tests/el_inner_html_1_arg_d.fql Normal file
View File

@ -0,0 +1,11 @@
LET url = @dynamic
LET doc = DOCUMENT(url, true)
LET el = ELEMENT(doc, "#root")
LET expected = `<div id="layout"><nav class="navbar navbar-expand-md navbar-dark bg-dark mb-4"><a class="navbar-brand active" aria-current="page" href="/">Ferret</a><button class="navbar-toggler" type="button"><span class="navbar-toggler-icon"></span></button><div class="collapse navbar-collapse"><ul class="navbar-nav mr-auto"><li class="nav-item"><a class="nav-link" href="/forms">Forms</a></li><li class="nav-item"><a class="nav-link" href="/navigation">Navigation</a></li><li class="nav-item"><a class="nav-link" href="/events">Events</a></li></ul></div></nav><main class="container"><div class="jumbotron"><div><h1>Welcome to Ferret E2E test page!</h1></div><div><p class="lead">It has several pages for testing different possibilities of the library</p></div></div></main></div>`
LET actual = INNER_HTML(el)
LET r1 = '(\s|\")'
LET r2 = '(\n|\s|\")'
RETURN EXPECT(REGEXP_REPLACE(expected, r1, ''), REGEXP_REPLACE(TRIM(actual), r2, ''))

11
e2e/tests/el_inner_text_1_arg.fql Normal file
View File

@ -0,0 +1,11 @@
LET url = @static + '/simple.html'
LET doc = DOCUMENT(url)
LET el = ELEMENT(doc, "body")
LET expected = `Hello world`
LET actual = INNER_TEXT(el)
LET r1 = '(\s|\")'
LET r2 = '(\n|\s|\")'
RETURN EXPECT(REGEXP_REPLACE(expected, r1, ''), REGEXP_REPLACE(TRIM(actual), r2, ''))

14
e2e/tests/el_inner_text_1_arg_d.fql Normal file
View File

@ -0,0 +1,14 @@
LET url = @dynamic
LET doc = DOCUMENT(url, true)
LET el = ELEMENT(doc, ".jumbotron")
LET expected = `
Welcome to Ferret E2E test page!
It has several pages for testing different possibilities of the library
`
LET actual = INNER_TEXT(el)
LET r1 = '(\n|\s)'
LET r2 = '(\n|\s)'
RETURN EXPECT(REGEXP_REPLACE(expected, r1, ''), REGEXP_REPLACE(TRIM(actual), r2, ''))

34
pkg/compiler/compiler_str_test.go
View File

@ -2,6 +2,8 @@ package compiler_test
import (
"context"
"encoding/json"
"fmt"
"github.com/MontFerret/ferret/pkg/compiler"
. "github.com/smartystreets/goconvey/convey"
"testing"
@ -20,6 +22,38 @@ BAR
So(string(out), ShouldEqual, `"\nFOO\nBAR\n"`)
})
Convey("Should be possible to use multi line string with nested strings", t, func() {
out := compiler.New().
MustCompile(fmt.Sprintf(`
RETURN %s<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Title</title>
</head>
<body>
Hello world
</body>
</html>%s
`, "`", "`")).
MustRun(context.Background())
out, err := json.Marshal(`<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Title</title>
</head>
<body>
Hello world
</body>
</html>`)
So(err, ShouldBeNil)
So(string(out), ShouldEqual, string(out))
})
}
func BenchmarkStringLiteral(b *testing.B) {

10
pkg/compiler/visitor.go
View File

@ -915,7 +915,15 @@ func (v *visitor) doVisitIntegerLiteral(ctx *fql.IntegerLiteralContext) (core.Ex
}
func (v *visitor) doVisitStringLiteral(ctx *fql.StringLiteralContext) (core.Expression, error) {
text := ctx.StringLiteral().GetText()
var text string
strLiteral := ctx.StringLiteral()
if strLiteral != nil {
text = strLiteral.GetText()
} else {
text = ctx.TemplateStringLiteral().GetText()
}
// remove extra quotes
return literals.NewStringLiteral(text[1 : len(text)-1]), nil

10
pkg/html/dynamic/element.go
View File

@ -444,6 +444,10 @@ func (el *HTMLElement) InnerText() values.String {
return values.EmptyString
}
if val == values.None {
return values.EmptyString
}
return val.(values.String)
}
@ -896,15 +900,15 @@ func (el *HTMLElement) loadInnerText() (core.Value, error) {
ctx, cancel := contextWithTimeout()
defer cancel()
text, err := eval.Property(ctx, el.client, el.id.objectID, "innerText")
text, err := loadInnerText(ctx, el.client, el.id)
if err == nil {
return text, nil
}
el.logError(err).Msg("failed to read 'innerText' property of remote object")
el.logError(err).Msg("failed to get get inner text from remote object")
// and just parse innerHTML
// and just parse cached innerHTML
}
h := el.InnerHTML()

66
pkg/html/dynamic/helpers.go
View File

@ -190,13 +190,75 @@ func loadInnerHTML(ctx context.Context, client *cdp.Client, id *HTMLElementIdent
objID = *repl.Object.ObjectID
}
res, err := eval.Property(ctx, client, objID, "innerHTML")
// not a document
if id.nodeID != 1 {
res, err := eval.Property(ctx, client, objID, "innerHTML")
if err != nil {
return "", err
}
return values.NewString(res.String()), err
}
repl, err := client.DOM.GetOuterHTML(ctx, dom.NewGetOuterHTMLArgs().SetObjectID(objID))
if err != nil {
return "", err
}
return values.NewString(res.String()), err
return values.NewString(repl.OuterHTML), nil
}
func loadInnerText(ctx context.Context, client *cdp.Client, id *HTMLElementIdentity) (values.String, error) {
var objID runtime.RemoteObjectID
if id.objectID != "" {
objID = id.objectID
} else if id.backendID > 0 {
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetBackendNodeID(id.backendID))
if err != nil {
return "", err
}
if repl.Object.ObjectID == nil {
return "", errors.New("unable to resolve node")
}
objID = *repl.Object.ObjectID
} else {
repl, err := client.DOM.ResolveNode(ctx, dom.NewResolveNodeArgs().SetNodeID(id.nodeID))
if err != nil {
return "", err
}
if repl.Object.ObjectID == nil {
return "", errors.New("unable to resolve node")
}
objID = *repl.Object.ObjectID
}
// not a document
if id.nodeID != 1 {
res, err := eval.Property(ctx, client, objID, "innerText")
if err != nil {
return "", err
}
return values.NewString(res.String()), err
}
repl, err := client.DOM.GetOuterHTML(ctx, dom.NewGetOuterHTMLArgs().SetObjectID(objID))
if err != nil {
return "", err
}
return parseInnerText(repl.OuterHTML)
}
func parseInnerText(innerHTML string) (values.String, error) {

22
pkg/runtime/program.go
View File

@ -4,8 +4,10 @@ import (
"context"
"github.com/MontFerret/ferret/pkg/html"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/pkg/errors"
"runtime"
)
type Program struct {
@ -30,6 +32,12 @@ func (p *Program) Source() string {
}
func (p *Program) Run(ctx context.Context, setters ...Option) (result []byte, err error) {
ctx = NewOptions().Apply(setters...).WithContext(ctx)
ctx = html.WithDynamicDriver(ctx)
ctx = html.WithStaticDriver(ctx)
logger := logging.FromContext(ctx)
defer func() {
if r := recover(); r != nil {
// find out exactly what the error was and set err
@ -42,18 +50,22 @@ func (p *Program) Run(ctx context.Context, setters ...Option) (result []byte, er
err = errors.New("unknown panic")
}
b := make([]byte, 0, 20)
runtime.Stack(b, true)
logger.Error().
Timestamp().
Err(err).
Str("stack", string(b)).
Msg("Panic")
result = nil
}
}()
scope, closeFn := core.NewRootScope()
defer closeFn()
ctx = NewOptions().Apply(setters...).WithContext(ctx)
ctx = html.WithDynamicDriver(ctx)
ctx = html.WithStaticDriver(ctx)
out, err := p.body.Exec(ctx, scope)
if err != nil {

13
pkg/stdlib/html/inner_html.go
View File

@ -7,12 +7,12 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/values"
)
// InnerHTML Returns inner HTML string of a matched element
// InnerHTML Returns inner HTML string of a given or matched by CSS selector element
// @param doc (Document|Element) - Parent document or element.
// @param selector (String) - String of CSS selector.
// @param selector (String, optional) - String of CSS selector.
// @returns (String) - Inner HTML string if an element found, otherwise empty string.
func InnerHTML(_ context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 2, 2)
err := core.ValidateArgs(args, 1, 2)
if err != nil {
return values.EmptyString, err
@ -24,13 +24,18 @@ func InnerHTML(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
node := args[0].(values.HTMLNode)
if len(args) == 1 {
return node.InnerHTML(), nil
}
err = core.ValidateType(args[1], core.StringType)
if err != nil {
return values.None, err
}
node := args[0].(values.HTMLNode)
selector := args[1].(values.String)
return node.InnerHTMLBySelector(selector), nil

15
pkg/stdlib/html/inner_text.go
View File

@ -7,12 +7,12 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/values"
)
// InnerText returns inner text of a matched element
// InnerText returns inner text string of a given or matched by CSS selector element
// @param doc (HTMLDocument|HTMLElement) - Parent document or element.
// @param selector (String) - String of CSS selector.
// @param selector (String, optional) - String of CSS selector.
// @returns (String) - Inner text if an element found, otherwise empty string.
func InnerText(_ context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 2, 2)
err := core.ValidateArgs(args, 1, 2)
if err != nil {
return values.EmptyString, err
@ -24,14 +24,19 @@ func InnerText(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
node := args[0].(values.HTMLNode)
if len(args) == 1 {
return node.InnerText(), nil
}
err = core.ValidateType(args[1], core.StringType)
if err != nil {
return values.None, err
}
doc := args[0].(values.HTMLNode)
selector := args[1].(values.String)
return doc.InnerTextBySelector(selector), nil
return node.InnerTextBySelector(selector), nil
}