mirror of
https://github.com/MontFerret/ferret.git
synced 2025-03-03 15:02:32 +02:00
#27 Added logging
This commit is contained in:
parent
4d4c6ceadd
commit
e427efd74e
1
.gitignore
vendored
1
.gitignore
vendored
@ -124,3 +124,4 @@ $RECYCLE.BIN/
|
||||
|
||||
vendor
|
||||
bin
|
||||
*.log
|
31
Gopkg.lock
generated
31
Gopkg.lock
generated
@ -41,6 +41,14 @@
|
||||
pruneopts = "UT"
|
||||
revision = "2b8494104d86337cdd41d0a49cbed8e4583c0ab4"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:ce579162ae1341f3e5ab30c0dce767f28b1eb6a81359aad01723f1ba6b4becdf"
|
||||
name = "github.com/gofrs/uuid"
|
||||
packages = ["."]
|
||||
pruneopts = "UT"
|
||||
revision = "370558f003bfe29580cd0f698d8640daccdcc45c"
|
||||
version = "v3.1.1"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:f14d1b50e0075fb00177f12a96dd7addf93d1e2883c25befd17285b779549795"
|
||||
@ -121,6 +129,14 @@
|
||||
revision = "75b0ecc5efcff27ac756a33ec71f0db75dc3d21c"
|
||||
version = "v0.19.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:c805e517269b0ba4c21ded5836019ed7d16953d4026cb7d00041d039c7906be9"
|
||||
name = "github.com/natefinch/lumberjack"
|
||||
packages = ["."]
|
||||
pruneopts = "UT"
|
||||
revision = "a96e63847dc3c67d17befa69c303767e2f84e54f"
|
||||
version = "v2.1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747"
|
||||
name = "github.com/pkg/errors"
|
||||
@ -129,6 +145,18 @@
|
||||
revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
|
||||
version = "v0.8.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:5bc8f93f977b72a7a5264725c3bab275e69de0cc3e5c0dc1ee56feb564c33f03"
|
||||
name = "github.com/rs/zerolog"
|
||||
packages = [
|
||||
".",
|
||||
"internal/cbor",
|
||||
"internal/json",
|
||||
]
|
||||
pruneopts = "UT"
|
||||
revision = "338f9bc14084d22cb8eeacd6492861f8449d715c"
|
||||
version = "v1.9.1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:4ca145a665316d3c020a39c0741780fa3636b9152b824206796c4dce541f4a24"
|
||||
name = "github.com/sethgrid/pester"
|
||||
@ -189,6 +217,7 @@
|
||||
"github.com/antlr/antlr4/runtime/Go/antlr",
|
||||
"github.com/chzyer/readline",
|
||||
"github.com/corpix/uarand",
|
||||
"github.com/gofrs/uuid",
|
||||
"github.com/mafredri/cdp",
|
||||
"github.com/mafredri/cdp/devtool",
|
||||
"github.com/mafredri/cdp/protocol/dom",
|
||||
@ -198,7 +227,9 @@
|
||||
"github.com/mafredri/cdp/protocol/target",
|
||||
"github.com/mafredri/cdp/rpcc",
|
||||
"github.com/mafredri/cdp/session",
|
||||
"github.com/natefinch/lumberjack",
|
||||
"github.com/pkg/errors",
|
||||
"github.com/rs/zerolog",
|
||||
"github.com/sethgrid/pester",
|
||||
"github.com/smartystreets/goconvey/convey",
|
||||
"golang.org/x/net/html",
|
||||
|
@ -42,5 +42,9 @@
|
||||
version = "1.4"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/PuerkitoBio/goquery"
|
||||
version = "1.4.1"
|
||||
name = "github.com/PuerkitoBio/goquery"
|
||||
version = "1.4.1"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/gofrs/uuid"
|
||||
version = "3.1.1"
|
@ -5,8 +5,11 @@ import (
|
||||
"fmt"
|
||||
"github.com/MontFerret/ferret/pkg/compiler"
|
||||
"github.com/MontFerret/ferret/pkg/runtime"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/logging"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func ExecFile(pathToFile string, opts Options) {
|
||||
@ -33,9 +36,25 @@ func Exec(query string, opts Options) {
|
||||
return
|
||||
}
|
||||
|
||||
l := NewLogger()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
c := make(chan os.Signal, 1)
|
||||
signal.Notify(c, syscall.SIGHUP)
|
||||
|
||||
go func() {
|
||||
for {
|
||||
<-c
|
||||
cancel()
|
||||
l.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
out, err := prog.Run(
|
||||
context.Background(),
|
||||
ctx,
|
||||
runtime.WithBrowser(opts.Cdp),
|
||||
runtime.WithLog(l),
|
||||
runtime.WithLogLevel(logging.DebugLevel),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
|
14
cmd/cli/logger.go
Normal file
14
cmd/cli/logger.go
Normal file
@ -0,0 +1,14 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"github.com/natefinch/lumberjack"
|
||||
)
|
||||
|
||||
func NewLogger() *lumberjack.Logger {
|
||||
l := &lumberjack.Logger{
|
||||
Filename: "./ferret.log",
|
||||
MaxSize: 100,
|
||||
}
|
||||
|
||||
return l
|
||||
}
|
@ -5,8 +5,12 @@ import (
|
||||
"fmt"
|
||||
"github.com/MontFerret/ferret/pkg/compiler"
|
||||
"github.com/MontFerret/ferret/pkg/runtime"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/logging"
|
||||
"github.com/chzyer/readline"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func Repl(version string, opts Options) {
|
||||
@ -32,6 +36,20 @@ func Repl(version string, opts Options) {
|
||||
|
||||
timer := NewTimer()
|
||||
|
||||
l := NewLogger()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
c := make(chan os.Signal, 1)
|
||||
signal.Notify(c, syscall.SIGHUP)
|
||||
|
||||
go func() {
|
||||
for {
|
||||
<-c
|
||||
cancel()
|
||||
l.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
for {
|
||||
line, err := rl.Readline()
|
||||
|
||||
@ -75,8 +93,10 @@ func Repl(version string, opts Options) {
|
||||
timer.Start()
|
||||
|
||||
out, err := program.Run(
|
||||
context.Background(),
|
||||
ctx,
|
||||
runtime.WithBrowser(opts.Cdp),
|
||||
runtime.WithLog(l),
|
||||
runtime.WithLogLevel(logging.DebugLevel),
|
||||
)
|
||||
|
||||
timer.Stop()
|
||||
|
@ -27,7 +27,7 @@ var (
|
||||
|
||||
conn = flag.String(
|
||||
"cdp",
|
||||
"http://127.0.0.1:9222",
|
||||
"http://0.0.0.0:9222",
|
||||
"set CDP address",
|
||||
)
|
||||
|
||||
|
@ -1,9 +0,0 @@
|
||||
LET g = DOCUMENT("https://www.google.com/", true)
|
||||
|
||||
INPUT(ELEMENT(g, 'input[name="q"]'), "ferret")
|
||||
|
||||
CLICK(g, 'input[name="btnK"]')
|
||||
|
||||
WAIT_NAVIGATION(g)
|
||||
|
||||
RETURN 1
|
@ -78,7 +78,7 @@ func (c *FqlCompiler) Compile(query string) (program *runtime.Program, err error
|
||||
}
|
||||
}()
|
||||
|
||||
l := newVisitor(c.funcs)
|
||||
l := newVisitor(query, c.funcs)
|
||||
|
||||
res := p.Visit(l).(*result)
|
||||
|
||||
|
@ -1688,39 +1688,24 @@ func TestForTernaryExpression(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
//func TestHtml(t *testing.T) {
|
||||
// Convey("Should load a document", t, func() {
|
||||
// c := compiler.New()
|
||||
//
|
||||
// prog, err := c.Compile(`
|
||||
//LET doc = DOCUMENT('https://soundcloud.com/charts/top', true)
|
||||
//
|
||||
//// TODO: We need a better way of waiting for page loading
|
||||
//// Something line WAIT_FOR(doc, selector)
|
||||
//SLEEP(2000)
|
||||
//
|
||||
//LET tracks = ELEMENTS(doc, '.chartTrack__details')
|
||||
//
|
||||
//LOG("found", LENGTH(tracks), "tracks")
|
||||
//
|
||||
//FOR track IN tracks
|
||||
// // LET username = ELEMENT(track, '.chartTrack__username')
|
||||
// // LET title = ELEMENT(track, '.chartTrack__title')
|
||||
//
|
||||
// // LOG("NODE", track.nodeName)
|
||||
//
|
||||
// SLEEP(500)
|
||||
//
|
||||
// RETURN track.innerHtml
|
||||
//
|
||||
// `)
|
||||
//
|
||||
// So(err, ShouldBeNil)
|
||||
//
|
||||
// out, err := prog.Run(context.Background(), runtime.WithBrowser("http://127.0.0.1:9222"))
|
||||
//
|
||||
// So(err, ShouldBeNil)
|
||||
//
|
||||
// So(string(out), ShouldEqual, `"int"`)
|
||||
// })
|
||||
//}
|
||||
func TestHtml(t *testing.T) {
|
||||
Convey("Should load a document", t, func() {
|
||||
c := compiler.New()
|
||||
|
||||
out, err := c.MustCompile(`
|
||||
LET doc = DOCUMENT("https://github.com/", true)
|
||||
LET btn = ELEMENT(doc, ".HeaderMenu a")
|
||||
|
||||
CLICK(btn)
|
||||
WAIT_NAVIGATION(doc)
|
||||
WAIT_ELEMENT(doc, '.IconNav')
|
||||
|
||||
RETURN INNER_HTML_ALL(doc, '.IconNav a')
|
||||
|
||||
`).Run(context.Background())
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(string(out), ShouldEqual, `"int"`)
|
||||
})
|
||||
}
|
||||
|
@ -18,13 +18,14 @@ import (
|
||||
|
||||
type visitor struct {
|
||||
*fql.BaseFqlParserVisitor
|
||||
|
||||
src string
|
||||
funcs map[string]core.Function
|
||||
}
|
||||
|
||||
func newVisitor(funcs map[string]core.Function) *visitor {
|
||||
func newVisitor(src string, funcs map[string]core.Function) *visitor {
|
||||
return &visitor{
|
||||
&fql.BaseFqlParserVisitor{},
|
||||
src,
|
||||
funcs,
|
||||
}
|
||||
}
|
||||
@ -38,7 +39,7 @@ func (v *visitor) VisitProgram(ctx *fql.ProgramContext) interface{} {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return runtime.NewProgram(block), nil
|
||||
return runtime.NewProgram(v.src, block)
|
||||
})
|
||||
}
|
||||
|
||||
|
23
pkg/runtime/logging/logger.go
Normal file
23
pkg/runtime/logging/logger.go
Normal file
@ -0,0 +1,23 @@
|
||||
package logging
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
type Level uint8
|
||||
|
||||
const (
|
||||
DebugLevel Level = iota
|
||||
InfoLevel
|
||||
WarnLevel
|
||||
ErrorLevel
|
||||
FatalLevel
|
||||
PanicLevel
|
||||
NoLevel
|
||||
Disabled
|
||||
)
|
||||
|
||||
func From(ctx context.Context) *zerolog.Logger {
|
||||
return zerolog.Ctx(ctx)
|
||||
}
|
@ -1,12 +1,21 @@
|
||||
package runtime
|
||||
|
||||
import "context"
|
||||
import (
|
||||
"context"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/logging"
|
||||
"github.com/gofrs/uuid"
|
||||
"github.com/rs/zerolog"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
type (
|
||||
Options struct {
|
||||
proxy string
|
||||
cdp string
|
||||
variables map[string]interface{}
|
||||
logWriter io.Writer
|
||||
logLevel zerolog.Level
|
||||
}
|
||||
|
||||
Option func(*Options)
|
||||
@ -14,8 +23,10 @@ type (
|
||||
|
||||
func newOptions() *Options {
|
||||
return &Options{
|
||||
cdp: "http://127.0.0.1:9222",
|
||||
cdp: "http://0.0.0.0:9222",
|
||||
variables: make(map[string]interface{}),
|
||||
logWriter: os.Stdout,
|
||||
logLevel: zerolog.ErrorLevel,
|
||||
}
|
||||
}
|
||||
|
||||
@ -38,10 +49,38 @@ func WithProxy(address string) Option {
|
||||
}
|
||||
}
|
||||
|
||||
func WithLog(writer io.Writer) Option {
|
||||
return func(options *Options) {
|
||||
options.logWriter = writer
|
||||
}
|
||||
}
|
||||
|
||||
func WithLogLevel(lvl logging.Level) Option {
|
||||
return func(options *Options) {
|
||||
options.logLevel = zerolog.Level(lvl)
|
||||
}
|
||||
}
|
||||
|
||||
func (opts *Options) withContext(parent context.Context) context.Context {
|
||||
return context.WithValue(
|
||||
ctx := context.WithValue(
|
||||
parent,
|
||||
"variables",
|
||||
opts.variables,
|
||||
)
|
||||
|
||||
id, err := uuid.NewV4()
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
logger := zerolog.New(opts.logWriter).
|
||||
With().
|
||||
Str("id", id.String()).
|
||||
Logger()
|
||||
logger.WithLevel(opts.logLevel)
|
||||
|
||||
ctx = logger.WithContext(ctx)
|
||||
|
||||
return ctx
|
||||
}
|
||||
|
@ -8,11 +8,24 @@ import (
|
||||
)
|
||||
|
||||
type Program struct {
|
||||
exp core.Expression
|
||||
src string
|
||||
body core.Expression
|
||||
}
|
||||
|
||||
func NewProgram(exp core.Expression) *Program {
|
||||
return &Program{exp}
|
||||
func NewProgram(src string, body core.Expression) (*Program, error) {
|
||||
if src == "" {
|
||||
return nil, core.Error(core.ErrMissedArgument, "source")
|
||||
}
|
||||
|
||||
if core.IsNil(body) {
|
||||
return nil, core.Error(core.ErrMissedArgument, "body")
|
||||
}
|
||||
|
||||
return &Program{src, body}, nil
|
||||
}
|
||||
|
||||
func (p *Program) Source() string {
|
||||
return p.src
|
||||
}
|
||||
|
||||
func (p *Program) Run(ctx context.Context, setters ...Option) ([]byte, error) {
|
||||
@ -30,7 +43,7 @@ func (p *Program) Run(ctx context.Context, setters ...Option) ([]byte, error) {
|
||||
ctx = driver.WithDynamicDriver(ctx, opts.cdp)
|
||||
ctx = driver.WithStaticDriver(ctx)
|
||||
|
||||
out, err := p.exp.Exec(ctx, scope)
|
||||
out, err := p.body.Exec(ctx, scope)
|
||||
|
||||
if err != nil {
|
||||
js, _ := values.None.MarshalJSON()
|
||||
|
@ -5,6 +5,7 @@ import (
|
||||
"crypto/sha512"
|
||||
"fmt"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/logging"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
"github.com/MontFerret/ferret/pkg/stdlib/html/driver/dynamic/eval"
|
||||
"github.com/MontFerret/ferret/pkg/stdlib/html/driver/dynamic/events"
|
||||
@ -15,12 +16,14 @@ import (
|
||||
"github.com/mafredri/cdp/protocol/page"
|
||||
"github.com/mafredri/cdp/rpcc"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type HtmlDocument struct {
|
||||
sync.Mutex
|
||||
logger *zerolog.Logger
|
||||
conn *rpcc.Conn
|
||||
client *cdp.Client
|
||||
events *events.EventBroker
|
||||
@ -93,7 +96,14 @@ func LoadHtmlDocument(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return NewHtmlDocument(conn, client, broker, root, innerHtml), nil
|
||||
return NewHtmlDocument(
|
||||
logging.From(ctx),
|
||||
conn,
|
||||
client,
|
||||
broker,
|
||||
root,
|
||||
innerHtml,
|
||||
), nil
|
||||
}
|
||||
|
||||
func getRootElement(client *cdp.Client) (dom.Node, values.String, error) {
|
||||
@ -117,6 +127,7 @@ func getRootElement(client *cdp.Client) (dom.Node, values.String, error) {
|
||||
}
|
||||
|
||||
func NewHtmlDocument(
|
||||
logger *zerolog.Logger,
|
||||
conn *rpcc.Conn,
|
||||
client *cdp.Client,
|
||||
broker *events.EventBroker,
|
||||
@ -124,10 +135,11 @@ func NewHtmlDocument(
|
||||
innerHtml values.String,
|
||||
) *HtmlDocument {
|
||||
doc := new(HtmlDocument)
|
||||
doc.logger = logger
|
||||
doc.conn = conn
|
||||
doc.client = client
|
||||
doc.events = broker
|
||||
doc.element = NewHtmlElement(client, broker, root.NodeID, root, innerHtml)
|
||||
doc.element = NewHtmlElement(doc.logger, client, broker, root.NodeID, root, innerHtml)
|
||||
doc.url = ""
|
||||
|
||||
if root.BaseURL != nil {
|
||||
@ -141,7 +153,11 @@ func NewHtmlDocument(
|
||||
updated, innerHtml, err := getRootElement(client)
|
||||
|
||||
if err != nil {
|
||||
// TODO: We need somehow log all errors outside of stdout
|
||||
doc.logger.Error().
|
||||
Timestamp().
|
||||
Err(err).
|
||||
Msg("failed to get root node after page load")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@ -149,7 +165,7 @@ func NewHtmlDocument(
|
||||
doc.element.Close()
|
||||
|
||||
// create a new root element wrapper
|
||||
doc.element = NewHtmlElement(client, broker, updated.NodeID, updated, innerHtml)
|
||||
doc.element = NewHtmlElement(doc.logger, client, broker, updated.NodeID, updated, innerHtml)
|
||||
doc.url = ""
|
||||
|
||||
if updated.BaseURL != nil {
|
||||
@ -226,11 +242,47 @@ func (doc *HtmlDocument) Close() error {
|
||||
doc.Lock()
|
||||
defer doc.Unlock()
|
||||
|
||||
doc.events.Stop()
|
||||
doc.events.Close()
|
||||
var err error
|
||||
|
||||
doc.element.Close()
|
||||
doc.client.Page.Close(context.Background())
|
||||
err = doc.events.Stop()
|
||||
|
||||
if err != nil {
|
||||
doc.logger.Warn().
|
||||
Timestamp().
|
||||
Str("url", doc.url.String()).
|
||||
Err(err).
|
||||
Msg("failed to stop event broker")
|
||||
}
|
||||
|
||||
err = doc.events.Close()
|
||||
|
||||
if err != nil {
|
||||
doc.logger.Warn().
|
||||
Timestamp().
|
||||
Str("url", doc.url.String()).
|
||||
Err(err).
|
||||
Msg("failed to close event broker")
|
||||
}
|
||||
|
||||
err = doc.element.Close()
|
||||
|
||||
if err != nil {
|
||||
doc.logger.Warn().
|
||||
Timestamp().
|
||||
Str("url", doc.url.String()).
|
||||
Err(err).
|
||||
Msg("failed to close root element")
|
||||
}
|
||||
|
||||
err = doc.client.Page.Close(context.Background())
|
||||
|
||||
if err != nil {
|
||||
doc.logger.Warn().
|
||||
Timestamp().
|
||||
Str("url", doc.url.String()).
|
||||
Err(err).
|
||||
Msg("failed to close browser page")
|
||||
}
|
||||
|
||||
return doc.conn.Close()
|
||||
}
|
||||
@ -354,13 +406,18 @@ func (doc *HtmlDocument) InnerHtmlBySelectorAll(selector values.String) (*values
|
||||
res, err := eval.Eval(
|
||||
doc.client,
|
||||
fmt.Sprintf(`
|
||||
var result = [];
|
||||
var elements = document.querySelectorAll(%s);
|
||||
|
||||
if (elements == null) {
|
||||
return [];
|
||||
return result;
|
||||
}
|
||||
|
||||
return elements.map(i => i.innerHtml);
|
||||
elements.forEach((i) => {
|
||||
result.push(i.innerHtml);
|
||||
});
|
||||
|
||||
return result;
|
||||
`, eval.ParamString(selector.String())),
|
||||
true,
|
||||
false,
|
||||
@ -408,13 +465,18 @@ func (doc *HtmlDocument) InnerTextBySelectorAll(selector values.String) (*values
|
||||
res, err := eval.Eval(
|
||||
doc.client,
|
||||
fmt.Sprintf(`
|
||||
var result = [];
|
||||
var elements = document.querySelectorAll(%s);
|
||||
|
||||
if (elements == null) {
|
||||
return [];
|
||||
return result;
|
||||
}
|
||||
|
||||
return elements.map(i => i.innerText);
|
||||
elements.forEach((i) => {
|
||||
result.push(i.innerText);
|
||||
});
|
||||
|
||||
return result;
|
||||
`, eval.ParamString(selector.String())),
|
||||
true,
|
||||
false,
|
||||
|
@ -13,6 +13,7 @@ import (
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/mafredri/cdp"
|
||||
"github.com/mafredri/cdp/protocol/dom"
|
||||
"github.com/rs/zerolog"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
@ -22,6 +23,7 @@ const DefaultTimeout = time.Second * 30
|
||||
|
||||
type HtmlElement struct {
|
||||
sync.Mutex
|
||||
logger *zerolog.Logger
|
||||
client *cdp.Client
|
||||
broker *events.EventBroker
|
||||
connected values.Boolean
|
||||
@ -31,12 +33,14 @@ type HtmlElement struct {
|
||||
innerHtml values.String
|
||||
innerText *common.LazyValue
|
||||
value core.Value
|
||||
rawAttrs []string
|
||||
attributes *common.LazyValue
|
||||
children []dom.NodeID
|
||||
loadedChildren *common.LazyValue
|
||||
}
|
||||
|
||||
func LoadElement(
|
||||
logger *zerolog.Logger,
|
||||
client *cdp.Client,
|
||||
broker *events.EventBroker,
|
||||
id dom.NodeID,
|
||||
@ -68,6 +72,7 @@ func LoadElement(
|
||||
}
|
||||
|
||||
return NewHtmlElement(
|
||||
logger,
|
||||
client,
|
||||
broker,
|
||||
id,
|
||||
@ -77,6 +82,7 @@ func LoadElement(
|
||||
}
|
||||
|
||||
func NewHtmlElement(
|
||||
logger *zerolog.Logger,
|
||||
client *cdp.Client,
|
||||
broker *events.EventBroker,
|
||||
id dom.NodeID,
|
||||
@ -84,6 +90,7 @@ func NewHtmlElement(
|
||||
innerHtml values.String,
|
||||
) *HtmlElement {
|
||||
el := new(HtmlElement)
|
||||
el.logger = logger
|
||||
el.client = client
|
||||
el.broker = broker
|
||||
el.connected = values.True
|
||||
@ -91,34 +98,11 @@ func NewHtmlElement(
|
||||
el.nodeType = values.NewInt(node.NodeType)
|
||||
el.nodeName = values.NewString(node.NodeName)
|
||||
el.innerHtml = innerHtml
|
||||
el.innerText = common.NewLazyValue(func() (core.Value, error) {
|
||||
h := el.InnerHtml()
|
||||
|
||||
if h == values.EmptyString {
|
||||
return h, nil
|
||||
}
|
||||
|
||||
buff := bytes.NewBuffer([]byte(h))
|
||||
|
||||
parsed, err := goquery.NewDocumentFromReader(buff)
|
||||
|
||||
if err != nil {
|
||||
return values.EmptyString, err
|
||||
}
|
||||
|
||||
return values.NewString(parsed.Text()), nil
|
||||
})
|
||||
el.attributes = common.NewLazyValue(func() (core.Value, error) {
|
||||
return parseAttrs(node.Attributes), nil
|
||||
})
|
||||
el.innerText = common.NewLazyValue(el.loadInnerText)
|
||||
el.rawAttrs = node.Attributes[:]
|
||||
el.attributes = common.NewLazyValue(el.loadAttrs)
|
||||
el.value = values.EmptyString
|
||||
el.loadedChildren = common.NewLazyValue(func() (core.Value, error) {
|
||||
if !el.IsConnected() {
|
||||
return values.NewArray(0), nil
|
||||
}
|
||||
|
||||
return loadNodes(client, broker, el.children)
|
||||
})
|
||||
el.loadedChildren = common.NewLazyValue(el.loadChildren)
|
||||
|
||||
if node.Value != nil {
|
||||
el.value = values.NewString(*node.Value)
|
||||
@ -205,11 +189,19 @@ func (el *HtmlElement) Unwrap() interface{} {
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Hash() int {
|
||||
el.Lock()
|
||||
defer el.Unlock()
|
||||
|
||||
h := sha512.New()
|
||||
|
||||
out, err := h.Write([]byte(strconv.Itoa(int(el.id))))
|
||||
out, err := h.Write([]byte(el.innerHtml))
|
||||
|
||||
if err != nil {
|
||||
el.logger.Error().
|
||||
Timestamp().
|
||||
Err(err).
|
||||
Msg("failed to calculate hash value")
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
@ -227,6 +219,11 @@ func (el *HtmlElement) Value() core.Value {
|
||||
val, err := eval.Property(ctx, el.client, el.id, "value")
|
||||
|
||||
if err != nil {
|
||||
el.logger.Error().
|
||||
Timestamp().
|
||||
Err(err).
|
||||
Msg("failed to get node value")
|
||||
|
||||
return el.value
|
||||
}
|
||||
|
||||
@ -309,12 +306,24 @@ func (el *HtmlElement) QuerySelector(selector values.String) core.Value {
|
||||
found, err := el.client.DOM.QuerySelector(ctx, selectorArgs)
|
||||
|
||||
if err != nil {
|
||||
el.logger.Error().
|
||||
Timestamp().
|
||||
Str("selector", selector.String()).
|
||||
Err(err).
|
||||
Msg("failed to retrieve a node by selector")
|
||||
|
||||
return values.None
|
||||
}
|
||||
|
||||
res, err := LoadElement(el.client, el.broker, found.NodeID)
|
||||
res, err := LoadElement(el.logger, el.client, el.broker, found.NodeID)
|
||||
|
||||
if err != nil {
|
||||
el.logger.Error().
|
||||
Timestamp().
|
||||
Str("selector", selector.String()).
|
||||
Err(err).
|
||||
Msg("failed to load a child node by selector")
|
||||
|
||||
return values.None
|
||||
}
|
||||
|
||||
@ -332,15 +341,27 @@ func (el *HtmlElement) QuerySelectorAll(selector values.String) core.Value {
|
||||
res, err := el.client.DOM.QuerySelectorAll(ctx, selectorArgs)
|
||||
|
||||
if err != nil {
|
||||
el.logger.Error().
|
||||
Timestamp().
|
||||
Str("selector", selector.String()).
|
||||
Err(err).
|
||||
Msg("failed to retrieve nodes by selector")
|
||||
|
||||
return values.None
|
||||
}
|
||||
|
||||
arr := values.NewArray(len(res.NodeIDs))
|
||||
|
||||
for _, id := range res.NodeIDs {
|
||||
childEl, err := LoadElement(el.client, el.broker, id)
|
||||
childEl, err := LoadElement(el.logger, el.client, el.broker, id)
|
||||
|
||||
if err != nil {
|
||||
el.logger.Error().
|
||||
Timestamp().
|
||||
Str("selector", selector.String()).
|
||||
Err(err).
|
||||
Msg("failed to load nodes by selector")
|
||||
|
||||
return values.None
|
||||
}
|
||||
|
||||
@ -389,6 +410,54 @@ func (el *HtmlElement) IsConnected() values.Boolean {
|
||||
return el.connected
|
||||
}
|
||||
|
||||
func (el *HtmlElement) loadInnerText() (core.Value, error) {
|
||||
h := el.InnerHtml()
|
||||
|
||||
if h == values.EmptyString {
|
||||
return h, nil
|
||||
}
|
||||
|
||||
buff := bytes.NewBuffer([]byte(h))
|
||||
|
||||
parsed, err := goquery.NewDocumentFromReader(buff)
|
||||
|
||||
if err != nil {
|
||||
el.logger.Error().
|
||||
Timestamp().
|
||||
Err(err).
|
||||
Int("id", int(el.id)).
|
||||
Msg("failed to parse inner html")
|
||||
|
||||
return values.EmptyString, err
|
||||
}
|
||||
|
||||
return values.NewString(parsed.Text()), nil
|
||||
}
|
||||
|
||||
func (el *HtmlElement) loadAttrs() (core.Value, error) {
|
||||
return parseAttrs(el.rawAttrs), nil
|
||||
}
|
||||
|
||||
func (el *HtmlElement) loadChildren() (core.Value, error) {
|
||||
if !el.IsConnected() {
|
||||
return values.NewArray(0), nil
|
||||
}
|
||||
|
||||
loaded, err := loadNodes(el.logger, el.client, el.broker, el.children)
|
||||
|
||||
if err != nil {
|
||||
el.logger.Error().
|
||||
Timestamp().
|
||||
Err(err).
|
||||
Int("id", int(el.id)).
|
||||
Msg("failed to load child nodes")
|
||||
|
||||
return values.None, err
|
||||
}
|
||||
|
||||
return loaded, nil
|
||||
}
|
||||
|
||||
func (el *HtmlElement) handlePageReload(message interface{}) {
|
||||
el.Close()
|
||||
}
|
||||
@ -484,6 +553,12 @@ func (el *HtmlElement) handleChildrenCountChanged(message interface{}) {
|
||||
node, err := el.client.DOM.DescribeNode(context.Background(), dom.NewDescribeNodeArgs())
|
||||
|
||||
if err != nil {
|
||||
el.logger.Error().
|
||||
Timestamp().
|
||||
Err(err).
|
||||
Int("id", int(el.id)).
|
||||
Msg("failed to update node")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@ -536,10 +611,15 @@ func (el *HtmlElement) handleChildInserted(message interface{}) {
|
||||
}
|
||||
|
||||
loadedArr := loaded.(*values.Array)
|
||||
|
||||
loadedEl, err := LoadElement(el.client, el.broker, nextId)
|
||||
loadedEl, err := LoadElement(el.logger, el.client, el.broker, nextId)
|
||||
|
||||
if err != nil {
|
||||
el.logger.Error().
|
||||
Timestamp().
|
||||
Err(err).
|
||||
Int("id", int(el.id)).
|
||||
Msg("failed to load an inserted node")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@ -548,6 +628,12 @@ func (el *HtmlElement) handleChildInserted(message interface{}) {
|
||||
newInnerHtml, err := loadInnerHtml(el.client, el.id)
|
||||
|
||||
if err != nil {
|
||||
el.logger.Error().
|
||||
Timestamp().
|
||||
Err(err).
|
||||
Int("id", int(el.id)).
|
||||
Msg("failed to update node")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@ -601,6 +687,12 @@ func (el *HtmlElement) handleChildDeleted(message interface{}) {
|
||||
newInnerHtml, err := loadInnerHtml(el.client, el.id)
|
||||
|
||||
if err != nil {
|
||||
el.logger.Error().
|
||||
Timestamp().
|
||||
Err(err).
|
||||
Int("id", int(el.id)).
|
||||
Msg("failed to update node")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -8,6 +8,7 @@ import (
|
||||
"github.com/mafredri/cdp"
|
||||
"github.com/mafredri/cdp/protocol/dom"
|
||||
"github.com/mafredri/cdp/protocol/page"
|
||||
"github.com/rs/zerolog"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@ -70,11 +71,11 @@ func createChildrenArray(nodes []dom.Node) []dom.NodeID {
|
||||
return children
|
||||
}
|
||||
|
||||
func loadNodes(client *cdp.Client, broker *events.EventBroker, nodes []dom.NodeID) (*values.Array, error) {
|
||||
func loadNodes(logger *zerolog.Logger, client *cdp.Client, broker *events.EventBroker, nodes []dom.NodeID) (*values.Array, error) {
|
||||
arr := values.NewArray(len(nodes))
|
||||
|
||||
for _, id := range nodes {
|
||||
child, err := LoadElement(client, broker, id)
|
||||
child, err := LoadElement(logger, client, broker, id)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -3,8 +3,8 @@ package utils
|
||||
import (
|
||||
"context"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/logging"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
@ -30,16 +30,16 @@ func Wait(_ context.Context, inputs ...core.Value) (core.Value, error) {
|
||||
return values.None, nil
|
||||
}
|
||||
|
||||
func Log(_ context.Context, inputs ...core.Value) (core.Value, error) {
|
||||
func Log(ctx context.Context, inputs ...core.Value) (core.Value, error) {
|
||||
args := make([]interface{}, 0, len(inputs)+1)
|
||||
|
||||
args = append(args, "LOG:")
|
||||
|
||||
for _, input := range inputs {
|
||||
args = append(args, input)
|
||||
}
|
||||
|
||||
log.Println(args...)
|
||||
logger := logging.From(ctx)
|
||||
|
||||
logger.Print(args...)
|
||||
|
||||
return values.None, nil
|
||||
}
|
||||
|
Loading…
x
Reference in New Issue
Block a user