From e427efd74ee9c5805b2b868ce3f63dc8de5724dc Mon Sep 17 00:00:00 2001 From: Tim Voronov Date: Fri, 28 Sep 2018 00:28:33 -0400 Subject: [PATCH] #27 Added logging --- .gitignore | 1 + Gopkg.lock | 31 ++++ Gopkg.toml | 8 +- cmd/cli/exec.go | 21 ++- cmd/cli/logger.go | 14 ++ cmd/cli/repl.go | 22 ++- cmd/main.go | 2 +- docs/examples/test.fql | 9 -- pkg/compiler/compiler.go | 2 +- pkg/compiler/compiler_test.go | 57 +++----- pkg/compiler/visitor.go | 7 +- pkg/runtime/logging/logger.go | 23 +++ pkg/runtime/options.go | 45 +++++- pkg/runtime/program.go | 21 ++- pkg/stdlib/html/driver/dynamic/document.go | 86 ++++++++++-- pkg/stdlib/html/driver/dynamic/element.go | 156 ++++++++++++++++----- pkg/stdlib/html/driver/dynamic/helpers.go | 5 +- pkg/stdlib/utils/utils.go | 10 +- 18 files changed, 408 insertions(+), 112 deletions(-) create mode 100644 cmd/cli/logger.go delete mode 100644 docs/examples/test.fql create mode 100644 pkg/runtime/logging/logger.go diff --git a/.gitignore b/.gitignore index 4c2a4ce8..3850f372 100644 --- a/.gitignore +++ b/.gitignore @@ -124,3 +124,4 @@ $RECYCLE.BIN/ vendor bin +*.log \ No newline at end of file diff --git a/Gopkg.lock b/Gopkg.lock index 0c0bd97d..a41259fd 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -41,6 +41,14 @@ pruneopts = "UT" revision = "2b8494104d86337cdd41d0a49cbed8e4583c0ab4" +[[projects]] + digest = "1:ce579162ae1341f3e5ab30c0dce767f28b1eb6a81359aad01723f1ba6b4becdf" + name = "github.com/gofrs/uuid" + packages = ["."] + pruneopts = "UT" + revision = "370558f003bfe29580cd0f698d8640daccdcc45c" + version = "v3.1.1" + [[projects]] branch = "master" digest = "1:f14d1b50e0075fb00177f12a96dd7addf93d1e2883c25befd17285b779549795" @@ -121,6 +129,14 @@ revision = "75b0ecc5efcff27ac756a33ec71f0db75dc3d21c" version = "v0.19.0" +[[projects]] + digest = "1:c805e517269b0ba4c21ded5836019ed7d16953d4026cb7d00041d039c7906be9" + name = "github.com/natefinch/lumberjack" + packages = ["."] + pruneopts = "UT" + revision = "a96e63847dc3c67d17befa69c303767e2f84e54f" + version = "v2.1" + [[projects]] digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747" name = "github.com/pkg/errors" @@ -129,6 +145,18 @@ revision = "645ef00459ed84a119197bfb8d8205042c6df63d" version = "v0.8.0" +[[projects]] + digest = "1:5bc8f93f977b72a7a5264725c3bab275e69de0cc3e5c0dc1ee56feb564c33f03" + name = "github.com/rs/zerolog" + packages = [ + ".", + "internal/cbor", + "internal/json", + ] + pruneopts = "UT" + revision = "338f9bc14084d22cb8eeacd6492861f8449d715c" + version = "v1.9.1" + [[projects]] digest = "1:4ca145a665316d3c020a39c0741780fa3636b9152b824206796c4dce541f4a24" name = "github.com/sethgrid/pester" @@ -189,6 +217,7 @@ "github.com/antlr/antlr4/runtime/Go/antlr", "github.com/chzyer/readline", "github.com/corpix/uarand", + "github.com/gofrs/uuid", "github.com/mafredri/cdp", "github.com/mafredri/cdp/devtool", "github.com/mafredri/cdp/protocol/dom", @@ -198,7 +227,9 @@ "github.com/mafredri/cdp/protocol/target", "github.com/mafredri/cdp/rpcc", "github.com/mafredri/cdp/session", + "github.com/natefinch/lumberjack", "github.com/pkg/errors", + "github.com/rs/zerolog", "github.com/sethgrid/pester", "github.com/smartystreets/goconvey/convey", "golang.org/x/net/html", diff --git a/Gopkg.toml b/Gopkg.toml index 29d34d70..5ae873d6 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -42,5 +42,9 @@ version = "1.4" [[constraint]] - name = "github.com/PuerkitoBio/goquery" - version = "1.4.1" \ No newline at end of file + name = "github.com/PuerkitoBio/goquery" + version = "1.4.1" + +[[constraint]] + name = "github.com/gofrs/uuid" + version = "3.1.1" \ No newline at end of file diff --git a/cmd/cli/exec.go b/cmd/cli/exec.go index 12510592..bb9fde47 100644 --- a/cmd/cli/exec.go +++ b/cmd/cli/exec.go @@ -5,8 +5,11 @@ import ( "fmt" "github.com/MontFerret/ferret/pkg/compiler" "github.com/MontFerret/ferret/pkg/runtime" + "github.com/MontFerret/ferret/pkg/runtime/logging" "io/ioutil" "os" + "os/signal" + "syscall" ) func ExecFile(pathToFile string, opts Options) { @@ -33,9 +36,25 @@ func Exec(query string, opts Options) { return } + l := NewLogger() + + ctx, cancel := context.WithCancel(context.Background()) + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGHUP) + + go func() { + for { + <-c + cancel() + l.Close() + } + }() + out, err := prog.Run( - context.Background(), + ctx, runtime.WithBrowser(opts.Cdp), + runtime.WithLog(l), + runtime.WithLogLevel(logging.DebugLevel), ) if err != nil { diff --git a/cmd/cli/logger.go b/cmd/cli/logger.go new file mode 100644 index 00000000..6ff6c084 --- /dev/null +++ b/cmd/cli/logger.go @@ -0,0 +1,14 @@ +package cli + +import ( + "github.com/natefinch/lumberjack" +) + +func NewLogger() *lumberjack.Logger { + l := &lumberjack.Logger{ + Filename: "./ferret.log", + MaxSize: 100, + } + + return l +} diff --git a/cmd/cli/repl.go b/cmd/cli/repl.go index 773e67cd..36b72c5f 100644 --- a/cmd/cli/repl.go +++ b/cmd/cli/repl.go @@ -5,8 +5,12 @@ import ( "fmt" "github.com/MontFerret/ferret/pkg/compiler" "github.com/MontFerret/ferret/pkg/runtime" + "github.com/MontFerret/ferret/pkg/runtime/logging" "github.com/chzyer/readline" + "os" + "os/signal" "strings" + "syscall" ) func Repl(version string, opts Options) { @@ -32,6 +36,20 @@ func Repl(version string, opts Options) { timer := NewTimer() + l := NewLogger() + + ctx, cancel := context.WithCancel(context.Background()) + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGHUP) + + go func() { + for { + <-c + cancel() + l.Close() + } + }() + for { line, err := rl.Readline() @@ -75,8 +93,10 @@ func Repl(version string, opts Options) { timer.Start() out, err := program.Run( - context.Background(), + ctx, runtime.WithBrowser(opts.Cdp), + runtime.WithLog(l), + runtime.WithLogLevel(logging.DebugLevel), ) timer.Stop() diff --git a/cmd/main.go b/cmd/main.go index d3b63d42..23dd7ec3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -27,7 +27,7 @@ var ( conn = flag.String( "cdp", - "http://127.0.0.1:9222", + "http://0.0.0.0:9222", "set CDP address", ) diff --git a/docs/examples/test.fql b/docs/examples/test.fql deleted file mode 100644 index c9d7ce4f..00000000 --- a/docs/examples/test.fql +++ /dev/null @@ -1,9 +0,0 @@ -LET g = DOCUMENT("https://www.google.com/", true) - -INPUT(ELEMENT(g, 'input[name="q"]'), "ferret") - -CLICK(g, 'input[name="btnK"]') - -WAIT_NAVIGATION(g) - -RETURN 1 \ No newline at end of file diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index c80f1c2f..df445299 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -78,7 +78,7 @@ func (c *FqlCompiler) Compile(query string) (program *runtime.Program, err error } }() - l := newVisitor(c.funcs) + l := newVisitor(query, c.funcs) res := p.Visit(l).(*result) diff --git a/pkg/compiler/compiler_test.go b/pkg/compiler/compiler_test.go index 83918f39..9da363ac 100644 --- a/pkg/compiler/compiler_test.go +++ b/pkg/compiler/compiler_test.go @@ -1688,39 +1688,24 @@ func TestForTernaryExpression(t *testing.T) { }) } -//func TestHtml(t *testing.T) { -// Convey("Should load a document", t, func() { -// c := compiler.New() -// -// prog, err := c.Compile(` -//LET doc = DOCUMENT('https://soundcloud.com/charts/top', true) -// -//// TODO: We need a better way of waiting for page loading -//// Something line WAIT_FOR(doc, selector) -//SLEEP(2000) -// -//LET tracks = ELEMENTS(doc, '.chartTrack__details') -// -//LOG("found", LENGTH(tracks), "tracks") -// -//FOR track IN tracks -// // LET username = ELEMENT(track, '.chartTrack__username') -// // LET title = ELEMENT(track, '.chartTrack__title') -// -// // LOG("NODE", track.nodeName) -// -// SLEEP(500) -// -// RETURN track.innerHtml -// -// `) -// -// So(err, ShouldBeNil) -// -// out, err := prog.Run(context.Background(), runtime.WithBrowser("http://127.0.0.1:9222")) -// -// So(err, ShouldBeNil) -// -// So(string(out), ShouldEqual, `"int"`) -// }) -//} +func TestHtml(t *testing.T) { + Convey("Should load a document", t, func() { + c := compiler.New() + + out, err := c.MustCompile(` +LET doc = DOCUMENT("https://github.com/", true) +LET btn = ELEMENT(doc, ".HeaderMenu a") + +CLICK(btn) +WAIT_NAVIGATION(doc) +WAIT_ELEMENT(doc, '.IconNav') + +RETURN INNER_HTML_ALL(doc, '.IconNav a') + + `).Run(context.Background()) + + So(err, ShouldBeNil) + + So(string(out), ShouldEqual, `"int"`) + }) +} diff --git a/pkg/compiler/visitor.go b/pkg/compiler/visitor.go index f89e718e..9329d6b9 100644 --- a/pkg/compiler/visitor.go +++ b/pkg/compiler/visitor.go @@ -18,13 +18,14 @@ import ( type visitor struct { *fql.BaseFqlParserVisitor - + src string funcs map[string]core.Function } -func newVisitor(funcs map[string]core.Function) *visitor { +func newVisitor(src string, funcs map[string]core.Function) *visitor { return &visitor{ &fql.BaseFqlParserVisitor{}, + src, funcs, } } @@ -38,7 +39,7 @@ func (v *visitor) VisitProgram(ctx *fql.ProgramContext) interface{} { return nil, err } - return runtime.NewProgram(block), nil + return runtime.NewProgram(v.src, block) }) } diff --git a/pkg/runtime/logging/logger.go b/pkg/runtime/logging/logger.go new file mode 100644 index 00000000..2701a025 --- /dev/null +++ b/pkg/runtime/logging/logger.go @@ -0,0 +1,23 @@ +package logging + +import ( + "context" + "github.com/rs/zerolog" +) + +type Level uint8 + +const ( + DebugLevel Level = iota + InfoLevel + WarnLevel + ErrorLevel + FatalLevel + PanicLevel + NoLevel + Disabled +) + +func From(ctx context.Context) *zerolog.Logger { + return zerolog.Ctx(ctx) +} diff --git a/pkg/runtime/options.go b/pkg/runtime/options.go index 6338e02e..570d0cbd 100644 --- a/pkg/runtime/options.go +++ b/pkg/runtime/options.go @@ -1,12 +1,21 @@ package runtime -import "context" +import ( + "context" + "github.com/MontFerret/ferret/pkg/runtime/logging" + "github.com/gofrs/uuid" + "github.com/rs/zerolog" + "io" + "os" +) type ( Options struct { proxy string cdp string variables map[string]interface{} + logWriter io.Writer + logLevel zerolog.Level } Option func(*Options) @@ -14,8 +23,10 @@ type ( func newOptions() *Options { return &Options{ - cdp: "http://127.0.0.1:9222", + cdp: "http://0.0.0.0:9222", variables: make(map[string]interface{}), + logWriter: os.Stdout, + logLevel: zerolog.ErrorLevel, } } @@ -38,10 +49,38 @@ func WithProxy(address string) Option { } } +func WithLog(writer io.Writer) Option { + return func(options *Options) { + options.logWriter = writer + } +} + +func WithLogLevel(lvl logging.Level) Option { + return func(options *Options) { + options.logLevel = zerolog.Level(lvl) + } +} + func (opts *Options) withContext(parent context.Context) context.Context { - return context.WithValue( + ctx := context.WithValue( parent, "variables", opts.variables, ) + + id, err := uuid.NewV4() + + if err != nil { + panic(err) + } + + logger := zerolog.New(opts.logWriter). + With(). + Str("id", id.String()). + Logger() + logger.WithLevel(opts.logLevel) + + ctx = logger.WithContext(ctx) + + return ctx } diff --git a/pkg/runtime/program.go b/pkg/runtime/program.go index 3c40fce7..0c04326f 100644 --- a/pkg/runtime/program.go +++ b/pkg/runtime/program.go @@ -8,11 +8,24 @@ import ( ) type Program struct { - exp core.Expression + src string + body core.Expression } -func NewProgram(exp core.Expression) *Program { - return &Program{exp} +func NewProgram(src string, body core.Expression) (*Program, error) { + if src == "" { + return nil, core.Error(core.ErrMissedArgument, "source") + } + + if core.IsNil(body) { + return nil, core.Error(core.ErrMissedArgument, "body") + } + + return &Program{src, body}, nil +} + +func (p *Program) Source() string { + return p.src } func (p *Program) Run(ctx context.Context, setters ...Option) ([]byte, error) { @@ -30,7 +43,7 @@ func (p *Program) Run(ctx context.Context, setters ...Option) ([]byte, error) { ctx = driver.WithDynamicDriver(ctx, opts.cdp) ctx = driver.WithStaticDriver(ctx) - out, err := p.exp.Exec(ctx, scope) + out, err := p.body.Exec(ctx, scope) if err != nil { js, _ := values.None.MarshalJSON() diff --git a/pkg/stdlib/html/driver/dynamic/document.go b/pkg/stdlib/html/driver/dynamic/document.go index cae40ff2..ea261bf7 100644 --- a/pkg/stdlib/html/driver/dynamic/document.go +++ b/pkg/stdlib/html/driver/dynamic/document.go @@ -5,6 +5,7 @@ import ( "crypto/sha512" "fmt" "github.com/MontFerret/ferret/pkg/runtime/core" + "github.com/MontFerret/ferret/pkg/runtime/logging" "github.com/MontFerret/ferret/pkg/runtime/values" "github.com/MontFerret/ferret/pkg/stdlib/html/driver/dynamic/eval" "github.com/MontFerret/ferret/pkg/stdlib/html/driver/dynamic/events" @@ -15,12 +16,14 @@ import ( "github.com/mafredri/cdp/protocol/page" "github.com/mafredri/cdp/rpcc" "github.com/pkg/errors" + "github.com/rs/zerolog" "sync" "time" ) type HtmlDocument struct { sync.Mutex + logger *zerolog.Logger conn *rpcc.Conn client *cdp.Client events *events.EventBroker @@ -93,7 +96,14 @@ func LoadHtmlDocument( return nil, err } - return NewHtmlDocument(conn, client, broker, root, innerHtml), nil + return NewHtmlDocument( + logging.From(ctx), + conn, + client, + broker, + root, + innerHtml, + ), nil } func getRootElement(client *cdp.Client) (dom.Node, values.String, error) { @@ -117,6 +127,7 @@ func getRootElement(client *cdp.Client) (dom.Node, values.String, error) { } func NewHtmlDocument( + logger *zerolog.Logger, conn *rpcc.Conn, client *cdp.Client, broker *events.EventBroker, @@ -124,10 +135,11 @@ func NewHtmlDocument( innerHtml values.String, ) *HtmlDocument { doc := new(HtmlDocument) + doc.logger = logger doc.conn = conn doc.client = client doc.events = broker - doc.element = NewHtmlElement(client, broker, root.NodeID, root, innerHtml) + doc.element = NewHtmlElement(doc.logger, client, broker, root.NodeID, root, innerHtml) doc.url = "" if root.BaseURL != nil { @@ -141,7 +153,11 @@ func NewHtmlDocument( updated, innerHtml, err := getRootElement(client) if err != nil { - // TODO: We need somehow log all errors outside of stdout + doc.logger.Error(). + Timestamp(). + Err(err). + Msg("failed to get root node after page load") + return } @@ -149,7 +165,7 @@ func NewHtmlDocument( doc.element.Close() // create a new root element wrapper - doc.element = NewHtmlElement(client, broker, updated.NodeID, updated, innerHtml) + doc.element = NewHtmlElement(doc.logger, client, broker, updated.NodeID, updated, innerHtml) doc.url = "" if updated.BaseURL != nil { @@ -226,11 +242,47 @@ func (doc *HtmlDocument) Close() error { doc.Lock() defer doc.Unlock() - doc.events.Stop() - doc.events.Close() + var err error - doc.element.Close() - doc.client.Page.Close(context.Background()) + err = doc.events.Stop() + + if err != nil { + doc.logger.Warn(). + Timestamp(). + Str("url", doc.url.String()). + Err(err). + Msg("failed to stop event broker") + } + + err = doc.events.Close() + + if err != nil { + doc.logger.Warn(). + Timestamp(). + Str("url", doc.url.String()). + Err(err). + Msg("failed to close event broker") + } + + err = doc.element.Close() + + if err != nil { + doc.logger.Warn(). + Timestamp(). + Str("url", doc.url.String()). + Err(err). + Msg("failed to close root element") + } + + err = doc.client.Page.Close(context.Background()) + + if err != nil { + doc.logger.Warn(). + Timestamp(). + Str("url", doc.url.String()). + Err(err). + Msg("failed to close browser page") + } return doc.conn.Close() } @@ -354,13 +406,18 @@ func (doc *HtmlDocument) InnerHtmlBySelectorAll(selector values.String) (*values res, err := eval.Eval( doc.client, fmt.Sprintf(` + var result = []; var elements = document.querySelectorAll(%s); if (elements == null) { - return []; + return result; } - return elements.map(i => i.innerHtml); + elements.forEach((i) => { + result.push(i.innerHtml); + }); + + return result; `, eval.ParamString(selector.String())), true, false, @@ -408,13 +465,18 @@ func (doc *HtmlDocument) InnerTextBySelectorAll(selector values.String) (*values res, err := eval.Eval( doc.client, fmt.Sprintf(` + var result = []; var elements = document.querySelectorAll(%s); if (elements == null) { - return []; + return result; } - return elements.map(i => i.innerText); + elements.forEach((i) => { + result.push(i.innerText); + }); + + return result; `, eval.ParamString(selector.String())), true, false, diff --git a/pkg/stdlib/html/driver/dynamic/element.go b/pkg/stdlib/html/driver/dynamic/element.go index 4f2c6b73..43aa1dee 100644 --- a/pkg/stdlib/html/driver/dynamic/element.go +++ b/pkg/stdlib/html/driver/dynamic/element.go @@ -13,6 +13,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/mafredri/cdp" "github.com/mafredri/cdp/protocol/dom" + "github.com/rs/zerolog" "strconv" "sync" "time" @@ -22,6 +23,7 @@ const DefaultTimeout = time.Second * 30 type HtmlElement struct { sync.Mutex + logger *zerolog.Logger client *cdp.Client broker *events.EventBroker connected values.Boolean @@ -31,12 +33,14 @@ type HtmlElement struct { innerHtml values.String innerText *common.LazyValue value core.Value + rawAttrs []string attributes *common.LazyValue children []dom.NodeID loadedChildren *common.LazyValue } func LoadElement( + logger *zerolog.Logger, client *cdp.Client, broker *events.EventBroker, id dom.NodeID, @@ -68,6 +72,7 @@ func LoadElement( } return NewHtmlElement( + logger, client, broker, id, @@ -77,6 +82,7 @@ func LoadElement( } func NewHtmlElement( + logger *zerolog.Logger, client *cdp.Client, broker *events.EventBroker, id dom.NodeID, @@ -84,6 +90,7 @@ func NewHtmlElement( innerHtml values.String, ) *HtmlElement { el := new(HtmlElement) + el.logger = logger el.client = client el.broker = broker el.connected = values.True @@ -91,34 +98,11 @@ func NewHtmlElement( el.nodeType = values.NewInt(node.NodeType) el.nodeName = values.NewString(node.NodeName) el.innerHtml = innerHtml - el.innerText = common.NewLazyValue(func() (core.Value, error) { - h := el.InnerHtml() - - if h == values.EmptyString { - return h, nil - } - - buff := bytes.NewBuffer([]byte(h)) - - parsed, err := goquery.NewDocumentFromReader(buff) - - if err != nil { - return values.EmptyString, err - } - - return values.NewString(parsed.Text()), nil - }) - el.attributes = common.NewLazyValue(func() (core.Value, error) { - return parseAttrs(node.Attributes), nil - }) + el.innerText = common.NewLazyValue(el.loadInnerText) + el.rawAttrs = node.Attributes[:] + el.attributes = common.NewLazyValue(el.loadAttrs) el.value = values.EmptyString - el.loadedChildren = common.NewLazyValue(func() (core.Value, error) { - if !el.IsConnected() { - return values.NewArray(0), nil - } - - return loadNodes(client, broker, el.children) - }) + el.loadedChildren = common.NewLazyValue(el.loadChildren) if node.Value != nil { el.value = values.NewString(*node.Value) @@ -205,11 +189,19 @@ func (el *HtmlElement) Unwrap() interface{} { } func (el *HtmlElement) Hash() int { + el.Lock() + defer el.Unlock() + h := sha512.New() - out, err := h.Write([]byte(strconv.Itoa(int(el.id)))) + out, err := h.Write([]byte(el.innerHtml)) if err != nil { + el.logger.Error(). + Timestamp(). + Err(err). + Msg("failed to calculate hash value") + return 0 } @@ -227,6 +219,11 @@ func (el *HtmlElement) Value() core.Value { val, err := eval.Property(ctx, el.client, el.id, "value") if err != nil { + el.logger.Error(). + Timestamp(). + Err(err). + Msg("failed to get node value") + return el.value } @@ -309,12 +306,24 @@ func (el *HtmlElement) QuerySelector(selector values.String) core.Value { found, err := el.client.DOM.QuerySelector(ctx, selectorArgs) if err != nil { + el.logger.Error(). + Timestamp(). + Str("selector", selector.String()). + Err(err). + Msg("failed to retrieve a node by selector") + return values.None } - res, err := LoadElement(el.client, el.broker, found.NodeID) + res, err := LoadElement(el.logger, el.client, el.broker, found.NodeID) if err != nil { + el.logger.Error(). + Timestamp(). + Str("selector", selector.String()). + Err(err). + Msg("failed to load a child node by selector") + return values.None } @@ -332,15 +341,27 @@ func (el *HtmlElement) QuerySelectorAll(selector values.String) core.Value { res, err := el.client.DOM.QuerySelectorAll(ctx, selectorArgs) if err != nil { + el.logger.Error(). + Timestamp(). + Str("selector", selector.String()). + Err(err). + Msg("failed to retrieve nodes by selector") + return values.None } arr := values.NewArray(len(res.NodeIDs)) for _, id := range res.NodeIDs { - childEl, err := LoadElement(el.client, el.broker, id) + childEl, err := LoadElement(el.logger, el.client, el.broker, id) if err != nil { + el.logger.Error(). + Timestamp(). + Str("selector", selector.String()). + Err(err). + Msg("failed to load nodes by selector") + return values.None } @@ -389,6 +410,54 @@ func (el *HtmlElement) IsConnected() values.Boolean { return el.connected } +func (el *HtmlElement) loadInnerText() (core.Value, error) { + h := el.InnerHtml() + + if h == values.EmptyString { + return h, nil + } + + buff := bytes.NewBuffer([]byte(h)) + + parsed, err := goquery.NewDocumentFromReader(buff) + + if err != nil { + el.logger.Error(). + Timestamp(). + Err(err). + Int("id", int(el.id)). + Msg("failed to parse inner html") + + return values.EmptyString, err + } + + return values.NewString(parsed.Text()), nil +} + +func (el *HtmlElement) loadAttrs() (core.Value, error) { + return parseAttrs(el.rawAttrs), nil +} + +func (el *HtmlElement) loadChildren() (core.Value, error) { + if !el.IsConnected() { + return values.NewArray(0), nil + } + + loaded, err := loadNodes(el.logger, el.client, el.broker, el.children) + + if err != nil { + el.logger.Error(). + Timestamp(). + Err(err). + Int("id", int(el.id)). + Msg("failed to load child nodes") + + return values.None, err + } + + return loaded, nil +} + func (el *HtmlElement) handlePageReload(message interface{}) { el.Close() } @@ -484,6 +553,12 @@ func (el *HtmlElement) handleChildrenCountChanged(message interface{}) { node, err := el.client.DOM.DescribeNode(context.Background(), dom.NewDescribeNodeArgs()) if err != nil { + el.logger.Error(). + Timestamp(). + Err(err). + Int("id", int(el.id)). + Msg("failed to update node") + return } @@ -536,10 +611,15 @@ func (el *HtmlElement) handleChildInserted(message interface{}) { } loadedArr := loaded.(*values.Array) - - loadedEl, err := LoadElement(el.client, el.broker, nextId) + loadedEl, err := LoadElement(el.logger, el.client, el.broker, nextId) if err != nil { + el.logger.Error(). + Timestamp(). + Err(err). + Int("id", int(el.id)). + Msg("failed to load an inserted node") + return } @@ -548,6 +628,12 @@ func (el *HtmlElement) handleChildInserted(message interface{}) { newInnerHtml, err := loadInnerHtml(el.client, el.id) if err != nil { + el.logger.Error(). + Timestamp(). + Err(err). + Int("id", int(el.id)). + Msg("failed to update node") + return } @@ -601,6 +687,12 @@ func (el *HtmlElement) handleChildDeleted(message interface{}) { newInnerHtml, err := loadInnerHtml(el.client, el.id) if err != nil { + el.logger.Error(). + Timestamp(). + Err(err). + Int("id", int(el.id)). + Msg("failed to update node") + return } diff --git a/pkg/stdlib/html/driver/dynamic/helpers.go b/pkg/stdlib/html/driver/dynamic/helpers.go index 8197bf25..70d98bed 100644 --- a/pkg/stdlib/html/driver/dynamic/helpers.go +++ b/pkg/stdlib/html/driver/dynamic/helpers.go @@ -8,6 +8,7 @@ import ( "github.com/mafredri/cdp" "github.com/mafredri/cdp/protocol/dom" "github.com/mafredri/cdp/protocol/page" + "github.com/rs/zerolog" "golang.org/x/sync/errgroup" ) @@ -70,11 +71,11 @@ func createChildrenArray(nodes []dom.Node) []dom.NodeID { return children } -func loadNodes(client *cdp.Client, broker *events.EventBroker, nodes []dom.NodeID) (*values.Array, error) { +func loadNodes(logger *zerolog.Logger, client *cdp.Client, broker *events.EventBroker, nodes []dom.NodeID) (*values.Array, error) { arr := values.NewArray(len(nodes)) for _, id := range nodes { - child, err := LoadElement(client, broker, id) + child, err := LoadElement(logger, client, broker, id) if err != nil { return nil, err diff --git a/pkg/stdlib/utils/utils.go b/pkg/stdlib/utils/utils.go index 4f0a95e9..6aec8258 100644 --- a/pkg/stdlib/utils/utils.go +++ b/pkg/stdlib/utils/utils.go @@ -3,8 +3,8 @@ package utils import ( "context" "github.com/MontFerret/ferret/pkg/runtime/core" + "github.com/MontFerret/ferret/pkg/runtime/logging" "github.com/MontFerret/ferret/pkg/runtime/values" - "log" "time" ) @@ -30,16 +30,16 @@ func Wait(_ context.Context, inputs ...core.Value) (core.Value, error) { return values.None, nil } -func Log(_ context.Context, inputs ...core.Value) (core.Value, error) { +func Log(ctx context.Context, inputs ...core.Value) (core.Value, error) { args := make([]interface{}, 0, len(inputs)+1) - args = append(args, "LOG:") - for _, input := range inputs { args = append(args, input) } - log.Println(args...) + logger := logging.From(ctx) + + logger.Print(args...) return values.None, nil }