mirror of
				https://github.com/MontFerret/ferret.git
				synced 2025-10-30 23:37:40 +02:00 
			
		
		
		
	#27 Added logging
This commit is contained in:
		
							
								
								
									
										1
									
								
								.gitignore
									
									
									
									
										vendored
									
									
								
							
							
						
						
									
										1
									
								
								.gitignore
									
									
									
									
										vendored
									
									
								
							| @@ -124,3 +124,4 @@ $RECYCLE.BIN/ | ||||
|  | ||||
| vendor | ||||
| bin | ||||
| *.log | ||||
							
								
								
									
										31
									
								
								Gopkg.lock
									
									
									
										generated
									
									
									
								
							
							
						
						
									
										31
									
								
								Gopkg.lock
									
									
									
										generated
									
									
									
								
							| @@ -41,6 +41,14 @@ | ||||
|   pruneopts = "UT" | ||||
|   revision = "2b8494104d86337cdd41d0a49cbed8e4583c0ab4" | ||||
|  | ||||
| [[projects]] | ||||
|   digest = "1:ce579162ae1341f3e5ab30c0dce767f28b1eb6a81359aad01723f1ba6b4becdf" | ||||
|   name = "github.com/gofrs/uuid" | ||||
|   packages = ["."] | ||||
|   pruneopts = "UT" | ||||
|   revision = "370558f003bfe29580cd0f698d8640daccdcc45c" | ||||
|   version = "v3.1.1" | ||||
|  | ||||
| [[projects]] | ||||
|   branch = "master" | ||||
|   digest = "1:f14d1b50e0075fb00177f12a96dd7addf93d1e2883c25befd17285b779549795" | ||||
| @@ -121,6 +129,14 @@ | ||||
|   revision = "75b0ecc5efcff27ac756a33ec71f0db75dc3d21c" | ||||
|   version = "v0.19.0" | ||||
|  | ||||
| [[projects]] | ||||
|   digest = "1:c805e517269b0ba4c21ded5836019ed7d16953d4026cb7d00041d039c7906be9" | ||||
|   name = "github.com/natefinch/lumberjack" | ||||
|   packages = ["."] | ||||
|   pruneopts = "UT" | ||||
|   revision = "a96e63847dc3c67d17befa69c303767e2f84e54f" | ||||
|   version = "v2.1" | ||||
|  | ||||
| [[projects]] | ||||
|   digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747" | ||||
|   name = "github.com/pkg/errors" | ||||
| @@ -129,6 +145,18 @@ | ||||
|   revision = "645ef00459ed84a119197bfb8d8205042c6df63d" | ||||
|   version = "v0.8.0" | ||||
|  | ||||
| [[projects]] | ||||
|   digest = "1:5bc8f93f977b72a7a5264725c3bab275e69de0cc3e5c0dc1ee56feb564c33f03" | ||||
|   name = "github.com/rs/zerolog" | ||||
|   packages = [ | ||||
|     ".", | ||||
|     "internal/cbor", | ||||
|     "internal/json", | ||||
|   ] | ||||
|   pruneopts = "UT" | ||||
|   revision = "338f9bc14084d22cb8eeacd6492861f8449d715c" | ||||
|   version = "v1.9.1" | ||||
|  | ||||
| [[projects]] | ||||
|   digest = "1:4ca145a665316d3c020a39c0741780fa3636b9152b824206796c4dce541f4a24" | ||||
|   name = "github.com/sethgrid/pester" | ||||
| @@ -189,6 +217,7 @@ | ||||
|     "github.com/antlr/antlr4/runtime/Go/antlr", | ||||
|     "github.com/chzyer/readline", | ||||
|     "github.com/corpix/uarand", | ||||
|     "github.com/gofrs/uuid", | ||||
|     "github.com/mafredri/cdp", | ||||
|     "github.com/mafredri/cdp/devtool", | ||||
|     "github.com/mafredri/cdp/protocol/dom", | ||||
| @@ -198,7 +227,9 @@ | ||||
|     "github.com/mafredri/cdp/protocol/target", | ||||
|     "github.com/mafredri/cdp/rpcc", | ||||
|     "github.com/mafredri/cdp/session", | ||||
|     "github.com/natefinch/lumberjack", | ||||
|     "github.com/pkg/errors", | ||||
|     "github.com/rs/zerolog", | ||||
|     "github.com/sethgrid/pester", | ||||
|     "github.com/smartystreets/goconvey/convey", | ||||
|     "golang.org/x/net/html", | ||||
|   | ||||
| @@ -42,5 +42,9 @@ | ||||
|   version = "1.4" | ||||
|  | ||||
| [[constraint]] | ||||
|     name = "github.com/PuerkitoBio/goquery" | ||||
|     version = "1.4.1" | ||||
|   name = "github.com/PuerkitoBio/goquery" | ||||
|   version = "1.4.1" | ||||
|  | ||||
| [[constraint]] | ||||
|   name = "github.com/gofrs/uuid" | ||||
|   version = "3.1.1" | ||||
| @@ -5,8 +5,11 @@ import ( | ||||
| 	"fmt" | ||||
| 	"github.com/MontFerret/ferret/pkg/compiler" | ||||
| 	"github.com/MontFerret/ferret/pkg/runtime" | ||||
| 	"github.com/MontFerret/ferret/pkg/runtime/logging" | ||||
| 	"io/ioutil" | ||||
| 	"os" | ||||
| 	"os/signal" | ||||
| 	"syscall" | ||||
| ) | ||||
|  | ||||
| func ExecFile(pathToFile string, opts Options) { | ||||
| @@ -33,9 +36,25 @@ func Exec(query string, opts Options) { | ||||
| 		return | ||||
| 	} | ||||
|  | ||||
| 	l := NewLogger() | ||||
|  | ||||
| 	ctx, cancel := context.WithCancel(context.Background()) | ||||
| 	c := make(chan os.Signal, 1) | ||||
| 	signal.Notify(c, syscall.SIGHUP) | ||||
|  | ||||
| 	go func() { | ||||
| 		for { | ||||
| 			<-c | ||||
| 			cancel() | ||||
| 			l.Close() | ||||
| 		} | ||||
| 	}() | ||||
|  | ||||
| 	out, err := prog.Run( | ||||
| 		context.Background(), | ||||
| 		ctx, | ||||
| 		runtime.WithBrowser(opts.Cdp), | ||||
| 		runtime.WithLog(l), | ||||
| 		runtime.WithLogLevel(logging.DebugLevel), | ||||
| 	) | ||||
|  | ||||
| 	if err != nil { | ||||
|   | ||||
							
								
								
									
										14
									
								
								cmd/cli/logger.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										14
									
								
								cmd/cli/logger.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,14 @@ | ||||
| package cli | ||||
|  | ||||
| import ( | ||||
| 	"github.com/natefinch/lumberjack" | ||||
| ) | ||||
|  | ||||
| func NewLogger() *lumberjack.Logger { | ||||
| 	l := &lumberjack.Logger{ | ||||
| 		Filename: "./ferret.log", | ||||
| 		MaxSize:  100, | ||||
| 	} | ||||
|  | ||||
| 	return l | ||||
| } | ||||
| @@ -5,8 +5,12 @@ import ( | ||||
| 	"fmt" | ||||
| 	"github.com/MontFerret/ferret/pkg/compiler" | ||||
| 	"github.com/MontFerret/ferret/pkg/runtime" | ||||
| 	"github.com/MontFerret/ferret/pkg/runtime/logging" | ||||
| 	"github.com/chzyer/readline" | ||||
| 	"os" | ||||
| 	"os/signal" | ||||
| 	"strings" | ||||
| 	"syscall" | ||||
| ) | ||||
|  | ||||
| func Repl(version string, opts Options) { | ||||
| @@ -32,6 +36,20 @@ func Repl(version string, opts Options) { | ||||
|  | ||||
| 	timer := NewTimer() | ||||
|  | ||||
| 	l := NewLogger() | ||||
|  | ||||
| 	ctx, cancel := context.WithCancel(context.Background()) | ||||
| 	c := make(chan os.Signal, 1) | ||||
| 	signal.Notify(c, syscall.SIGHUP) | ||||
|  | ||||
| 	go func() { | ||||
| 		for { | ||||
| 			<-c | ||||
| 			cancel() | ||||
| 			l.Close() | ||||
| 		} | ||||
| 	}() | ||||
|  | ||||
| 	for { | ||||
| 		line, err := rl.Readline() | ||||
|  | ||||
| @@ -75,8 +93,10 @@ func Repl(version string, opts Options) { | ||||
| 		timer.Start() | ||||
|  | ||||
| 		out, err := program.Run( | ||||
| 			context.Background(), | ||||
| 			ctx, | ||||
| 			runtime.WithBrowser(opts.Cdp), | ||||
| 			runtime.WithLog(l), | ||||
| 			runtime.WithLogLevel(logging.DebugLevel), | ||||
| 		) | ||||
|  | ||||
| 		timer.Stop() | ||||
|   | ||||
| @@ -27,7 +27,7 @@ var ( | ||||
|  | ||||
| 	conn = flag.String( | ||||
| 		"cdp", | ||||
| 		"http://127.0.0.1:9222", | ||||
| 		"http://0.0.0.0:9222", | ||||
| 		"set CDP address", | ||||
| 	) | ||||
|  | ||||
|   | ||||
| @@ -1,9 +0,0 @@ | ||||
| LET g = DOCUMENT("https://www.google.com/", true) | ||||
|  | ||||
| INPUT(ELEMENT(g, 'input[name="q"]'), "ferret") | ||||
|  | ||||
| CLICK(g, 'input[name="btnK"]') | ||||
|  | ||||
| WAIT_NAVIGATION(g) | ||||
|  | ||||
| RETURN 1 | ||||
| @@ -78,7 +78,7 @@ func (c *FqlCompiler) Compile(query string) (program *runtime.Program, err error | ||||
| 		} | ||||
| 	}() | ||||
|  | ||||
| 	l := newVisitor(c.funcs) | ||||
| 	l := newVisitor(query, c.funcs) | ||||
|  | ||||
| 	res := p.Visit(l).(*result) | ||||
|  | ||||
|   | ||||
| @@ -1688,39 +1688,24 @@ func TestForTernaryExpression(t *testing.T) { | ||||
| 	}) | ||||
| } | ||||
|  | ||||
| //func TestHtml(t *testing.T) { | ||||
| //	Convey("Should load a document", t, func() { | ||||
| //		c := compiler.New() | ||||
| // | ||||
| //		prog, err := c.Compile(` | ||||
| //LET doc = DOCUMENT('https://soundcloud.com/charts/top', true) | ||||
| // | ||||
| //// TODO: We need a better way of waiting for page loading | ||||
| //// Something line WAIT_FOR(doc, selector) | ||||
| //SLEEP(2000) | ||||
| // | ||||
| //LET tracks = ELEMENTS(doc, '.chartTrack__details') | ||||
| // | ||||
| //LOG("found", LENGTH(tracks), "tracks") | ||||
| // | ||||
| //FOR track IN tracks | ||||
| //    // LET username = ELEMENT(track, '.chartTrack__username') | ||||
| //    // LET title = ELEMENT(track, '.chartTrack__title') | ||||
| // | ||||
| //    // LOG("NODE", track.nodeName) | ||||
| // | ||||
| //    SLEEP(500) | ||||
| // | ||||
| //    RETURN track.innerHtml | ||||
| // | ||||
| //		`) | ||||
| // | ||||
| //		So(err, ShouldBeNil) | ||||
| // | ||||
| //		out, err := prog.Run(context.Background(), runtime.WithBrowser("http://127.0.0.1:9222")) | ||||
| // | ||||
| //		So(err, ShouldBeNil) | ||||
| // | ||||
| //		So(string(out), ShouldEqual, `"int"`) | ||||
| //	}) | ||||
| //} | ||||
| func TestHtml(t *testing.T) { | ||||
| 	Convey("Should load a document", t, func() { | ||||
| 		c := compiler.New() | ||||
|  | ||||
| 		out, err := c.MustCompile(` | ||||
| LET doc = DOCUMENT("https://github.com/", true) | ||||
| LET btn = ELEMENT(doc, ".HeaderMenu a") | ||||
|  | ||||
| CLICK(btn) | ||||
| WAIT_NAVIGATION(doc) | ||||
| WAIT_ELEMENT(doc, '.IconNav') | ||||
|  | ||||
| RETURN INNER_HTML_ALL(doc, '.IconNav a') | ||||
|  | ||||
| 		`).Run(context.Background()) | ||||
|  | ||||
| 		So(err, ShouldBeNil) | ||||
|  | ||||
| 		So(string(out), ShouldEqual, `"int"`) | ||||
| 	}) | ||||
| } | ||||
|   | ||||
| @@ -18,13 +18,14 @@ import ( | ||||
|  | ||||
| type visitor struct { | ||||
| 	*fql.BaseFqlParserVisitor | ||||
|  | ||||
| 	src   string | ||||
| 	funcs map[string]core.Function | ||||
| } | ||||
|  | ||||
| func newVisitor(funcs map[string]core.Function) *visitor { | ||||
| func newVisitor(src string, funcs map[string]core.Function) *visitor { | ||||
| 	return &visitor{ | ||||
| 		&fql.BaseFqlParserVisitor{}, | ||||
| 		src, | ||||
| 		funcs, | ||||
| 	} | ||||
| } | ||||
| @@ -38,7 +39,7 @@ func (v *visitor) VisitProgram(ctx *fql.ProgramContext) interface{} { | ||||
| 			return nil, err | ||||
| 		} | ||||
|  | ||||
| 		return runtime.NewProgram(block), nil | ||||
| 		return runtime.NewProgram(v.src, block) | ||||
| 	}) | ||||
| } | ||||
|  | ||||
|   | ||||
							
								
								
									
										23
									
								
								pkg/runtime/logging/logger.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										23
									
								
								pkg/runtime/logging/logger.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,23 @@ | ||||
| package logging | ||||
|  | ||||
| import ( | ||||
| 	"context" | ||||
| 	"github.com/rs/zerolog" | ||||
| ) | ||||
|  | ||||
| type Level uint8 | ||||
|  | ||||
| const ( | ||||
| 	DebugLevel Level = iota | ||||
| 	InfoLevel | ||||
| 	WarnLevel | ||||
| 	ErrorLevel | ||||
| 	FatalLevel | ||||
| 	PanicLevel | ||||
| 	NoLevel | ||||
| 	Disabled | ||||
| ) | ||||
|  | ||||
| func From(ctx context.Context) *zerolog.Logger { | ||||
| 	return zerolog.Ctx(ctx) | ||||
| } | ||||
| @@ -1,12 +1,21 @@ | ||||
| package runtime | ||||
|  | ||||
| import "context" | ||||
| import ( | ||||
| 	"context" | ||||
| 	"github.com/MontFerret/ferret/pkg/runtime/logging" | ||||
| 	"github.com/gofrs/uuid" | ||||
| 	"github.com/rs/zerolog" | ||||
| 	"io" | ||||
| 	"os" | ||||
| ) | ||||
|  | ||||
| type ( | ||||
| 	Options struct { | ||||
| 		proxy     string | ||||
| 		cdp       string | ||||
| 		variables map[string]interface{} | ||||
| 		logWriter io.Writer | ||||
| 		logLevel  zerolog.Level | ||||
| 	} | ||||
|  | ||||
| 	Option func(*Options) | ||||
| @@ -14,8 +23,10 @@ type ( | ||||
|  | ||||
| func newOptions() *Options { | ||||
| 	return &Options{ | ||||
| 		cdp:       "http://127.0.0.1:9222", | ||||
| 		cdp:       "http://0.0.0.0:9222", | ||||
| 		variables: make(map[string]interface{}), | ||||
| 		logWriter: os.Stdout, | ||||
| 		logLevel:  zerolog.ErrorLevel, | ||||
| 	} | ||||
| } | ||||
|  | ||||
| @@ -38,10 +49,38 @@ func WithProxy(address string) Option { | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func WithLog(writer io.Writer) Option { | ||||
| 	return func(options *Options) { | ||||
| 		options.logWriter = writer | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func WithLogLevel(lvl logging.Level) Option { | ||||
| 	return func(options *Options) { | ||||
| 		options.logLevel = zerolog.Level(lvl) | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func (opts *Options) withContext(parent context.Context) context.Context { | ||||
| 	return context.WithValue( | ||||
| 	ctx := context.WithValue( | ||||
| 		parent, | ||||
| 		"variables", | ||||
| 		opts.variables, | ||||
| 	) | ||||
|  | ||||
| 	id, err := uuid.NewV4() | ||||
|  | ||||
| 	if err != nil { | ||||
| 		panic(err) | ||||
| 	} | ||||
|  | ||||
| 	logger := zerolog.New(opts.logWriter). | ||||
| 		With(). | ||||
| 		Str("id", id.String()). | ||||
| 		Logger() | ||||
| 	logger.WithLevel(opts.logLevel) | ||||
|  | ||||
| 	ctx = logger.WithContext(ctx) | ||||
|  | ||||
| 	return ctx | ||||
| } | ||||
|   | ||||
| @@ -8,11 +8,24 @@ import ( | ||||
| ) | ||||
|  | ||||
| type Program struct { | ||||
| 	exp core.Expression | ||||
| 	src  string | ||||
| 	body core.Expression | ||||
| } | ||||
|  | ||||
| func NewProgram(exp core.Expression) *Program { | ||||
| 	return &Program{exp} | ||||
| func NewProgram(src string, body core.Expression) (*Program, error) { | ||||
| 	if src == "" { | ||||
| 		return nil, core.Error(core.ErrMissedArgument, "source") | ||||
| 	} | ||||
|  | ||||
| 	if core.IsNil(body) { | ||||
| 		return nil, core.Error(core.ErrMissedArgument, "body") | ||||
| 	} | ||||
|  | ||||
| 	return &Program{src, body}, nil | ||||
| } | ||||
|  | ||||
| func (p *Program) Source() string { | ||||
| 	return p.src | ||||
| } | ||||
|  | ||||
| func (p *Program) Run(ctx context.Context, setters ...Option) ([]byte, error) { | ||||
| @@ -30,7 +43,7 @@ func (p *Program) Run(ctx context.Context, setters ...Option) ([]byte, error) { | ||||
| 	ctx = driver.WithDynamicDriver(ctx, opts.cdp) | ||||
| 	ctx = driver.WithStaticDriver(ctx) | ||||
|  | ||||
| 	out, err := p.exp.Exec(ctx, scope) | ||||
| 	out, err := p.body.Exec(ctx, scope) | ||||
|  | ||||
| 	if err != nil { | ||||
| 		js, _ := values.None.MarshalJSON() | ||||
|   | ||||
| @@ -5,6 +5,7 @@ import ( | ||||
| 	"crypto/sha512" | ||||
| 	"fmt" | ||||
| 	"github.com/MontFerret/ferret/pkg/runtime/core" | ||||
| 	"github.com/MontFerret/ferret/pkg/runtime/logging" | ||||
| 	"github.com/MontFerret/ferret/pkg/runtime/values" | ||||
| 	"github.com/MontFerret/ferret/pkg/stdlib/html/driver/dynamic/eval" | ||||
| 	"github.com/MontFerret/ferret/pkg/stdlib/html/driver/dynamic/events" | ||||
| @@ -15,12 +16,14 @@ import ( | ||||
| 	"github.com/mafredri/cdp/protocol/page" | ||||
| 	"github.com/mafredri/cdp/rpcc" | ||||
| 	"github.com/pkg/errors" | ||||
| 	"github.com/rs/zerolog" | ||||
| 	"sync" | ||||
| 	"time" | ||||
| ) | ||||
|  | ||||
| type HtmlDocument struct { | ||||
| 	sync.Mutex | ||||
| 	logger  *zerolog.Logger | ||||
| 	conn    *rpcc.Conn | ||||
| 	client  *cdp.Client | ||||
| 	events  *events.EventBroker | ||||
| @@ -93,7 +96,14 @@ func LoadHtmlDocument( | ||||
| 		return nil, err | ||||
| 	} | ||||
|  | ||||
| 	return NewHtmlDocument(conn, client, broker, root, innerHtml), nil | ||||
| 	return NewHtmlDocument( | ||||
| 		logging.From(ctx), | ||||
| 		conn, | ||||
| 		client, | ||||
| 		broker, | ||||
| 		root, | ||||
| 		innerHtml, | ||||
| 	), nil | ||||
| } | ||||
|  | ||||
| func getRootElement(client *cdp.Client) (dom.Node, values.String, error) { | ||||
| @@ -117,6 +127,7 @@ func getRootElement(client *cdp.Client) (dom.Node, values.String, error) { | ||||
| } | ||||
|  | ||||
| func NewHtmlDocument( | ||||
| 	logger *zerolog.Logger, | ||||
| 	conn *rpcc.Conn, | ||||
| 	client *cdp.Client, | ||||
| 	broker *events.EventBroker, | ||||
| @@ -124,10 +135,11 @@ func NewHtmlDocument( | ||||
| 	innerHtml values.String, | ||||
| ) *HtmlDocument { | ||||
| 	doc := new(HtmlDocument) | ||||
| 	doc.logger = logger | ||||
| 	doc.conn = conn | ||||
| 	doc.client = client | ||||
| 	doc.events = broker | ||||
| 	doc.element = NewHtmlElement(client, broker, root.NodeID, root, innerHtml) | ||||
| 	doc.element = NewHtmlElement(doc.logger, client, broker, root.NodeID, root, innerHtml) | ||||
| 	doc.url = "" | ||||
|  | ||||
| 	if root.BaseURL != nil { | ||||
| @@ -141,7 +153,11 @@ func NewHtmlDocument( | ||||
| 		updated, innerHtml, err := getRootElement(client) | ||||
|  | ||||
| 		if err != nil { | ||||
| 			// TODO: We need somehow log all errors outside of stdout | ||||
| 			doc.logger.Error(). | ||||
| 				Timestamp(). | ||||
| 				Err(err). | ||||
| 				Msg("failed to get root node after page load") | ||||
|  | ||||
| 			return | ||||
| 		} | ||||
|  | ||||
| @@ -149,7 +165,7 @@ func NewHtmlDocument( | ||||
| 		doc.element.Close() | ||||
|  | ||||
| 		// create a new root element wrapper | ||||
| 		doc.element = NewHtmlElement(client, broker, updated.NodeID, updated, innerHtml) | ||||
| 		doc.element = NewHtmlElement(doc.logger, client, broker, updated.NodeID, updated, innerHtml) | ||||
| 		doc.url = "" | ||||
|  | ||||
| 		if updated.BaseURL != nil { | ||||
| @@ -226,11 +242,47 @@ func (doc *HtmlDocument) Close() error { | ||||
| 	doc.Lock() | ||||
| 	defer doc.Unlock() | ||||
|  | ||||
| 	doc.events.Stop() | ||||
| 	doc.events.Close() | ||||
| 	var err error | ||||
|  | ||||
| 	doc.element.Close() | ||||
| 	doc.client.Page.Close(context.Background()) | ||||
| 	err = doc.events.Stop() | ||||
|  | ||||
| 	if err != nil { | ||||
| 		doc.logger.Warn(). | ||||
| 			Timestamp(). | ||||
| 			Str("url", doc.url.String()). | ||||
| 			Err(err). | ||||
| 			Msg("failed to stop event broker") | ||||
| 	} | ||||
|  | ||||
| 	err = doc.events.Close() | ||||
|  | ||||
| 	if err != nil { | ||||
| 		doc.logger.Warn(). | ||||
| 			Timestamp(). | ||||
| 			Str("url", doc.url.String()). | ||||
| 			Err(err). | ||||
| 			Msg("failed to close event broker") | ||||
| 	} | ||||
|  | ||||
| 	err = doc.element.Close() | ||||
|  | ||||
| 	if err != nil { | ||||
| 		doc.logger.Warn(). | ||||
| 			Timestamp(). | ||||
| 			Str("url", doc.url.String()). | ||||
| 			Err(err). | ||||
| 			Msg("failed to close root element") | ||||
| 	} | ||||
|  | ||||
| 	err = doc.client.Page.Close(context.Background()) | ||||
|  | ||||
| 	if err != nil { | ||||
| 		doc.logger.Warn(). | ||||
| 			Timestamp(). | ||||
| 			Str("url", doc.url.String()). | ||||
| 			Err(err). | ||||
| 			Msg("failed to close browser page") | ||||
| 	} | ||||
|  | ||||
| 	return doc.conn.Close() | ||||
| } | ||||
| @@ -354,13 +406,18 @@ func (doc *HtmlDocument) InnerHtmlBySelectorAll(selector values.String) (*values | ||||
| 	res, err := eval.Eval( | ||||
| 		doc.client, | ||||
| 		fmt.Sprintf(` | ||||
| 			var result = []; | ||||
| 			var elements = document.querySelectorAll(%s); | ||||
|  | ||||
| 			if (elements == null) { | ||||
| 				return []; | ||||
| 				return result; | ||||
| 			} | ||||
|  | ||||
| 			return elements.map(i => i.innerHtml); | ||||
| 			elements.forEach((i) => { | ||||
| 				result.push(i.innerHtml); | ||||
| 			}); | ||||
|  | ||||
| 			return result; | ||||
| 		`, eval.ParamString(selector.String())), | ||||
| 		true, | ||||
| 		false, | ||||
| @@ -408,13 +465,18 @@ func (doc *HtmlDocument) InnerTextBySelectorAll(selector values.String) (*values | ||||
| 	res, err := eval.Eval( | ||||
| 		doc.client, | ||||
| 		fmt.Sprintf(` | ||||
| 			var result = []; | ||||
| 			var elements = document.querySelectorAll(%s); | ||||
|  | ||||
| 			if (elements == null) { | ||||
| 				return []; | ||||
| 				return result; | ||||
| 			} | ||||
|  | ||||
| 			return elements.map(i => i.innerText); | ||||
| 			elements.forEach((i) => { | ||||
| 				result.push(i.innerText); | ||||
| 			}); | ||||
|  | ||||
| 			return result; | ||||
| 		`, eval.ParamString(selector.String())), | ||||
| 		true, | ||||
| 		false, | ||||
|   | ||||
| @@ -13,6 +13,7 @@ import ( | ||||
| 	"github.com/PuerkitoBio/goquery" | ||||
| 	"github.com/mafredri/cdp" | ||||
| 	"github.com/mafredri/cdp/protocol/dom" | ||||
| 	"github.com/rs/zerolog" | ||||
| 	"strconv" | ||||
| 	"sync" | ||||
| 	"time" | ||||
| @@ -22,6 +23,7 @@ const DefaultTimeout = time.Second * 30 | ||||
|  | ||||
| type HtmlElement struct { | ||||
| 	sync.Mutex | ||||
| 	logger         *zerolog.Logger | ||||
| 	client         *cdp.Client | ||||
| 	broker         *events.EventBroker | ||||
| 	connected      values.Boolean | ||||
| @@ -31,12 +33,14 @@ type HtmlElement struct { | ||||
| 	innerHtml      values.String | ||||
| 	innerText      *common.LazyValue | ||||
| 	value          core.Value | ||||
| 	rawAttrs       []string | ||||
| 	attributes     *common.LazyValue | ||||
| 	children       []dom.NodeID | ||||
| 	loadedChildren *common.LazyValue | ||||
| } | ||||
|  | ||||
| func LoadElement( | ||||
| 	logger *zerolog.Logger, | ||||
| 	client *cdp.Client, | ||||
| 	broker *events.EventBroker, | ||||
| 	id dom.NodeID, | ||||
| @@ -68,6 +72,7 @@ func LoadElement( | ||||
| 	} | ||||
|  | ||||
| 	return NewHtmlElement( | ||||
| 		logger, | ||||
| 		client, | ||||
| 		broker, | ||||
| 		id, | ||||
| @@ -77,6 +82,7 @@ func LoadElement( | ||||
| } | ||||
|  | ||||
| func NewHtmlElement( | ||||
| 	logger *zerolog.Logger, | ||||
| 	client *cdp.Client, | ||||
| 	broker *events.EventBroker, | ||||
| 	id dom.NodeID, | ||||
| @@ -84,6 +90,7 @@ func NewHtmlElement( | ||||
| 	innerHtml values.String, | ||||
| ) *HtmlElement { | ||||
| 	el := new(HtmlElement) | ||||
| 	el.logger = logger | ||||
| 	el.client = client | ||||
| 	el.broker = broker | ||||
| 	el.connected = values.True | ||||
| @@ -91,34 +98,11 @@ func NewHtmlElement( | ||||
| 	el.nodeType = values.NewInt(node.NodeType) | ||||
| 	el.nodeName = values.NewString(node.NodeName) | ||||
| 	el.innerHtml = innerHtml | ||||
| 	el.innerText = common.NewLazyValue(func() (core.Value, error) { | ||||
| 		h := el.InnerHtml() | ||||
|  | ||||
| 		if h == values.EmptyString { | ||||
| 			return h, nil | ||||
| 		} | ||||
|  | ||||
| 		buff := bytes.NewBuffer([]byte(h)) | ||||
|  | ||||
| 		parsed, err := goquery.NewDocumentFromReader(buff) | ||||
|  | ||||
| 		if err != nil { | ||||
| 			return values.EmptyString, err | ||||
| 		} | ||||
|  | ||||
| 		return values.NewString(parsed.Text()), nil | ||||
| 	}) | ||||
| 	el.attributes = common.NewLazyValue(func() (core.Value, error) { | ||||
| 		return parseAttrs(node.Attributes), nil | ||||
| 	}) | ||||
| 	el.innerText = common.NewLazyValue(el.loadInnerText) | ||||
| 	el.rawAttrs = node.Attributes[:] | ||||
| 	el.attributes = common.NewLazyValue(el.loadAttrs) | ||||
| 	el.value = values.EmptyString | ||||
| 	el.loadedChildren = common.NewLazyValue(func() (core.Value, error) { | ||||
| 		if !el.IsConnected() { | ||||
| 			return values.NewArray(0), nil | ||||
| 		} | ||||
|  | ||||
| 		return loadNodes(client, broker, el.children) | ||||
| 	}) | ||||
| 	el.loadedChildren = common.NewLazyValue(el.loadChildren) | ||||
|  | ||||
| 	if node.Value != nil { | ||||
| 		el.value = values.NewString(*node.Value) | ||||
| @@ -205,11 +189,19 @@ func (el *HtmlElement) Unwrap() interface{} { | ||||
| } | ||||
|  | ||||
| func (el *HtmlElement) Hash() int { | ||||
| 	el.Lock() | ||||
| 	defer el.Unlock() | ||||
|  | ||||
| 	h := sha512.New() | ||||
|  | ||||
| 	out, err := h.Write([]byte(strconv.Itoa(int(el.id)))) | ||||
| 	out, err := h.Write([]byte(el.innerHtml)) | ||||
|  | ||||
| 	if err != nil { | ||||
| 		el.logger.Error(). | ||||
| 			Timestamp(). | ||||
| 			Err(err). | ||||
| 			Msg("failed to calculate hash value") | ||||
|  | ||||
| 		return 0 | ||||
| 	} | ||||
|  | ||||
| @@ -227,6 +219,11 @@ func (el *HtmlElement) Value() core.Value { | ||||
| 	val, err := eval.Property(ctx, el.client, el.id, "value") | ||||
|  | ||||
| 	if err != nil { | ||||
| 		el.logger.Error(). | ||||
| 			Timestamp(). | ||||
| 			Err(err). | ||||
| 			Msg("failed to get node value") | ||||
|  | ||||
| 		return el.value | ||||
| 	} | ||||
|  | ||||
| @@ -309,12 +306,24 @@ func (el *HtmlElement) QuerySelector(selector values.String) core.Value { | ||||
| 	found, err := el.client.DOM.QuerySelector(ctx, selectorArgs) | ||||
|  | ||||
| 	if err != nil { | ||||
| 		el.logger.Error(). | ||||
| 			Timestamp(). | ||||
| 			Str("selector", selector.String()). | ||||
| 			Err(err). | ||||
| 			Msg("failed to retrieve a node by selector") | ||||
|  | ||||
| 		return values.None | ||||
| 	} | ||||
|  | ||||
| 	res, err := LoadElement(el.client, el.broker, found.NodeID) | ||||
| 	res, err := LoadElement(el.logger, el.client, el.broker, found.NodeID) | ||||
|  | ||||
| 	if err != nil { | ||||
| 		el.logger.Error(). | ||||
| 			Timestamp(). | ||||
| 			Str("selector", selector.String()). | ||||
| 			Err(err). | ||||
| 			Msg("failed to load a child node by selector") | ||||
|  | ||||
| 		return values.None | ||||
| 	} | ||||
|  | ||||
| @@ -332,15 +341,27 @@ func (el *HtmlElement) QuerySelectorAll(selector values.String) core.Value { | ||||
| 	res, err := el.client.DOM.QuerySelectorAll(ctx, selectorArgs) | ||||
|  | ||||
| 	if err != nil { | ||||
| 		el.logger.Error(). | ||||
| 			Timestamp(). | ||||
| 			Str("selector", selector.String()). | ||||
| 			Err(err). | ||||
| 			Msg("failed to retrieve nodes by selector") | ||||
|  | ||||
| 		return values.None | ||||
| 	} | ||||
|  | ||||
| 	arr := values.NewArray(len(res.NodeIDs)) | ||||
|  | ||||
| 	for _, id := range res.NodeIDs { | ||||
| 		childEl, err := LoadElement(el.client, el.broker, id) | ||||
| 		childEl, err := LoadElement(el.logger, el.client, el.broker, id) | ||||
|  | ||||
| 		if err != nil { | ||||
| 			el.logger.Error(). | ||||
| 				Timestamp(). | ||||
| 				Str("selector", selector.String()). | ||||
| 				Err(err). | ||||
| 				Msg("failed to load nodes by selector") | ||||
|  | ||||
| 			return values.None | ||||
| 		} | ||||
|  | ||||
| @@ -389,6 +410,54 @@ func (el *HtmlElement) IsConnected() values.Boolean { | ||||
| 	return el.connected | ||||
| } | ||||
|  | ||||
| func (el *HtmlElement) loadInnerText() (core.Value, error) { | ||||
| 	h := el.InnerHtml() | ||||
|  | ||||
| 	if h == values.EmptyString { | ||||
| 		return h, nil | ||||
| 	} | ||||
|  | ||||
| 	buff := bytes.NewBuffer([]byte(h)) | ||||
|  | ||||
| 	parsed, err := goquery.NewDocumentFromReader(buff) | ||||
|  | ||||
| 	if err != nil { | ||||
| 		el.logger.Error(). | ||||
| 			Timestamp(). | ||||
| 			Err(err). | ||||
| 			Int("id", int(el.id)). | ||||
| 			Msg("failed to parse inner html") | ||||
|  | ||||
| 		return values.EmptyString, err | ||||
| 	} | ||||
|  | ||||
| 	return values.NewString(parsed.Text()), nil | ||||
| } | ||||
|  | ||||
| func (el *HtmlElement) loadAttrs() (core.Value, error) { | ||||
| 	return parseAttrs(el.rawAttrs), nil | ||||
| } | ||||
|  | ||||
| func (el *HtmlElement) loadChildren() (core.Value, error) { | ||||
| 	if !el.IsConnected() { | ||||
| 		return values.NewArray(0), nil | ||||
| 	} | ||||
|  | ||||
| 	loaded, err := loadNodes(el.logger, el.client, el.broker, el.children) | ||||
|  | ||||
| 	if err != nil { | ||||
| 		el.logger.Error(). | ||||
| 			Timestamp(). | ||||
| 			Err(err). | ||||
| 			Int("id", int(el.id)). | ||||
| 			Msg("failed to load child nodes") | ||||
|  | ||||
| 		return values.None, err | ||||
| 	} | ||||
|  | ||||
| 	return loaded, nil | ||||
| } | ||||
|  | ||||
| func (el *HtmlElement) handlePageReload(message interface{}) { | ||||
| 	el.Close() | ||||
| } | ||||
| @@ -484,6 +553,12 @@ func (el *HtmlElement) handleChildrenCountChanged(message interface{}) { | ||||
| 	node, err := el.client.DOM.DescribeNode(context.Background(), dom.NewDescribeNodeArgs()) | ||||
|  | ||||
| 	if err != nil { | ||||
| 		el.logger.Error(). | ||||
| 			Timestamp(). | ||||
| 			Err(err). | ||||
| 			Int("id", int(el.id)). | ||||
| 			Msg("failed to update node") | ||||
|  | ||||
| 		return | ||||
| 	} | ||||
|  | ||||
| @@ -536,10 +611,15 @@ func (el *HtmlElement) handleChildInserted(message interface{}) { | ||||
| 	} | ||||
|  | ||||
| 	loadedArr := loaded.(*values.Array) | ||||
|  | ||||
| 	loadedEl, err := LoadElement(el.client, el.broker, nextId) | ||||
| 	loadedEl, err := LoadElement(el.logger, el.client, el.broker, nextId) | ||||
|  | ||||
| 	if err != nil { | ||||
| 		el.logger.Error(). | ||||
| 			Timestamp(). | ||||
| 			Err(err). | ||||
| 			Int("id", int(el.id)). | ||||
| 			Msg("failed to load an inserted node") | ||||
|  | ||||
| 		return | ||||
| 	} | ||||
|  | ||||
| @@ -548,6 +628,12 @@ func (el *HtmlElement) handleChildInserted(message interface{}) { | ||||
| 	newInnerHtml, err := loadInnerHtml(el.client, el.id) | ||||
|  | ||||
| 	if err != nil { | ||||
| 		el.logger.Error(). | ||||
| 			Timestamp(). | ||||
| 			Err(err). | ||||
| 			Int("id", int(el.id)). | ||||
| 			Msg("failed to update node") | ||||
|  | ||||
| 		return | ||||
| 	} | ||||
|  | ||||
| @@ -601,6 +687,12 @@ func (el *HtmlElement) handleChildDeleted(message interface{}) { | ||||
| 	newInnerHtml, err := loadInnerHtml(el.client, el.id) | ||||
|  | ||||
| 	if err != nil { | ||||
| 		el.logger.Error(). | ||||
| 			Timestamp(). | ||||
| 			Err(err). | ||||
| 			Int("id", int(el.id)). | ||||
| 			Msg("failed to update node") | ||||
|  | ||||
| 		return | ||||
| 	} | ||||
|  | ||||
|   | ||||
| @@ -8,6 +8,7 @@ import ( | ||||
| 	"github.com/mafredri/cdp" | ||||
| 	"github.com/mafredri/cdp/protocol/dom" | ||||
| 	"github.com/mafredri/cdp/protocol/page" | ||||
| 	"github.com/rs/zerolog" | ||||
| 	"golang.org/x/sync/errgroup" | ||||
| ) | ||||
|  | ||||
| @@ -70,11 +71,11 @@ func createChildrenArray(nodes []dom.Node) []dom.NodeID { | ||||
| 	return children | ||||
| } | ||||
|  | ||||
| func loadNodes(client *cdp.Client, broker *events.EventBroker, nodes []dom.NodeID) (*values.Array, error) { | ||||
| func loadNodes(logger *zerolog.Logger, client *cdp.Client, broker *events.EventBroker, nodes []dom.NodeID) (*values.Array, error) { | ||||
| 	arr := values.NewArray(len(nodes)) | ||||
|  | ||||
| 	for _, id := range nodes { | ||||
| 		child, err := LoadElement(client, broker, id) | ||||
| 		child, err := LoadElement(logger, client, broker, id) | ||||
|  | ||||
| 		if err != nil { | ||||
| 			return nil, err | ||||
|   | ||||
| @@ -3,8 +3,8 @@ package utils | ||||
| import ( | ||||
| 	"context" | ||||
| 	"github.com/MontFerret/ferret/pkg/runtime/core" | ||||
| 	"github.com/MontFerret/ferret/pkg/runtime/logging" | ||||
| 	"github.com/MontFerret/ferret/pkg/runtime/values" | ||||
| 	"log" | ||||
| 	"time" | ||||
| ) | ||||
|  | ||||
| @@ -30,16 +30,16 @@ func Wait(_ context.Context, inputs ...core.Value) (core.Value, error) { | ||||
| 	return values.None, nil | ||||
| } | ||||
|  | ||||
| func Log(_ context.Context, inputs ...core.Value) (core.Value, error) { | ||||
| func Log(ctx context.Context, inputs ...core.Value) (core.Value, error) { | ||||
| 	args := make([]interface{}, 0, len(inputs)+1) | ||||
|  | ||||
| 	args = append(args, "LOG:") | ||||
|  | ||||
| 	for _, input := range inputs { | ||||
| 		args = append(args, input) | ||||
| 	} | ||||
|  | ||||
| 	log.Println(args...) | ||||
| 	logger := logging.From(ctx) | ||||
|  | ||||
| 	logger.Print(args...) | ||||
|  | ||||
| 	return values.None, nil | ||||
| } | ||||
|   | ||||
		Reference in New Issue
	
	Block a user