Decoupled runtime and HTML driver initialization (#198)

* Decoupled runtime and HTML driver initialization

* Updates

README.md

@@ -219,14 +219,16 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/MontFerret/ferret/pkg/compiler"
 	"os"
+	
+    "github.com/MontFerret/ferret/pkg/compiler"
+    "github.com/MontFerret/ferret/pkg/html"
 )
 
 type Topic struct {
 	Name        string `json:"name"`
 	Description string `json:"description"`
-	Url         string `json:"url"`
+	URL         string `json:"url"`
 }
 
 func main() {
@@ -238,7 +240,7 @@ func main() {
 	}
 
 	for _, topic := range topics {
-		fmt.Println(fmt.Sprintf("%s: %s %s", topic.Name, topic.Description, topic.Url))
+		fmt.Println(fmt.Sprintf("%s: %s %s", topic.Name, topic.Description, topic.URL))
 	}
 }
 
@@ -267,7 +269,16 @@ func getTopTenTrendingTopics() ([]*Topic, error) {
 		return nil, err
 	}
 
-	out, err := program.Run(context.Background())
+	// create a root context
+	ctx := context.Background()
+
+	// enable HTML drivers
+	// by default, Ferret Runtime knows nothing about HTML drivers
+	// all HTML manipulations are done via functions from standard library
+	ctx = html.WithDynamicDriver(ctx)
+	ctx = html.WithStaticDriver(ctx)
+
+	out, err := program.Run(ctx)
 
 	if err != nil {
 		return nil, err
@@ -283,6 +294,7 @@ func getTopTenTrendingTopics() ([]*Topic, error) {
 
 	return res, nil
 }
+
 ```
 
 ## Extensibility
@@ -296,10 +308,12 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
+	"strings"
+
 	"github.com/MontFerret/ferret/pkg/compiler"
 	"github.com/MontFerret/ferret/pkg/runtime/core"
 	"github.com/MontFerret/ferret/pkg/runtime/values"
-	"os"
 )
 
 func main() {
@@ -319,7 +333,7 @@ func getStrings() ([]string, error) {
 	// function implements is a type of a function that ferret supports as a runtime function
 	transform := func(ctx context.Context, args ...core.Value) (core.Value, error) {
 		// it's just a helper function which helps to validate a number of passed args
-		err := core.ValidateArgs(args, 1)
+		err := core.ValidateArgs(args, 1, 1)
 
 		if err != nil {
 			// it's recommended to return built-in None type, instead of nil
@@ -336,7 +350,7 @@ func getStrings() ([]string, error) {
 		// cast to built-in string type
 		str := args[0].(values.String)
 
-		return str.Concat(values.NewString("_ferret")).ToUpper(), nil
+		return values.NewString(strings.ToUpper(str.String() + "_ferret")), nil
 	}
 
 	query := `
@@ -346,7 +360,10 @@ func getStrings() ([]string, error) {
 	`
 
 	comp := compiler.New()
-	comp.RegisterFunction("transform", transform)
+
+	if err := comp.RegisterFunction("transform", transform); err != nil {
+		return nil, err
+	}
 
 	program, err := comp.Compile(query)
 

cli/exec.go

@@ -38,7 +38,7 @@ func Exec(query string, opts Options) {
 
 	l := NewLogger()
 
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(opts.WithContext(context.Background()))
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, syscall.SIGHUP)
 
@@ -59,12 +59,9 @@ func Exec(query string, opts Options) {
 
 	out, err := prog.Run(
 		ctx,
-		runtime.WithBrowser(opts.Cdp),
 		runtime.WithLog(l),
 		runtime.WithLogLevel(logging.DebugLevel),
 		runtime.WithParams(opts.Params),
-		runtime.WithProxy(opts.Proxy),
-		runtime.WithUserAgent(opts.UserAgent),
 	)
 
 	if opts.ShowTime {

cli/options.go

@@ -1,5 +1,12 @@
 package cli
 
+import (
+	"context"
+	"github.com/MontFerret/ferret/pkg/html"
+	"github.com/MontFerret/ferret/pkg/html/dynamic"
+	"github.com/MontFerret/ferret/pkg/html/static"
+)
+
 type Options struct {
 	Cdp       string
 	Params    map[string]interface{}
@@ -7,3 +14,20 @@ type Options struct {
 	UserAgent string
 	ShowTime  bool
 }
+
+func (opts Options) WithContext(ctx context.Context) context.Context {
+	ctx = html.WithDynamicDriver(
+		ctx,
+		dynamic.WithCDP(opts.Cdp),
+		dynamic.WithProxy(opts.Proxy),
+		dynamic.WithUserAgent(opts.UserAgent),
+	)
+
+	ctx = html.WithStaticDriver(
+		ctx,
+		static.WithProxy(opts.Proxy),
+		static.WithUserAgent(opts.UserAgent),
+	)
+
+	return ctx
+}

cli/repl.go

@@ -42,7 +42,7 @@ func Repl(version string, opts Options) {
 
 	l := NewLogger()
 
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(opts.WithContext(context.Background()))
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, syscall.SIGHUP)
 
@@ -110,12 +110,9 @@ func Repl(version string, opts Options) {
 
 		out, err := program.Run(
 			ctx,
-			runtime.WithBrowser(opts.Cdp),
 			runtime.WithLog(l),
 			runtime.WithLogLevel(logging.DebugLevel),
 			runtime.WithParams(opts.Params),
-			runtime.WithProxy(opts.Proxy),
-			runtime.WithUserAgent(opts.UserAgent),
 		)
 
 		if err != nil {

e2e/runner/runner.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"encoding/json"
 	"github.com/MontFerret/ferret/pkg/compiler"
+	"github.com/MontFerret/ferret/pkg/html"
+	"github.com/MontFerret/ferret/pkg/html/dynamic"
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -131,10 +133,16 @@ func (r *Runner) runQuery(c *compiler.FqlCompiler, name, script string) Result {
 		}
 	}
 
+	ctx := context.Background()
+	ctx = html.WithDynamicDriver(
+		ctx,
+		dynamic.WithCDP(r.settings.CDPAddress),
+	)
+	ctx = html.WithStaticDriver(ctx)
+
 	out, err := p.Run(
-		context.Background(),
+		ctx,
 		runtime.WithLog(os.Stdout),
-		runtime.WithBrowser(r.settings.CDPAddress),
 		runtime.WithParam("static", r.settings.StaticServerAddress),
 		runtime.WithParam("dynamic", r.settings.DynamicServerAddress),
 	)

examples/embedded.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/MontFerret/ferret/pkg/compiler"
 	"os"
+
+	"github.com/MontFerret/ferret/pkg/compiler"
+	"github.com/MontFerret/ferret/pkg/html"
 )
 
 type Topic struct {
@@ -52,7 +54,16 @@ func getTopTenTrendingTopics() ([]*Topic, error) {
 		return nil, err
 	}
 
-	out, err := program.Run(context.Background())
+	// create a root context
+	ctx := context.Background()
+
+	// enable HTML drivers
+	// by default, Ferret Runtime knows nothing about HTML drivers
+	// all HTML manipulations are done via functions from standard library
+	ctx = html.WithDynamicDriver(ctx)
+	ctx = html.WithStaticDriver(ctx)
+
+	out, err := program.Run(ctx)
 
 	if err != nil {
 		return nil, err

examples/extensible.go

@@ -4,10 +4,12 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
+	"strings"
+
 	"github.com/MontFerret/ferret/pkg/compiler"
 	"github.com/MontFerret/ferret/pkg/runtime/core"
 	"github.com/MontFerret/ferret/pkg/runtime/values"
-	"os"
 )
 
 func main() {
@@ -27,7 +29,7 @@ func getStrings() ([]string, error) {
 	// function implements is a type of a function that ferret supports as a runtime function
 	transform := func(ctx context.Context, args ...core.Value) (core.Value, error) {
 		// it's just a helper function which helps to validate a number of passed args
-		err := core.ValidateArgs(args, 1)
+		err := core.ValidateArgs(args, 1, 1)
 
 		if err != nil {
 			// it's recommended to return built-in None type, instead of nil
@@ -44,7 +46,7 @@ func getStrings() ([]string, error) {
 		// cast to built-in string type
 		str := args[0].(values.String)
 
-		return str.Concat(values.NewString("_ferret")).ToUpper(), nil
+		return values.NewString(strings.ToUpper(str.String() + "_ferret")), nil
 	}
 
 	query := `
@@ -54,7 +56,10 @@ func getStrings() ([]string, error) {
 	`
 
 	comp := compiler.New()
-	comp.RegisterFunction("transform", transform)
+
+	if err := comp.RegisterFunction("transform", transform); err != nil {
+		return nil, err
+	}
 
 	program, err := comp.Compile(query)
 

pkg/html/common/ua.go

@@ -1,16 +1,17 @@
 package common
 
 import (
-	"github.com/MontFerret/ferret/pkg/runtime/env"
 	"github.com/corpix/uarand"
 )
 
+const RandomUserAgent = "*"
+
 func GetUserAgent(val string) string {
 	if val == "" {
 		return val
 	}
 
-	if val != env.RandomUserAgent {
+	if val != RandomUserAgent {
 		return val
 	}
 

pkg/html/driver.go

@@ -6,15 +6,10 @@ import (
 	"github.com/MontFerret/ferret/pkg/html/dynamic"
 	"github.com/MontFerret/ferret/pkg/html/static"
 	"github.com/MontFerret/ferret/pkg/runtime/core"
-	"github.com/MontFerret/ferret/pkg/runtime/env"
 	"github.com/MontFerret/ferret/pkg/runtime/values"
 )
 
-type (
+type DriverName string
-	DriverName    string
-	dynamicCtxKey struct{}
-	staticCtxKey  struct{}
-)
 
 const (
 	Dynamic DriverName = "dynamic"
@@ -26,67 +21,21 @@ type Driver interface {
 	Close() error
 }
 
-func ToContext(ctx context.Context, name DriverName, drv Driver) context.Context {
-	var key interface{}
-
-	switch name {
-	case Dynamic:
-		key = dynamicCtxKey{}
-	case Static:
-		key = staticCtxKey{}
-	default:
-		return ctx
-	}
-
-	return context.WithValue(ctx, key, drv)
-}
-
 func FromContext(ctx context.Context, name DriverName) (Driver, error) {
-	var key interface{}
-
 	switch name {
 	case Dynamic:
-		key = dynamicCtxKey{}
+		return dynamic.FromContext(ctx)
 	case Static:
-		key = staticCtxKey{}
+		return static.FromContext(ctx)
 	default:
 		return nil, core.Error(core.ErrInvalidArgument, fmt.Sprintf("%s driver", name))
 	}
-
-	val := ctx.Value(key)
-
-	drv, ok := val.(Driver)
-
-	if ok {
-		return drv, nil
-	}
-
-	return nil, core.Error(core.ErrNotFound, fmt.Sprintf("%s driver", name))
 }
 
-func WithDynamicDriver(ctx context.Context) context.Context {
+func WithDynamicDriver(ctx context.Context, opts ...dynamic.Option) context.Context {
-	e := env.FromContext(ctx)
+	return dynamic.WithContext(ctx, dynamic.NewDriver(opts...))
-
-	return context.WithValue(
-		ctx,
-		dynamicCtxKey{},
-		dynamic.NewDriver(
-			e.CDPAddress,
-			dynamic.WithProxy(e.ProxyAddress),
-			dynamic.WithUserAgent(e.UserAgent),
-		),
-	)
 }
 
-func WithStaticDriver(ctx context.Context) context.Context {
+func WithStaticDriver(ctx context.Context, opts ...static.Option) context.Context {
-	e := env.FromContext(ctx)
+	return static.WithContext(ctx, static.NewDriver(opts...))
-
-	return context.WithValue(
-		ctx,
-		staticCtxKey{},
-		static.NewDriver(
-			static.WithProxy(e.ProxyAddress),
-			static.WithUserAgent(e.UserAgent),
-		),
-	)
 }

pkg/html/dynamic/driver.go

@@ -2,6 +2,9 @@ package dynamic
 
 import (
 	"context"
+	"github.com/MontFerret/ferret/pkg/runtime/core"
+	"sync"
+
 	"github.com/MontFerret/ferret/pkg/html/common"
 	"github.com/MontFerret/ferret/pkg/runtime/logging"
 	"github.com/MontFerret/ferret/pkg/runtime/values"
@@ -13,28 +16,47 @@ import (
 	"github.com/mafredri/cdp/rpcc"
 	"github.com/mafredri/cdp/session"
 	"github.com/pkg/errors"
-	"sync"
 )
 
-type Driver struct {
+type (
-	sync.Mutex
+	ctxKey struct{}
-	dev       *devtool.DevTools
+
-	conn      *rpcc.Conn
+	Driver struct {
-	client    *cdp.Client
+		sync.Mutex
-	session   *session.Manager
+		dev       *devtool.DevTools
-	contextID target.BrowserContextID
+		conn      *rpcc.Conn
-	options   *Options
+		client    *cdp.Client
+		session   *session.Manager
+		contextID target.BrowserContextID
+		options   *Options
+	}
+)
+
+func WithContext(ctx context.Context, drv *Driver) context.Context {
+	return context.WithValue(
+		ctx,
+		ctxKey{},
+		drv,
+	)
 }
 
-func NewDriver(address string, opts ...Option) *Driver {
+func FromContext(ctx context.Context) (*Driver, error) {
-	drv := new(Driver)
+	val := ctx.Value(ctxKey{})
-	drv.dev = devtool.New(address)
-	drv.options = new(Options)
 
-	for _, opt := range opts {
+	drv, ok := val.(*Driver)
-		opt(drv.options)
+
+	if !ok {
+		return nil, core.Error(core.ErrNotFound, "dynamic HTML Driver")
 	}
 
+	return drv, nil
+}
+
+func NewDriver(opts ...Option) *Driver {
+	drv := new(Driver)
+	drv.options = newOptions(opts)
+	drv.dev = devtool.New(drv.options.cdp)
+
 	return drv
 }
 

pkg/html/dynamic/options.go

@@ -4,11 +4,29 @@ type (
 	Options struct {
 		proxy     string
 		userAgent string
+		cdp       string
 	}
 
 	Option func(opts *Options)
 )
 
+func newOptions(setters []Option) *Options {
+	opts := new(Options)
+	opts.cdp = "http://127.0.0.1:9222"
+
+	for _, setter := range setters {
+		setter(opts)
+	}
+
+	return opts
+}
+
+func WithCDP(address string) Option {
+	return func(opts *Options) {
+		opts.cdp = address
+	}
+}
+
 func WithProxy(address string) Option {
 	return func(opts *Options) {
 		opts.proxy = address

pkg/html/static/driver.go

@@ -3,6 +3,7 @@ package static
 import (
 	"bytes"
 	"context"
+	"github.com/MontFerret/ferret/pkg/runtime/core"
 	"net/http"
 	"net/url"
 
@@ -14,22 +15,38 @@ import (
 	"github.com/sethgrid/pester"
 )
 
-type Driver struct {
+type (
-	client  *pester.Client
+	ctxKey struct{}
-	options *Options
+
+	Driver struct {
+		client  *pester.Client
+		options *Options
+	}
+)
+
+func WithContext(ctx context.Context, drv *Driver) context.Context {
+	return context.WithValue(
+		ctx,
+		ctxKey{},
+		drv,
+	)
+}
+
+func FromContext(ctx context.Context) (*Driver, error) {
+	val := ctx.Value(ctxKey{})
+
+	drv, ok := val.(*Driver)
+
+	if !ok {
+		return nil, core.Error(core.ErrNotFound, "static HTML Driver")
+	}
+
+	return drv, nil
 }
 
 func NewDriver(opts ...Option) *Driver {
 	drv := new(Driver)
-	drv.options = &Options{
+	drv.options = newOptions(opts)
-		concurrency: 3,
-		maxRetries:  5,
-		backoff:     pester.ExponentialBackoff,
-	}
-
-	for _, opt := range opts {
-		opt(drv.options)
-	}
 
 	if drv.options.proxy == "" {
 		drv.client = pester.New()

pkg/html/static/options.go

@@ -15,6 +15,19 @@ type (
 	}
 )
 
+func newOptions(setters []Option) *Options {
+	opts := new(Options)
+	opts.backoff = pester.ExponentialBackoff
+	opts.concurrency = 3
+	opts.maxRetries = 5
+
+	for _, setter := range setters {
+		setter(opts)
+	}
+
+	return opts
+}
+
 func WithDefaultBackoff() Option {
 	return func(opts *Options) {
 		opts.backoff = pester.DefaultBackoff

pkg/runtime/env/env.go

@@ -1,31 +0,0 @@
-package env
-
-import "context"
-
-type (
-	ctxKey struct{}
-
-	Environment struct {
-		CDPAddress   string
-		ProxyAddress string
-		UserAgent    string
-	}
-)
-
-const RandomUserAgent = "*"
-
-func WithContext(ctx context.Context, e Environment) context.Context {
-	return context.WithValue(ctx, ctxKey{}, e)
-}
-
-func FromContext(ctx context.Context) Environment {
-	res := ctx.Value(ctxKey{})
-
-	val, ok := res.(Environment)
-
-	if !ok {
-		return Environment{}
-	}
-
-	return val
-}

pkg/runtime/options.go

@@ -2,35 +2,37 @@ package runtime
 
 import (
 	"context"
-	"github.com/MontFerret/ferret/pkg/runtime/core"
-	"github.com/MontFerret/ferret/pkg/runtime/env"
-	"github.com/MontFerret/ferret/pkg/runtime/logging"
-	"github.com/MontFerret/ferret/pkg/runtime/values"
 	"io"
 	"os"
+
+	"github.com/MontFerret/ferret/pkg/runtime/core"
+	"github.com/MontFerret/ferret/pkg/runtime/logging"
+	"github.com/MontFerret/ferret/pkg/runtime/values"
 )
 
 type (
 	Options struct {
-		proxy     string
+		params  map[string]core.Value
-		cdp       string
+		logging *logging.Options
-		params    map[string]core.Value
-		logging   *logging.Options
-		userAgent string
 	}
 
 	Option func(*Options)
 )
 
-func NewOptions() *Options {
+func NewOptions(setters []Option) *Options {
-	return &Options{
+	opts := &Options{
-		cdp:    "http://127.0.0.1:9222",
 		params: make(map[string]core.Value),
 		logging: &logging.Options{
 			Writer: os.Stdout,
 			Level:  logging.ErrorLevel,
 		},
 	}
+
+	for _, setter := range setters {
+		setter(opts)
+	}
+
+	return opts
 }
 
 func WithParam(name string, value interface{}) Option {
@@ -47,30 +49,6 @@ func WithParams(params map[string]interface{}) Option {
 	}
 }
 
-func WithBrowser(address string) Option {
-	return func(options *Options) {
-		options.cdp = address
-	}
-}
-
-func WithProxy(address string) Option {
-	return func(options *Options) {
-		options.proxy = address
-	}
-}
-
-func WithUserAgent(value string) Option {
-	return func(options *Options) {
-		options.userAgent = value
-	}
-}
-
-func WithRandomUserAgent() Option {
-	return func(options *Options) {
-		options.userAgent = env.RandomUserAgent
-	}
-}
-
 func WithLog(writer io.Writer) Option {
 	return func(options *Options) {
 		options.logging.Writer = writer
@@ -83,22 +61,9 @@ func WithLogLevel(lvl logging.Level) Option {
 	}
 }
 
-func (opts *Options) Apply(setters ...Option) *Options {
-	for _, setter := range setters {
-		setter(opts)
-	}
-
-	return opts
-}
-
 func (opts *Options) WithContext(parent context.Context) context.Context {
 	ctx := core.ParamsWith(parent, opts.params)
 	ctx = logging.WithContext(ctx, opts.logging)
-	ctx = env.WithContext(ctx, env.Environment{
-		CDPAddress:   opts.cdp,
-		ProxyAddress: opts.proxy,
-		UserAgent:    opts.userAgent,
-	})
 
 	return ctx
 }

pkg/runtime/program.go

@@ -2,7 +2,6 @@ package runtime
 
 import (
 	"context"
-	"github.com/MontFerret/ferret/pkg/html"
 	"github.com/MontFerret/ferret/pkg/runtime/core"
 	"github.com/MontFerret/ferret/pkg/runtime/logging"
 	"github.com/MontFerret/ferret/pkg/runtime/values"
@@ -32,9 +31,7 @@ func (p *Program) Source() string {
 }
 
 func (p *Program) Run(ctx context.Context, setters ...Option) (result []byte, err error) {
-	ctx = NewOptions().Apply(setters...).WithContext(ctx)
+	ctx = NewOptions(setters).WithContext(ctx)
-	ctx = html.WithDynamicDriver(ctx)
-	ctx = html.WithStaticDriver(ctx)
 
 	logger := logging.FromContext(ctx)