Decoupled runtime and HTML driver initialization (#198)

* Decoupled runtime and HTML driver initialization

* Updates

pkg/html/common/ua.go

@@ -1,16 +1,17 @@
 package common
 
 import (
-	"github.com/MontFerret/ferret/pkg/runtime/env"
 	"github.com/corpix/uarand"
 )
 
+const RandomUserAgent = "*"
+
 func GetUserAgent(val string) string {
 	if val == "" {
 		return val
 	}
 
-	if val != env.RandomUserAgent {
+	if val != RandomUserAgent {
 		return val
 	}
 

pkg/html/driver.go

@@ -6,15 +6,10 @@ import (
 	"github.com/MontFerret/ferret/pkg/html/dynamic"
 	"github.com/MontFerret/ferret/pkg/html/static"
 	"github.com/MontFerret/ferret/pkg/runtime/core"
-	"github.com/MontFerret/ferret/pkg/runtime/env"
 	"github.com/MontFerret/ferret/pkg/runtime/values"
 )
 
-type (
-	DriverName    string
-	dynamicCtxKey struct{}
-	staticCtxKey  struct{}
-)
+type DriverName string
 
 const (
 	Dynamic DriverName = "dynamic"
@@ -26,67 +21,21 @@ type Driver interface {
 	Close() error
 }
 
-func ToContext(ctx context.Context, name DriverName, drv Driver) context.Context {
-	var key interface{}
-
-	switch name {
-	case Dynamic:
-		key = dynamicCtxKey{}
-	case Static:
-		key = staticCtxKey{}
-	default:
-		return ctx
-	}
-
-	return context.WithValue(ctx, key, drv)
-}
-
 func FromContext(ctx context.Context, name DriverName) (Driver, error) {
-	var key interface{}
-
 	switch name {
 	case Dynamic:
-		key = dynamicCtxKey{}
+		return dynamic.FromContext(ctx)
 	case Static:
-		key = staticCtxKey{}
+		return static.FromContext(ctx)
 	default:
 		return nil, core.Error(core.ErrInvalidArgument, fmt.Sprintf("%s driver", name))
 	}
-
-	val := ctx.Value(key)
-
-	drv, ok := val.(Driver)
-
-	if ok {
-		return drv, nil
-	}
-
-	return nil, core.Error(core.ErrNotFound, fmt.Sprintf("%s driver", name))
 }
 
-func WithDynamicDriver(ctx context.Context) context.Context {
-	e := env.FromContext(ctx)
-
-	return context.WithValue(
-		ctx,
-		dynamicCtxKey{},
-		dynamic.NewDriver(
-			e.CDPAddress,
-			dynamic.WithProxy(e.ProxyAddress),
-			dynamic.WithUserAgent(e.UserAgent),
-		),
-	)
+func WithDynamicDriver(ctx context.Context, opts ...dynamic.Option) context.Context {
+	return dynamic.WithContext(ctx, dynamic.NewDriver(opts...))
 }
 
-func WithStaticDriver(ctx context.Context) context.Context {
-	e := env.FromContext(ctx)
-
-	return context.WithValue(
-		ctx,
-		staticCtxKey{},
-		static.NewDriver(
-			static.WithProxy(e.ProxyAddress),
-			static.WithUserAgent(e.UserAgent),
-		),
-	)
+func WithStaticDriver(ctx context.Context, opts ...static.Option) context.Context {
+	return static.WithContext(ctx, static.NewDriver(opts...))
 }

pkg/html/dynamic/driver.go

@@ -2,6 +2,9 @@ package dynamic
 
 import (
 	"context"
+	"github.com/MontFerret/ferret/pkg/runtime/core"
+	"sync"
+
 	"github.com/MontFerret/ferret/pkg/html/common"
 	"github.com/MontFerret/ferret/pkg/runtime/logging"
 	"github.com/MontFerret/ferret/pkg/runtime/values"
@@ -13,28 +16,47 @@ import (
 	"github.com/mafredri/cdp/rpcc"
 	"github.com/mafredri/cdp/session"
 	"github.com/pkg/errors"
-	"sync"
 )
 
-type Driver struct {
-	sync.Mutex
-	dev       *devtool.DevTools
-	conn      *rpcc.Conn
-	client    *cdp.Client
-	session   *session.Manager
-	contextID target.BrowserContextID
-	options   *Options
+type (
+	ctxKey struct{}
+
+	Driver struct {
+		sync.Mutex
+		dev       *devtool.DevTools
+		conn      *rpcc.Conn
+		client    *cdp.Client
+		session   *session.Manager
+		contextID target.BrowserContextID
+		options   *Options
+	}
+)
+
+func WithContext(ctx context.Context, drv *Driver) context.Context {
+	return context.WithValue(
+		ctx,
+		ctxKey{},
+		drv,
+	)
 }
 
-func NewDriver(address string, opts ...Option) *Driver {
-	drv := new(Driver)
-	drv.dev = devtool.New(address)
-	drv.options = new(Options)
+func FromContext(ctx context.Context) (*Driver, error) {
+	val := ctx.Value(ctxKey{})
 
-	for _, opt := range opts {
-		opt(drv.options)
+	drv, ok := val.(*Driver)
+
+	if !ok {
+		return nil, core.Error(core.ErrNotFound, "dynamic HTML Driver")
 	}
 
+	return drv, nil
+}
+
+func NewDriver(opts ...Option) *Driver {
+	drv := new(Driver)
+	drv.options = newOptions(opts)
+	drv.dev = devtool.New(drv.options.cdp)
+
 	return drv
 }
 

pkg/html/dynamic/options.go

@@ -4,11 +4,29 @@ type (
 	Options struct {
 		proxy     string
 		userAgent string
+		cdp       string
 	}
 
 	Option func(opts *Options)
 )
 
+func newOptions(setters []Option) *Options {
+	opts := new(Options)
+	opts.cdp = "http://127.0.0.1:9222"
+
+	for _, setter := range setters {
+		setter(opts)
+	}
+
+	return opts
+}
+
+func WithCDP(address string) Option {
+	return func(opts *Options) {
+		opts.cdp = address
+	}
+}
+
 func WithProxy(address string) Option {
 	return func(opts *Options) {
 		opts.proxy = address

pkg/html/static/driver.go

@@ -3,6 +3,7 @@ package static
 import (
 	"bytes"
 	"context"
+	"github.com/MontFerret/ferret/pkg/runtime/core"
 	"net/http"
 	"net/url"
 
@@ -14,22 +15,38 @@ import (
 	"github.com/sethgrid/pester"
 )
 
-type Driver struct {
-	client  *pester.Client
-	options *Options
+type (
+	ctxKey struct{}
+
+	Driver struct {
+		client  *pester.Client
+		options *Options
+	}
+)
+
+func WithContext(ctx context.Context, drv *Driver) context.Context {
+	return context.WithValue(
+		ctx,
+		ctxKey{},
+		drv,
+	)
+}
+
+func FromContext(ctx context.Context) (*Driver, error) {
+	val := ctx.Value(ctxKey{})
+
+	drv, ok := val.(*Driver)
+
+	if !ok {
+		return nil, core.Error(core.ErrNotFound, "static HTML Driver")
+	}
+
+	return drv, nil
 }
 
 func NewDriver(opts ...Option) *Driver {
 	drv := new(Driver)
-	drv.options = &Options{
-		concurrency: 3,
-		maxRetries:  5,
-		backoff:     pester.ExponentialBackoff,
-	}
-
-	for _, opt := range opts {
-		opt(drv.options)
-	}
+	drv.options = newOptions(opts)
 
 	if drv.options.proxy == "" {
 		drv.client = pester.New()

pkg/html/static/options.go

@@ -15,6 +15,19 @@ type (
 	}
 )
 
+func newOptions(setters []Option) *Options {
+	opts := new(Options)
+	opts.backoff = pester.ExponentialBackoff
+	opts.concurrency = 3
+	opts.maxRetries = 5
+
+	for _, setter := range setters {
+		setter(opts)
+	}
+
+	return opts
+}
+
 func WithDefaultBackoff() Option {
 	return func(opts *Options) {
 		opts.backoff = pester.DefaultBackoff

pkg/runtime/env/env.go

@@ -1,31 +0,0 @@
-package env
-
-import "context"
-
-type (
-	ctxKey struct{}
-
-	Environment struct {
-		CDPAddress   string
-		ProxyAddress string
-		UserAgent    string
-	}
-)
-
-const RandomUserAgent = "*"
-
-func WithContext(ctx context.Context, e Environment) context.Context {
-	return context.WithValue(ctx, ctxKey{}, e)
-}
-
-func FromContext(ctx context.Context) Environment {
-	res := ctx.Value(ctxKey{})
-
-	val, ok := res.(Environment)
-
-	if !ok {
-		return Environment{}
-	}
-
-	return val
-}

pkg/runtime/options.go

@@ -2,35 +2,37 @@ package runtime
 
 import (
 	"context"
-	"github.com/MontFerret/ferret/pkg/runtime/core"
-	"github.com/MontFerret/ferret/pkg/runtime/env"
-	"github.com/MontFerret/ferret/pkg/runtime/logging"
-	"github.com/MontFerret/ferret/pkg/runtime/values"
 	"io"
 	"os"
+
+	"github.com/MontFerret/ferret/pkg/runtime/core"
+	"github.com/MontFerret/ferret/pkg/runtime/logging"
+	"github.com/MontFerret/ferret/pkg/runtime/values"
 )
 
 type (
 	Options struct {
-		proxy     string
-		cdp       string
-		params    map[string]core.Value
-		logging   *logging.Options
-		userAgent string
+		params  map[string]core.Value
+		logging *logging.Options
 	}
 
 	Option func(*Options)
 )
 
-func NewOptions() *Options {
-	return &Options{
-		cdp:    "http://127.0.0.1:9222",
+func NewOptions(setters []Option) *Options {
+	opts := &Options{
 		params: make(map[string]core.Value),
 		logging: &logging.Options{
 			Writer: os.Stdout,
 			Level:  logging.ErrorLevel,
 		},
 	}
+
+	for _, setter := range setters {
+		setter(opts)
+	}
+
+	return opts
 }
 
 func WithParam(name string, value interface{}) Option {
@@ -47,30 +49,6 @@ func WithParams(params map[string]interface{}) Option {
 	}
 }
 
-func WithBrowser(address string) Option {
-	return func(options *Options) {
-		options.cdp = address
-	}
-}
-
-func WithProxy(address string) Option {
-	return func(options *Options) {
-		options.proxy = address
-	}
-}
-
-func WithUserAgent(value string) Option {
-	return func(options *Options) {
-		options.userAgent = value
-	}
-}
-
-func WithRandomUserAgent() Option {
-	return func(options *Options) {
-		options.userAgent = env.RandomUserAgent
-	}
-}
-
 func WithLog(writer io.Writer) Option {
 	return func(options *Options) {
 		options.logging.Writer = writer
@@ -83,22 +61,9 @@ func WithLogLevel(lvl logging.Level) Option {
 	}
 }
 
-func (opts *Options) Apply(setters ...Option) *Options {
-	for _, setter := range setters {
-		setter(opts)
-	}
-
-	return opts
-}
-
 func (opts *Options) WithContext(parent context.Context) context.Context {
 	ctx := core.ParamsWith(parent, opts.params)
 	ctx = logging.WithContext(ctx, opts.logging)
-	ctx = env.WithContext(ctx, env.Environment{
-		CDPAddress:   opts.cdp,
-		ProxyAddress: opts.proxy,
-		UserAgent:    opts.userAgent,
-	})
 
 	return ctx
 }

pkg/runtime/program.go

@@ -2,7 +2,6 @@ package runtime
 
 import (
 	"context"
-	"github.com/MontFerret/ferret/pkg/html"
 	"github.com/MontFerret/ferret/pkg/runtime/core"
 	"github.com/MontFerret/ferret/pkg/runtime/logging"
 	"github.com/MontFerret/ferret/pkg/runtime/values"
@@ -32,9 +31,7 @@ func (p *Program) Source() string {
 }
 
 func (p *Program) Run(ctx context.Context, setters ...Option) (result []byte, err error) {
-	ctx = NewOptions().Apply(setters...).WithContext(ctx)
-	ctx = html.WithDynamicDriver(ctx)
-	ctx = html.WithStaticDriver(ctx)
+	ctx = NewOptions(setters).WithContext(ctx)
 
 	logger := logging.FromContext(ctx)