go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2025-07-17 01:32:22 +02:00
Code Issues Releases Activity

Decoupled runtime and HTML driver initialization (#198)

Browse Source 
* Decoupled runtime and HTML driver initialization

* Updates
This commit is contained in:
Tim Voronov
2018-11-30 19:30:55 -05:00
committed by GitHub
parent 0ce0426b55
commit 39e379f0f2
16 changed files with 205 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
README.md
View File

@ -219,14 +219,16 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/MontFerret/ferret/pkg/compiler"
"os"
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/html"
)
type Topic struct {
Name string `json:"name"`
Description string `json:"description"`
Url string `json:"url"`
URL string `json:"url"`
}
func main() {
@ -238,7 +240,7 @@ func main() {
}
for _, topic := range topics {
fmt.Println(fmt.Sprintf("%s: %s %s", topic.Name, topic.Description, topic.Url))
fmt.Println(fmt.Sprintf("%s: %s %s", topic.Name, topic.Description, topic.URL))
}
}
@ -267,7 +269,16 @@ func getTopTenTrendingTopics() ([]*Topic, error) {
return nil, err
}
out, err := program.Run(context.Background())
// create a root context
ctx := context.Background()
// enable HTML drivers
// by default, Ferret Runtime knows nothing about HTML drivers
// all HTML manipulations are done via functions from standard library
ctx = html.WithDynamicDriver(ctx)
ctx = html.WithStaticDriver(ctx)
out, err := program.Run(ctx)
if err != nil {
return nil, err
@ -283,6 +294,7 @@ func getTopTenTrendingTopics() ([]*Topic, error) {
return res, nil
}
```
## Extensibility
@ -296,10 +308,12 @@ import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"os"
)
func main() {
@ -319,7 +333,7 @@ func getStrings() ([]string, error) {
// function implements is a type of a function that ferret supports as a runtime function
transform := func(ctx context.Context, args ...core.Value) (core.Value, error) {
// it's just a helper function which helps to validate a number of passed args
err := core.ValidateArgs(args, 1)
err := core.ValidateArgs(args, 1, 1)
if err != nil {
// it's recommended to return built-in None type, instead of nil
@ -336,7 +350,7 @@ func getStrings() ([]string, error) {
// cast to built-in string type
str := args[0].(values.String)
return str.Concat(values.NewString("_ferret")).ToUpper(), nil
return values.NewString(strings.ToUpper(str.String() + "_ferret")), nil
}
query := `
@ -346,7 +360,10 @@ func getStrings() ([]string, error) {
`
comp := compiler.New()
comp.RegisterFunction("transform", transform)
if err := comp.RegisterFunction("transform", transform); err != nil {
return nil, err
}
program, err := comp.Compile(query)

5
cli/exec.go
View File

@ -38,7 +38,7 @@ func Exec(query string, opts Options) {
l := NewLogger()
ctx, cancel := context.WithCancel(context.Background())
ctx, cancel := context.WithCancel(opts.WithContext(context.Background()))
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP)
@ -59,12 +59,9 @@ func Exec(query string, opts Options) {
out, err := prog.Run(
ctx,
runtime.WithBrowser(opts.Cdp),
runtime.WithLog(l),
runtime.WithLogLevel(logging.DebugLevel),
runtime.WithParams(opts.Params),
runtime.WithProxy(opts.Proxy),
runtime.WithUserAgent(opts.UserAgent),
)
if opts.ShowTime {

24
cli/options.go
View File

@ -1,5 +1,12 @@
package cli
import (
"context"
"github.com/MontFerret/ferret/pkg/html"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/html/static"
)
type Options struct {
Cdp string
Params map[string]interface{}
@ -7,3 +14,20 @@ type Options struct {
UserAgent string
ShowTime bool
}
func (opts Options) WithContext(ctx context.Context) context.Context {
ctx = html.WithDynamicDriver(
ctx,
dynamic.WithCDP(opts.Cdp),
dynamic.WithProxy(opts.Proxy),
dynamic.WithUserAgent(opts.UserAgent),
)
ctx = html.WithStaticDriver(
ctx,
static.WithProxy(opts.Proxy),
static.WithUserAgent(opts.UserAgent),
)
return ctx
}

5
cli/repl.go
View File

@ -42,7 +42,7 @@ func Repl(version string, opts Options) {
l := NewLogger()
ctx, cancel := context.WithCancel(context.Background())
ctx, cancel := context.WithCancel(opts.WithContext(context.Background()))
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP)
@ -110,12 +110,9 @@ func Repl(version string, opts Options) {
out, err := program.Run(
ctx,
runtime.WithBrowser(opts.Cdp),
runtime.WithLog(l),
runtime.WithLogLevel(logging.DebugLevel),
runtime.WithParams(opts.Params),
runtime.WithProxy(opts.Proxy),
runtime.WithUserAgent(opts.UserAgent),
)
if err != nil {

12
e2e/runner/runner.go
View File

@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/html"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime"
"github.com/pkg/errors"
"github.com/rs/zerolog"
@ -131,10 +133,16 @@ func (r *Runner) runQuery(c *compiler.FqlCompiler, name, script string) Result {
}
}
ctx := context.Background()
ctx = html.WithDynamicDriver(
ctx,
dynamic.WithCDP(r.settings.CDPAddress),
)
ctx = html.WithStaticDriver(ctx)
out, err := p.Run(
context.Background(),
ctx,
runtime.WithLog(os.Stdout),
runtime.WithBrowser(r.settings.CDPAddress),
runtime.WithParam("static", r.settings.StaticServerAddress),
runtime.WithParam("dynamic", r.settings.DynamicServerAddress),
)

15
examples/embedded.go
View File

@ -4,8 +4,10 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/MontFerret/ferret/pkg/compiler"
"os"
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/html"
)
type Topic struct {
@ -52,7 +54,16 @@ func getTopTenTrendingTopics() ([]*Topic, error) {
return nil, err
}
out, err := program.Run(context.Background())
// create a root context
ctx := context.Background()
// enable HTML drivers
// by default, Ferret Runtime knows nothing about HTML drivers
// all HTML manipulations are done via functions from standard library
ctx = html.WithDynamicDriver(ctx)
ctx = html.WithStaticDriver(ctx)
out, err := program.Run(ctx)
if err != nil {
return nil, err

13
examples/extensible.go
View File

@ -4,10 +4,12 @@ import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"os"
)
func main() {
@ -27,7 +29,7 @@ func getStrings() ([]string, error) {
// function implements is a type of a function that ferret supports as a runtime function
transform := func(ctx context.Context, args ...core.Value) (core.Value, error) {
// it's just a helper function which helps to validate a number of passed args
err := core.ValidateArgs(args, 1)
err := core.ValidateArgs(args, 1, 1)
if err != nil {
// it's recommended to return built-in None type, instead of nil
@ -44,7 +46,7 @@ func getStrings() ([]string, error) {
// cast to built-in string type
str := args[0].(values.String)
return str.Concat(values.NewString("_ferret")).ToUpper(), nil
return values.NewString(strings.ToUpper(str.String() + "_ferret")), nil
}
query := `
@ -54,7 +56,10 @@ func getStrings() ([]string, error) {
`
comp := compiler.New()
comp.RegisterFunction("transform", transform)
if err := comp.RegisterFunction("transform", transform); err != nil {
return nil, err
}
program, err := comp.Compile(query)

5
pkg/html/common/ua.go
View File

@ -1,16 +1,17 @@
package common
import (
"github.com/MontFerret/ferret/pkg/runtime/env"
"github.com/corpix/uarand"
)
const RandomUserAgent = "*"
func GetUserAgent(val string) string {
if val == "" {
return val
}
if val != env.RandomUserAgent {
if val != RandomUserAgent {
return val
}

65
pkg/html/driver.go
View File

@ -6,15 +6,10 @@ import (
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/html/static"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/env"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type (
DriverName string
dynamicCtxKey struct{}
staticCtxKey struct{}
)
type DriverName string
const (
Dynamic DriverName = "dynamic"
@ -26,67 +21,21 @@ type Driver interface {
Close() error
}
func ToContext(ctx context.Context, name DriverName, drv Driver) context.Context {
var key interface{}
switch name {
case Dynamic:
key = dynamicCtxKey{}
case Static:
key = staticCtxKey{}
default:
return ctx
}
return context.WithValue(ctx, key, drv)
}
func FromContext(ctx context.Context, name DriverName) (Driver, error) {
var key interface{}
switch name {
case Dynamic:
key = dynamicCtxKey{}
return dynamic.FromContext(ctx)
case Static:
key = staticCtxKey{}
return static.FromContext(ctx)
default:
return nil, core.Error(core.ErrInvalidArgument, fmt.Sprintf("%s driver", name))
}
val := ctx.Value(key)
drv, ok := val.(Driver)
if ok {
return drv, nil
}
return nil, core.Error(core.ErrNotFound, fmt.Sprintf("%s driver", name))
}
func WithDynamicDriver(ctx context.Context) context.Context {
e := env.FromContext(ctx)
return context.WithValue(
ctx,
dynamicCtxKey{},
dynamic.NewDriver(
e.CDPAddress,
dynamic.WithProxy(e.ProxyAddress),
dynamic.WithUserAgent(e.UserAgent),
),
)
func WithDynamicDriver(ctx context.Context, opts ...dynamic.Option) context.Context {
return dynamic.WithContext(ctx, dynamic.NewDriver(opts...))
}
func WithStaticDriver(ctx context.Context) context.Context {
e := env.FromContext(ctx)
return context.WithValue(
ctx,
staticCtxKey{},
static.NewDriver(
static.WithProxy(e.ProxyAddress),
static.WithUserAgent(e.UserAgent),
),
)
func WithStaticDriver(ctx context.Context, opts ...static.Option) context.Context {
return static.WithContext(ctx, static.NewDriver(opts...))
}

52
pkg/html/dynamic/driver.go
View File

@ -2,6 +2,9 @@ package dynamic
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
"sync"
"github.com/MontFerret/ferret/pkg/html/common"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
@ -13,28 +16,47 @@ import (
"github.com/mafredri/cdp/rpcc"
"github.com/mafredri/cdp/session"
"github.com/pkg/errors"
"sync"
)
type Driver struct {
sync.Mutex
dev *devtool.DevTools
conn *rpcc.Conn
client *cdp.Client
session *session.Manager
contextID target.BrowserContextID
options *Options
type (
ctxKey struct{}
Driver struct {
sync.Mutex
dev *devtool.DevTools
conn *rpcc.Conn
client *cdp.Client
session *session.Manager
contextID target.BrowserContextID
options *Options
}
)
func WithContext(ctx context.Context, drv *Driver) context.Context {
return context.WithValue(
ctx,
ctxKey{},
drv,
)
}
func NewDriver(address string, opts ...Option) *Driver {
drv := new(Driver)
drv.dev = devtool.New(address)
drv.options = new(Options)
func FromContext(ctx context.Context) (*Driver, error) {
val := ctx.Value(ctxKey{})
for _, opt := range opts {
opt(drv.options)
drv, ok := val.(*Driver)
if !ok {
return nil, core.Error(core.ErrNotFound, "dynamic HTML Driver")
}
return drv, nil
}
func NewDriver(opts ...Option) *Driver {
drv := new(Driver)
drv.options = newOptions(opts)
drv.dev = devtool.New(drv.options.cdp)
return drv
}

18
pkg/html/dynamic/options.go
View File

@ -4,11 +4,29 @@ type (
Options struct {
proxy string
userAgent string
cdp string
}
Option func(opts *Options)
)
func newOptions(setters []Option) *Options {
opts := new(Options)
opts.cdp = "http://127.0.0.1:9222"
for _, setter := range setters {
setter(opts)
}
return opts
}
func WithCDP(address string) Option {
return func(opts *Options) {
opts.cdp = address
}
}
func WithProxy(address string) Option {
return func(opts *Options) {
opts.proxy = address

41
pkg/html/static/driver.go
View File

@ -3,6 +3,7 @@ package static
import (
"bytes"
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
"net/http"
"net/url"
@ -14,22 +15,38 @@ import (
"github.com/sethgrid/pester"
)
type Driver struct {
client *pester.Client
options *Options
type (
ctxKey struct{}
Driver struct {
client *pester.Client
options *Options
}
)
func WithContext(ctx context.Context, drv *Driver) context.Context {
return context.WithValue(
ctx,
ctxKey{},
drv,
)
}
func FromContext(ctx context.Context) (*Driver, error) {
val := ctx.Value(ctxKey{})
drv, ok := val.(*Driver)
if !ok {
return nil, core.Error(core.ErrNotFound, "static HTML Driver")
}
return drv, nil
}
func NewDriver(opts ...Option) *Driver {
drv := new(Driver)
drv.options = &Options{
concurrency: 3,
maxRetries: 5,
backoff: pester.ExponentialBackoff,
}
for _, opt := range opts {
opt(drv.options)
}
drv.options = newOptions(opts)
if drv.options.proxy == "" {
drv.client = pester.New()

13
pkg/html/static/options.go
View File

@ -15,6 +15,19 @@ type (
}
)
func newOptions(setters []Option) *Options {
opts := new(Options)
opts.backoff = pester.ExponentialBackoff
opts.concurrency = 3
opts.maxRetries = 5
for _, setter := range setters {
setter(opts)
}
return opts
}
func WithDefaultBackoff() Option {
return func(opts *Options) {
opts.backoff = pester.DefaultBackoff

31
pkg/runtime/env/env.go vendored
View File

@ -1,31 +0,0 @@
package env
import "context"
type (
ctxKey struct{}
Environment struct {
CDPAddress string
ProxyAddress string
UserAgent string
}
)
const RandomUserAgent = "*"
func WithContext(ctx context.Context, e Environment) context.Context {
return context.WithValue(ctx, ctxKey{}, e)
}
func FromContext(ctx context.Context) Environment {
res := ctx.Value(ctxKey{})
val, ok := res.(Environment)
if !ok {
return Environment{}
}
return val
}

63
pkg/runtime/options.go
View File

@ -2,35 +2,37 @@ package runtime
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/env"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
"io"
"os"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type (
Options struct {
proxy string
cdp string
params map[string]core.Value
logging *logging.Options
userAgent string
params map[string]core.Value
logging *logging.Options
}
Option func(*Options)
)
func NewOptions() *Options {
return &Options{
cdp: "http://127.0.0.1:9222",
func NewOptions(setters []Option) *Options {
opts := &Options{
params: make(map[string]core.Value),
logging: &logging.Options{
Writer: os.Stdout,
Level: logging.ErrorLevel,
},
}
for _, setter := range setters {
setter(opts)
}
return opts
}
func WithParam(name string, value interface{}) Option {
@ -47,30 +49,6 @@ func WithParams(params map[string]interface{}) Option {
}
}
func WithBrowser(address string) Option {
return func(options *Options) {
options.cdp = address
}
}
func WithProxy(address string) Option {
return func(options *Options) {
options.proxy = address
}
}
func WithUserAgent(value string) Option {
return func(options *Options) {
options.userAgent = value
}
}
func WithRandomUserAgent() Option {
return func(options *Options) {
options.userAgent = env.RandomUserAgent
}
}
func WithLog(writer io.Writer) Option {
return func(options *Options) {
options.logging.Writer = writer
@ -83,22 +61,9 @@ func WithLogLevel(lvl logging.Level) Option {
}
}
func (opts *Options) Apply(setters ...Option) *Options {
for _, setter := range setters {
setter(opts)
}
return opts
}
func (opts *Options) WithContext(parent context.Context) context.Context {
ctx := core.ParamsWith(parent, opts.params)
ctx = logging.WithContext(ctx, opts.logging)
ctx = env.WithContext(ctx, env.Environment{
CDPAddress: opts.cdp,
ProxyAddress: opts.proxy,
UserAgent: opts.userAgent,
})
return ctx
}

5
pkg/runtime/program.go
View File

@ -2,7 +2,6 @@ package runtime
import (
"context"
"github.com/MontFerret/ferret/pkg/html"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
@ -32,9 +31,7 @@ func (p *Program) Source() string {
}
func (p *Program) Run(ctx context.Context, setters ...Option) (result []byte, err error) {
ctx = NewOptions().Apply(setters...).WithContext(ctx)
ctx = html.WithDynamicDriver(ctx)
ctx = html.WithStaticDriver(ctx)
ctx = NewOptions(setters).WithContext(ctx)
logger := logging.FromContext(ctx)