1
0
mirror of https://github.com/MontFerret/ferret.git synced 2025-01-18 03:22:02 +02:00
* Renamed DOCUMENT to PAGE

* Added PageLoadParams

* Added PageLoadParams

* Renamed LoadPageParams -> PageLoadParams

* Added support for context.Done() (#201)

* Bug/#189 operators precedence (#202)

* Fixed math operators precedence

* Fixed logical operators precedence

* Fixed array operator

* Added support for parentheses to enforce a different operator evaluation order

* Feature/#200 drivers (#209)

* Added new interfaces

* Renamed dynamic to cdp driver

* Renamed drivers

* Added ELEMENT_EXISTS function (#210)

* Renamed back PAGE to DOCUMENT (#211)

* Added Getter and Setter interfaces
This commit is contained in:
Tim Voronov 2018-12-21 23:14:41 -05:00 committed by GitHub
parent 6bc4b3e0e3
commit 5620be211c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
92 changed files with 2953 additions and 1656 deletions

View File

@ -38,7 +38,16 @@ func Exec(query string, opts Options) {
l := NewLogger()
ctx, cancel := context.WithCancel(opts.WithContext(context.Background()))
ctx, err := opts.WithContext(context.Background())
if err != nil {
fmt.Println("Failed to register HTML drivers")
fmt.Println(err)
os.Exit(1)
return
}
ctx, cancel := context.WithCancel(ctx)
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP)

View File

@ -2,9 +2,9 @@ package cli
import (
"context"
"github.com/MontFerret/ferret/pkg/html"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/html/static"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp"
"github.com/MontFerret/ferret/pkg/drivers/http"
)
type Options struct {
@ -15,19 +15,29 @@ type Options struct {
ShowTime bool
}
func (opts Options) WithContext(ctx context.Context) context.Context {
ctx = html.WithDynamicDriver(
func (opts Options) WithContext(ctx context.Context) (context.Context, error) {
var err error
ctx = drivers.WithDynamic(
ctx,
dynamic.WithCDP(opts.Cdp),
dynamic.WithProxy(opts.Proxy),
dynamic.WithUserAgent(opts.UserAgent),
cdp.NewDriver(
cdp.WithAddress(opts.Cdp),
cdp.WithProxy(opts.Proxy),
cdp.WithUserAgent(opts.UserAgent),
),
)
ctx = html.WithStaticDriver(
if err != nil {
return ctx, err
}
ctx = drivers.WithStatic(
ctx,
static.WithProxy(opts.Proxy),
static.WithUserAgent(opts.UserAgent),
http.NewDriver(
http.WithProxy(opts.Proxy),
http.WithUserAgent(opts.UserAgent),
),
)
return ctx
return ctx, err
}

View File

@ -42,7 +42,16 @@ func Repl(version string, opts Options) {
l := NewLogger()
ctx, cancel := context.WithCancel(opts.WithContext(context.Background()))
ctx, err := opts.WithContext(context.Background())
if err != nil {
fmt.Println("Failed to register HTML drivers")
fmt.Println(err)
os.Exit(1)
return
}
ctx, cancel := context.WithCancel(ctx)
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP)

View File

@ -3,12 +3,12 @@ package main
import (
"flag"
"fmt"
"os"
"path/filepath"
"github.com/MontFerret/ferret/e2e/runner"
"github.com/MontFerret/ferret/e2e/server"
"github.com/rs/zerolog"
"os"
"path/filepath"
"regexp"
)
var (
@ -29,6 +29,12 @@ var (
"http://0.0.0.0:9222",
"address of remote Chrome instance",
)
filter = flag.String(
"filter",
"",
"regexp expression to filter out tests",
)
)
func main() {
@ -48,6 +54,19 @@ func main() {
Dir: filepath.Join(*pagesDir, "dynamic"),
})
var filterR *regexp.Regexp
if *filter != "" {
r, err := regexp.Compile(*filter)
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
filterR = r
}
go func() {
if err := static.Start(); err != nil {
logger.Info().Timestamp().Msg("shutting down the static pages server")
@ -79,6 +98,7 @@ func main() {
DynamicServerAddress: fmt.Sprintf("http://0.0.0.0:%d", dynamicPort),
CDPAddress: *cdp,
Dir: *testsDir,
Filter: filterR,
})
err := r.Run()

View File

@ -4,14 +4,16 @@ import (
"context"
"encoding/json"
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/html"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp"
"github.com/MontFerret/ferret/pkg/drivers/http"
"github.com/MontFerret/ferret/pkg/runtime"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"time"
)
@ -21,6 +23,7 @@ type (
DynamicServerAddress string
CDPAddress string
Dir string
Filter *regexp.Regexp
}
Result struct {
@ -102,7 +105,15 @@ func (r *Runner) runQueries(dir string) ([]Result, error) {
// read scripts
for _, f := range files {
fName := filepath.Join(dir, f.Name())
n := f.Name()
if r.settings.Filter != nil {
if r.settings.Filter.Match([]byte(n)) != true {
continue
}
}
fName := filepath.Join(dir, n)
b, err := ioutil.ReadFile(fName)
if err != nil {
@ -134,11 +145,12 @@ func (r *Runner) runQuery(c *compiler.FqlCompiler, name, script string) Result {
}
ctx := context.Background()
ctx = html.WithDynamicDriver(
ctx = drivers.WithDynamic(
ctx,
dynamic.WithCDP(r.settings.CDPAddress),
cdp.NewDriver(cdp.WithAddress(r.settings.CDPAddress)),
)
ctx = html.WithStaticDriver(ctx)
ctx = drivers.WithStatic(ctx, http.NewDriver())
out, err := p.Run(
ctx,

View File

@ -0,0 +1,10 @@
LET url = @static + '/overview.html'
LET doc = DOCUMENT(url)
LET expectedP = TRUE
LET actualP = ELEMENT_EXISTS(doc, '.section-nav')
LET expectedN = FALSE
LET actualN = ELEMENT_EXISTS(doc, '.foo-bar')
RETURN EXPECT(expectedP + expectedN, actualP + expectedN)

View File

@ -0,0 +1,10 @@
LET url = @dynamic
LET doc = DOCUMENT(url)
LET expectedP = TRUE
LET actualP = ELEMENT_EXISTS(doc, '.text-center')
LET expectedN = FALSE
LET actualN = ELEMENT_EXISTS(doc, '.foo-bar')
RETURN EXPECT(expectedP + expectedN, actualP + expectedN)

View File

@ -0,0 +1,12 @@
LET url = @static + '/value.html'
LET doc = DOCUMENT(url)
LET el = ELEMENT(doc, "#listings_table")
LET expectedP = TRUE
LET actualP = ELEMENT_EXISTS(el, '.odd')
LET expectedN = FALSE
LET actualN = ELEMENT_EXISTS(el, '.foo-bar')
RETURN EXPECT(expectedP + expectedN, actualP + expectedN)

View File

@ -0,0 +1,12 @@
LET url = @dynamic
LET doc = DOCUMENT(url)
LET el = ELEMENT(doc, "#root")
LET expectedP = TRUE
LET actualP = ELEMENT_EXISTS(el, '.jumbotron')
LET expectedN = FALSE
LET actualN = ELEMENT_EXISTS(el, '.foo-bar')
RETURN EXPECT(expectedP + expectedN, actualP + expectedN)

View File

@ -0,0 +1,6 @@
LET url = @dynamic
LET doc = DOCUMENT(url, {
dynamic: true
})
RETURN EXPECT(doc.url, url)

View File

@ -4,10 +4,12 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/MontFerret/ferret/pkg/drivers/cdp"
"github.com/MontFerret/ferret/pkg/drivers/http"
"os"
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/html"
"github.com/MontFerret/ferret/pkg/drivers"
)
type Topic struct {
@ -60,8 +62,8 @@ func getTopTenTrendingTopics() ([]*Topic, error) {
// enable HTML drivers
// by default, Ferret Runtime knows nothing about HTML drivers
// all HTML manipulations are done via functions from standard library
ctx = html.WithDynamicDriver(ctx)
ctx = html.WithStaticDriver(ctx)
ctx = drivers.WithDynamic(ctx, cdp.NewDriver())
ctx = drivers.WithStatic(ctx, http.NewDriver())
out, err := program.Run(ctx)

View File

@ -1,5 +1,4 @@
LET doc = DOCUMENT("http://getbootstrap.com/docs/4.1/components/collapse/", true)
LET el = ELEMENT(doc, "#collapseTwo")
CLICK(doc, "#headingTwo > h5 > button")

View File

@ -0,0 +1,65 @@
package compiler_test
import (
"context"
"github.com/MontFerret/ferret/pkg/compiler"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func TestPrecedence(t *testing.T) {
Convey("Math operators", t, func() {
Convey("2 + 2 * 2", func() {
c := compiler.New()
p := c.MustCompile(`RETURN 2 + 2 * 2`)
out := p.MustRun(context.Background())
So(string(out), ShouldEqual, "6")
})
Convey("2 * 2 + 2", func() {
c := compiler.New()
p := c.MustCompile(`RETURN 2 * 2 + 2`)
out := p.MustRun(context.Background())
So(string(out), ShouldEqual, "6")
})
Convey("2 * (2 + 2)", func() {
c := compiler.New()
p := c.MustCompile(`RETURN 2 * (2 + 2)`)
out := p.MustRun(context.Background())
So(string(out), ShouldEqual, "8")
})
})
Convey("Logical", t, func() {
Convey("TRUE OR TRUE AND FALSE", func() {
c := compiler.New()
p := c.MustCompile(`RETURN TRUE OR TRUE AND FALSE`)
out := p.MustRun(context.Background())
So(string(out), ShouldEqual, "true")
})
Convey("FALSE AND TRUE OR TRUE", func() {
c := compiler.New()
p := c.MustCompile(`RETURN FALSE AND TRUE OR TRUE`)
out := p.MustRun(context.Background())
So(string(out), ShouldEqual, "true")
})
})
}

View File

@ -352,10 +352,16 @@ func (v *visitor) doVisitFilterClause(ctx *fql.FilterClauseContext, scope *scope
return operators.NewEqualityOperator(v.getSourceMap(ctx), left, right, equalityOp.GetText())
}
logicalOp := exp.LogicalOperator()
logicalAndOp := exp.LogicalAndOperator()
if logicalOp != nil {
return operators.NewLogicalOperator(v.getSourceMap(ctx), left, right, logicalOp.GetText())
if logicalAndOp != nil {
return operators.NewLogicalOperator(v.getSourceMap(ctx), left, right, logicalAndOp.GetText())
}
logicalOrOp := exp.LogicalOrOperator()
if logicalOrOp != nil {
return operators.NewLogicalOperator(v.getSourceMap(ctx), left, right, logicalOrOp.GetText())
}
} else {
// should be unary operator
@ -1078,7 +1084,21 @@ func (v *visitor) doVisitAllExpressions(contexts []fql.IExpressionContext, scope
}
func (v *visitor) doVisitMathOperator(ctx *fql.ExpressionContext, scope *scope) (core.OperatorExpression, error) {
mathOp := ctx.MathOperator().(*fql.MathOperatorContext)
var operator operators.MathOperatorType
multiCtx := ctx.MultiplicativeOperator()
if multiCtx != nil {
operator = operators.MathOperatorType(multiCtx.GetText())
} else {
additiveCtx := ctx.AdditiveOperator()
if additiveCtx == nil {
return nil, ErrInvalidToken
}
operator = operators.MathOperatorType(additiveCtx.GetText())
}
exps, err := v.doVisitAllExpressions(ctx.AllExpression(), scope)
if err != nil {
@ -1089,10 +1109,10 @@ func (v *visitor) doVisitMathOperator(ctx *fql.ExpressionContext, scope *scope)
right := exps[1]
return operators.NewMathOperator(
v.getSourceMap(mathOp),
v.getSourceMap(ctx),
left,
right,
operators.MathOperatorType(mathOp.GetText()),
operator,
)
}
@ -1115,7 +1135,22 @@ func (v *visitor) doVisitUnaryOperator(ctx *fql.ExpressionContext, scope *scope)
}
func (v *visitor) doVisitLogicalOperator(ctx *fql.ExpressionContext, scope *scope) (core.OperatorExpression, error) {
logicalOp := ctx.LogicalOperator().(*fql.LogicalOperatorContext)
var operator string
logicalAndOp := ctx.LogicalAndOperator()
if logicalAndOp != nil {
operator = logicalAndOp.GetText()
} else {
logicalOrOp := ctx.LogicalOrOperator()
if logicalOrOp == nil {
return nil, ErrInvalidToken
}
operator = logicalOrOp.GetText()
}
exps, err := v.doVisitAllExpressions(ctx.AllExpression(), scope)
if err != nil {
@ -1125,7 +1160,7 @@ func (v *visitor) doVisitLogicalOperator(ctx *fql.ExpressionContext, scope *scop
left := exps[0]
right := exps[1]
return operators.NewLogicalOperator(v.getSourceMap(logicalOp), left, right, logicalOp.GetText())
return operators.NewLogicalOperator(v.getSourceMap(ctx), left, right, operator)
}
func (v *visitor) doVisitEqualityOperator(ctx *fql.ExpressionContext, scope *scope) (core.OperatorExpression, error) {
@ -1206,13 +1241,83 @@ func (v *visitor) doVisitArrayOperator(ctx *fql.ExpressionContext, scope *scope)
)
}
func (v *visitor) doVisitExpressionGroup(ctx *fql.ExpressionGroupContext, scope *scope) (core.Expression, error) {
exp := ctx.Expression()
if exp == nil {
return nil, ErrInvalidToken
}
return v.doVisitExpression(exp.(*fql.ExpressionContext), scope)
}
func (v *visitor) doVisitExpression(ctx *fql.ExpressionContext, scope *scope) (core.Expression, error) {
seq := ctx.ExpressionGroup()
if seq != nil {
return v.doVisitExpressionGroup(seq.(*fql.ExpressionGroupContext), scope)
}
member := ctx.MemberExpression()
if member != nil {
return v.doVisitMemberExpression(member.(*fql.MemberExpressionContext), scope)
}
funCall := ctx.FunctionCallExpression()
if funCall != nil {
return v.doVisitFunctionCallExpression(funCall.(*fql.FunctionCallExpressionContext), scope)
}
notOp := ctx.UnaryOperator()
if notOp != nil {
return v.doVisitUnaryOperator(ctx, scope)
}
multiOp := ctx.MultiplicativeOperator()
if multiOp != nil {
return v.doVisitMathOperator(ctx, scope)
}
addOp := ctx.AdditiveOperator()
if addOp != nil {
return v.doVisitMathOperator(ctx, scope)
}
arrOp := ctx.ArrayOperator()
if arrOp != nil {
return v.doVisitArrayOperator(ctx, scope)
}
equalityOp := ctx.EqualityOperator()
if equalityOp != nil {
return v.doVisitEqualityOperator(ctx, scope)
}
inOp := ctx.InOperator()
if inOp != nil {
return v.doVisitInOperator(ctx, scope)
}
logicalAndOp := ctx.LogicalAndOperator()
if logicalAndOp != nil {
return v.doVisitLogicalOperator(ctx, scope)
}
logicalOrOp := ctx.LogicalOrOperator()
if logicalOrOp != nil {
return v.doVisitLogicalOperator(ctx, scope)
}
variable := ctx.Variable()
if variable != nil {
@ -1255,54 +1360,12 @@ func (v *visitor) doVisitExpression(ctx *fql.ExpressionContext, scope *scope) (c
return v.doVisitObjectLiteral(obj.(*fql.ObjectLiteralContext), scope)
}
funCall := ctx.FunctionCallExpression()
if funCall != nil {
return v.doVisitFunctionCallExpression(funCall.(*fql.FunctionCallExpressionContext), scope)
}
member := ctx.MemberExpression()
if member != nil {
return v.doVisitMemberExpression(member.(*fql.MemberExpressionContext), scope)
}
none := ctx.NoneLiteral()
if none != nil {
return v.doVisitNoneLiteral(none.(*fql.NoneLiteralContext))
}
arrOp := ctx.ArrayOperator()
if arrOp != nil {
return v.doVisitArrayOperator(ctx, scope)
}
inOp := ctx.InOperator()
if inOp != nil {
return v.doVisitInOperator(ctx, scope)
}
equalityOp := ctx.EqualityOperator()
if equalityOp != nil {
return v.doVisitEqualityOperator(ctx, scope)
}
logicalOp := ctx.LogicalOperator()
if logicalOp != nil {
return v.doVisitLogicalOperator(ctx, scope)
}
mathOp := ctx.MathOperator()
if mathOp != nil {
return v.doVisitMathOperator(ctx, scope)
}
questionCtx := ctx.QuestionMark()
if questionCtx != nil {

View File

@ -1,4 +1,4 @@
package dynamic
package cdp
import (
"context"
@ -8,8 +8,8 @@ import (
"sync"
"time"
"github.com/MontFerret/ferret/pkg/html/dynamic/eval"
"github.com/MontFerret/ferret/pkg/html/dynamic/events"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
@ -24,16 +24,6 @@ import (
const BlankPageURL = "about:blank"
type (
ScreenshotFormat string
ScreenshotArgs struct {
X float64
Y float64
Width float64
Height float64
Format ScreenshotFormat
Quality int
}
HTMLDocument struct {
sync.Mutex
logger *zerolog.Logger
@ -45,11 +35,6 @@ type (
}
)
const (
ScreenshotFormatPNG ScreenshotFormat = "png"
ScreenshotFormatJPEG ScreenshotFormat = "jpeg"
)
func handleLoadError(logger *zerolog.Logger, client *cdp.Client) {
err := client.Page.Close(context.Background())
@ -58,12 +43,6 @@ func handleLoadError(logger *zerolog.Logger, client *cdp.Client) {
}
}
func IsScreenshotFormatValid(format string) bool {
value := ScreenshotFormat(format)
return value == ScreenshotFormatPNG || value == ScreenshotFormatJPEG
}
func LoadHTMLDocument(
ctx context.Context,
conn *rpcc.Conn,
@ -390,6 +369,13 @@ func (doc *HTMLDocument) CountBySelector(selector values.String) values.Int {
return doc.element.CountBySelector(selector)
}
func (doc *HTMLDocument) ExistsBySelector(selector values.String) values.Boolean {
doc.Lock()
defer doc.Unlock()
return doc.element.ExistsBySelector(selector)
}
func (doc *HTMLDocument) ClickBySelector(selector values.String) (values.Boolean, error) {
res, err := eval.Eval(
doc.client,
@ -806,23 +792,71 @@ func (doc *HTMLDocument) NavigateForward(skip values.Int, timeout values.Int) (v
return values.True, nil
}
func (doc *HTMLDocument) PrintToPDF(params *page.PrintToPDFArgs) (core.Value, error) {
func (doc *HTMLDocument) PrintToPDF(params values.HTMLPDFParams) (values.Binary, error) {
ctx := context.Background()
reply, err := doc.client.Page.PrintToPDF(ctx, params)
args := page.NewPrintToPDFArgs()
args.
SetLandscape(bool(params.Landscape)).
SetDisplayHeaderFooter(bool(params.DisplayHeaderFooter)).
SetPrintBackground(bool(params.PrintBackground)).
SetIgnoreInvalidPageRanges(bool(params.IgnoreInvalidPageRanges)).
SetPreferCSSPageSize(bool(params.PreferCSSPageSize))
if params.Scale > 0 {
args.SetScale(float64(params.Scale))
}
if params.PaperWidth > 0 {
args.SetPaperWidth(float64(params.PaperWidth))
}
if params.PaperHeight > 0 {
args.SetPaperHeight(float64(params.PaperHeight))
}
if params.MarginTop > 0 {
args.SetMarginTop(float64(params.MarginTop))
}
if params.MarginBottom > 0 {
args.SetMarginBottom(float64(params.MarginBottom))
}
if params.MarginRight > 0 {
args.SetMarginRight(float64(params.MarginRight))
}
if params.MarginLeft > 0 {
args.SetMarginLeft(float64(params.MarginLeft))
}
if params.PageRanges != values.EmptyString {
args.SetPageRanges(string(params.PageRanges))
}
if params.HeaderTemplate != values.EmptyString {
args.SetHeaderTemplate(string(params.HeaderTemplate))
}
if params.FooterTemplate != values.EmptyString {
args.SetFooterTemplate(string(params.FooterTemplate))
}
reply, err := doc.client.Page.PrintToPDF(ctx, args)
if err != nil {
return values.None, err
return values.NewBinary([]byte{}), err
}
return values.NewBinary(reply.Data), nil
}
func (doc *HTMLDocument) CaptureScreenshot(params *ScreenshotArgs) (core.Value, error) {
func (doc *HTMLDocument) CaptureScreenshot(params values.HTMLScreenshotParams) (values.Binary, error) {
ctx := context.Background()
metrics, err := doc.client.Page.GetLayoutMetrics(ctx)
if params.Format == ScreenshotFormatJPEG && params.Quality < 0 && params.Quality > 100 {
if params.Format == values.HTMLScreenshotFormatJPEG && params.Quality < 0 && params.Quality > 100 {
params.Quality = 100
}
@ -835,32 +869,33 @@ func (doc *HTMLDocument) CaptureScreenshot(params *ScreenshotArgs) (core.Value,
}
if params.Width <= 0 {
params.Width = float64(metrics.LayoutViewport.ClientWidth) - params.X
params.Width = values.Float(metrics.LayoutViewport.ClientWidth) - params.X
}
if params.Height <= 0 {
params.Height = float64(metrics.LayoutViewport.ClientHeight) - params.Y
params.Height = values.Float(metrics.LayoutViewport.ClientHeight) - params.Y
}
clip := page.Viewport{
X: params.X,
Y: params.Y,
Width: params.Width,
Height: params.Height,
X: float64(params.X),
Y: float64(params.Y),
Width: float64(params.Width),
Height: float64(params.Height),
Scale: 1.0,
}
format := string(params.Format)
screenshotArgs := page.CaptureScreenshotArgs{
quality := int(params.Quality)
args := page.CaptureScreenshotArgs{
Format: &format,
Quality: &params.Quality,
Quality: &quality,
Clip: &clip,
}
reply, err := doc.client.Page.CaptureScreenshot(ctx, &screenshotArgs)
reply, err := doc.client.Page.CaptureScreenshot(ctx, &args)
if err != nil {
return values.None, err
return values.NewBinary([]byte{}), err
}
return values.NewBinary(reply.Data), nil

View File

@ -1,11 +1,10 @@
package dynamic
package cdp
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
"sync"
"github.com/MontFerret/ferret/pkg/html/common"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/mafredri/cdp"
@ -18,49 +17,25 @@ import (
"github.com/pkg/errors"
)
type (
ctxKey struct{}
Driver struct {
sync.Mutex
dev *devtool.DevTools
conn *rpcc.Conn
client *cdp.Client
session *session.Manager
contextID target.BrowserContextID
options *Options
}
)
func WithContext(ctx context.Context, drv *Driver) context.Context {
return context.WithValue(
ctx,
ctxKey{},
drv,
)
}
func FromContext(ctx context.Context) (*Driver, error) {
val := ctx.Value(ctxKey{})
drv, ok := val.(*Driver)
if !ok {
return nil, core.Error(core.ErrNotFound, "dynamic HTML Driver")
}
return drv, nil
type Driver struct {
sync.Mutex
dev *devtool.DevTools
conn *rpcc.Conn
client *cdp.Client
session *session.Manager
contextID target.BrowserContextID
options *Options
}
func NewDriver(opts ...Option) *Driver {
drv := new(Driver)
drv.options = newOptions(opts)
drv.dev = devtool.New(drv.options.cdp)
drv.dev = devtool.New(drv.options.address)
return drv
}
func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (values.HTMLNode, error) {
func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (values.DHTMLDocument, error) {
logger := logging.FromContext(ctx)
err := drv.init(ctx)

View File

@ -1,4 +1,4 @@
package dynamic
package cdp
import (
"context"
@ -11,9 +11,9 @@ import (
"sync"
"time"
"github.com/MontFerret/ferret/pkg/html/common"
"github.com/MontFerret/ferret/pkg/html/dynamic/eval"
"github.com/MontFerret/ferret/pkg/html/dynamic/events"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/mafredri/cdp"
@ -679,6 +679,33 @@ func (el *HTMLElement) CountBySelector(selector values.String) values.Int {
return values.NewInt(len(res.NodeIDs))
}
func (el *HTMLElement) ExistsBySelector(selector values.String) values.Boolean {
if !el.IsConnected() {
return values.False
}
ctx, cancel := contextWithTimeout()
defer cancel()
// TODO: Can we use RemoteObjectID or BackendID instead of NodeId?
selectorArgs := dom.NewQuerySelectorArgs(el.id.nodeID, selector.String())
res, err := el.client.DOM.QuerySelector(ctx, selectorArgs)
if err != nil {
el.logError(err).
Str("selector", selector.String()).
Msg("failed to retrieve nodes by selector")
return values.False
}
if res.NodeID == 0 {
return values.False
}
return values.True
}
func (el *HTMLElement) WaitForClass(class values.String, timeout values.Int) error {
task := events.NewWaitTask(
func() (core.Value, error) {

View File

@ -1,7 +1,7 @@
package events_test
import (
"github.com/MontFerret/ferret/pkg/html/dynamic/events"
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/page"
. "github.com/smartystreets/goconvey/convey"

View File

@ -3,7 +3,7 @@ package events
import (
"context"
"fmt"
"github.com/MontFerret/ferret/pkg/html/dynamic/eval"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/runtime"

View File

@ -1,7 +1,7 @@
package events
import (
"github.com/MontFerret/ferret/pkg/html/dynamic/eval"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/mafredri/cdp"

View File

@ -1,12 +1,12 @@
package dynamic
package cdp
import (
"bytes"
"context"
"errors"
"github.com/MontFerret/ferret/pkg/html/common"
"github.com/MontFerret/ferret/pkg/html/dynamic/eval"
"github.com/MontFerret/ferret/pkg/html/dynamic/events"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/PuerkitoBio/goquery"
"github.com/mafredri/cdp"

View File

@ -1,18 +1,20 @@
package dynamic
package cdp
type (
Options struct {
proxy string
userAgent string
cdp string
address string
}
Option func(opts *Options)
)
const DefaultAddress = "http://127.0.0.1:9222"
func newOptions(setters []Option) *Options {
opts := new(Options)
opts.cdp = "http://127.0.0.1:9222"
opts.address = DefaultAddress
for _, setter := range setters {
setter(opts)
@ -21,9 +23,9 @@ func newOptions(setters []Option) *Options {
return opts
}
func WithCDP(address string) Option {
func WithAddress(address string) Option {
return func(opts *Options) {
opts.cdp = address
opts.address = address
}
}

65
pkg/drivers/driver.go Normal file
View File

@ -0,0 +1,65 @@
package drivers
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"io"
)
type (
staticCtxKey struct{}
dynamicCtxKey struct{}
Static interface {
io.Closer
GetDocument(ctx context.Context, url values.String) (values.HTMLDocument, error)
ParseDocument(ctx context.Context, str values.String) (values.HTMLDocument, error)
}
Dynamic interface {
io.Closer
GetDocument(ctx context.Context, url values.String) (values.DHTMLDocument, error)
}
)
func StaticFrom(ctx context.Context) (Static, error) {
val := ctx.Value(staticCtxKey{})
drv, ok := val.(Static)
if !ok {
return nil, core.Error(core.ErrNotFound, "HTML Driver")
}
return drv, nil
}
func DynamicFrom(ctx context.Context) (Dynamic, error) {
val := ctx.Value(dynamicCtxKey{})
drv, ok := val.(Dynamic)
if !ok {
return nil, core.Error(core.ErrNotFound, "DHTML Driver")
}
return drv, nil
}
func WithStatic(ctx context.Context, drv Static) context.Context {
return context.WithValue(
ctx,
staticCtxKey{},
drv,
)
}
func WithDynamic(ctx context.Context, drv Dynamic) context.Context {
return context.WithValue(
ctx,
dynamicCtxKey{},
drv,
)
}

View File

@ -1,4 +1,4 @@
package static
package http
import (
"github.com/MontFerret/ferret/pkg/runtime/core"

View File

@ -1,8 +1,8 @@
package static_test
package http_test
import (
"bytes"
"github.com/MontFerret/ferret/pkg/html/static"
"github.com/MontFerret/ferret/pkg/drivers/http"
"github.com/PuerkitoBio/goquery"
. "github.com/smartystreets/goconvey/convey"
"testing"
@ -228,7 +228,7 @@ func TestDocument(t *testing.T) {
So(err, ShouldBeNil)
el, err := static.NewHTMLElement(doc.Selection)
el, err := http.NewHTMLElement(doc.Selection)
So(err, ShouldBeNil)

View File

@ -1,13 +1,13 @@
package static
package http
import (
"bytes"
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"net/http"
"net/url"
"github.com/MontFerret/ferret/pkg/html/common"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/PuerkitoBio/goquery"
"github.com/corpix/uarand"
@ -15,33 +15,9 @@ import (
"github.com/sethgrid/pester"
)
type (
ctxKey struct{}
Driver struct {
client *pester.Client
options *Options
}
)
func WithContext(ctx context.Context, drv *Driver) context.Context {
return context.WithValue(
ctx,
ctxKey{},
drv,
)
}
func FromContext(ctx context.Context) (*Driver, error) {
val := ctx.Value(ctxKey{})
drv, ok := val.(*Driver)
if !ok {
return nil, core.Error(core.ErrNotFound, "static HTML Driver")
}
return drv, nil
type Driver struct {
client *pester.Client
options *Options
}
func NewDriver(opts ...Option) *Driver {
@ -80,7 +56,7 @@ func newClientWithProxy(options *Options) (*http.Client, error) {
return &http.Client{Transport: tr}, nil
}
func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (values.HTMLNode, error) {
func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (values.HTMLDocument, error) {
u := targetURL.String()
req, err := http.NewRequest(http.MethodGet, u, nil)
@ -97,6 +73,12 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (va
ua := common.GetUserAgent(drv.options.userAgent)
logger := logging.FromContext(ctx)
logger.
Debug().
Str("user-agent", ua).
Msg("using User-Agent")
// use custom user agent
if ua != "" {
req.Header.Set("User-Agent", uarand.GetRandom())
@ -119,7 +101,7 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (va
return NewHTMLDocument(u, doc)
}
func (drv *Driver) ParseDocument(_ context.Context, str values.String) (values.HTMLNode, error) {
func (drv *Driver) ParseDocument(_ context.Context, str values.String) (values.HTMLDocument, error) {
buf := bytes.NewBuffer([]byte(str))
doc, err := goquery.NewDocumentFromReader(buf)

View File

@ -1,10 +1,10 @@
package static
package http
import (
"encoding/json"
"hash/fnv"
"github.com/MontFerret/ferret/pkg/html/common"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/PuerkitoBio/goquery"
@ -248,6 +248,16 @@ func (el *HTMLElement) CountBySelector(selector values.String) values.Int {
return values.NewInt(selection.Size())
}
func (el *HTMLElement) ExistsBySelector(selector values.String) values.Boolean {
selection := el.selection.Closest(selector.String())
if selection == nil {
return values.False
}
return values.True
}
func (el *HTMLElement) parseAttrs() *values.Object {
obj := values.NewObject()

View File

@ -1,8 +1,8 @@
package static_test
package http_test
import (
"bytes"
"github.com/MontFerret/ferret/pkg/html/static"
"github.com/MontFerret/ferret/pkg/drivers/http"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/PuerkitoBio/goquery"
. "github.com/smartystreets/goconvey/convey"
@ -251,7 +251,7 @@ func TestElement(t *testing.T) {
So(err, ShouldBeNil)
el, err := static.NewHTMLElement(doc.Find("body"))
el, err := http.NewHTMLElement(doc.Find("body"))
So(err, ShouldBeNil)
@ -267,7 +267,7 @@ func TestElement(t *testing.T) {
So(err, ShouldBeNil)
el, err := static.NewHTMLElement(doc.Find("body"))
el, err := http.NewHTMLElement(doc.Find("body"))
So(err, ShouldBeNil)
@ -291,7 +291,7 @@ func TestElement(t *testing.T) {
So(err, ShouldBeNil)
el, err := static.NewHTMLElement(doc.Find("body"))
el, err := http.NewHTMLElement(doc.Find("body"))
So(err, ShouldBeNil)
@ -316,7 +316,7 @@ func TestElement(t *testing.T) {
So(err, ShouldBeNil)
el, err := static.NewHTMLElement(doc.Find("#q"))
el, err := http.NewHTMLElement(doc.Find("#q"))
So(err, ShouldBeNil)
@ -343,7 +343,7 @@ func TestElement(t *testing.T) {
So(err, ShouldBeNil)
el, err := static.NewHTMLElement(doc.Find("h2"))
el, err := http.NewHTMLElement(doc.Find("h2"))
So(err, ShouldBeNil)
@ -370,7 +370,7 @@ func TestElement(t *testing.T) {
So(err, ShouldBeNil)
el, err := static.NewHTMLElement(doc.Find("#content"))
el, err := http.NewHTMLElement(doc.Find("#content"))
So(err, ShouldBeNil)
@ -386,7 +386,7 @@ func TestElement(t *testing.T) {
So(err, ShouldBeNil)
el, err := static.NewHTMLElement(doc.Selection)
el, err := http.NewHTMLElement(doc.Selection)
So(err, ShouldBeNil)
@ -408,7 +408,7 @@ func TestElement(t *testing.T) {
So(err, ShouldBeNil)
el, err := static.NewHTMLElement(doc.Selection)
el, err := http.NewHTMLElement(doc.Selection)
So(err, ShouldBeNil)

View File

@ -1,11 +1,12 @@
package static
package http
import (
"github.com/sethgrid/pester"
)
type (
Option func(opts *Options)
Option func(opts *Options)
Options struct {
backoff pester.BackoffStrategy
maxRetries int

View File

@ -1,41 +0,0 @@
package html
import (
"context"
"fmt"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/html/static"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type DriverName string
const (
Dynamic DriverName = "dynamic"
Static DriverName = "static"
)
type Driver interface {
GetDocument(ctx context.Context, url values.String) (values.HTMLNode, error)
Close() error
}
func FromContext(ctx context.Context, name DriverName) (Driver, error) {
switch name {
case Dynamic:
return dynamic.FromContext(ctx)
case Static:
return static.FromContext(ctx)
default:
return nil, core.Error(core.ErrInvalidArgument, fmt.Sprintf("%s driver", name))
}
}
func WithDynamicDriver(ctx context.Context, opts ...dynamic.Option) context.Context {
return dynamic.WithContext(ctx, dynamic.NewDriver(opts...))
}
func WithStaticDriver(ctx context.Context, opts ...static.Option) context.Context {
return static.WithContext(ctx, static.NewDriver(opts...))
}

View File

@ -27,13 +27,13 @@ Lte: '<=';
Neq: '!=';
// Arithmetic operators
Multi: '*';
Div: '/';
Mod: '%';
Plus: '+';
Minus: '-';
MinusMinus: '--';
PlusPlus: '++';
Multi: '*';
Div: '/';
Mod: '%';
// Logical operators
And: 'AND' | '&&';

View File

@ -18,13 +18,13 @@ Eq=17
Gte=18
Lte=19
Neq=20
Plus=21
Minus=22
MinusMinus=23
PlusPlus=24
Multi=25
Div=26
Mod=27
Multi=21
Div=22
Mod=23
Plus=24
Minus=25
MinusMinus=26
PlusPlus=27
And=28
Or=29
Range=30
@ -76,13 +76,13 @@ FloatLiteral=62
'>='=18
'<='=19
'!='=20
'+'=21
'-'=22
'--'=23
'++'=24
'*'=25
'/'=26
'%'=27
'*'=21
'/'=22
'%'=23
'+'=24
'-'=25
'--'=26
'++'=27
'='=31
'?'=32
'!~'=33

View File

@ -203,8 +203,8 @@ propertyName
| stringLiteral
;
expressionSequence
: expression (Comma expression)*
expressionGroup
: OpenParen expression CloseParen
;
functionCallExpression
@ -217,13 +217,15 @@ arguments
expression
: unaryOperator expression
| expression equalityOperator expression
| expression logicalOperator expression
| expression mathOperator expression
| expression multiplicativeOperator expression
| expression additiveOperator expression
| functionCallExpression
| OpenParen expressionSequence CloseParen
| expressionGroup
| expression arrayOperator (inOperator | equalityOperator) expression
| expression inOperator expression
| expression equalityOperator expression
| expression logicalAndOperator expression
| expression logicalOrOperator expression
| expression QuestionMark expression? Colon expression
| rangeOperator
| stringLiteral
@ -264,19 +266,25 @@ equalityOperator
| Neq
;
logicalOperator
logicalAndOperator
: And
| Or
;
mathOperator
: Plus
| Minus
| Multi
logicalOrOperator
: Or
;
multiplicativeOperator
: Multi
| Div
| Mod
;
additiveOperator
: Plus
| Minus
;
unaryOperator
: Not
| Plus

File diff suppressed because one or more lines are too long

View File

@ -18,13 +18,13 @@ Eq=17
Gte=18
Lte=19
Neq=20
Plus=21
Minus=22
MinusMinus=23
PlusPlus=24
Multi=25
Div=26
Mod=27
Multi=21
Div=22
Mod=23
Plus=24
Minus=25
MinusMinus=26
PlusPlus=27
And=28
Or=29
Range=30
@ -76,13 +76,13 @@ FloatLiteral=62
'>='=18
'<='=19
'!='=20
'+'=21
'-'=22
'--'=23
'++'=24
'*'=25
'/'=26
'%'=27
'*'=21
'/'=22
'%'=23
'+'=24
'-'=25
'--'=26
'++'=27
'='=31
'?'=32
'!~'=33

File diff suppressed because one or more lines are too long

View File

@ -18,13 +18,13 @@ Eq=17
Gte=18
Lte=19
Neq=20
Plus=21
Minus=22
MinusMinus=23
PlusPlus=24
Multi=25
Div=26
Mod=27
Multi=21
Div=22
Mod=23
Plus=24
Minus=25
MinusMinus=26
PlusPlus=27
And=28
Or=29
Range=30
@ -76,13 +76,13 @@ FloatLiteral=62
'>='=18
'<='=19
'!='=20
'+'=21
'-'=22
'--'=23
'++'=24
'*'=25
'/'=26
'%'=27
'*'=21
'/'=22
'%'=23
'+'=24
'-'=25
'--'=26
'++'=27
'='=31
'?'=32
'!~'=33

View File

@ -35,8 +35,8 @@ var serializedLexerAtn = []uint16{
7, 3, 7, 3, 8, 3, 8, 3, 9, 3, 9, 3, 10, 3, 10, 3, 11, 3, 11, 3, 12, 3,
12, 3, 13, 3, 13, 3, 14, 3, 14, 3, 15, 3, 15, 3, 16, 3, 16, 3, 17, 3, 17,
3, 18, 3, 18, 3, 18, 3, 19, 3, 19, 3, 19, 3, 20, 3, 20, 3, 20, 3, 21, 3,
21, 3, 21, 3, 22, 3, 22, 3, 23, 3, 23, 3, 24, 3, 24, 3, 24, 3, 25, 3, 25,
3, 25, 3, 26, 3, 26, 3, 27, 3, 27, 3, 28, 3, 28, 3, 29, 3, 29, 3, 29, 3,
21, 3, 21, 3, 22, 3, 22, 3, 23, 3, 23, 3, 24, 3, 24, 3, 25, 3, 25, 3, 26,
3, 26, 3, 27, 3, 27, 3, 27, 3, 28, 3, 28, 3, 28, 3, 29, 3, 29, 3, 29, 3,
29, 3, 29, 5, 29, 237, 10, 29, 3, 30, 3, 30, 3, 30, 3, 30, 5, 30, 243,
10, 30, 3, 31, 3, 31, 3, 31, 3, 32, 3, 32, 3, 33, 3, 33, 3, 34, 3, 34,
3, 34, 3, 35, 3, 35, 3, 35, 3, 36, 3, 36, 3, 36, 3, 36, 3, 37, 3, 37, 3,
@ -102,8 +102,8 @@ var serializedLexerAtn = []uint16{
3, 2, 2, 2, 27, 195, 3, 2, 2, 2, 29, 197, 3, 2, 2, 2, 31, 199, 3, 2, 2,
2, 33, 201, 3, 2, 2, 2, 35, 203, 3, 2, 2, 2, 37, 206, 3, 2, 2, 2, 39, 209,
3, 2, 2, 2, 41, 212, 3, 2, 2, 2, 43, 215, 3, 2, 2, 2, 45, 217, 3, 2, 2,
2, 47, 219, 3, 2, 2, 2, 49, 222, 3, 2, 2, 2, 51, 225, 3, 2, 2, 2, 53, 227,
3, 2, 2, 2, 55, 229, 3, 2, 2, 2, 57, 236, 3, 2, 2, 2, 59, 242, 3, 2, 2,
2, 47, 219, 3, 2, 2, 2, 49, 221, 3, 2, 2, 2, 51, 223, 3, 2, 2, 2, 53, 225,
3, 2, 2, 2, 55, 228, 3, 2, 2, 2, 57, 236, 3, 2, 2, 2, 59, 242, 3, 2, 2,
2, 61, 244, 3, 2, 2, 2, 63, 247, 3, 2, 2, 2, 65, 249, 3, 2, 2, 2, 67, 251,
3, 2, 2, 2, 69, 254, 3, 2, 2, 2, 71, 257, 3, 2, 2, 2, 73, 261, 3, 2, 2,
2, 75, 268, 3, 2, 2, 2, 77, 277, 3, 2, 2, 2, 79, 284, 3, 2, 2, 2, 81, 289,
@ -139,11 +139,11 @@ var serializedLexerAtn = []uint16{
36, 3, 2, 2, 2, 206, 207, 7, 64, 2, 2, 207, 208, 7, 63, 2, 2, 208, 38,
3, 2, 2, 2, 209, 210, 7, 62, 2, 2, 210, 211, 7, 63, 2, 2, 211, 40, 3, 2,
2, 2, 212, 213, 7, 35, 2, 2, 213, 214, 7, 63, 2, 2, 214, 42, 3, 2, 2, 2,
215, 216, 7, 45, 2, 2, 216, 44, 3, 2, 2, 2, 217, 218, 7, 47, 2, 2, 218,
46, 3, 2, 2, 2, 219, 220, 7, 47, 2, 2, 220, 221, 7, 47, 2, 2, 221, 48,
3, 2, 2, 2, 222, 223, 7, 45, 2, 2, 223, 224, 7, 45, 2, 2, 224, 50, 3, 2,
2, 2, 225, 226, 7, 44, 2, 2, 226, 52, 3, 2, 2, 2, 227, 228, 7, 49, 2, 2,
228, 54, 3, 2, 2, 2, 229, 230, 7, 39, 2, 2, 230, 56, 3, 2, 2, 2, 231, 232,
215, 216, 7, 44, 2, 2, 216, 44, 3, 2, 2, 2, 217, 218, 7, 49, 2, 2, 218,
46, 3, 2, 2, 2, 219, 220, 7, 39, 2, 2, 220, 48, 3, 2, 2, 2, 221, 222, 7,
45, 2, 2, 222, 50, 3, 2, 2, 2, 223, 224, 7, 47, 2, 2, 224, 52, 3, 2, 2,
2, 225, 226, 7, 47, 2, 2, 226, 227, 7, 47, 2, 2, 227, 54, 3, 2, 2, 2, 228,
229, 7, 45, 2, 2, 229, 230, 7, 45, 2, 2, 230, 56, 3, 2, 2, 2, 231, 232,
7, 67, 2, 2, 232, 233, 7, 80, 2, 2, 233, 237, 7, 70, 2, 2, 234, 235, 7,
40, 2, 2, 235, 237, 7, 40, 2, 2, 236, 231, 3, 2, 2, 2, 236, 234, 3, 2,
2, 2, 237, 58, 3, 2, 2, 2, 238, 239, 7, 81, 2, 2, 239, 243, 7, 84, 2, 2,
@ -261,8 +261,8 @@ var lexerModeNames = []string{
var lexerLiteralNames = []string{
"", "", "", "", "", "':'", "';'", "'.'", "','", "'['", "']'", "'('", "')'",
"'{'", "'}'", "'>'", "'<'", "'=='", "'>='", "'<='", "'!='", "'+'", "'-'",
"'--'", "'++'", "'*'", "'/'", "'%'", "", "", "", "'='", "'?'", "'!~'",
"'{'", "'}'", "'>'", "'<'", "'=='", "'>='", "'<='", "'!='", "'*'", "'/'",
"'%'", "'+'", "'-'", "'--'", "'++'", "", "", "", "'='", "'?'", "'!~'",
"'=~'", "'FOR'", "'RETURN'", "'DISTINCT'", "'FILTER'", "'SORT'", "'LIMIT'",
"'LET'", "'COLLECT'", "", "'NONE'", "'NULL'", "", "'INTO'", "'KEEP'", "'WITH'",
"'COUNT'", "'ALL'", "'ANY'", "'AGGREGATE'", "'LIKE'", "", "'IN'", "'@'",
@ -272,7 +272,7 @@ var lexerSymbolicNames = []string{
"", "MultiLineComment", "SingleLineComment", "WhiteSpaces", "LineTerminator",
"Colon", "SemiColon", "Dot", "Comma", "OpenBracket", "CloseBracket", "OpenParen",
"CloseParen", "OpenBrace", "CloseBrace", "Gt", "Lt", "Eq", "Gte", "Lte",
"Neq", "Plus", "Minus", "MinusMinus", "PlusPlus", "Multi", "Div", "Mod",
"Neq", "Multi", "Div", "Mod", "Plus", "Minus", "MinusMinus", "PlusPlus",
"And", "Or", "Range", "Assign", "QuestionMark", "RegexNotMatch", "RegexMatch",
"For", "Return", "Distinct", "Filter", "Sort", "Limit", "Let", "Collect",
"SortDirection", "None", "Null", "BooleanLiteral", "Into", "Keep", "With",
@ -284,7 +284,7 @@ var lexerRuleNames = []string{
"MultiLineComment", "SingleLineComment", "WhiteSpaces", "LineTerminator",
"Colon", "SemiColon", "Dot", "Comma", "OpenBracket", "CloseBracket", "OpenParen",
"CloseParen", "OpenBrace", "CloseBrace", "Gt", "Lt", "Eq", "Gte", "Lte",
"Neq", "Plus", "Minus", "MinusMinus", "PlusPlus", "Multi", "Div", "Mod",
"Neq", "Multi", "Div", "Mod", "Plus", "Minus", "MinusMinus", "PlusPlus",
"And", "Or", "Range", "Assign", "QuestionMark", "RegexNotMatch", "RegexMatch",
"For", "Return", "Distinct", "Filter", "Sort", "Limit", "Let", "Collect",
"SortDirection", "None", "Null", "BooleanLiteral", "Into", "Keep", "With",
@ -349,13 +349,13 @@ const (
FqlLexerGte = 18
FqlLexerLte = 19
FqlLexerNeq = 20
FqlLexerPlus = 21
FqlLexerMinus = 22
FqlLexerMinusMinus = 23
FqlLexerPlusPlus = 24
FqlLexerMulti = 25
FqlLexerDiv = 26
FqlLexerMod = 27
FqlLexerMulti = 21
FqlLexerDiv = 22
FqlLexerMod = 23
FqlLexerPlus = 24
FqlLexerMinus = 25
FqlLexerMinusMinus = 26
FqlLexerPlusPlus = 27
FqlLexerAnd = 28
FqlLexerOr = 29
FqlLexerRange = 30

File diff suppressed because it is too large Load Diff

View File

@ -274,11 +274,11 @@ func (s *BaseFqlParserListener) EnterPropertyName(ctx *PropertyNameContext) {}
// ExitPropertyName is called when production propertyName is exited.
func (s *BaseFqlParserListener) ExitPropertyName(ctx *PropertyNameContext) {}
// EnterExpressionSequence is called when production expressionSequence is entered.
func (s *BaseFqlParserListener) EnterExpressionSequence(ctx *ExpressionSequenceContext) {}
// EnterExpressionGroup is called when production expressionGroup is entered.
func (s *BaseFqlParserListener) EnterExpressionGroup(ctx *ExpressionGroupContext) {}
// ExitExpressionSequence is called when production expressionSequence is exited.
func (s *BaseFqlParserListener) ExitExpressionSequence(ctx *ExpressionSequenceContext) {}
// ExitExpressionGroup is called when production expressionGroup is exited.
func (s *BaseFqlParserListener) ExitExpressionGroup(ctx *ExpressionGroupContext) {}
// EnterFunctionCallExpression is called when production functionCallExpression is entered.
func (s *BaseFqlParserListener) EnterFunctionCallExpression(ctx *FunctionCallExpressionContext) {}
@ -322,17 +322,29 @@ func (s *BaseFqlParserListener) EnterEqualityOperator(ctx *EqualityOperatorConte
// ExitEqualityOperator is called when production equalityOperator is exited.
func (s *BaseFqlParserListener) ExitEqualityOperator(ctx *EqualityOperatorContext) {}
// EnterLogicalOperator is called when production logicalOperator is entered.
func (s *BaseFqlParserListener) EnterLogicalOperator(ctx *LogicalOperatorContext) {}
// EnterLogicalAndOperator is called when production logicalAndOperator is entered.
func (s *BaseFqlParserListener) EnterLogicalAndOperator(ctx *LogicalAndOperatorContext) {}
// ExitLogicalOperator is called when production logicalOperator is exited.
func (s *BaseFqlParserListener) ExitLogicalOperator(ctx *LogicalOperatorContext) {}
// ExitLogicalAndOperator is called when production logicalAndOperator is exited.
func (s *BaseFqlParserListener) ExitLogicalAndOperator(ctx *LogicalAndOperatorContext) {}
// EnterMathOperator is called when production mathOperator is entered.
func (s *BaseFqlParserListener) EnterMathOperator(ctx *MathOperatorContext) {}
// EnterLogicalOrOperator is called when production logicalOrOperator is entered.
func (s *BaseFqlParserListener) EnterLogicalOrOperator(ctx *LogicalOrOperatorContext) {}
// ExitMathOperator is called when production mathOperator is exited.
func (s *BaseFqlParserListener) ExitMathOperator(ctx *MathOperatorContext) {}
// ExitLogicalOrOperator is called when production logicalOrOperator is exited.
func (s *BaseFqlParserListener) ExitLogicalOrOperator(ctx *LogicalOrOperatorContext) {}
// EnterMultiplicativeOperator is called when production multiplicativeOperator is entered.
func (s *BaseFqlParserListener) EnterMultiplicativeOperator(ctx *MultiplicativeOperatorContext) {}
// ExitMultiplicativeOperator is called when production multiplicativeOperator is exited.
func (s *BaseFqlParserListener) ExitMultiplicativeOperator(ctx *MultiplicativeOperatorContext) {}
// EnterAdditiveOperator is called when production additiveOperator is entered.
func (s *BaseFqlParserListener) EnterAdditiveOperator(ctx *AdditiveOperatorContext) {}
// ExitAdditiveOperator is called when production additiveOperator is exited.
func (s *BaseFqlParserListener) ExitAdditiveOperator(ctx *AdditiveOperatorContext) {}
// EnterUnaryOperator is called when production unaryOperator is entered.
func (s *BaseFqlParserListener) EnterUnaryOperator(ctx *UnaryOperatorContext) {}

View File

@ -175,7 +175,7 @@ func (v *BaseFqlParserVisitor) VisitPropertyName(ctx *PropertyNameContext) inter
return v.VisitChildren(ctx)
}
func (v *BaseFqlParserVisitor) VisitExpressionSequence(ctx *ExpressionSequenceContext) interface{} {
func (v *BaseFqlParserVisitor) VisitExpressionGroup(ctx *ExpressionGroupContext) interface{} {
return v.VisitChildren(ctx)
}
@ -207,11 +207,19 @@ func (v *BaseFqlParserVisitor) VisitEqualityOperator(ctx *EqualityOperatorContex
return v.VisitChildren(ctx)
}
func (v *BaseFqlParserVisitor) VisitLogicalOperator(ctx *LogicalOperatorContext) interface{} {
func (v *BaseFqlParserVisitor) VisitLogicalAndOperator(ctx *LogicalAndOperatorContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BaseFqlParserVisitor) VisitMathOperator(ctx *MathOperatorContext) interface{} {
func (v *BaseFqlParserVisitor) VisitLogicalOrOperator(ctx *LogicalOrOperatorContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BaseFqlParserVisitor) VisitMultiplicativeOperator(ctx *MultiplicativeOperatorContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BaseFqlParserVisitor) VisitAdditiveOperator(ctx *AdditiveOperatorContext) interface{} {
return v.VisitChildren(ctx)
}

View File

@ -133,8 +133,8 @@ type FqlParserListener interface {
// EnterPropertyName is called when entering the propertyName production.
EnterPropertyName(c *PropertyNameContext)
// EnterExpressionSequence is called when entering the expressionSequence production.
EnterExpressionSequence(c *ExpressionSequenceContext)
// EnterExpressionGroup is called when entering the expressionGroup production.
EnterExpressionGroup(c *ExpressionGroupContext)
// EnterFunctionCallExpression is called when entering the functionCallExpression production.
EnterFunctionCallExpression(c *FunctionCallExpressionContext)
@ -157,11 +157,17 @@ type FqlParserListener interface {
// EnterEqualityOperator is called when entering the equalityOperator production.
EnterEqualityOperator(c *EqualityOperatorContext)
// EnterLogicalOperator is called when entering the logicalOperator production.
EnterLogicalOperator(c *LogicalOperatorContext)
// EnterLogicalAndOperator is called when entering the logicalAndOperator production.
EnterLogicalAndOperator(c *LogicalAndOperatorContext)
// EnterMathOperator is called when entering the mathOperator production.
EnterMathOperator(c *MathOperatorContext)
// EnterLogicalOrOperator is called when entering the logicalOrOperator production.
EnterLogicalOrOperator(c *LogicalOrOperatorContext)
// EnterMultiplicativeOperator is called when entering the multiplicativeOperator production.
EnterMultiplicativeOperator(c *MultiplicativeOperatorContext)
// EnterAdditiveOperator is called when entering the additiveOperator production.
EnterAdditiveOperator(c *AdditiveOperatorContext)
// EnterUnaryOperator is called when entering the unaryOperator production.
EnterUnaryOperator(c *UnaryOperatorContext)
@ -292,8 +298,8 @@ type FqlParserListener interface {
// ExitPropertyName is called when exiting the propertyName production.
ExitPropertyName(c *PropertyNameContext)
// ExitExpressionSequence is called when exiting the expressionSequence production.
ExitExpressionSequence(c *ExpressionSequenceContext)
// ExitExpressionGroup is called when exiting the expressionGroup production.
ExitExpressionGroup(c *ExpressionGroupContext)
// ExitFunctionCallExpression is called when exiting the functionCallExpression production.
ExitFunctionCallExpression(c *FunctionCallExpressionContext)
@ -316,11 +322,17 @@ type FqlParserListener interface {
// ExitEqualityOperator is called when exiting the equalityOperator production.
ExitEqualityOperator(c *EqualityOperatorContext)
// ExitLogicalOperator is called when exiting the logicalOperator production.
ExitLogicalOperator(c *LogicalOperatorContext)
// ExitLogicalAndOperator is called when exiting the logicalAndOperator production.
ExitLogicalAndOperator(c *LogicalAndOperatorContext)
// ExitMathOperator is called when exiting the mathOperator production.
ExitMathOperator(c *MathOperatorContext)
// ExitLogicalOrOperator is called when exiting the logicalOrOperator production.
ExitLogicalOrOperator(c *LogicalOrOperatorContext)
// ExitMultiplicativeOperator is called when exiting the multiplicativeOperator production.
ExitMultiplicativeOperator(c *MultiplicativeOperatorContext)
// ExitAdditiveOperator is called when exiting the additiveOperator production.
ExitAdditiveOperator(c *AdditiveOperatorContext)
// ExitUnaryOperator is called when exiting the unaryOperator production.
ExitUnaryOperator(c *UnaryOperatorContext)

View File

@ -133,8 +133,8 @@ type FqlParserVisitor interface {
// Visit a parse tree produced by FqlParser#propertyName.
VisitPropertyName(ctx *PropertyNameContext) interface{}
// Visit a parse tree produced by FqlParser#expressionSequence.
VisitExpressionSequence(ctx *ExpressionSequenceContext) interface{}
// Visit a parse tree produced by FqlParser#expressionGroup.
VisitExpressionGroup(ctx *ExpressionGroupContext) interface{}
// Visit a parse tree produced by FqlParser#functionCallExpression.
VisitFunctionCallExpression(ctx *FunctionCallExpressionContext) interface{}
@ -157,11 +157,17 @@ type FqlParserVisitor interface {
// Visit a parse tree produced by FqlParser#equalityOperator.
VisitEqualityOperator(ctx *EqualityOperatorContext) interface{}
// Visit a parse tree produced by FqlParser#logicalOperator.
VisitLogicalOperator(ctx *LogicalOperatorContext) interface{}
// Visit a parse tree produced by FqlParser#logicalAndOperator.
VisitLogicalAndOperator(ctx *LogicalAndOperatorContext) interface{}
// Visit a parse tree produced by FqlParser#mathOperator.
VisitMathOperator(ctx *MathOperatorContext) interface{}
// Visit a parse tree produced by FqlParser#logicalOrOperator.
VisitLogicalOrOperator(ctx *LogicalOrOperatorContext) interface{}
// Visit a parse tree produced by FqlParser#multiplicativeOperator.
VisitMultiplicativeOperator(ctx *MultiplicativeOperatorContext) interface{}
// Visit a parse tree produced by FqlParser#additiveOperator.
VisitAdditiveOperator(ctx *AdditiveOperatorContext) interface{}
// Visit a parse tree produced by FqlParser#unaryOperator.
VisitUnaryOperator(ctx *UnaryOperatorContext) interface{}

View File

@ -47,12 +47,24 @@ func (s *RootScope) Close() error {
s.closed = true
var errors []error
// close all values implemented io.Close
for _, c := range s.disposables {
c.Close()
if err := c.Close(); err != nil {
if errors == nil {
errors = make([]error, 0, len(s.disposables))
}
errors = append(errors, err)
}
}
return nil
if errors == nil {
return nil
}
return Errors(errors...)
}
func newScope(root *RootScope, parent *Scope) *Scope {

View File

@ -55,6 +55,20 @@ type Value interface {
Copy() Value
}
// Getter represents an interface of
// complex types that needs to be used to read values by path.
// The interface is created to let user-defined types be used in dot notation data access.
type Getter interface {
GetIn(path []Value) (Value, error)
}
// Setter represents an interface of
// complex types that needs to be used to write values by path.
// The interface is created to let user-defined types be used in dot notation assignment.
type Setter interface {
SetIn(path []Value, value Value) error
}
// IsTypeOf return true when value's type
// is equal to check type.
// Returns false, otherwise.

View File

@ -4,7 +4,7 @@ import (
"testing"
"time"
"github.com/MontFerret/ferret/pkg/html/static"
"github.com/MontFerret/ferret/pkg/drivers/http"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
. "github.com/smartystreets/goconvey/convey"
@ -37,8 +37,8 @@ func TestIsTypeOf(t *testing.T) {
So(core.IsTypeOf(values.NewDateTime(time.Now()), core.DateTimeType), ShouldBeTrue)
So(core.IsTypeOf(values.NewArray(1), core.ArrayType), ShouldBeTrue)
So(core.IsTypeOf(values.NewObject(), core.ObjectType), ShouldBeTrue)
So(core.IsTypeOf(&static.HTMLElement{}, core.HTMLElementType), ShouldBeTrue)
So(core.IsTypeOf(&static.HTMLDocument{}, core.HTMLDocumentType), ShouldBeTrue)
So(core.IsTypeOf(&http.HTMLElement{}, core.HTMLElementType), ShouldBeTrue)
So(core.IsTypeOf(&http.HTMLDocument{}, core.HTMLDocumentType), ShouldBeTrue)
So(core.IsTypeOf(values.NewBinary([]byte{}), core.BinaryType), ShouldBeTrue)
})
}
@ -54,8 +54,8 @@ func TestValidateType(t *testing.T) {
So(core.ValidateType(values.NewDateTime(time.Now()), core.DateTimeType), ShouldBeNil)
So(core.ValidateType(values.NewArray(1), core.ArrayType), ShouldBeNil)
So(core.ValidateType(values.NewObject(), core.ObjectType), ShouldBeNil)
So(core.ValidateType(&static.HTMLElement{}, core.HTMLElementType), ShouldBeNil)
So(core.ValidateType(&static.HTMLDocument{}, core.HTMLDocumentType), ShouldBeNil)
So(core.ValidateType(&http.HTMLElement{}, core.HTMLElementType), ShouldBeNil)
So(core.ValidateType(&http.HTMLDocument{}, core.HTMLDocumentType), ShouldBeNil)
So(core.ValidateType(values.NewBinary([]byte{}), core.BinaryType), ShouldBeNil)
})
@ -69,8 +69,8 @@ func TestValidateType(t *testing.T) {
So(core.ValidateType(values.NewDateTime(time.Now()), core.BooleanType), ShouldBeError)
So(core.ValidateType(values.NewArray(1), core.StringType), ShouldBeError)
So(core.ValidateType(values.NewObject(), core.BooleanType), ShouldBeError)
So(core.ValidateType(&static.HTMLElement{}, core.ArrayType), ShouldBeError)
So(core.ValidateType(&static.HTMLDocument{}, core.HTMLElementType), ShouldBeError)
So(core.ValidateType(&http.HTMLElement{}, core.ArrayType), ShouldBeError)
So(core.ValidateType(&http.HTMLDocument{}, core.HTMLElementType), ShouldBeError)
So(core.ValidateType(values.NewBinary([]byte{}), core.NoneType), ShouldBeError)
})
}

View File

@ -28,23 +28,33 @@ func (exp *BlockExpression) Add(stmt core.Expression) {
}
func (exp *BlockExpression) Exec(ctx context.Context, scope *core.Scope) (core.Value, error) {
for _, stmt := range exp.statements {
_, err := stmt.Exec(ctx, scope)
select {
case <-ctx.Done():
return values.None, core.ErrTerminated
default:
for _, stmt := range exp.statements {
_, err := stmt.Exec(ctx, scope)
if err != nil {
return values.None, err
if err != nil {
return values.None, err
}
}
}
return values.None, nil
return values.None, nil
}
}
func (exp *BlockExpression) Iterate(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
iter, err := exp.values.Iterate(ctx, scope)
select {
case <-ctx.Done():
return nil, core.ErrTerminated
default:
iter, err := exp.values.Iterate(ctx, scope)
if err != nil {
return nil, err
if err != nil {
return nil, err
}
return collections.NewTapIterator(iter, exp)
}
return collections.NewTapIterator(iter, exp)
}

View File

@ -0,0 +1,158 @@
package expressions_test
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/collections"
"testing"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/expressions"
"github.com/MontFerret/ferret/pkg/runtime/values"
. "github.com/smartystreets/goconvey/convey"
)
type IterableFn func(ctx context.Context, scope *core.Scope) (collections.Iterator, error)
func (f IterableFn) Iterate(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
return f(ctx, scope)
}
type ExpressionFn func(ctx context.Context, scope *core.Scope) (core.Value, error)
func (f ExpressionFn) Exec(ctx context.Context, scope *core.Scope) (core.Value, error) {
return f(ctx, scope)
}
func TestBlock(t *testing.T) {
newExp := func(values []core.Value) (*expressions.BlockExpression, error) {
iter, err := collections.NewDefaultSliceIterator(values)
if err != nil {
return nil, err
}
return expressions.NewBlockExpression(IterableFn(func(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
return iter, nil
}))
}
Convey("Should create a block expression", t, func() {
s, err := newExp(make([]core.Value, 0, 10))
So(err, ShouldBeNil)
So(s, ShouldHaveSameTypeAs, &expressions.BlockExpression{})
})
Convey("Should add a new expression of a default type", t, func() {
s, _ := newExp(make([]core.Value, 0, 10))
sourceMap := core.NewSourceMap("test", 1, 1)
exp, err := expressions.NewVariableExpression(sourceMap, "testExp")
So(err, ShouldBeNil)
s.Add(exp)
})
Convey("Should exec a block expression", t, func() {
s, _ := newExp(make([]core.Value, 0, 10))
sourceMap := core.NewSourceMap("test", 1, 1)
exp, err := expressions.NewVariableDeclarationExpression(sourceMap, "test", ExpressionFn(func(ctx context.Context, scope *core.Scope) (core.Value, error) {
return values.NewString("value"), nil
}))
So(err, ShouldBeNil)
s.Add(exp)
rootScope, _ := core.NewRootScope()
scope := rootScope.Fork()
_, err = s.Exec(context.Background(), scope)
So(err, ShouldBeNil)
val, err := scope.GetVariable("test")
So(err, ShouldBeNil)
So(val, ShouldEqual, "value")
})
Convey("Should not exec a nil block expression", t, func() {
s, _ := newExp(make([]core.Value, 0, 10))
sourceMap := core.NewSourceMap("test", 1, 1)
exp, err := expressions.NewVariableExpression(sourceMap, "test")
So(err, ShouldBeNil)
s.Add(exp)
So(err, ShouldBeNil)
rootScope, fn := core.NewRootScope()
scope := rootScope.Fork()
scope.SetVariable("test", values.NewString("value"))
fn()
value, err := s.Exec(context.Background(), scope)
So(err, ShouldBeNil)
So(value, ShouldHaveSameTypeAs, values.None)
})
Convey("Should return an iterator", t, func() {
s, _ := newExp([]core.Value{
values.NewInt(1),
values.NewInt(2),
values.NewInt(3),
})
sourceMap := core.NewSourceMap("test", 1, 1)
exp, _ := expressions.NewVariableExpression(sourceMap, "test")
s.Add(exp)
rootScope, _ := core.NewRootScope()
scope := rootScope.Fork()
scope.SetVariable("test", values.NewString("value"))
iter, err := s.Iterate(context.Background(), scope)
So(err, ShouldBeNil)
items, err := collections.ToSlice(context.Background(), scope, iter)
So(err, ShouldBeNil)
So(items, ShouldHaveLength, 3)
})
Convey("Should stop an execution when context is cancelled", t, func() {
s, _ := newExp(make([]core.Value, 0, 10))
sourceMap := core.NewSourceMap("test", 1, 1)
exp, _ := expressions.NewVariableExpression(sourceMap, "test")
s.Add(exp)
rootScope, _ := core.NewRootScope()
scope := rootScope.Fork()
scope.SetVariable("test", values.NewString("value"))
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := s.Exec(ctx, scope)
So(err, ShouldEqual, core.ErrTerminated)
})
Convey("Should stop an execution when context is cancelled 2", t, func() {
s, _ := newExp([]core.Value{
values.NewInt(1),
values.NewInt(2),
values.NewInt(3),
})
sourceMap := core.NewSourceMap("test", 1, 1)
exp, _ := expressions.NewVariableExpression(sourceMap, "test")
s.Add(exp)
rootScope, _ := core.NewRootScope()
scope := rootScope.Fork()
scope.SetVariable("test", values.NewString("value"))
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := s.Iterate(ctx, scope)
So(err, ShouldEqual, core.ErrTerminated)
})
}

View File

@ -33,6 +33,12 @@ func (b *BodyExpression) Add(exp core.Expression) error {
}
func (b *BodyExpression) Exec(ctx context.Context, scope *core.Scope) (core.Value, error) {
select {
case <-ctx.Done():
return values.None, core.ErrTerminated
default:
}
for _, exp := range b.statements {
if _, err := exp.Exec(ctx, scope); err != nil {
return values.None, err

View File

@ -10,15 +10,13 @@ import (
. "github.com/smartystreets/goconvey/convey"
)
func TestNewBodyExpression(t *testing.T) {
func TestBody(t *testing.T) {
Convey("Should create a block expression", t, func() {
s := expressions.NewBodyExpression(1)
So(s, ShouldHaveSameTypeAs, &expressions.BodyExpression{})
})
}
func TestBlockExpressionAddVariableExpression(t *testing.T) {
Convey("Should add a new expression of a default type", t, func() {
s := expressions.NewBodyExpression(0)
@ -29,9 +27,7 @@ func TestBlockExpressionAddVariableExpression(t *testing.T) {
err = s.Add(exp)
So(err, ShouldBeNil)
})
}
func TestBlockExpressionAddReturnExpression(t *testing.T) {
Convey("Should add a new Return expression", t, func() {
s := expressions.NewBodyExpression(0)
@ -45,9 +41,7 @@ func TestBlockExpressionAddReturnExpression(t *testing.T) {
err = s.Add(exp)
So(err, ShouldBeNil)
})
}
func TestBlockExpressionAddReturnExpressionFailed(t *testing.T) {
Convey("Should not add an already defined Return expression", t, func() {
s := expressions.NewBodyExpression(0)
@ -65,9 +59,7 @@ func TestBlockExpressionAddReturnExpressionFailed(t *testing.T) {
So(err, ShouldBeError)
So(err.Error(), ShouldEqual, "invalid operation: return expression is already defined")
})
}
func TestBlockExpressionExec(t *testing.T) {
Convey("Should exec a block expression", t, func() {
s := expressions.NewBodyExpression(1)
@ -91,9 +83,7 @@ func TestBlockExpressionExec(t *testing.T) {
So(value, ShouldNotBeNil)
So(value, ShouldEqual, "value")
})
}
func TestBlockExpressionExecNonFound(t *testing.T) {
Convey("Should not found a missing statement", t, func() {
s := expressions.NewBodyExpression(1)
@ -117,9 +107,7 @@ func TestBlockExpressionExecNonFound(t *testing.T) {
So(err, ShouldHaveSameTypeAs, core.ErrNotFound)
So(value, ShouldHaveSameTypeAs, values.None)
})
}
func TestBlockExpressionExecNilExpression(t *testing.T) {
Convey("Should not exec a nil block expression", t, func() {
s := expressions.NewBodyExpression(1)
@ -139,4 +127,21 @@ func TestBlockExpressionExecNilExpression(t *testing.T) {
So(err, ShouldBeNil)
So(value, ShouldHaveSameTypeAs, values.None)
})
Convey("Should stop an execution when context is cancelled", t, func() {
s := expressions.NewBodyExpression(1)
sourceMap := core.NewSourceMap("test", 1, 1)
exp, _ := expressions.NewVariableExpression(sourceMap, "test")
s.Add(exp)
rootScope, _ := core.NewRootScope()
scope := rootScope.Fork()
scope.SetVariable("test", values.NewString("value"))
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := s.Exec(ctx, scope)
So(err, ShouldEqual, core.ErrTerminated)
})
}

View File

@ -33,43 +33,48 @@ func NewDataSource(
}
func (ds *DataSource) Iterate(ctx context.Context, scope *core.Scope) (collections.Iterator, error) {
data, err := ds.exp.Exec(ctx, scope)
if err != nil {
return nil, core.SourceError(ds.src, err)
}
switch data.Type() {
case core.ArrayType:
return collections.NewIndexedIterator(ds.valVariable, ds.keyVariable, data.(collections.IndexedCollection))
case core.ObjectType:
return collections.NewKeyedIterator(ds.valVariable, ds.keyVariable, data.(collections.KeyedCollection))
case core.HTMLElementType, core.HTMLDocumentType:
return collections.NewHTMLNodeIterator(ds.valVariable, ds.keyVariable, data.(values.HTMLNode))
select {
case <-ctx.Done():
return nil, core.ErrTerminated
default:
// fallback to user defined types
switch data.(type) {
case collections.IterableCollection:
collection := data.(collections.IterableCollection)
iterator, err := collection.Iterate(ctx)
data, err := ds.exp.Exec(ctx, scope)
if err != nil {
return nil, err
}
if err != nil {
return nil, core.SourceError(ds.src, err)
}
return collections.NewCollectionIterator(ds.valVariable, ds.keyVariable, iterator)
case collections.KeyedCollection:
switch data.Type() {
case core.ArrayType:
return collections.NewIndexedIterator(ds.valVariable, ds.keyVariable, data.(collections.IndexedCollection))
case collections.IndexedCollection:
case core.ObjectType:
return collections.NewKeyedIterator(ds.valVariable, ds.keyVariable, data.(collections.KeyedCollection))
case core.HTMLElementType, core.HTMLDocumentType:
return collections.NewHTMLNodeIterator(ds.valVariable, ds.keyVariable, data.(values.HTMLNode))
default:
return nil, core.TypeError(
data.Type(),
core.ArrayType,
core.ObjectType,
core.HTMLDocumentType,
core.HTMLElementType,
)
// fallback to user defined types
switch data.(type) {
case collections.IterableCollection:
collection := data.(collections.IterableCollection)
iterator, err := collection.Iterate(ctx)
if err != nil {
return nil, err
}
return collections.NewCollectionIterator(ds.valVariable, ds.keyVariable, iterator)
case collections.KeyedCollection:
return collections.NewIndexedIterator(ds.valVariable, ds.keyVariable, data.(collections.IndexedCollection))
case collections.IndexedCollection:
return collections.NewKeyedIterator(ds.valVariable, ds.keyVariable, data.(collections.KeyedCollection))
default:
return nil, core.TypeError(
data.Type(),
core.ArrayType,
core.ObjectType,
core.HTMLDocumentType,
core.HTMLElementType,
)
}
}
}
}

View File

@ -122,5 +122,40 @@ func TestDataSource(t *testing.T) {
So(pos, ShouldEqual, int(arr.Length()))
})
Convey("Should stop an execution when context is cancelled", func() {
arr := values.NewArrayWith(
values.NewInt(1),
values.NewInt(2),
values.NewInt(3),
values.NewInt(4),
values.NewInt(5),
values.NewInt(6),
values.NewInt(7),
values.NewInt(8),
values.NewInt(9),
values.NewInt(10),
)
ds, err := expressions.NewDataSource(
core.SourceMap{},
collections.DefaultValueVar,
collections.DefaultKeyVar,
TestDataSourceExpression(func(ctx context.Context, scope *core.Scope) (core.Value, error) {
return &testIterableCollection{arr}, nil
}),
)
So(err, ShouldBeNil)
rootScope, _ := core.NewRootScope()
scope := rootScope.Fork()
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err = ds.Iterate(ctx, scope)
So(err, ShouldEqual, core.ErrTerminated)
})
})
}

View File

@ -108,73 +108,78 @@ func (e *ForExpression) AddStatement(stmt core.Expression) error {
}
func (e *ForExpression) Exec(ctx context.Context, scope *core.Scope) (core.Value, error) {
iterator, err := e.dataSource.Iterate(ctx, scope)
if err != nil {
return values.None, err
}
// Hash map for a check for uniqueness
var hashTable map[uint64]bool
if e.distinct {
hashTable = make(map[uint64]bool)
}
res := values.NewArray(10)
for {
nextScope, err := iterator.Next(ctx, scope)
if err != nil {
return values.None, core.SourceError(e.src, err)
}
// no data anymore
if nextScope == nil {
break
}
out, err := e.predicate.Exec(ctx, nextScope)
select {
case <-ctx.Done():
return values.None, core.ErrTerminated
default:
iterator, err := e.dataSource.Iterate(ctx, scope)
if err != nil {
return values.None, err
}
var add bool
// Hash map for a check for uniqueness
var hashTable map[uint64]bool
// The result shouldn't be distinct
// Just add the output
if !e.distinct {
add = true
} else {
// We need to check whether the value already exists in the result set
hash := out.Hash()
_, exists := hashTable[hash]
if e.distinct {
hashTable = make(map[uint64]bool)
}
if !exists {
hashTable[hash] = true
res := values.NewArray(10)
for {
nextScope, err := iterator.Next(ctx, scope)
if err != nil {
return values.None, core.SourceError(e.src, err)
}
// no data anymore
if nextScope == nil {
break
}
out, err := e.predicate.Exec(ctx, nextScope)
if err != nil {
return values.None, err
}
var add bool
// The result shouldn't be distinct
// Just add the output
if !e.distinct {
add = true
}
}
if add {
if !e.spread {
res.Push(out)
} else {
elements, ok := out.(*values.Array)
// We need to check whether the value already exists in the result set
hash := out.Hash()
_, exists := hashTable[hash]
if !ok {
return values.None, core.Error(core.ErrInvalidOperation, "spread of non-array value")
if !exists {
hashTable[hash] = true
add = true
}
}
elements.ForEach(func(i core.Value, _ int) bool {
res.Push(i)
if add {
if !e.spread {
res.Push(out)
} else {
elements, ok := out.(*values.Array)
return true
})
if !ok {
return values.None, core.Error(core.ErrInvalidOperation, "spread of non-array value")
}
elements.ForEach(func(i core.Value, _ int) bool {
res.Push(i)
return true
})
}
}
}
}
return res, nil
return res, nil
}
}

View File

@ -33,30 +33,35 @@ func (e *FunctionCallExpression) Function() core.Function {
}
func (e *FunctionCallExpression) Exec(ctx context.Context, scope *core.Scope) (core.Value, error) {
var out core.Value
var err error
select {
case <-ctx.Done():
return values.None, core.ErrTerminated
default:
var out core.Value
var err error
if len(e.args) == 0 {
out, err = e.fun(ctx)
} else {
args := make([]core.Value, len(e.args))
if len(e.args) == 0 {
out, err = e.fun(ctx)
} else {
args := make([]core.Value, len(e.args))
for idx, arg := range e.args {
out, err := arg.Exec(ctx, scope)
for idx, arg := range e.args {
out, err := arg.Exec(ctx, scope)
if err != nil {
return values.None, core.SourceError(e.src, err)
if err != nil {
return values.None, core.SourceError(e.src, err)
}
args[idx] = out
}
args[idx] = out
out, err = e.fun(ctx, args...)
}
out, err = e.fun(ctx, args...)
}
if err != nil {
return values.None, core.SourceError(e.src, err)
}
if err != nil {
return values.None, core.SourceError(e.src, err)
return out, nil
}
return out, nil
}

View File

@ -0,0 +1,89 @@
package expressions_test
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/expressions"
"github.com/MontFerret/ferret/pkg/runtime/expressions/literals"
"github.com/MontFerret/ferret/pkg/runtime/values"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func TestFunctionCallExpression(t *testing.T) {
Convey(".Exec", t, func() {
Convey("Should execute an underlying function without arguments", func() {
f, err := expressions.NewFunctionCallExpression(
core.SourceMap{},
func(ctx context.Context, args ...core.Value) (value core.Value, e error) {
So(args, ShouldHaveLength, 0)
return values.True, nil
},
)
So(err, ShouldBeNil)
rootScope, _ := core.NewRootScope()
out, err := f.Exec(context.Background(), rootScope.Fork())
So(err, ShouldBeNil)
So(out, ShouldEqual, values.True)
})
Convey("Should execute an underlying function with arguments", func() {
args := []core.Expression{
literals.NewIntLiteral(1),
literals.NewStringLiteral("foo"),
}
f, err := expressions.NewFunctionCallExpression(
core.SourceMap{},
func(ctx context.Context, args ...core.Value) (value core.Value, e error) {
So(args, ShouldHaveLength, len(args))
return values.True, nil
},
args...,
)
So(err, ShouldBeNil)
rootScope, _ := core.NewRootScope()
out, err := f.Exec(context.Background(), rootScope.Fork())
So(err, ShouldBeNil)
So(out, ShouldEqual, values.True)
})
Convey("Should stop an execution when context is cancelled", func() {
args := []core.Expression{
literals.NewIntLiteral(1),
literals.NewStringLiteral("foo"),
}
f, err := expressions.NewFunctionCallExpression(
core.SourceMap{},
func(ctx context.Context, args ...core.Value) (value core.Value, e error) {
So(args, ShouldHaveLength, len(args))
return values.True, nil
},
args...,
)
So(err, ShouldBeNil)
rootScope, _ := core.NewRootScope()
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err = f.Exec(ctx, rootScope.Fork())
So(err, ShouldEqual, core.ErrTerminated)
})
})
}

View File

@ -8,8 +8,10 @@ import (
type (
MathOperatorType string
MathOperator struct {
MathOperator struct {
*baseOperator
opType MathOperatorType
fn OperatorFunc
leftOnly bool
}
@ -55,11 +57,16 @@ func NewMathOperator(
return &MathOperator{
&baseOperator{src, left, right},
operator,
fn,
leftOnly,
}, nil
}
func (operator *MathOperator) Type() MathOperatorType {
return operator.opType
}
func (operator *MathOperator) Exec(ctx context.Context, scope *core.Scope) (core.Value, error) {
left, err := operator.left.Exec(ctx, scope)

View File

@ -28,11 +28,16 @@ func NewReturnExpression(
}
func (e *ReturnExpression) Exec(ctx context.Context, scope *core.Scope) (core.Value, error) {
val, err := e.predicate.Exec(ctx, scope)
select {
case <-ctx.Done():
return values.None, core.ErrTerminated
default:
val, err := e.predicate.Exec(ctx, scope)
if err != nil {
return values.None, core.SourceError(e.src, err)
if err != nil {
return values.None, core.SourceError(e.src, err)
}
return val, nil
}
return val, nil
}

View File

@ -2,15 +2,16 @@ package expressions_test
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/expressions/literals"
"github.com/MontFerret/ferret/pkg/runtime/values"
"testing"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/expressions"
"github.com/MontFerret/ferret/pkg/runtime/values"
. "github.com/smartystreets/goconvey/convey"
)
func TestNewReturnExpression(t *testing.T) {
func TestReturnExpression(t *testing.T) {
Convey("Should create a return expression", t, func() {
sourceMap := core.NewSourceMap("test", 1, 10)
predicate, err := expressions.NewVariableExpression(sourceMap, "testExp")
@ -28,9 +29,7 @@ func TestNewReturnExpression(t *testing.T) {
So(err, ShouldBeError)
So(exp, ShouldBeNil)
})
}
func TestReturnExpressionExec(t *testing.T) {
Convey("Should exec a return expression with an existing predicate", t, func() {
sourceMap := core.NewSourceMap("test", 1, 1)
predicate, err := expressions.NewVariableExpression(sourceMap, "test")
@ -68,4 +67,21 @@ func TestReturnExpressionExec(t *testing.T) {
So(err, ShouldHaveSameTypeAs, core.ErrNotFound)
So(value, ShouldHaveSameTypeAs, values.None)
})
Convey("Should stop an execution when context is cancelled", t, func() {
sourceMap := core.NewSourceMap("test", 1, 1)
predicate := literals.NewIntLiteral(1)
exp, err := expressions.NewReturnExpression(sourceMap, predicate)
So(err, ShouldBeNil)
rootScope, _ := core.NewRootScope()
scope := rootScope.Fork()
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err = exp.Exec(ctx, scope)
So(err, ShouldEqual, core.ErrTerminated)
})
}

View File

@ -61,7 +61,14 @@ func (p *Program) Run(ctx context.Context, setters ...Option) (result []byte, er
}()
scope, closeFn := core.NewRootScope()
defer closeFn()
defer func() {
if err := closeFn(); err != nil {
logger.Error().
Timestamp().
Err(err).
Msg("Closing root scope")
}
}()
out, err := p.body.Exec(ctx, scope)

View File

@ -8,6 +8,11 @@ import (
"testing"
)
type Result struct {
Value []byte
Error error
}
func TestProgram(t *testing.T) {
Convey("Should recover from panic", t, func() {
c := compiler.New()
@ -22,4 +27,28 @@ func TestProgram(t *testing.T) {
So(err, ShouldBeError)
So(err.Error(), ShouldEqual, "test")
})
Convey("Should stop an execution when context is cancelled", t, func() {
c := compiler.New()
p := c.MustCompile(`WAIT(1000) RETURN TRUE`)
out := make(chan Result)
ctx, cancel := context.WithCancel(context.Background())
go func() {
v, err := p.Run(ctx)
out <- Result{
Value: v,
Error: err,
}
}()
cancel()
o := <-out
So(o.Error, ShouldEqual, core.ErrTerminated)
})
}

View File

@ -9,37 +9,35 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/core"
)
type Binary struct {
values []byte
type Binary []byte
func NewBinary(values []byte) Binary {
return Binary(values)
}
func NewBinary(values []byte) *Binary {
return &Binary{values}
}
func NewBinaryFrom(stream io.Reader) (*Binary, error) {
func NewBinaryFrom(stream io.Reader) (Binary, error) {
values, err := ioutil.ReadAll(stream)
if err != nil {
return nil, err
}
return &Binary{values}, nil
return Binary(values), nil
}
func (b *Binary) MarshalJSON() ([]byte, error) {
return json.Marshal(b.values)
func (b Binary) MarshalJSON() ([]byte, error) {
return json.Marshal([]byte(b))
}
func (b *Binary) Type() core.Type {
func (b Binary) Type() core.Type {
return core.BinaryType
}
func (b *Binary) String() string {
return string(b.values)
func (b Binary) String() string {
return string(b)
}
func (b *Binary) Compare(other core.Value) int {
func (b Binary) Compare(other core.Value) int {
// TODO: Lame comparison, need to think more about it
switch other.Type() {
case core.BooleanType:
@ -59,28 +57,28 @@ func (b *Binary) Compare(other core.Value) int {
}
}
func (b *Binary) Unwrap() interface{} {
return b.values
func (b Binary) Unwrap() interface{} {
return []byte(b)
}
func (b *Binary) Hash() uint64 {
func (b Binary) Hash() uint64 {
h := fnv.New64a()
h.Write([]byte(b.Type().String()))
h.Write([]byte(":"))
h.Write(b.values)
h.Write(b)
return h.Sum64()
}
func (b *Binary) Copy() core.Value {
c := make([]byte, len(b.values))
func (b Binary) Copy() core.Value {
c := make([]byte, len(b))
copy(c, b.values)
copy(c, b)
return NewBinary(c)
}
func (b *Binary) Length() Int {
return NewInt(len(b.values))
func (b Binary) Length() Int {
return NewInt(len(b))
}

View File

@ -19,7 +19,7 @@ func GetIn(from core.Value, byPath []core.Value) (core.Value, error) {
var result = from
var err error
for _, segment := range byPath {
for i, segment := range byPath {
if result == None || result == nil {
break
}
@ -92,6 +92,12 @@ func GetIn(from core.Value, byPath []core.Value) (core.Value, error) {
}
default:
getter, ok := result.(core.Getter)
if ok {
return getter.GetIn(byPath[i:])
}
return None, core.TypeError(
from.Type(),
core.ArrayType,
@ -144,11 +150,19 @@ func SetIn(to core.Value, byPath []core.Value, value core.Value) error {
if isTarget == false {
current = parent.Get(segment.(Int))
} else {
parent.Set(segment.(Int), value)
if err := parent.Set(segment.(Int), value); err != nil {
return err
}
}
break
default:
setter, ok := parent.(core.Setter)
if ok {
return setter.SetIn(byPath[idx:0], value)
}
// redefine parent
isArray := segmentType == core.IntType
@ -169,12 +183,16 @@ func SetIn(to core.Value, byPath []core.Value, value core.Value) error {
parent = arr
if isTarget {
arr.Set(segment.(Int), value)
if err := arr.Set(segment.(Int), value); err != nil {
return err
}
}
}
// set new parent
SetIn(to, byPath[0:idx-1], parent)
if err := SetIn(to, byPath[0:idx-1], parent); err != nil {
return err
}
if isTarget == false {
current = None

View File

@ -1,8 +1,64 @@
package values
import "github.com/MontFerret/ferret/pkg/runtime/core"
import (
"github.com/MontFerret/ferret/pkg/runtime/core"
"io"
)
const (
HTMLScreenshotFormatPNG HTMLScreenshotFormat = "png"
HTMLScreenshotFormatJPEG HTMLScreenshotFormat = "jpeg"
)
type (
// HTMLPDFParams represents the arguments for PrintToPDF function.
HTMLPDFParams struct {
// Paper orientation. Defaults to false.
Landscape Boolean
// Display header and footer. Defaults to false.
DisplayHeaderFooter Boolean
// Print background graphics. Defaults to false.
PrintBackground Boolean
// Scale of the webpage rendering. Defaults to 1.
Scale Float
// Paper width in inches. Defaults to 8.5 inches.
PaperWidth Float
// Paper height in inches. Defaults to 11 inches.
PaperHeight Float
// Top margin in inches. Defaults to 1cm (~0.4 inches).
MarginTop Float
// Bottom margin in inches. Defaults to 1cm (~0.4 inches).
MarginBottom Float
// Left margin in inches. Defaults to 1cm (~0.4 inches).
MarginLeft Float
// Right margin in inches. Defaults to 1cm (~0.4 inches).
MarginRight Float
// Paper ranges to print, e.g., '1-5, 8, 11-13'. Defaults to the empty string, which means print all pages.
PageRanges String
// Whether to silently ignore invalid but successfully parsed page ranges, such as '3-2'. Defaults to false.
IgnoreInvalidPageRanges Boolean
// HTML template for the print header. Should be valid HTML markup with following classes used to inject printing values into them: - `date`: formatted print date - `title`: document title - `url`: document location - `pageNumber`: current page number - `totalPages`: total pages in the document
// For example, `<span class=title></span>` would generate span containing the title.
HeaderTemplate String
// HTML template for the print footer. Should use the same format as the `headerTemplate`.
FooterTemplate String
// Whether or not to prefer page size as defined by css.
// Defaults to false, in which case the content will be scaled to fit the paper size.
PreferCSSPageSize Boolean
}
HTMLScreenshotFormat string
HTMLScreenshotParams struct {
X Float
Y Float
Width Float
Height Float
Format HTMLScreenshotFormat
Quality Int
}
// HTMLNode is a HTML Node
HTMLNode interface {
core.Value
@ -39,11 +95,97 @@ type (
InnerTextBySelectorAll(selector String) *Array
CountBySelector(selector String) Int
ExistsBySelector(selector String) Boolean
}
DHTMLNode interface {
HTMLNode
io.Closer
Click() (Boolean, error)
Input(value core.Value, delay Int) error
Select(value *Array) (*Array, error)
ScrollIntoView() error
Hover() error
WaitForClass(class String, timeout Int) error
}
// HTMLDocument is a HTML Document
HTMLDocument interface {
HTMLNode
URL() core.Value
}
// DHTMLDocument is a Dynamic HTML Document
DHTMLDocument interface {
HTMLDocument
io.Closer
Navigate(url String, timeout Int) error
NavigateBack(skip Int, timeout Int) (Boolean, error)
NavigateForward(skip Int, timeout Int) (Boolean, error)
ClickBySelector(selector String) (Boolean, error)
ClickBySelectorAll(selector String) (Boolean, error)
InputBySelector(selector String, value core.Value, delay Int) (Boolean, error)
SelectBySelector(selector String, value *Array) (*Array, error)
HoverBySelector(selector String) error
PrintToPDF(params HTMLPDFParams) (Binary, error)
CaptureScreenshot(params HTMLScreenshotParams) (Binary, error)
ScrollTop() error
ScrollBottom() error
ScrollBySelector(selector String) error
WaitForNavigation(timeout Int) error
WaitForSelector(selector String, timeout Int) error
WaitForClass(selector, class String, timeout Int) error
WaitForClassAll(selector, class String, timeout Int) error
}
)
func IsHTMLScreenshotFormatValid(format string) bool {
value := HTMLScreenshotFormat(format)
return value == HTMLScreenshotFormatPNG || value == HTMLScreenshotFormatJPEG
}
func NewDefaultHTMLPDFParams() HTMLPDFParams {
return HTMLPDFParams{
Landscape: False,
DisplayHeaderFooter: False,
PrintBackground: False,
Scale: Float(1),
PaperWidth: Float(8.5),
PaperHeight: Float(11),
MarginTop: Float(0.4),
MarginBottom: Float(0.4),
MarginLeft: Float(0.4),
MarginRight: Float(0.4),
PageRanges: EmptyString,
IgnoreInvalidPageRanges: False,
HeaderTemplate: EmptyString,
FooterTemplate: EmptyString,
PreferCSSPageSize: False,
}
}

View File

@ -3,7 +3,6 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -28,7 +27,7 @@ func Click(_ context.Context, args ...core.Value) (core.Value, error) {
return values.False, err
}
el, ok := arg1.(*dynamic.HTMLElement)
el, ok := arg1.(values.DHTMLNode)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
@ -47,7 +46,7 @@ func Click(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc, ok := arg1.(*dynamic.HTMLDocument)
doc, ok := arg1.(values.DHTMLDocument)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)

View File

@ -3,7 +3,6 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -28,7 +27,7 @@ func ClickAll(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc, ok := arg1.(*dynamic.HTMLDocument)
doc, ok := arg1.(values.DHTMLDocument)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)

View File

@ -2,27 +2,28 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"time"
)
type LoadDocumentArgs struct {
Dynamic bool
type DocumentLoadParams struct {
Dynamic values.Boolean
Timeout time.Duration
}
// Page loads a HTML document by a given url.
// Document loads a HTML document by a given url.
// By default, loads a document by http call - resulted document does not support any interactions.
// If passed "true" as a second argument, headless browser is used for loading the document which support interactions.
// @param url (String) - Target url string. If passed "about:blank" for dynamic document - it will open an empty page.
// @param dynamicOrTimeout (Boolean|Int, optional) - If boolean value is passed, it indicates whether to use dynamic document.
// If integer values is passed it sets a custom timeout.
// @param timeout (Int, optional) - Sets a custom timeout.
// @param isDynamicOrParams (Boolean|DocumentLoadParams) - Either a boolean value that indicates whether to use dynamic page
// or an object with the following properties :
// dynamic (Boolean) - Optional, indicates whether to use dynamic page.
// timeout (Int) - Optional, Document load timeout.
// @returns (HTMLDocument) - Returns loaded HTML document.
func Document(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 1, 3)
err := core.ValidateArgs(args, 1, 2)
if err != nil {
return values.None, err
@ -36,23 +37,35 @@ func Document(ctx context.Context, args ...core.Value) (core.Value, error) {
url := args[0].(values.String)
params, err := parseLoadDocumentArgs(args)
var params DocumentLoadParams
if err != nil {
return values.None, err
if len(args) == 1 {
params = newDefaultDocLoadParams()
} else {
p, err := newDocLoadParams(args[1])
if err != nil {
return values.None, err
}
params = p
}
var drv html.Driver
ctx, cancel := context.WithTimeout(ctx, params.Timeout)
defer cancel()
if params.Dynamic == false {
drv, err = html.FromContext(ctx, html.Static)
} else {
drv, err = html.FromContext(ctx, html.Dynamic)
if params.Dynamic {
drv, err := drivers.DynamicFrom(ctx)
if err != nil {
return values.None, err
}
return drv.GetDocument(ctx, url)
}
drv, err := drivers.StaticFrom(ctx)
if err != nil {
return values.None, err
}
@ -60,39 +73,46 @@ func Document(ctx context.Context, args ...core.Value) (core.Value, error) {
return drv.GetDocument(ctx, url)
}
func parseLoadDocumentArgs(args []core.Value) (LoadDocumentArgs, error) {
res := LoadDocumentArgs{
func newDefaultDocLoadParams() DocumentLoadParams {
return DocumentLoadParams{
Dynamic: false,
Timeout: time.Second * 30,
}
}
if len(args) == 3 {
err := core.ValidateType(args[1], core.BooleanType)
func newDocLoadParams(arg core.Value) (DocumentLoadParams, error) {
res := newDefaultDocLoadParams()
if err != nil {
if err := core.ValidateType(arg, core.BooleanType, core.ObjectType); err != nil {
return res, err
}
if arg.Type() == core.BooleanType {
res.Dynamic = arg.(values.Boolean)
return res, nil
}
obj := arg.(*values.Object)
isDynamic, exists := obj.Get(values.NewString("dynamic"))
if exists {
if err := core.ValidateType(isDynamic, core.BooleanType); err != nil {
return res, err
}
res.Dynamic = bool(args[1].(values.Boolean))
res.Dynamic = isDynamic.(values.Boolean)
}
err = core.ValidateType(args[2], core.IntType)
timeout, exists := obj.Get(values.NewString("timeout"))
if err != nil {
if exists {
if err := core.ValidateType(timeout, core.IntType); err != nil {
return res, err
}
res.Timeout = time.Duration(args[2].(values.Int)) * time.Millisecond
} else if len(args) == 2 {
err := core.ValidateType(args[1], core.BooleanType, core.IntType)
if err != nil {
return res, err
}
if args[1].Type() == core.BooleanType {
res.Dynamic = bool(args[1].(values.Boolean))
} else {
res.Timeout = time.Duration(args[1].(values.Int)) * time.Millisecond
}
res.Timeout = time.Duration(timeout.(values.Int)) + time.Millisecond
}
return res, nil

View File

@ -0,0 +1,44 @@
package html
import (
"context"
"io/ioutil"
"net/http"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
// Download a resource from the given URL.
// @param URL (String) - URL to download.
// @returns data (Binary) - Returns a base64 encoded string in binary format.
func Download(_ context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 1, 1)
if err != nil {
return values.None, err
}
arg1 := args[0]
err = core.ValidateType(arg1, core.StringType)
if err != nil {
return values.None, err
}
resp, err := http.Get(arg1.String())
if err != nil {
return values.None, err
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return values.None, err
}
return values.NewBinary(data), nil
}

View File

@ -0,0 +1,22 @@
package html
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
// ElementExists returns a boolean value indicating whether there is an element matched by selector.
// @param docOrEl (HTMLDocument|HTMLElement) - Parent document or element.
// @param selector (String) - CSS selector.
// @returns (Boolean) - A boolean value indicating whether there is an element matched by selector.
func ElementExists(_ context.Context, args ...core.Value) (core.Value, error) {
el, selector, err := queryArgs(args)
if err != nil {
return values.None, err
}
return el.ExistsBySelector(selector), nil
}

View File

@ -2,7 +2,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -33,7 +33,7 @@ func Hover(_ context.Context, args ...core.Value) (core.Value, error) {
}
// Document with a selector
doc, ok := args[0].(*dynamic.HTMLDocument)
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)
@ -45,7 +45,7 @@ func Hover(_ context.Context, args ...core.Value) (core.Value, error) {
}
// Element
el, ok := args[0].(*dynamic.HTMLElement)
el, ok := args[0].(values.DHTMLNode)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)

View File

@ -3,7 +3,6 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -29,8 +28,8 @@ func Input(_ context.Context, args ...core.Value) (core.Value, error) {
}
switch args[0].(type) {
case *dynamic.HTMLDocument:
doc, ok := arg1.(*dynamic.HTMLDocument)
case values.DHTMLDocument:
doc, ok := arg1.(values.DHTMLDocument)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
@ -50,6 +49,7 @@ func Input(_ context.Context, args ...core.Value) (core.Value, error) {
arg4 := args[3]
err = core.ValidateType(arg4, core.IntType)
if err != nil {
return values.False, err
}
@ -58,8 +58,8 @@ func Input(_ context.Context, args ...core.Value) (core.Value, error) {
}
return doc.InputBySelector(arg2.(values.String), args[2], delay)
case *dynamic.HTMLElement:
el, ok := arg1.(*dynamic.HTMLElement)
case values.DHTMLNode:
el, ok := arg1.(values.DHTMLNode)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
@ -71,6 +71,7 @@ func Input(_ context.Context, args ...core.Value) (core.Value, error) {
arg3 := args[2]
err = core.ValidateType(arg3, core.IntType)
if err != nil {
return values.False, err
}

View File

@ -1,7 +1,10 @@
package html
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/pkg/errors"
)
@ -16,12 +19,13 @@ func NewLib() map[string]core.Function {
"CLICK": Click,
"CLICK_ALL": ClickAll,
"DOCUMENT": Document,
"DOCUMENT_PARSE": DocumentParse,
"DOWNLOAD": Download,
"ELEMENT": Element,
"ELEMENT_EXISTS": ElementExists,
"ELEMENTS": Elements,
"ELEMENTS_COUNT": ElementsCount,
"HOVER": Hover,
"HTML_PARSE": Parse,
"INNER_HTML": InnerHTML,
"INNER_HTML_ALL": InnerHTMLAll,
"INNER_TEXT": InnerText,
@ -43,3 +47,31 @@ func NewLib() map[string]core.Function {
"WAIT_NAVIGATION": WaitNavigation,
}
}
func ValidateDocument(ctx context.Context, value core.Value) (core.Value, error) {
err := core.ValidateType(value, core.HTMLDocumentType, core.StringType)
if err != nil {
return values.None, err
}
var doc values.DHTMLDocument
var ok bool
if value.Type() == core.StringType {
buf, err := Document(ctx, value, values.NewBoolean(true))
if err != nil {
return values.None, err
}
doc, ok = buf.(values.DHTMLDocument)
} else {
doc, ok = value.(values.DHTMLDocument)
}
if !ok {
return nil, ErrNotDynamic
}
return doc, nil
}

View File

@ -3,7 +3,6 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -33,7 +32,7 @@ func Navigate(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc, ok := args[0].(*dynamic.HTMLDocument)
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)

View File

@ -3,7 +3,6 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -28,7 +27,7 @@ func NavigateBack(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc, ok := args[0].(*dynamic.HTMLDocument)
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)

View File

@ -3,7 +3,6 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -28,7 +27,7 @@ func NavigateForward(_ context.Context, args ...core.Value) (core.Value, error)
return values.None, err
}
doc, ok := args[0].(*dynamic.HTMLDocument)
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)

View File

@ -2,7 +2,6 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
@ -20,7 +19,7 @@ func Pagination(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc, ok := args[0].(*dynamic.HTMLDocument)
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
@ -39,12 +38,12 @@ func Pagination(_ context.Context, args ...core.Value) (core.Value, error) {
type (
Paging struct {
document *dynamic.HTMLDocument
document values.DHTMLDocument
selector values.String
}
PagingIterator struct {
document *dynamic.HTMLDocument
document values.DHTMLDocument
selector values.String
pos values.Int
}

View File

@ -3,17 +3,16 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html"
"github.com/MontFerret/ferret/pkg/html/static"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
// DocumentParse parses a given HTML string and returns a HTML document.
// Parse parses a given HTML string and returns a HTML document.
// Returned HTML document is always static.
// @param html (String) - Target HTML string.
// @returns (HTMLDocument) - Parsed HTML static document.
func DocumentParse(ctx context.Context, args ...core.Value) (core.Value, error) {
func Parse(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 1, 1)
if err != nil {
@ -26,7 +25,7 @@ func DocumentParse(ctx context.Context, args ...core.Value) (core.Value, error)
return values.None, err
}
drv, err := html.FromContext(ctx, html.Static)
drv, err := drivers.StaticFrom(ctx)
if err != nil {
return values.None, err
@ -34,5 +33,5 @@ func DocumentParse(ctx context.Context, args ...core.Value) (core.Value, error)
str := args[0].(values.String)
return drv.(*static.Driver).ParseDocument(ctx, str)
return drv.ParseDocument(ctx, str)
}

View File

@ -3,172 +3,19 @@ package html
import (
"context"
"fmt"
"io/ioutil"
"net/http"
"regexp"
"github.com/mafredri/cdp/protocol/page"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
func ValidateDocument(ctx context.Context, value core.Value) (core.Value, error) {
err := core.ValidateType(value, core.HTMLDocumentType, core.StringType)
if err != nil {
return values.None, err
}
var doc *dynamic.HTMLDocument
var ok bool
if value.Type() == core.StringType {
buf, err := Document(ctx, value, values.NewBoolean(true))
if err != nil {
return values.None, err
}
doc, ok = buf.(*dynamic.HTMLDocument)
} else {
doc, ok = value.(*dynamic.HTMLDocument)
}
if !ok {
return nil, core.Error(core.ErrInvalidType, "expected dynamic document")
}
return doc, nil
}
// Screenshot take a screenshot of the current page.
// @param source (Document) - Document.
// @param params (Object) - Optional, An object containing the following properties :
// x (Float|Int) - Optional, X position of the viewport.
// x (Float|Int) - Optional,Y position of the viewport.
// width (Float|Int) - Optional, Width of the viewport.
// height (Float|Int) - Optional, Height of the viewport.
// format (String) - Optional, Either "jpeg" or "png".
// quality (Int) - Optional, Quality, in [0, 100], only for jpeg format.
// @returns data (Binary) - Returns a base64 encoded string in binary format.
func Screenshot(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 1, 2)
if err != nil {
return values.None, err
}
arg1 := args[0]
err = core.ValidateType(arg1, core.HTMLDocumentType, core.StringType)
if err != nil {
return values.None, err
}
val, err := ValidateDocument(ctx, arg1)
if err != nil {
return values.None, err
}
doc := val.(*dynamic.HTMLDocument)
defer doc.Close()
screenshotParams := &dynamic.ScreenshotArgs{
X: 0,
Y: 0,
Width: -1,
Height: -1,
Format: "jpeg",
Quality: 100,
}
if len(args) == 2 {
arg2 := args[1]
err = core.ValidateType(arg2, core.ObjectType)
if err != nil {
return values.None, err
}
params, ok := arg2.(*values.Object)
if !ok {
return values.None, core.Error(core.ErrInvalidType, "expected object")
}
format, found := params.Get("format")
if found {
err = core.ValidateType(format, core.StringType)
if err != nil {
return values.None, err
}
if !dynamic.IsScreenshotFormatValid(format.String()) {
return values.None, core.Error(
core.ErrInvalidArgument,
fmt.Sprintf("format is not valid, expected jpeg or png, but got %s", format.String()))
}
screenshotParams.Format = dynamic.ScreenshotFormat(format.String())
}
x, found := params.Get("x")
if found {
err = core.ValidateType(x, core.FloatType, core.IntType)
if err != nil {
return values.None, err
}
if x.Type() == core.IntType {
x = values.Float(x.(values.Int))
}
screenshotParams.X = x.Unwrap().(float64)
}
y, found := params.Get("y")
if found {
err = core.ValidateType(y, core.FloatType, core.IntType)
if err != nil {
return values.None, err
}
if y.Type() == core.IntType {
y = values.Float(y.(values.Int))
}
screenshotParams.Y = y.Unwrap().(float64)
}
width, found := params.Get("width")
if found {
err = core.ValidateType(width, core.FloatType, core.IntType)
if err != nil {
return values.None, err
}
if width.Type() == core.IntType {
width = values.Float(width.(values.Int))
}
screenshotParams.Width = width.Unwrap().(float64)
}
height, found := params.Get("height")
if found {
err = core.ValidateType(height, core.FloatType, core.IntType)
if err != nil {
return values.None, err
}
if height.Type() == core.IntType {
height = values.Float(height.(values.Int))
}
screenshotParams.Height = height.Unwrap().(float64)
}
quality, found := params.Get("quality")
if found {
err = core.ValidateType(quality, core.IntType)
if err != nil {
return values.None, err
}
screenshotParams.Quality = quality.Unwrap().(int)
}
}
scr, err := doc.CaptureScreenshot(screenshotParams)
if err != nil {
return values.None, err
}
return scr, nil
}
func ValidatePageRanges(pageRanges string) (bool, error) {
match, err := regexp.Match(`^(([1-9][0-9]*|[1-9][0-9]*)(\s*-\s*|\s*,\s*|))*$`, []byte(pageRanges))
if err != nil {
return false, err
}
return match, nil
}
@ -193,211 +40,261 @@ func ValidatePageRanges(pageRanges string) (bool, error) {
// @returns data (Binary) - Returns a base64 encoded string in binary format.
func PDF(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 1, 2)
if err != nil {
return values.None, err
}
arg1 := args[0]
val, err := ValidateDocument(ctx, arg1)
if err != nil {
return values.None, err
}
doc := val.(*dynamic.HTMLDocument)
doc := val.(values.DHTMLDocument)
defer doc.Close()
pdfParams := page.NewPrintToPDFArgs()
pdfParams := values.HTMLPDFParams{}
if len(args) == 2 {
arg2 := args[1]
err = core.ValidateType(arg2, core.ObjectType)
if err != nil {
return values.None, err
}
params, ok := arg2.(*values.Object)
if !ok {
return values.None, core.Error(core.ErrInvalidType, "expected object")
}
landscape, found := params.Get("landscape")
if found {
err = core.ValidateType(landscape, core.BooleanType)
if err != nil {
return values.None, err
}
pdfParams.SetLandscape(landscape.Unwrap().(bool))
pdfParams.Landscape = landscape.(values.Boolean)
}
displayHeaderFooter, found := params.Get("displayHeaderFooter")
if found {
err = core.ValidateType(displayHeaderFooter, core.BooleanType)
if err != nil {
return values.None, err
}
pdfParams.SetDisplayHeaderFooter(displayHeaderFooter.Unwrap().(bool))
pdfParams.DisplayHeaderFooter = displayHeaderFooter.(values.Boolean)
}
printBackground, found := params.Get("printBackground")
if found {
err = core.ValidateType(printBackground, core.BooleanType)
if err != nil {
return values.None, err
}
pdfParams.SetPrintBackground(printBackground.Unwrap().(bool))
pdfParams.PrintBackground = printBackground.(values.Boolean)
}
scale, found := params.Get("scale")
if found {
err = core.ValidateType(scale, core.FloatType, core.IntType)
if err != nil {
return values.None, err
}
if scale.Type() == core.IntType {
scale = values.Float(scale.(values.Int))
pdfParams.Scale = values.Float(scale.(values.Int))
} else {
pdfParams.Scale = scale.(values.Float)
}
pdfParams.SetScale(scale.Unwrap().(float64))
}
paperWidth, found := params.Get("paperWidth")
if found {
err = core.ValidateType(paperWidth, core.FloatType, core.IntType)
if err != nil {
return values.None, err
}
if paperWidth.Type() == core.IntType {
paperWidth = values.Float(paperWidth.(values.Int))
pdfParams.PaperWidth = values.Float(paperWidth.(values.Int))
} else {
pdfParams.PaperWidth = paperWidth.(values.Float)
}
pdfParams.SetPaperWidth(paperWidth.Unwrap().(float64))
}
paperHeight, found := params.Get("paperHeight")
if found {
err = core.ValidateType(paperHeight, core.FloatType, core.IntType)
if err != nil {
return values.None, err
}
if paperHeight.Type() == core.IntType {
paperHeight = values.Float(paperHeight.(values.Int))
pdfParams.PaperHeight = values.Float(paperHeight.(values.Int))
} else {
pdfParams.PaperHeight = paperHeight.(values.Float)
}
pdfParams.SetPaperHeight(paperHeight.Unwrap().(float64))
}
marginTop, found := params.Get("marginTop")
if found {
err = core.ValidateType(marginTop, core.FloatType, core.IntType)
if err != nil {
return values.None, err
}
if marginTop.Type() == core.IntType {
marginTop = values.Float(marginTop.(values.Int))
pdfParams.MarginTop = values.Float(marginTop.(values.Int))
} else {
pdfParams.MarginTop = marginTop.(values.Float)
}
pdfParams.SetMarginTop(marginTop.Unwrap().(float64))
}
marginBottom, found := params.Get("marginBottom")
if found {
err = core.ValidateType(marginBottom, core.FloatType, core.IntType)
if err != nil {
return values.None, err
}
if marginBottom.Type() == core.IntType {
marginBottom = values.Float(marginBottom.(values.Int))
pdfParams.MarginBottom = values.Float(marginBottom.(values.Int))
} else {
pdfParams.MarginBottom = marginBottom.(values.Float)
}
pdfParams.SetMarginBottom(marginBottom.Unwrap().(float64))
}
marginLeft, found := params.Get("marginLeft")
if found {
err = core.ValidateType(marginLeft, core.FloatType, core.IntType)
if err != nil {
return values.None, err
}
if marginLeft.Type() == core.IntType {
marginLeft = values.Float(marginLeft.(values.Int))
pdfParams.MarginLeft = values.Float(marginLeft.(values.Int))
} else {
pdfParams.MarginLeft = marginLeft.(values.Float)
}
pdfParams.SetMarginLeft(marginLeft.Unwrap().(float64))
}
marginRight, found := params.Get("marginRight")
if found {
err = core.ValidateType(marginRight, core.FloatType, core.IntType)
if err != nil {
return values.None, err
}
if marginRight.Type() == core.IntType {
marginRight = values.Float(marginRight.(values.Int))
pdfParams.MarginRight = values.Float(marginRight.(values.Int))
} else {
pdfParams.MarginRight = marginRight.(values.Float)
}
pdfParams.SetMarginRight(marginRight.Unwrap().(float64))
}
pageRanges, found := params.Get("pageRanges")
if found {
err = core.ValidateType(pageRanges, core.StringType)
if err != nil {
return values.None, err
}
validate, err := ValidatePageRanges(pageRanges.String())
if err != nil {
return values.None, err
}
if !validate {
return values.None, core.Error(core.ErrInvalidArgument, fmt.Sprintf(`page ranges "%s", not valid`, pageRanges.String()))
}
pdfParams.SetPageRanges(pageRanges.String())
pdfParams.PageRanges = pageRanges.(values.String)
}
ignoreInvalidPageRanges, found := params.Get("ignoreInvalidPageRanges")
if found {
err = core.ValidateType(ignoreInvalidPageRanges, core.BooleanType)
if err != nil {
return values.None, err
}
pdfParams.SetIgnoreInvalidPageRanges(ignoreInvalidPageRanges.Unwrap().(bool))
pdfParams.IgnoreInvalidPageRanges = ignoreInvalidPageRanges.(values.Boolean)
}
headerTemplate, found := params.Get("headerTemplate")
if found {
err = core.ValidateType(headerTemplate, core.StringType)
if err != nil {
return values.None, err
}
pdfParams.SetHeaderTemplate(headerTemplate.String())
pdfParams.HeaderTemplate = headerTemplate.(values.String)
}
footerTemplate, found := params.Get("footerTemplate")
if found {
err = core.ValidateType(footerTemplate, core.StringType)
if err != nil {
return values.None, err
}
pdfParams.SetFooterTemplate(footerTemplate.String())
pdfParams.FooterTemplate = footerTemplate.(values.String)
}
preferCSSPageSize, found := params.Get("preferCSSPageSize")
if found {
err = core.ValidateType(preferCSSPageSize, core.BooleanType)
if err != nil {
return values.None, err
}
pdfParams.SetPreferCSSPageSize(preferCSSPageSize.Unwrap().(bool))
pdfParams.PreferCSSPageSize = preferCSSPageSize.(values.Boolean)
}
}
pdf, err := doc.PrintToPDF(pdfParams)
if err != nil {
return values.None, err
}
return pdf, nil
}
// Download a ressource from the given URL.
// @param URL (String) - URL to download.
// @returns data (Binary) - Returns a base64 encoded string in binary format.
func Download(_ context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 1, 1)
if err != nil {
return values.None, err
}
arg1 := args[0]
err = core.ValidateType(arg1, core.StringType)
if err != nil {
return values.None, err
}
resp, err := http.Get(arg1.String())
if err != nil {
return values.None, err
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return values.None, err
}
return values.NewBinary(data), nil
}

View File

@ -0,0 +1,163 @@
package html
import (
"context"
"fmt"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
// Screenshot takes a screenshot of the current page.
// @param source (Document) - Document.
// @param params (Object) - Optional, An object containing the following properties :
// x (Float|Int) - Optional, X position of the viewport.
// x (Float|Int) - Optional,Y position of the viewport.
// width (Float|Int) - Optional, Width of the viewport.
// height (Float|Int) - Optional, Height of the viewport.
// format (String) - Optional, Either "jpeg" or "png".
// quality (Int) - Optional, Quality, in [0, 100], only for jpeg format.
// @returns data (Binary) - Returns a base64 encoded string in binary format.
func Screenshot(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 1, 2)
if err != nil {
return values.None, err
}
arg1 := args[0]
err = core.ValidateType(arg1, core.HTMLDocumentType, core.StringType)
if err != nil {
return values.None, err
}
val, err := ValidateDocument(ctx, arg1)
if err != nil {
return values.None, err
}
doc := val.(values.DHTMLDocument)
defer doc.Close()
screenshotParams := values.HTMLScreenshotParams{
X: 0,
Y: 0,
Width: -1,
Height: -1,
Format: values.HTMLScreenshotFormatJPEG,
Quality: 100,
}
if len(args) == 2 {
arg2 := args[1]
err = core.ValidateType(arg2, core.ObjectType)
if err != nil {
return values.None, err
}
params, ok := arg2.(*values.Object)
if !ok {
return values.None, core.Error(core.ErrInvalidType, "expected object")
}
format, found := params.Get("format")
if found {
err = core.ValidateType(format, core.StringType)
if err != nil {
return values.None, err
}
if !values.IsHTMLScreenshotFormatValid(format.String()) {
return values.None, core.Error(
core.ErrInvalidArgument,
fmt.Sprintf("format is not valid, expected jpeg or png, but got %s", format.String()))
}
screenshotParams.Format = values.HTMLScreenshotFormat(format.String())
}
x, found := params.Get("x")
if found {
err = core.ValidateType(x, core.FloatType, core.IntType)
if err != nil {
return values.None, err
}
if x.Type() == core.IntType {
screenshotParams.X = values.Float(x.(values.Int))
}
}
y, found := params.Get("y")
if found {
err = core.ValidateType(y, core.FloatType, core.IntType)
if err != nil {
return values.None, err
}
if y.Type() == core.IntType {
screenshotParams.Y = values.Float(y.(values.Int))
}
}
width, found := params.Get("width")
if found {
err = core.ValidateType(width, core.FloatType, core.IntType)
if err != nil {
return values.None, err
}
if width.Type() == core.IntType {
screenshotParams.Width = values.Float(width.(values.Int))
}
}
height, found := params.Get("height")
if found {
err = core.ValidateType(height, core.FloatType, core.IntType)
if err != nil {
return values.None, err
}
if height.Type() == core.IntType {
screenshotParams.Height = values.Float(height.(values.Int))
}
}
quality, found := params.Get("quality")
if found {
err = core.ValidateType(quality, core.IntType)
if err != nil {
return values.None, err
}
screenshotParams.Quality = quality.(values.Int)
}
}
scr, err := doc.CaptureScreenshot(screenshotParams)
if err != nil {
return values.None, err
}
return scr, nil
}

View File

@ -2,7 +2,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -22,7 +22,7 @@ func ScrollBottom(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc, ok := args[0].(*dynamic.HTMLDocument)
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)

View File

@ -2,7 +2,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -32,7 +32,7 @@ func ScrollInto(_ context.Context, args ...core.Value) (core.Value, error) {
}
// Document with a selector
doc, ok := args[0].(*dynamic.HTMLDocument)
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)
@ -44,7 +44,7 @@ func ScrollInto(_ context.Context, args ...core.Value) (core.Value, error) {
}
// Element
el, ok := args[0].(*dynamic.HTMLElement)
el, ok := args[0].(values.DHTMLNode)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)

View File

@ -2,7 +2,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -22,7 +22,7 @@ func ScrollTop(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc, ok := args[0].(*dynamic.HTMLDocument)
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)

View File

@ -2,7 +2,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -27,8 +27,8 @@ func Select(_ context.Context, args ...core.Value) (core.Value, error) {
}
switch args[0].(type) {
case *dynamic.HTMLDocument:
doc, ok := arg1.(*dynamic.HTMLDocument)
case values.DHTMLDocument:
doc, ok := arg1.(values.DHTMLDocument)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
@ -50,8 +50,8 @@ func Select(_ context.Context, args ...core.Value) (core.Value, error) {
}
return doc.SelectBySelector(arg2.(values.String), arg3.(*values.Array))
case *dynamic.HTMLElement:
el, ok := arg1.(*dynamic.HTMLElement)
case values.DHTMLNode:
el, ok := arg1.(values.DHTMLNode)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)

View File

@ -3,7 +3,6 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -42,7 +41,7 @@ func WaitClass(_ context.Context, args ...core.Value) (core.Value, error) {
// lets figure out what is passed as 1st argument
switch args[0].(type) {
case *dynamic.HTMLDocument:
case values.DHTMLDocument:
// revalidate args with more accurate amount
err := core.ValidateArgs(args, 3, 4)
@ -57,7 +56,7 @@ func WaitClass(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc, ok := args[0].(*dynamic.HTMLDocument)
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)
@ -77,8 +76,8 @@ func WaitClass(_ context.Context, args ...core.Value) (core.Value, error) {
}
return values.None, doc.WaitForClass(selector, class, timeout)
case *dynamic.HTMLElement:
el, ok := args[0].(*dynamic.HTMLElement)
case values.DHTMLNode:
el, ok := args[0].(values.DHTMLNode)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)

View File

@ -3,7 +3,6 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -42,7 +41,7 @@ func WaitClassAll(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc, ok := args[0].(*dynamic.HTMLDocument)
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)

View File

@ -3,7 +3,6 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -40,7 +39,7 @@ func WaitElement(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc, ok := arg.(*dynamic.HTMLDocument)
doc, ok := arg.(values.DHTMLDocument)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)

View File

@ -3,7 +3,6 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/html/dynamic"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
@ -25,7 +24,7 @@ func WaitNavigation(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc, ok := args[0].(*dynamic.HTMLDocument)
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)