go/ferret
go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2025-01-16 03:21:03 +02:00
Code Issues Releases Activity

Refactoring/externalized html (#234)

Browse Source 
* Externalized HTML drivers

* Fixed unit tests

* Updated logging

* Added support to set default driver

* Updated GetIn and SetIn helpers
This commit is contained in:
Tim Voronov 2019-02-19 18:10:18 -05:00 committed by GitHub
parent f8e061cc80
commit 34c8c02258
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
62 changed files with 1356 additions and 1054 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
cli/options.go
View File

@ -2,6 +2,7 @@ package cli
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp"
"github.com/MontFerret/ferret/pkg/drivers/http"
@ -16,9 +17,16 @@ type Options struct {
}
func (opts Options) WithContext(ctx context.Context) (context.Context, error) {
var err error
ctx = drivers.WithContext(
ctx,
http.NewDriver(
http.WithProxy(opts.Proxy),
http.WithUserAgent(opts.UserAgent),
),
drivers.AsDefault(),
)
ctx = drivers.WithDynamic(
ctx = drivers.WithContext(
ctx,
cdp.NewDriver(
cdp.WithAddress(opts.Cdp),
@ -27,17 +35,5 @@ func (opts Options) WithContext(ctx context.Context) (context.Context, error) {
),
)
if err != nil {
return ctx, err
}
ctx = drivers.WithStatic(
ctx,
http.NewDriver(
http.WithProxy(opts.Proxy),
http.WithUserAgent(opts.UserAgent),
),
)
return ctx, err
return ctx, nil
}

1
e2e/runner/lib.go
View File

@ -3,6 +3,7 @@ package runner
import (
"context"
"fmt"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)

23
e2e/runner/runner.go
View File

@ -3,6 +3,12 @@ package runner
import (
"context"
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"time"
"github.com/MontFerret/ferret/pkg/compiler"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp"
@ -10,11 +16,6 @@ import (
"github.com/MontFerret/ferret/pkg/runtime"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"time"
)
type (
@ -145,16 +146,22 @@ func (r *Runner) runQuery(c *compiler.FqlCompiler, name, script string) Result {
}
ctx := context.Background()
ctx = drivers.WithDynamic(
ctx = drivers.WithContext(
ctx,
cdp.NewDriver(cdp.WithAddress(r.settings.CDPAddress)),
)
ctx = drivers.WithStatic(ctx, http.NewDriver())
ctx = drivers.WithContext(
ctx,
http.NewDriver(),
drivers.AsDefault(),
)
r.logger.Info().Timestamp().Str("name", name).Msg("Running test")
out, err := p.Run(
ctx,
runtime.WithLog(os.Stdout),
runtime.WithLog(zerolog.ConsoleWriter{Out: os.Stdout}),
runtime.WithParam("static", r.settings.StaticServerAddress),
runtime.WithParam("dynamic", r.settings.DynamicServerAddress),
)

154
pkg/drivers/cdp/document.go
View File

@ -7,8 +7,10 @@ import (
"sync"
"time"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
@ -24,17 +26,15 @@ import (
const BlankPageURL = "about:blank"
type (
HTMLDocument struct {
sync.Mutex
logger *zerolog.Logger
conn *rpcc.Conn
client *cdp.Client
events *events.EventBroker
url values.String
element *HTMLElement
}
)
type HTMLDocument struct {
sync.Mutex
logger *zerolog.Logger
conn *rpcc.Conn
client *cdp.Client
events *events.EventBroker
url values.String
element *HTMLElement
}
func handleLoadError(logger *zerolog.Logger, client *cdp.Client) {
err := client.Page.Close(context.Background())
@ -49,7 +49,7 @@ func LoadHTMLDocument(
conn *rpcc.Conn,
client *cdp.Client,
url string,
) (*HTMLDocument, error) {
) (drivers.HTMLDocument, error) {
logger := logging.FromContext(ctx)
if conn == nil {
@ -143,7 +143,7 @@ func (doc *HTMLDocument) MarshalJSON() ([]byte, error) {
}
func (doc *HTMLDocument) Type() core.Type {
return types.HTMLDocument
return drivers.HTMLDocumentType
}
func (doc *HTMLDocument) String() string {
@ -181,13 +181,26 @@ func (doc *HTMLDocument) Compare(other core.Value) int64 {
doc.Lock()
defer doc.Unlock()
if other.Type() == types.HTMLDocument {
other := other.(*HTMLDocument)
switch other.Type() {
case drivers.HTMLDocumentType:
other := other.(drivers.HTMLDocument)
return doc.url.Compare(other.url)
return doc.url.Compare(other.GetURL())
default:
return drivers.Compare(doc.Type(), other.Type())
}
}
return types.Compare(other.Type(), types.HTMLDocument)
func (doc *HTMLDocument) Iterate(ctx context.Context) (core.Iterator, error) {
return doc.element.Iterate(ctx)
}
func (doc *HTMLDocument) GetIn(ctx context.Context, path []core.Value) (core.Value, error) {
return common.GetInDocument(ctx, doc, path)
}
func (doc *HTMLDocument) SetIn(ctx context.Context, path []core.Value, value core.Value) error {
return common.SetInDocument(ctx, doc, path, value)
}
func (doc *HTMLDocument) Close() error {
@ -260,41 +273,6 @@ func (doc *HTMLDocument) Length() values.Int {
return doc.element.Length()
}
func (doc *HTMLDocument) InnerText() values.String {
doc.Lock()
defer doc.Unlock()
return doc.element.InnerText()
}
func (doc *HTMLDocument) InnerHTML() values.String {
doc.Lock()
defer doc.Unlock()
return doc.element.InnerHTML()
}
func (doc *HTMLDocument) Value() core.Value {
doc.Lock()
defer doc.Unlock()
return doc.element.Value()
}
func (doc *HTMLDocument) GetAttributes() core.Value {
doc.Lock()
defer doc.Unlock()
return doc.element.GetAttributes()
}
func (doc *HTMLDocument) GetAttribute(name values.String) core.Value {
doc.Lock()
defer doc.Unlock()
return doc.element.GetAttribute(name)
}
func (doc *HTMLDocument) GetChildNodes() core.Value {
doc.Lock()
defer doc.Unlock()
@ -323,39 +301,22 @@ func (doc *HTMLDocument) QuerySelectorAll(selector values.String) core.Value {
return doc.element.QuerySelectorAll(selector)
}
func (doc *HTMLDocument) URL() core.Value {
func (doc *HTMLDocument) DocumentElement() drivers.HTMLElement {
doc.Lock()
defer doc.Unlock()
return doc.element
}
func (doc *HTMLDocument) GetURL() core.Value {
doc.Lock()
defer doc.Unlock()
return doc.url
}
func (doc *HTMLDocument) InnerHTMLBySelector(selector values.String) values.String {
doc.Lock()
defer doc.Unlock()
return doc.element.InnerHTMLBySelector(selector)
}
func (doc *HTMLDocument) InnerHTMLBySelectorAll(selector values.String) *values.Array {
doc.Lock()
defer doc.Unlock()
return doc.element.InnerHTMLBySelectorAll(selector)
}
func (doc *HTMLDocument) InnerTextBySelector(selector values.String) values.String {
doc.Lock()
defer doc.Unlock()
return doc.element.InnerTextBySelector(selector)
}
func (doc *HTMLDocument) InnerTextBySelectorAll(selector values.String) *values.Array {
doc.Lock()
defer doc.Unlock()
return doc.element.InnerTextBySelectorAll(selector)
func (doc *HTMLDocument) SetURL(url values.String) error {
return doc.Navigate(url, values.Int(DefaultTimeout))
}
func (doc *HTMLDocument) CountBySelector(selector values.String) values.Int {
@ -377,14 +338,11 @@ func (doc *HTMLDocument) ClickBySelector(selector values.String) (values.Boolean
doc.client,
fmt.Sprintf(`
var el = document.querySelector(%s);
if (el == null) {
return false;
}
var evt = new window.MouseEvent('click', { bubbles: true, cancelable: true });
el.dispatchEvent(evt);
return true;
`, eval.ParamString(selector.String())),
true,
@ -407,16 +365,13 @@ func (doc *HTMLDocument) ClickBySelectorAll(selector values.String) (values.Bool
doc.client,
fmt.Sprintf(`
var elements = document.querySelectorAll(%s);
if (elements == null) {
return false;
}
elements.forEach((el) => {
var evt = new window.MouseEvent('click', { bubbles: true, cancelable: true });
el.dispatchEvent(evt);
});
return true;
`, eval.ParamString(selector.String())),
true,
@ -483,20 +438,15 @@ func (doc *HTMLDocument) SelectBySelector(selector values.String, value *values.
doc.client,
fmt.Sprintf(`
var element = document.querySelector(%s);
if (element == null) {
return [];
}
var values = %s;
if (element.nodeName.toLowerCase() !== 'select') {
throw new Error('Element is not a <select> element.');
}
var options = Array.from(element.options);
element.value = undefined;
for (var option of options) {
option.selected = values.includes(option.value);
@ -504,7 +454,6 @@ func (doc *HTMLDocument) SelectBySelector(selector values.String, value *values.
break;
}
}
element.dispatchEvent(new Event('input', { 'bubbles': true, cancelable: true }));
element.dispatchEvent(new Event('change', { 'bubbles': true, cancelable: true }));
@ -574,11 +523,9 @@ func (doc *HTMLDocument) WaitForSelector(selector values.String, timeout values.
doc.client,
fmt.Sprintf(`
var el = document.querySelector(%s);
if (el != null) {
return true;
}
// null means we need to repeat
return null;
`, eval.ParamString(selector.String())),
@ -591,19 +538,16 @@ func (doc *HTMLDocument) WaitForSelector(selector values.String, timeout values.
return err
}
func (doc *HTMLDocument) WaitForClass(selector, class values.String, timeout values.Int) error {
func (doc *HTMLDocument) WaitForClassBySelector(selector, class values.String, timeout values.Int) error {
task := events.NewEvalWaitTask(
doc.client,
fmt.Sprintf(`
var el = document.querySelector(%s);
if (el == null) {
return false;
}
var className = %s;
var found = el.className.split(' ').find(i => i === className);
if (found != null) {
return true;
}
@ -623,27 +567,22 @@ func (doc *HTMLDocument) WaitForClass(selector, class values.String, timeout val
return err
}
func (doc *HTMLDocument) WaitForClassAll(selector, class values.String, timeout values.Int) error {
func (doc *HTMLDocument) WaitForClassBySelectorAll(selector, class values.String, timeout values.Int) error {
task := events.NewEvalWaitTask(
doc.client,
fmt.Sprintf(`
var elements = document.querySelectorAll(%s);
if (elements == null || elements.length === 0) {
return false;
}
var className = %s;
var foundCount = 0;
elements.forEach((el) => {
var found = el.className.split(' ').find(i => i === className);
if (found != null) {
foundCount++;
}
});
if (foundCount === elements.length) {
return true;
}
@ -788,7 +727,7 @@ func (doc *HTMLDocument) NavigateForward(skip values.Int, timeout values.Int) (v
return values.True, nil
}
func (doc *HTMLDocument) PrintToPDF(params values.HTMLPDFParams) (values.Binary, error) {
func (doc *HTMLDocument) PrintToPDF(params drivers.PDFParams) (values.Binary, error) {
ctx := context.Background()
args := page.NewPrintToPDFArgs()
@ -848,11 +787,11 @@ func (doc *HTMLDocument) PrintToPDF(params values.HTMLPDFParams) (values.Binary,
return values.NewBinary(reply.Data), nil
}
func (doc *HTMLDocument) CaptureScreenshot(params values.HTMLScreenshotParams) (values.Binary, error) {
func (doc *HTMLDocument) CaptureScreenshot(params drivers.ScreenshotParams) (values.Binary, error) {
ctx := context.Background()
metrics, err := doc.client.Page.GetLayoutMetrics(ctx)
if params.Format == values.HTMLScreenshotFormatJPEG && params.Quality < 0 && params.Quality > 100 {
if params.Format == drivers.ScreenshotFormatJPEG && params.Quality < 0 && params.Quality > 100 {
params.Quality = 100
}
@ -924,15 +863,12 @@ func (doc *HTMLDocument) ScrollBottom() error {
func (doc *HTMLDocument) ScrollBySelector(selector values.String) error {
_, err := eval.Eval(doc.client, fmt.Sprintf(`
var el = document.querySelector(%s);
if (el == null) {
throw new Error("element not found");
}
el.scrollIntoView({
behavior: 'instant'
});
return true;
`, eval.ParamString(selector.String()),
), false, false)

19
pkg/drivers/cdp/driver.go
View File

@ -4,6 +4,7 @@ import (
"context"
"sync"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
@ -17,6 +18,8 @@ import (
"github.com/pkg/errors"
)
const DriverName = "cdp"
type Driver struct {
sync.Mutex
dev *devtool.DevTools
@ -35,7 +38,11 @@ func NewDriver(opts ...Option) *Driver {
return drv
}
func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (values.DHTMLDocument, error) {
func (drv *Driver) Name() string {
return DriverName
}
func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (drivers.HTMLDocument, error) {
logger := logging.FromContext(ctx)
err := drv.init(ctx)
@ -43,8 +50,9 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (va
if err != nil {
logger.
Error().
Timestamp().
Err(err).
Str("driver", "dynamic").
Str("driver", DriverName).
Msg("failed to initialize the driver")
return nil, err
@ -64,8 +72,9 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (va
if err != nil {
logger.
Error().
Timestamp().
Err(err).
Str("driver", "dynamic").
Str("driver", DriverName).
Msg("failed to create a browser target")
return nil, err
@ -77,8 +86,9 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (va
if err != nil {
logger.
Error().
Timestamp().
Err(err).
Str("driver", "dynamic").
Str("driver", DriverName).
Msg("failed to establish a connection")
return nil, err
@ -111,6 +121,7 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (va
logger.
Debug().
Timestamp().
Str("user-agent", ua).
Msg("using User-Agent")

132
pkg/drivers/cdp/element.go
View File

@ -10,6 +10,7 @@ import (
"sync"
"time"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
"github.com/MontFerret/ferret/pkg/drivers/common"
@ -196,7 +197,7 @@ func (el *HTMLElement) Close() error {
}
func (el *HTMLElement) Type() core.Type {
return types.HTMLElement
return drivers.HTMLElementType
}
func (el *HTMLElement) MarshalJSON() ([]byte, error) {
@ -214,24 +215,14 @@ func (el *HTMLElement) String() string {
}
func (el *HTMLElement) Compare(other core.Value) int64 {
if other.Type() == types.HTMLElement {
other := other.(*HTMLElement)
switch other.Type() {
case drivers.HTMLElementType:
other := other.(drivers.HTMLElement)
id := int(el.id.backendID)
otherID := int(other.id.backendID)
if id == otherID {
return 0
}
if id > otherID {
return 1
}
return -1
return el.InnerHTML().Compare(other.InnerHTML())
default:
return drivers.Compare(el.Type(), other.Type())
}
return types.Compare(other.Type(), types.HTMLElement)
}
func (el *HTMLElement) Unwrap() interface{} {
@ -251,7 +242,23 @@ func (el *HTMLElement) Hash() uint64 {
return h.Sum64()
}
func (el *HTMLElement) Value() core.Value {
func (el *HTMLElement) Copy() core.Value {
return values.None
}
func (el *HTMLElement) Iterate(_ context.Context) (core.Iterator, error) {
return common.NewIterator(el)
}
func (el *HTMLElement) GetIn(ctx context.Context, path []core.Value) (core.Value, error) {
return common.GetInElement(ctx, el, path)
}
func (el *HTMLElement) SetIn(ctx context.Context, path []core.Value, value core.Value) error {
return common.SetInElement(ctx, el, path, value)
}
func (el *HTMLElement) GetValue() core.Value {
if !el.IsConnected() {
return el.value
}
@ -272,12 +279,16 @@ func (el *HTMLElement) Value() core.Value {
return val
}
func (el *HTMLElement) Copy() core.Value {
return values.None
}
func (el *HTMLElement) SetValue(value core.Value) error {
if !el.IsConnected() {
// TODO: Return an error
return nil
}
func (el *HTMLElement) Length() values.Int {
return values.NewInt(len(el.children))
ctx, cancel := contextWithTimeout()
defer cancel()
return el.client.DOM.SetNodeValue(ctx, dom.NewSetNodeValueArgs(el.id.nodeID, value.String()))
}
func (el *HTMLElement) NodeType() values.Int {
@ -288,6 +299,10 @@ func (el *HTMLElement) NodeName() values.String {
return el.nodeName
}
func (el *HTMLElement) Length() values.Int {
return values.NewInt(len(el.children))
}
func (el *HTMLElement) GetAttributes() core.Value {
val, err := el.attributes.Read()
@ -315,6 +330,13 @@ func (el *HTMLElement) GetAttribute(name values.String) core.Value {
return val
}
func (el *HTMLElement) SetAttribute(name, value values.String) error {
return el.client.DOM.SetAttributeValue(
context.Background(),
dom.NewSetAttributeValueArgs(el.id.nodeID, string(name), string(value)),
)
}
func (el *HTMLElement) GetChildNodes() core.Value {
val, err := el.loadedChildren.Read()
@ -482,7 +504,7 @@ func (el *HTMLElement) InnerTextBySelector(selector values.String) values.String
el.logError(err).
Int("childNodeID", int(childNodeID)).
Str("selector", selector.String()).
Msg("failed to resolve remote object for child element")
Msg("failed to resolve remote object for child el")
return values.EmptyString
}
@ -491,7 +513,7 @@ func (el *HTMLElement) InnerTextBySelector(selector values.String) values.String
el.logError(err).
Int("childNodeID", int(childNodeID)).
Str("selector", selector.String()).
Msg("failed to resolve remote object for child element")
Msg("failed to resolve remote object for child el")
return values.EmptyString
}
@ -504,7 +526,7 @@ func (el *HTMLElement) InnerTextBySelector(selector values.String) values.String
el.logError(err).
Str("childObjectID", string(objID)).
Str("selector", selector.String()).
Msg("failed to load inner text for found child element")
Msg("failed to load inner text for found child el")
return values.EmptyString
}
@ -545,7 +567,7 @@ func (el *HTMLElement) InnerTextBySelectorAll(selector values.String) *values.Ar
Int("index", idx).
Int("childNodeID", int(id)).
Str("selector", selector.String()).
Msg("failed to resolve remote object for child element")
Msg("failed to resolve remote object for child el")
continue
}
@ -562,7 +584,7 @@ func (el *HTMLElement) InnerTextBySelectorAll(selector values.String) *values.Ar
el.logError(err).
Str("childObjectID", string(objID)).
Str("selector", selector.String()).
Msg("failed to load inner text for found child element")
Msg("failed to load inner text for found child el")
continue
}
@ -606,7 +628,7 @@ func (el *HTMLElement) InnerHTMLBySelector(selector values.String) values.String
if err != nil {
el.logError(err).
Str("selector", selector.String()).
Msg("failed to load inner HTML for found child element")
Msg("failed to load inner HTML for found child el")
return values.EmptyString
}
@ -640,7 +662,7 @@ func (el *HTMLElement) InnerHTMLBySelectorAll(selector values.String) *values.Ar
if err != nil {
el.logError(err).
Str("selector", selector.String()).
Msg("failed to load inner HTML for found child element")
Msg("failed to load inner HTML for found child el")
// return what we have
return arr
@ -777,7 +799,7 @@ func (el *HTMLElement) Select(value *values.Array) (*values.Array, error) {
var attrID = "data-ferret-select"
if el.NodeName() != "SELECT" {
return nil, core.Error(core.ErrInvalidOperation, "Element is not a <select> element.")
return nil, core.Error(core.ErrInvalidOperation, "element is not a <select> element.")
}
id, err := uuid.NewV4()
@ -798,31 +820,25 @@ func (el *HTMLElement) Select(value *values.Array) (*values.Array, error) {
res, err := eval.Eval(
el.client,
fmt.Sprintf(`
var element = document.querySelector('[%s="%s"]');
if (element == null) {
var el = document.querySelector('[%s="%s"]');
if (el == null) {
return [];
}
var values = %s;
if (element.nodeName.toLowerCase() !== 'select') {
throw new Error('Element is not a <select> element.');
if (el.nodeName.toLowerCase() !== 'select') {
throw new Error('element is not a <select> element.');
}
var options = Array.from(element.options);
element.value = undefined;
var options = Array.from(el.options);
el.value = undefined;
for (var option of options) {
option.selected = values.includes(option.value);
if (option.selected && !element.multiple) {
if (option.selected && !el.multiple) {
break;
}
}
element.dispatchEvent(new Event('input', { 'bubbles': true }));
element.dispatchEvent(new Event('change', { 'bubbles': true }));
el.dispatchEvent(new Event('input', { 'bubbles': true }));
el.dispatchEvent(new Event('change', { 'bubbles': true }));
return options.filter(option => option.selected).map(option => option.value);
`,
@ -869,11 +885,9 @@ func (el *HTMLElement) ScrollIntoView() error {
_, err = eval.Eval(el.client, fmt.Sprintf(`
var el = document.querySelector('[%s="%s"]');
if (el == null) {
throw new Error('element not found');
}
el.scrollIntoView({
behavior: 'instant',
inline: 'center',
@ -929,7 +943,7 @@ func (el *HTMLElement) loadInnerText() (core.Value, error) {
return text, nil
}
el.logError(err).Msg("failed to get get inner text from remote object")
el.logError(err).Msg("failed to get inner text from remote object")
// and just parse cached innerHTML
}
@ -976,7 +990,7 @@ func (el *HTMLElement) loadChildren() (core.Value, error) {
)
if err != nil {
el.logError(err).Msg("failed to load child nodes")
el.logError(err).Msg("failed to load child elements")
continue
}
@ -999,14 +1013,14 @@ func (el *HTMLElement) handleAttrModified(message interface{}) {
return
}
// it's not for this element
// it's not for this el
if reply.NodeID != el.id.nodeID {
return
}
el.attributes.Write(func(v core.Value, err error) {
if err != nil {
el.logError(err).Msg("failed to update node")
el.logError(err).Msg("failed to update element")
return
}
@ -1029,7 +1043,7 @@ func (el *HTMLElement) handleAttrRemoved(message interface{}) {
return
}
// it's not for this element
// it's not for this el
if reply.NodeID != el.id.nodeID {
return
}
@ -1042,7 +1056,7 @@ func (el *HTMLElement) handleAttrRemoved(message interface{}) {
el.attributes.Write(func(v core.Value, err error) {
if err != nil {
el.logError(err).Msg("failed to update node")
el.logError(err).Msg("failed to update element")
return
}
@ -1077,7 +1091,7 @@ func (el *HTMLElement) handleChildrenCountChanged(message interface{}) {
)
if err != nil {
el.logError(err).Msg("failed to update node")
el.logError(err).Msg("failed to update element")
return
}
@ -1137,7 +1151,7 @@ func (el *HTMLElement) handleChildInserted(message interface{}) {
loadedEl, err := LoadElement(ctx, el.logger, el.client, el.events, nextID, emptyBackendID)
if err != nil {
el.logError(err).Msg("failed to load an inserted node")
el.logError(err).Msg("failed to load an inserted element")
return
}
@ -1147,7 +1161,7 @@ func (el *HTMLElement) handleChildInserted(message interface{}) {
newInnerHTML, err := loadInnerHTML(ctx, el.client, el.id)
if err != nil {
el.logError(err).Msg("failed to update node")
el.logError(err).Msg("failed to update element")
return
}
@ -1198,7 +1212,7 @@ func (el *HTMLElement) handleChildRemoved(message interface{}) {
Timestamp().
Err(err).
Int("nodeID", int(el.id.nodeID)).
Msg("failed to update node")
Msg("failed to update element")
return
}
@ -1216,7 +1230,7 @@ func (el *HTMLElement) handleChildRemoved(message interface{}) {
Timestamp().
Err(err).
Int("nodeID", int(el.id.nodeID)).
Msg("failed to update node")
Msg("failed to update element")
return
}

2
pkg/drivers/cdp/helpers.go
View File

@ -4,7 +4,6 @@ import (
"bytes"
"context"
"errors"
"golang.org/x/sync/errgroup"
"math"
"strings"
@ -17,6 +16,7 @@ import (
"github.com/mafredri/cdp/protocol/dom"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/protocol/runtime"
"golang.org/x/sync/errgroup"
)
type (

8
pkg/drivers/common/errors.go Normal file
View File

@ -0,0 +1,8 @@
package common
import "github.com/MontFerret/ferret/pkg/runtime/core"
var (
ErrReadOnly = core.Error(core.ErrInvalidOperation, "read only")
ErrInvalidPath = core.Error(core.ErrInvalidOperation, "invalid path")
)

101
pkg/drivers/common/getter.go Normal file
View File

@ -0,0 +1,101 @@
package common
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
func GetInDocument(ctx context.Context, doc drivers.HTMLDocument, path []core.Value) (core.Value, error) {
if path == nil || len(path) == 0 {
return values.None, nil
}
segment := path[0]
if segment.Type() == types.String {
segment := segment.(values.String)
switch segment {
case "url", "URL":
return doc.GetURL(), nil
case "body":
return doc.QuerySelector("body"), nil
case "head":
return doc.QuerySelector("head"), nil
default:
return GetInNode(ctx, doc.DocumentElement(), path)
}
}
return GetInNode(ctx, doc.DocumentElement(), path)
}
func GetInElement(ctx context.Context, el drivers.HTMLElement, path []core.Value) (core.Value, error) {
if path == nil || len(path) == 0 {
return values.None, nil
}
segment := path[0]
if segment.Type() == types.String {
segment := segment.(values.String)
switch segment {
case "innerText":
return el.InnerText(), nil
case "innerHTML":
return el.InnerHTML(), nil
case "value":
return el.GetValue(), nil
case "attributes":
return el.GetAttributes(), nil
default:
return GetInNode(ctx, el, path)
}
}
return GetInNode(ctx, el, path)
}
func GetInNode(ctx context.Context, node drivers.HTMLNode, path []core.Value) (core.Value, error) {
if path == nil || len(path) == 0 {
return values.None, nil
}
nt := node.Type()
segment := path[0]
st := segment.Type()
if st == types.Int {
if nt == drivers.HTMLElementType || nt == drivers.HTMLDocumentType {
re := node.(drivers.HTMLNode)
return re.GetChildNode(segment.(values.Int)), nil
}
return values.GetIn(ctx, node, path[0:])
}
if st == types.String {
segment := segment.(values.String)
switch segment {
case "nodeType":
return node.NodeType(), nil
case "nodeName":
return node.NodeName(), nil
case "children":
return node.GetChildNodes(), nil
case "length":
return node.Length(), nil
default:
return values.None, nil
}
}
return values.None, core.TypeError(st, types.Int, types.String)
}

37
pkg/drivers/common/iterator.go Normal file
View File

@ -0,0 +1,37 @@
package common
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type Iterator struct {
node drivers.HTMLElement
pos values.Int
}
func NewIterator(
node drivers.HTMLElement,
) (core.Iterator, error) {
if node == nil {
return nil, core.Error(core.ErrMissedArgument, "result")
}
return &Iterator{node, 0}, nil
}
func (iterator *Iterator) Next(_ context.Context) (value core.Value, key core.Value, err error) {
if iterator.node.Length() > iterator.pos {
idx := iterator.pos
val := iterator.node.GetChildNode(idx)
iterator.pos++
return val, idx, nil
}
return values.None, values.None, nil
}

17
pkg/drivers/common/path.go Normal file
View File

@ -0,0 +1,17 @@
package common
import (
"strings"
"github.com/MontFerret/ferret/pkg/runtime/core"
)
func PathToString(path []core.Value) string {
spath := make([]string, 0, len(path))
for i, s := range path {
spath[i] = s.String()
}
return strings.Join(spath, ".")
}

90
pkg/drivers/common/setter.go Normal file
View File

@ -0,0 +1,90 @@
package common
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
func SetInDocument(ctx context.Context, doc drivers.HTMLDocument, path []core.Value, value core.Value) error {
if path == nil || len(path) == 0 {
return nil
}
segment := path[0]
if segment.Type() == types.String {
segment := segment.(values.String)
switch segment {
case "url", "URL":
return doc.SetURL(values.NewString(value.String()))
default:
return SetInNode(ctx, doc, path, value)
}
}
return SetInNode(ctx, doc, path, value)
}
func SetInElement(ctx context.Context, el drivers.HTMLElement, path []core.Value, value core.Value) error {
if path == nil || len(path) == 0 {
return nil
}
segment := path[0]
if segment.Type() == types.String {
segment := segment.(values.String)
switch segment {
case "attributes":
if len(path) > 1 {
attrName := path[1]
return el.SetAttribute(values.NewString(attrName.String()), values.NewString(value.String()))
}
err := core.ValidateType(value, types.Object)
if err != nil {
return err
}
obj := value.(*values.Object)
obj.ForEach(func(value core.Value, key string) bool {
err = el.SetAttribute(values.NewString(key), values.NewString(value.String()))
return err == nil
})
return err
case "value":
if len(path) > 1 {
return core.Error(ErrInvalidPath, PathToString(path[1:]))
}
return el.SetValue(value)
}
}
return SetInNode(ctx, el, path, value)
}
func SetInNode(_ context.Context, _ drivers.HTMLNode, path []core.Value, _ core.Value) error {
if path == nil || len(path) == 0 {
return nil
}
segment := path[0]
st := segment.Type()
if st == types.Int {
return core.Error(core.ErrInvalidOperation, "children are read-only")
}
return core.Error(ErrReadOnly, PathToString(path))
}

75
pkg/drivers/driver.go
View File

@ -9,58 +9,61 @@ import (
)
type (
staticCtxKey struct{}
ctxKey struct{}
dynamicCtxKey struct{}
Static interface {
io.Closer
GetDocument(ctx context.Context, url values.String) (values.HTMLDocument, error)
ParseDocument(ctx context.Context, str values.String) (values.HTMLDocument, error)
ctxValue struct {
opts *options
drivers map[string]Driver
}
Dynamic interface {
Driver interface {
io.Closer
GetDocument(ctx context.Context, url values.String) (values.DHTMLDocument, error)
Name() string
GetDocument(ctx context.Context, url values.String) (HTMLDocument, error)
}
)
func StaticFrom(ctx context.Context) (Static, error) {
val := ctx.Value(staticCtxKey{})
func WithContext(ctx context.Context, drv Driver, opts ...Option) context.Context {
ctx, value := resolveValue(ctx)
drv, ok := val.(Static)
value.drivers[drv.Name()] = drv
if !ok {
return nil, core.Error(core.ErrNotFound, "HTML Driver")
for _, opt := range opts {
opt(drv, value.opts)
}
return ctx
}
func FromContext(ctx context.Context, name string) (Driver, error) {
_, value := resolveValue(ctx)
if name == "" {
name = value.opts.defaultDriver
}
drv, exists := value.drivers[name]
if !exists {
return nil, core.Error(core.ErrNotFound, name)
}
return drv, nil
}
func DynamicFrom(ctx context.Context) (Dynamic, error) {
val := ctx.Value(dynamicCtxKey{})
drv, ok := val.(Dynamic)
func resolveValue(ctx context.Context) (context.Context, *ctxValue) {
key := ctxKey{}
v := ctx.Value(key)
value, ok := v.(*ctxValue)
if !ok {
return nil, core.Error(core.ErrNotFound, "DHTML Driver")
value = &ctxValue{
opts: &options{},
drivers: make(map[string]Driver),
}
return context.WithValue(ctx, key, value), value
}
return drv, nil
}
func WithStatic(ctx context.Context, drv Static) context.Context {
return context.WithValue(
ctx,
staticCtxKey{},
drv,
)
}
func WithDynamic(ctx context.Context, drv Dynamic) context.Context {
return context.WithValue(
ctx,
dynamicCtxKey{},
drv,
)
return ctx, value
}

204
pkg/drivers/http/document.go
View File

@ -1,21 +1,26 @@
package http
import (
"context"
"hash/fnv"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
"github.com/PuerkitoBio/goquery"
)
type HTMLDocument struct {
*HTMLElement
url values.String
url values.String
docNode *goquery.Document
element drivers.HTMLElement
}
func NewHTMLDocument(
url string,
node *goquery.Document,
) (*HTMLDocument, error) {
) (drivers.HTMLDocument, error) {
if url == "" {
return nil, core.Error(core.ErrMissedArgument, "document url")
}
@ -30,23 +35,200 @@ func NewHTMLDocument(
return nil, err
}
return &HTMLDocument{el, values.NewString(url)}, nil
return &HTMLDocument{values.NewString(url), node, el}, nil
}
func (doc *HTMLDocument) MarshalJSON() ([]byte, error) {
return doc.element.MarshalJSON()
}
func (doc *HTMLDocument) Type() core.Type {
return types.HTMLDocument
return drivers.HTMLDocumentType
}
func (doc *HTMLDocument) String() string {
str, err := doc.docNode.Html()
if err != nil {
return ""
}
return str
}
func (doc *HTMLDocument) Compare(other core.Value) int64 {
if other.Type() == types.HTMLDocument {
otherDoc := other.(values.HTMLDocument)
switch other.Type() {
case drivers.HTMLElementType:
otherDoc := other.(drivers.HTMLDocument)
return doc.url.Compare(otherDoc.URL())
return doc.url.Compare(otherDoc.GetURL())
default:
return drivers.Compare(doc.Type(), other.Type())
}
}
func (doc *HTMLDocument) Unwrap() interface{} {
return doc.docNode
}
func (doc *HTMLDocument) Hash() uint64 {
h := fnv.New64a()
h.Write([]byte(doc.Type().String()))
h.Write([]byte(":"))
h.Write([]byte(doc.url))
return h.Sum64()
}
func (doc *HTMLDocument) Copy() core.Value {
cp, err := NewHTMLDocument(string(doc.url), doc.docNode)
if err != nil {
return values.None
}
return types.Compare(other.Type(), types.HTMLDocument)
return cp
}
func (doc *HTMLDocument) URL() core.Value {
func (doc *HTMLDocument) Clone() core.Value {
cp, err := NewHTMLDocument(string(doc.url), goquery.CloneDocument(doc.docNode))
if err != nil {
return values.None
}
return cp
}
func (doc *HTMLDocument) Length() values.Int {
return values.NewInt(doc.docNode.Length())
}
func (doc *HTMLDocument) Iterate(_ context.Context) (core.Iterator, error) {
return common.NewIterator(doc.element)
}
func (doc *HTMLDocument) GetIn(ctx context.Context, path []core.Value) (core.Value, error) {
return common.GetInDocument(ctx, doc, path)
}
func (doc *HTMLDocument) SetIn(ctx context.Context, path []core.Value, value core.Value) error {
return common.SetInDocument(ctx, doc, path, value)
}
func (doc *HTMLDocument) NodeType() values.Int {
return 9
}
func (doc *HTMLDocument) NodeName() values.String {
return "#document"
}
func (doc *HTMLDocument) GetChildNodes() core.Value {
return doc.element.GetChildNodes()
}
func (doc *HTMLDocument) GetChildNode(idx values.Int) core.Value {
return doc.element.GetChildNode(idx)
}
func (doc *HTMLDocument) QuerySelector(selector values.String) core.Value {
return doc.element.QuerySelector(selector)
}
func (doc *HTMLDocument) QuerySelectorAll(selector values.String) core.Value {
return doc.element.QuerySelectorAll(selector)
}
func (doc *HTMLDocument) CountBySelector(selector values.String) values.Int {
return doc.element.CountBySelector(selector)
}
func (doc *HTMLDocument) ExistsBySelector(selector values.String) values.Boolean {
return doc.element.ExistsBySelector(selector)
}
func (doc *HTMLDocument) DocumentElement() drivers.HTMLElement {
return doc.element
}
func (doc *HTMLDocument) GetURL() core.Value {
return doc.url
}
func (doc *HTMLDocument) SetURL(_ values.String) error {
return core.ErrInvalidOperation
}
func (doc *HTMLDocument) Navigate(_ values.String, _ values.Int) error {
return core.ErrNotSupported
}
func (doc *HTMLDocument) NavigateBack(_ values.Int, _ values.Int) (values.Boolean, error) {
return false, core.ErrNotSupported
}
func (doc *HTMLDocument) NavigateForward(_ values.Int, _ values.Int) (values.Boolean, error) {
return false, core.ErrNotSupported
}
func (doc *HTMLDocument) ClickBySelector(_ values.String) (values.Boolean, error) {
return false, core.ErrNotSupported
}
func (doc *HTMLDocument) ClickBySelectorAll(_ values.String) (values.Boolean, error) {
return false, core.ErrNotSupported
}
func (doc *HTMLDocument) InputBySelector(_ values.String, _ core.Value, _ values.Int) (values.Boolean, error) {
return false, core.ErrNotSupported
}
func (doc *HTMLDocument) SelectBySelector(_ values.String, _ *values.Array) (*values.Array, error) {
return nil, core.ErrNotSupported
}
func (doc *HTMLDocument) HoverBySelector(_ values.String) error {
return core.ErrNotSupported
}
func (doc *HTMLDocument) PrintToPDF(_ drivers.PDFParams) (values.Binary, error) {
return nil, core.ErrNotSupported
}
func (doc *HTMLDocument) CaptureScreenshot(_ drivers.ScreenshotParams) (values.Binary, error) {
return nil, core.ErrNotSupported
}
func (doc *HTMLDocument) ScrollTop() error {
return core.ErrNotSupported
}
func (doc *HTMLDocument) ScrollBottom() error {
return core.ErrNotSupported
}
func (doc *HTMLDocument) ScrollBySelector(_ values.String) error {
return core.ErrNotSupported
}
func (doc *HTMLDocument) WaitForNavigation(_ values.Int) error {
return core.ErrNotSupported
}
func (doc *HTMLDocument) WaitForSelector(_ values.String, _ values.Int) error {
return core.ErrNotSupported
}
func (doc *HTMLDocument) WaitForClassBySelector(_, _ values.String, _ values.Int) error {
return core.ErrNotSupported
}
func (doc *HTMLDocument) WaitForClassBySelectorAll(_, _ values.String, _ values.Int) error {
return core.ErrNotSupported
}
func (doc *HTMLDocument) Close() error {
return nil
}

12
pkg/drivers/http/driver.go
View File

@ -6,6 +6,7 @@ import (
"net/http"
"net/url"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
@ -15,6 +16,8 @@ import (
"github.com/sethgrid/pester"
)
const DriverName = "http"
type Driver struct {
client *pester.Client
options *Options
@ -56,7 +59,11 @@ func newClientWithProxy(options *Options) (*http.Client, error) {
return &http.Client{Transport: tr}, nil
}
func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (values.HTMLDocument, error) {
func (drv *Driver) Name() string {
return DriverName
}
func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (drivers.HTMLDocument, error) {
u := targetURL.String()
req, err := http.NewRequest(http.MethodGet, u, nil)
@ -76,6 +83,7 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (va
logger := logging.FromContext(ctx)
logger.
Debug().
Timestamp().
Str("user-agent", ua).
Msg("using User-Agent")
@ -101,7 +109,7 @@ func (drv *Driver) GetDocument(ctx context.Context, targetURL values.String) (va
return NewHTMLDocument(u, doc)
}
func (drv *Driver) ParseDocument(_ context.Context, str values.String) (values.HTMLDocument, error) {
func (drv *Driver) ParseDocument(_ context.Context, str values.String) (drivers.HTMLDocument, error) {
buf := bytes.NewBuffer([]byte(str))
doc, err := goquery.NewDocumentFromReader(buf)

204
pkg/drivers/http/element.go
View File

@ -1,13 +1,14 @@
package http
import (
"context"
"encoding/json"
"hash/fnv"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/common"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
"github.com/PuerkitoBio/goquery"
)
@ -17,7 +18,7 @@ type HTMLElement struct {
children *values.Array
}
func NewHTMLElement(node *goquery.Selection) (*HTMLElement, error) {
func NewHTMLElement(node *goquery.Selection) (drivers.HTMLElement, error) {
if node == nil {
return nil, core.Error(core.ErrMissedArgument, "element selection")
}
@ -25,33 +26,35 @@ func NewHTMLElement(node *goquery.Selection) (*HTMLElement, error) {
return &HTMLElement{node, nil, nil}, nil
}
func (el *HTMLElement) MarshalJSON() ([]byte, error) {
return json.Marshal(el.InnerText().String())
func (nd *HTMLElement) MarshalJSON() ([]byte, error) {
return json.Marshal(nd.InnerText().String())
}
func (el *HTMLElement) Type() core.Type {
return types.HTMLElement
func (nd *HTMLElement) Type() core.Type {
return drivers.HTMLElementType
}
func (el *HTMLElement) String() string {
return el.InnerHTML().String()
func (nd *HTMLElement) String() string {
return nd.InnerHTML().String()
}
func (el *HTMLElement) Compare(other core.Value) int64 {
if other.Type() == types.HTMLElement {
// TODO: complete the comparison
return -1
func (nd *HTMLElement) Compare(other core.Value) int64 {
switch other.Type() {
case drivers.HTMLElementType:
other := other.(drivers.HTMLElement)
return nd.InnerHTML().Compare(other.InnerHTML())
default:
return drivers.Compare(nd.Type(), other.Type())
}
return types.Compare(other.Type(), types.HTMLElement)
}
func (el *HTMLElement) Unwrap() interface{} {
return el.selection
func (nd *HTMLElement) Unwrap() interface{} {
return nd.selection
}
func (el *HTMLElement) Hash() uint64 {
str, err := el.selection.Html()
func (nd *HTMLElement) Hash() uint64 {
str, err := nd.selection.Html()
if err != nil {
return 0
@ -59,21 +62,21 @@ func (el *HTMLElement) Hash() uint64 {
h := fnv.New64a()
h.Write([]byte(el.Type().String()))
h.Write([]byte(nd.Type().String()))
h.Write([]byte(":"))
h.Write([]byte(str))
return h.Sum64()
}
func (el *HTMLElement) Copy() core.Value {
c, _ := NewHTMLElement(el.selection.Clone())
func (nd *HTMLElement) Copy() core.Value {
c, _ := NewHTMLElement(nd.selection.Clone())
return c
}
func (el *HTMLElement) NodeType() values.Int {
nodes := el.selection.Nodes
func (nd *HTMLElement) NodeType() values.Int {
nodes := nd.selection.Nodes
if len(nodes) == 0 {
return 0
@ -82,20 +85,24 @@ func (el *HTMLElement) NodeType() values.Int {
return values.NewInt(common.ToHTMLType(nodes[0].Type))
}
func (el *HTMLElement) NodeName() values.String {
return values.NewString(goquery.NodeName(el.selection))
func (nd *HTMLElement) Close() error {
return nil
}
func (el *HTMLElement) Length() values.Int {
if el.children == nil {
el.children = el.parseChildren()
func (nd *HTMLElement) NodeName() values.String {
return values.NewString(goquery.NodeName(nd.selection))
}
func (nd *HTMLElement) Length() values.Int {
if nd.children == nil {
nd.children = nd.parseChildren()
}
return el.children.Length()
return nd.children.Length()
}
func (el *HTMLElement) Value() core.Value {
val, ok := el.selection.Attr("value")
func (nd *HTMLElement) GetValue() core.Value {
val, ok := nd.selection.Attr("value")
if ok {
return values.NewString(val)
@ -104,12 +111,18 @@ func (el *HTMLElement) Value() core.Value {
return values.EmptyString
}
func (el *HTMLElement) InnerText() values.String {
return values.NewString(el.selection.Text())
func (nd *HTMLElement) SetValue(value core.Value) error {
nd.selection.SetAttr("value", value.String())
return nil
}
func (el *HTMLElement) InnerHTML() values.String {
h, err := el.selection.Html()
func (nd *HTMLElement) InnerText() values.String {
return values.NewString(nd.selection.Text())
}
func (nd *HTMLElement) InnerHTML() values.String {
h, err := nd.selection.Html()
if err != nil {
return values.EmptyString
@ -118,16 +131,16 @@ func (el *HTMLElement) InnerHTML() values.String {
return values.NewString(h)
}
func (el *HTMLElement) GetAttributes() core.Value {
if el.attrs == nil {
el.attrs = el.parseAttrs()
func (nd *HTMLElement) GetAttributes() core.Value {
if nd.attrs == nil {
nd.attrs = nd.parseAttrs()
}
return el.attrs
return nd.attrs
}
func (el *HTMLElement) GetAttribute(name values.String) core.Value {
v, ok := el.selection.Attr(name.String())
func (nd *HTMLElement) GetAttribute(name values.String) core.Value {
v, ok := nd.selection.Attr(name.String())
if ok {
return values.NewString(v)
@ -136,24 +149,30 @@ func (el *HTMLElement) GetAttribute(name values.String) core.Value {
return values.None
}
func (el *HTMLElement) GetChildNodes() core.Value {
if el.children == nil {
el.children = el.parseChildren()
}
func (nd *HTMLElement) SetAttribute(name, value values.String) error {
nd.selection.SetAttr(string(name), string(value))
return el.children
return nil
}
func (el *HTMLElement) GetChildNode(idx values.Int) core.Value {
if el.children == nil {
el.children = el.parseChildren()
func (nd *HTMLElement) GetChildNodes() core.Value {
if nd.children == nil {
nd.children = nd.parseChildren()
}
return el.children.Get(idx)
return nd.children
}
func (el *HTMLElement) QuerySelector(selector values.String) core.Value {
selection := el.selection.Find(selector.String())
func (nd *HTMLElement) GetChildNode(idx values.Int) core.Value {
if nd.children == nil {
nd.children = nd.parseChildren()
}
return nd.children.Get(idx)
}
func (nd *HTMLElement) QuerySelector(selector values.String) core.Value {
selection := nd.selection.Find(selector.String())
if selection == nil {
return values.None
@ -168,8 +187,8 @@ func (el *HTMLElement) QuerySelector(selector values.String) core.Value {
return res
}
func (el *HTMLElement) QuerySelectorAll(selector values.String) core.Value {
selection := el.selection.Find(selector.String())
func (nd *HTMLElement) QuerySelectorAll(selector values.String) core.Value {
selection := nd.selection.Find(selector.String())
if selection == nil {
return values.None
@ -188,8 +207,8 @@ func (el *HTMLElement) QuerySelectorAll(selector values.String) core.Value {
return arr
}
func (el *HTMLElement) InnerHTMLBySelector(selector values.String) values.String {
selection := el.selection.Find(selector.String())
func (nd *HTMLElement) InnerHTMLBySelector(selector values.String) values.String {
selection := nd.selection.Find(selector.String())
str, err := selection.Html()
@ -201,8 +220,8 @@ func (el *HTMLElement) InnerHTMLBySelector(selector values.String) values.String
return values.NewString(str)
}
func (el *HTMLElement) InnerHTMLBySelectorAll(selector values.String) *values.Array {
selection := el.selection.Find(selector.String())
func (nd *HTMLElement) InnerHTMLBySelectorAll(selector values.String) *values.Array {
selection := nd.selection.Find(selector.String())
arr := values.NewArray(selection.Length())
selection.Each(func(_ int, selection *goquery.Selection) {
@ -217,14 +236,14 @@ func (el *HTMLElement) InnerHTMLBySelectorAll(selector values.String) *values.Ar
return arr
}
func (el *HTMLElement) InnerTextBySelector(selector values.String) values.String {
selection := el.selection.Find(selector.String())
func (nd *HTMLElement) InnerTextBySelector(selector values.String) values.String {
selection := nd.selection.Find(selector.String())
return values.NewString(selection.Text())
}
func (el *HTMLElement) InnerTextBySelectorAll(selector values.String) *values.Array {
selection := el.selection.Find(selector.String())
func (nd *HTMLElement) InnerTextBySelectorAll(selector values.String) *values.Array {
selection := nd.selection.Find(selector.String())
arr := values.NewArray(selection.Length())
selection.Each(func(_ int, selection *goquery.Selection) {
@ -234,8 +253,8 @@ func (el *HTMLElement) InnerTextBySelectorAll(selector values.String) *values.Ar
return arr
}
func (el *HTMLElement) CountBySelector(selector values.String) values.Int {
selection := el.selection.Find(selector.String())
func (nd *HTMLElement) CountBySelector(selector values.String) values.Int {
selection := nd.selection.Find(selector.String())
if selection == nil {
return values.ZeroInt
@ -244,8 +263,8 @@ func (el *HTMLElement) CountBySelector(selector values.String) values.Int {
return values.NewInt(selection.Size())
}
func (el *HTMLElement) ExistsBySelector(selector values.String) values.Boolean {
selection := el.selection.Closest(selector.String())
func (nd *HTMLElement) ExistsBySelector(selector values.String) values.Boolean {
selection := nd.selection.Closest(selector.String())
if selection == nil {
return values.False
@ -254,11 +273,47 @@ func (el *HTMLElement) ExistsBySelector(selector values.String) values.Boolean {
return values.True
}
func (el *HTMLElement) parseAttrs() *values.Object {
func (nd *HTMLElement) GetIn(ctx context.Context, path []core.Value) (core.Value, error) {
return common.GetInElement(ctx, nd, path)
}
func (nd *HTMLElement) SetIn(ctx context.Context, path []core.Value, value core.Value) error {
return common.SetInElement(ctx, nd, path, value)
}
func (nd *HTMLElement) Iterate(_ context.Context) (core.Iterator, error) {
return common.NewIterator(nd)
}
func (nd *HTMLElement) Click() (values.Boolean, error) {
return false, core.ErrNotSupported
}
func (nd *HTMLElement) Input(_ core.Value, _ values.Int) error {
return core.ErrNotSupported
}
func (nd *HTMLElement) Select(_ *values.Array) (*values.Array, error) {
return nil, core.ErrNotSupported
}
func (nd *HTMLElement) ScrollIntoView() error {
return core.ErrNotSupported
}
func (nd *HTMLElement) Hover() error {
return core.ErrNotSupported
}
func (nd *HTMLElement) WaitForClass(_ values.String, _ values.Int) error {
return core.ErrNotSupported
}
func (nd *HTMLElement) parseAttrs() *values.Object {
obj := values.NewObject()
for _, name := range common.Attributes {
val, ok := el.selection.Attr(name)
val, ok := nd.selection.Attr(name)
if ok {
obj.Set(values.NewString(name), values.NewString(val))
@ -268,21 +323,12 @@ func (el *HTMLElement) parseAttrs() *values.Object {
return obj
}
func (el *HTMLElement) parseChildren() *values.Array {
children := el.selection.Children()
func (nd *HTMLElement) parseChildren() *values.Array {
children := nd.selection.Children()
arr := values.NewArray(10)
children.Each(func(i int, selection *goquery.Selection) {
//name := goquery.NodeName(selection)
//if name != "#text" && name != "#comment" {
// child, err := NewHTMLElement(selection)
//
// if err == nil {
// arr.Push(child)
// }
//}
child, err := NewHTMLElement(selection)
if err == nil {

5
pkg/drivers/http/element_test.go
View File

@ -2,6 +2,7 @@ package http_test
import (
"bytes"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/http"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/PuerkitoBio/goquery"
@ -320,7 +321,7 @@ func TestElement(t *testing.T) {
So(err, ShouldBeNil)
v := el.Value()
v := el.GetValue()
So(v, ShouldEqual, "find")
})
@ -394,7 +395,7 @@ func TestElement(t *testing.T) {
So(found, ShouldNotEqual, values.None)
v := found.(values.HTMLNode).NodeName()
v := found.(drivers.HTMLNode).NodeName()
So(err, ShouldBeNil)

15
pkg/drivers/options.go Normal file
View File

@ -0,0 +1,15 @@
package drivers
type (
options struct {
defaultDriver string
}
Option func(drv Driver, opts *options)
)
func AsDefault() Option {
return func(drv Driver, opts *options) {
opts.defaultDriver = drv.Name()
}
}

39
pkg/drivers/pdf.go Normal file
View File

@ -0,0 +1,39 @@
package drivers
import "github.com/MontFerret/ferret/pkg/runtime/values"
// PDFParams represents the arguments for PrintToPDF function.
type PDFParams struct {
// Paper orientation. Defaults to false.
Landscape values.Boolean
// Display header and footer. Defaults to false.
DisplayHeaderFooter values.Boolean
// Print background graphics. Defaults to false.
PrintBackground values.Boolean
// Scale of the webpage rendering. Defaults to 1.
Scale values.Float
// Paper width in inches. Defaults to 8.5 inches.
PaperWidth values.Float
// Paper height in inches. Defaults to 11 inches.
PaperHeight values.Float
// Top margin in inches. Defaults to 1cm (~0.4 inches).
MarginTop values.Float
// Bottom margin in inches. Defaults to 1cm (~0.4 inches).
MarginBottom values.Float
// Left margin in inches. Defaults to 1cm (~0.4 inches).
MarginLeft values.Float
// Right margin in inches. Defaults to 1cm (~0.4 inches).
MarginRight values.Float
// Paper ranges to print, e.g., '1-5, 8, 11-13'. Defaults to the empty string, which means print all pages.
PageRanges values.String
// Whether to silently ignore invalid but successfully parsed page ranges, such as '3-2'. Defaults to false.
IgnoreInvalidPageRanges values.Boolean
// HTML template for the print header. Should be valid HTML markup with following classes used to inject printing values into them: - `date`: formatted print date - `title`: document title - `url`: document location - `pageNumber`: current page number - `totalPages`: total pages in the document
// For example, `<span class=title></span>` would generate span containing the title.
HeaderTemplate values.String
// HTML template for the print footer. Should use the same format as the `headerTemplate`.
FooterTemplate values.String
// Whether or not to prefer page size as defined by css.
// Defaults to false, in which case the content will be scaled to fit the paper size.
PreferCSSPageSize values.Boolean
}

47
pkg/drivers/screenshot.go Normal file
View File

@ -0,0 +1,47 @@
package drivers
import "github.com/MontFerret/ferret/pkg/runtime/values"
const (
ScreenshotFormatPNG ScreenshotFormat = "png"
ScreenshotFormatJPEG ScreenshotFormat = "jpeg"
)
type (
ScreenshotFormat string
ScreenshotParams struct {
X values.Float
Y values.Float
Width values.Float
Height values.Float
Format ScreenshotFormat
Quality values.Int
}
)
func IsScreenshotFormatValid(format string) bool {
value := ScreenshotFormat(format)
return value == ScreenshotFormatPNG || value == ScreenshotFormatJPEG
}
func NewDefaultHTMLPDFParams() PDFParams {
return PDFParams{
Landscape: values.False,
DisplayHeaderFooter: values.False,
PrintBackground: values.False,
Scale: values.Float(1),
PaperWidth: values.Float(8.5),
PaperHeight: values.Float(11),
MarginTop: values.Float(0.4),
MarginBottom: values.Float(0.4),
MarginLeft: values.Float(0.4),
MarginRight: values.Float(0.4),
PageRanges: values.EmptyString,
IgnoreInvalidPageRanges: values.False,
HeaderTemplate: values.EmptyString,
FooterTemplate: values.EmptyString,
PreferCSSPageSize: values.False,
}
}

38
pkg/drivers/type.go Normal file
View File

@ -0,0 +1,38 @@
package drivers
import "github.com/MontFerret/ferret/pkg/runtime/core"
var (
HTMLElementType = core.NewType("HTMLElement")
HTMLDocumentType = core.NewType("HTMLDocument")
)
// Comparison table of builtin types
var typeComparisonTable = map[core.Type]uint64{
HTMLElementType: 0,
HTMLDocumentType: 1,
}
func Compare(first, second core.Type) int64 {
f, ok := typeComparisonTable[first]
if !ok {
return -1
}
s, ok := typeComparisonTable[second]
if !ok {
return -1
}
if f == s {
return 0
}
if f > s {
return 1
}
return -1
}

124
pkg/drivers/value.go Normal file
View File

@ -0,0 +1,124 @@
package drivers
import (
"io"
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type (
// Node is an interface from which a number of DOM API object types inherit.
// It allows those types to be treated similarly;
// for example, inheriting the same set of methods, or being tested in the same way.
HTMLNode interface {
core.Value
core.Iterable
core.Getter
core.Setter
collections.Measurable
io.Closer
NodeType() values.Int
NodeName() values.String
GetChildNodes() core.Value
GetChildNode(idx values.Int) core.Value
QuerySelector(selector values.String) core.Value
QuerySelectorAll(selector values.String) core.Value
CountBySelector(selector values.String) values.Int
ExistsBySelector(selector values.String) values.Boolean
}
// HTMLElement is the most general base interface which most objects in a Document implement.
HTMLElement interface {
HTMLNode
InnerText() values.String
InnerHTML() values.String
GetValue() core.Value
SetValue(value core.Value) error
GetAttributes() core.Value
GetAttribute(name values.String) core.Value
SetAttribute(name, value values.String) error
InnerHTMLBySelector(selector values.String) values.String
InnerHTMLBySelectorAll(selector values.String) *values.Array
InnerTextBySelector(selector values.String) values.String
InnerTextBySelectorAll(selector values.String) *values.Array
Click() (values.Boolean, error)
Input(value core.Value, delay values.Int) error
Select(value *values.Array) (*values.Array, error)
ScrollIntoView() error
Hover() error
WaitForClass(class values.String, timeout values.Int) error
}
// The Document interface represents any web page loaded in the browser
// and serves as an entry point into the web page's content, which is the DOM tree.
HTMLDocument interface {
HTMLNode
DocumentElement() HTMLElement
GetURL() core.Value
SetURL(url values.String) error
Navigate(url values.String, timeout values.Int) error
NavigateBack(skip values.Int, timeout values.Int) (values.Boolean, error)
NavigateForward(skip values.Int, timeout values.Int) (values.Boolean, error)
ClickBySelector(selector values.String) (values.Boolean, error)
ClickBySelectorAll(selector values.String) (values.Boolean, error)
InputBySelector(selector values.String, value core.Value, delay values.Int) (values.Boolean, error)
SelectBySelector(selector values.String, value *values.Array) (*values.Array, error)
HoverBySelector(selector values.String) error
PrintToPDF(params PDFParams) (values.Binary, error)
CaptureScreenshot(params ScreenshotParams) (values.Binary, error)
ScrollTop() error
ScrollBottom() error
ScrollBySelector(selector values.String) error
WaitForNavigation(timeout values.Int) error
WaitForSelector(selector values.String, timeout values.Int) error
WaitForClassBySelector(selector, class values.String, timeout values.Int) error
WaitForClassBySelectorAll(selector, class values.String, timeout values.Int) error
}
)

55
pkg/runtime/collections/html.go
View File

@ -1,55 +0,0 @@
package collections
import (
"context"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
)
type HTMLNodeIterator struct {
valVar string
keyVar string
values values.HTMLNode
pos int
}
func NewHTMLNodeIterator(
valVar,
keyVar string,
values values.HTMLNode,
) (Iterator, error) {
if valVar == "" {
return nil, core.Error(core.ErrMissedArgument, "value variable")
}
if values == nil {
return nil, core.Error(core.ErrMissedArgument, "result")
}
return &HTMLNodeIterator{valVar, keyVar, values, 0}, nil
}
func (iterator *HTMLNodeIterator) Next(_ context.Context, scope *core.Scope) (*core.Scope, error) {
if iterator.values.Length() > values.NewInt(iterator.pos) {
idx := values.NewInt(iterator.pos)
val := iterator.values.GetChildNode(idx)
iterator.pos++
nextScope := scope.Fork()
if err := nextScope.SetVariable(iterator.valVar, val); err != nil {
return nil, err
}
if iterator.keyVar != "" {
if err := nextScope.SetVariable(iterator.keyVar, idx); err != nil {
return nil, err
}
}
return nextScope, nil
}
return nil, nil
}

1
pkg/runtime/core/errors.go
View File

@ -18,6 +18,7 @@ var (
ErrUnexpected = errors.New("unexpected error")
ErrTimeout = errors.New("operation timed out")
ErrNotImplemented = errors.New("not implemented")
ErrNotSupported = errors.New("not supported")
)
const typeErrorTemplate = "expected %s, but got %s"

5
pkg/runtime/expressions/data_source.go
View File

@ -5,7 +5,6 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/collections"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
@ -50,8 +49,6 @@ func (ds *DataSource) Iterate(ctx context.Context, scope *core.Scope) (collectio
return collections.NewIndexedIterator(ds.valVariable, ds.keyVariable, data.(collections.IndexedCollection))
case types.Object:
return collections.NewKeyedIterator(ds.valVariable, ds.keyVariable, data.(collections.KeyedCollection))
case types.HTMLElement, types.HTMLDocument:
return collections.NewHTMLNodeIterator(ds.valVariable, ds.keyVariable, data.(values.HTMLNode))
default:
// fallback to user defined types
switch collection := data.(type) {
@ -72,8 +69,6 @@ func (ds *DataSource) Iterate(ctx context.Context, scope *core.Scope) (collectio
data.Type(),
types.Array,
types.Object,
types.HTMLDocument,
types.HTMLElement,
core.NewType("Iterable"),
)
}

189
pkg/runtime/values/helpers.go
View File

@ -19,93 +19,39 @@ func GetIn(ctx context.Context, from core.Value, byPath []core.Value) (core.Valu
}
var result = from
var err error
for i, segment := range byPath {
if result == None || result == nil {
break
}
segmentType := segment.Type()
segType := segment.Type()
switch result.Type() {
case types.Object:
obj := result.(*Object)
if segmentType != types.String {
return nil, core.TypeError(segmentType, types.String)
switch segVal := result.(type) {
case *Object:
if segType != types.String {
return nil, core.TypeError(segType, types.String)
}
result, _ = obj.Get(segment.(String))
result, _ = segVal.Get(segment.(String))
break
case types.Array:
arr := result.(*Array)
if segmentType != types.Int {
return nil, core.TypeError(segmentType, types.Int)
case *Array:
if segType != types.Int {
return nil, core.TypeError(segType, types.Int)
}
result = arr.Get(segment.(Int))
result = segVal.Get(segment.(Int))
break
case types.HTMLElement, types.HTMLDocument:
el := result.(HTMLNode)
if segmentType == types.Int {
result = el.GetChildNode(segment.(Int))
} else if segmentType == types.String {
strSegment := segment.(String)
switch strSegment {
case "nodeType":
result = el.NodeType()
case "nodeName":
result = el.NodeName()
case "innerText":
result = el.InnerText()
case "innerHTML":
result = el.InnerHTML()
case "value":
result = el.Value()
case "attributes":
result = el.GetAttributes()
case "children":
result = el.GetChildNodes()
case "length":
result = el.Length()
case "url":
if result.Type() == types.HTMLDocument {
doc, ok := result.(HTMLDocument)
if ok {
result = doc.URL()
}
}
default:
result = None
}
if err != nil {
return None, err
}
} else {
return nil, core.TypeError(segmentType, types.Int, types.String)
}
case core.Getter:
return segVal.GetIn(ctx, byPath[i:])
default:
getter, ok := result.(core.Getter)
if ok {
return getter.GetIn(ctx, byPath[i:])
}
return None, core.TypeError(
from.Type(),
types.Array,
types.Object,
types.HTMLDocument,
types.HTMLElement,
core.NewType("Getter"),
)
}
}
@ -127,46 +73,38 @@ func SetIn(ctx context.Context, to core.Value, byPath []core.Value, value core.V
isTarget := target == idx
segmentType := segment.Type()
switch parent.Type() {
case types.Object:
parent := parent.(*Object)
switch parVal := parent.(type) {
case *Object:
if segmentType != types.String {
return core.TypeError(segmentType, types.String)
}
if isTarget == false {
current, _ = parent.Get(segment.(String))
current, _ = parVal.Get(segment.(String))
} else {
parent.Set(segment.(String), value)
parVal.Set(segment.(String), value)
}
break
case types.Array:
case *Array:
if segmentType != types.Int {
return core.TypeError(segmentType, types.Int)
}
parent := parent.(*Array)
if isTarget == false {
current = parent.Get(segment.(Int))
current = parVal.Get(segment.(Int))
} else {
if err := parent.Set(segment.(Int), value); err != nil {
if err := parVal.Set(segment.(Int), value); err != nil {
return err
}
}
break
case core.Setter:
return parVal.SetIn(ctx, byPath[idx:], value)
default:
setter, ok := parent.(core.Setter)
if ok {
return setter.SetIn(ctx, byPath[idx:], value)
}
// redefine parent
isArray := segmentType == types.Int
isArray := segmentType.Equals(types.Int)
// it's not an index
if isArray == false {
@ -199,6 +137,8 @@ func SetIn(ctx context.Context, to core.Value, byPath []core.Value, value core.V
if isTarget == false {
current = None
}
break
}
}
@ -300,17 +240,6 @@ func Unmarshal(value json.RawMessage) (core.Value, error) {
return Parse(o), nil
}
func IsCloneable(value core.Value) Boolean {
switch value.Type() {
case types.Array:
return NewBoolean(true)
case types.Object:
return NewBoolean(true)
default:
return NewBoolean(false)
}
}
func ToBoolean(input core.Value) core.Value {
switch input.Type() {
case types.Boolean:
@ -328,55 +257,53 @@ func ToBoolean(input core.Value) core.Value {
}
}
func ToArray(input core.Value) core.Value {
switch input.Type() {
case types.Boolean,
types.Int,
types.Float,
types.String,
types.DateTime:
func ToArray(ctx context.Context, input core.Value) (core.Value, error) {
switch value := input.(type) {
case Boolean,
Int,
Float,
String,
DateTime:
return NewArrayWith(input)
case types.HTMLElement,
types.HTMLDocument:
val := input.(HTMLNode)
attrs := val.GetAttributes()
return NewArrayWith(value), nil
case *Array:
return value.Copy(), nil
case *Object:
arr := NewArray(int(value.Length()))
obj, ok := attrs.(*Object)
if !ok {
return NewArray(0)
}
arr := NewArray(int(obj.Length()))
obj.ForEach(func(value core.Value, key string) bool {
value.ForEach(func(value core.Value, key string) bool {
arr.Push(value)
return true
})
return obj
case types.Array:
return input.Copy()
case types.Object:
obj, ok := input.(*Object)
return value, nil
case core.Iterable:
iterator, err := value.Iterate(ctx)
if !ok {
return NewArray(0)
if err != nil {
return None, err
}
arr := NewArray(int(obj.Length()))
arr := NewArray(10)
obj.ForEach(func(value core.Value, key string) bool {
arr.Push(value)
for {
val, _, err := iterator.Next(ctx)
return true
})
if err != nil {
return None, err
}
return obj
if val == None {
break
}
arr.Push(val)
}
return arr, nil
default:
return NewArray(0)
return NewArray(0), nil
}
}

191
pkg/runtime/values/html.go
View File

@ -1,191 +0,0 @@
package values
import (
"github.com/MontFerret/ferret/pkg/runtime/core"
"io"
)
const (
HTMLScreenshotFormatPNG HTMLScreenshotFormat = "png"
HTMLScreenshotFormatJPEG HTMLScreenshotFormat = "jpeg"
)
type (
// HTMLPDFParams represents the arguments for PrintToPDF function.
HTMLPDFParams struct {
// Paper orientation. Defaults to false.
Landscape Boolean
// Display header and footer. Defaults to false.
DisplayHeaderFooter Boolean
// Print background graphics. Defaults to false.
PrintBackground Boolean
// Scale of the webpage rendering. Defaults to 1.
Scale Float
// Paper width in inches. Defaults to 8.5 inches.
PaperWidth Float
// Paper height in inches. Defaults to 11 inches.
PaperHeight Float
// Top margin in inches. Defaults to 1cm (~0.4 inches).
MarginTop Float
// Bottom margin in inches. Defaults to 1cm (~0.4 inches).
MarginBottom Float
// Left margin in inches. Defaults to 1cm (~0.4 inches).
MarginLeft Float
// Right margin in inches. Defaults to 1cm (~0.4 inches).
MarginRight Float
// Paper ranges to print, e.g., '1-5, 8, 11-13'. Defaults to the empty string, which means print all pages.
PageRanges String
// Whether to silently ignore invalid but successfully parsed page ranges, such as '3-2'. Defaults to false.
IgnoreInvalidPageRanges Boolean
// HTML template for the print header. Should be valid HTML markup with following classes used to inject printing values into them: - `date`: formatted print date - `title`: document title - `url`: document location - `pageNumber`: current page number - `totalPages`: total pages in the document
// For example, `<span class=title></span>` would generate span containing the title.
HeaderTemplate String
// HTML template for the print footer. Should use the same format as the `headerTemplate`.
FooterTemplate String
// Whether or not to prefer page size as defined by css.
// Defaults to false, in which case the content will be scaled to fit the paper size.
PreferCSSPageSize Boolean
}
HTMLScreenshotFormat string
HTMLScreenshotParams struct {
X Float
Y Float
Width Float
Height Float
Format HTMLScreenshotFormat
Quality Int
}
// HTMLNode is a HTML Node
HTMLNode interface {
core.Value
NodeType() Int
NodeName() String
Length() Int
InnerText() String
InnerHTML() String
Value() core.Value
GetAttributes() core.Value
GetAttribute(name String) core.Value
GetChildNodes() core.Value
GetChildNode(idx Int) core.Value
QuerySelector(selector String) core.Value
QuerySelectorAll(selector String) core.Value
InnerHTMLBySelector(selector String) String
InnerHTMLBySelectorAll(selector String) *Array
InnerTextBySelector(selector String) String
InnerTextBySelectorAll(selector String) *Array
CountBySelector(selector String) Int
ExistsBySelector(selector String) Boolean
}
DHTMLNode interface {
HTMLNode
io.Closer
Click() (Boolean, error)
Input(value core.Value, delay Int) error
Select(value *Array) (*Array, error)
ScrollIntoView() error
Hover() error
WaitForClass(class String, timeout Int) error
}
// HTMLDocument is a HTML Document
HTMLDocument interface {
HTMLNode
URL() core.Value
}
// DHTMLDocument is a Dynamic HTML Document
DHTMLDocument interface {
HTMLDocument
io.Closer
Navigate(url String, timeout Int) error
NavigateBack(skip Int, timeout Int) (Boolean, error)
NavigateForward(skip Int, timeout Int) (Boolean, error)
ClickBySelector(selector String) (Boolean, error)
ClickBySelectorAll(selector String) (Boolean, error)
InputBySelector(selector String, value core.Value, delay Int) (Boolean, error)
SelectBySelector(selector String, value *Array) (*Array, error)
HoverBySelector(selector String) error
PrintToPDF(params HTMLPDFParams) (Binary, error)
CaptureScreenshot(params HTMLScreenshotParams) (Binary, error)
ScrollTop() error
ScrollBottom() error
ScrollBySelector(selector String) error
WaitForNavigation(timeout Int) error
WaitForSelector(selector String, timeout Int) error
WaitForClass(selector, class String, timeout Int) error
WaitForClassAll(selector, class String, timeout Int) error
}
)
func IsHTMLScreenshotFormatValid(format string) bool {
value := HTMLScreenshotFormat(format)
return value == HTMLScreenshotFormatPNG || value == HTMLScreenshotFormatJPEG
}
func NewDefaultHTMLPDFParams() HTMLPDFParams {
return HTMLPDFParams{
Landscape: False,
DisplayHeaderFooter: False,
PrintBackground: False,
Scale: Float(1),
PaperWidth: Float(8.5),
PaperHeight: Float(11),
MarginTop: Float(0.4),
MarginBottom: Float(0.4),
MarginLeft: Float(0.4),
MarginRight: Float(0.4),
PageRanges: EmptyString,
IgnoreInvalidPageRanges: False,
HeaderTemplate: EmptyString,
FooterTemplate: EmptyString,
PreferCSSPageSize: False,
}
}

20
pkg/runtime/values/types/helpers.go
View File

@ -4,17 +4,15 @@ import "github.com/MontFerret/ferret/pkg/runtime/core"
// Comparison table of builtin types
var typeComparisonTable = map[core.Type]uint64{
None: 0,
Boolean: 1,
Int: 2,
Float: 3,
String: 4,
DateTime: 5,
Array: 6,
Object: 7,
HTMLElement: 8,
HTMLDocument: 9,
Binary: 10,
None: 0,
Boolean: 1,
Int: 2,
Float: 3,
String: 4,
DateTime: 5,
Array: 6,
Object: 7,
Binary: 8,
}
func Compare(first, second core.Type) int64 {

20
pkg/runtime/values/types/types.go
View File

@ -3,15 +3,13 @@ package types
import "github.com/MontFerret/ferret/pkg/runtime/core"
var (
None = core.NewType("none")
Boolean = core.NewType("boolean")
Int = core.NewType("int")
Float = core.NewType("float")
String = core.NewType("string")
DateTime = core.NewType("date_time")
Array = core.NewType("array")
Object = core.NewType("object")
Binary = core.NewType("binary")
HTMLElement = core.NewType("HTMLElement")
HTMLDocument = core.NewType("HTMLDocument")
None = core.NewType("none")
Boolean = core.NewType("boolean")
Int = core.NewType("int")
Float = core.NewType("float")
String = core.NewType("string")
DateTime = core.NewType("date_time")
Array = core.NewType("array")
Object = core.NewType("object")
Binary = core.NewType("binary")
)

2
pkg/stdlib/collections/length.go
View File

@ -25,8 +25,6 @@ func Length(_ context.Context, inputs ...core.Value) (core.Value, error) {
types.String,
types.Array,
types.Object,
types.HTMLElement,
types.HTMLDocument,
types.Binary,
core.NewType("Measurable"),
)

24
pkg/stdlib/html/click.go
View File

@ -5,7 +5,6 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// Click dispatches click event on a given element
@ -20,38 +19,23 @@ func Click(_ context.Context, args ...core.Value) (core.Value, error) {
// CLICK(el)
if len(args) == 1 {
arg1 := args[0]
err := core.ValidateType(arg1, types.HTMLElement)
el, err := toElement(args[0])
if err != nil {
return values.False, err
}
el, ok := arg1.(values.DHTMLNode)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
}
return el.Click()
}
// CLICK(doc, selector)
arg1 := args[0]
selector := args[1].String()
err = core.ValidateType(arg1, types.HTMLDocument)
doc, err := toDocument(args[0])
if err != nil {
return values.None, err
return values.False, err
}
doc, ok := arg1.(values.DHTMLDocument)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
}
selector := args[1].String()
return doc.ClickBySelector(values.NewString(selector))
}

10
pkg/stdlib/html/click_all.go
View File

@ -3,9 +3,9 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// ClickAll dispatches click event on all matched element
@ -22,16 +22,16 @@ func ClickAll(_ context.Context, args ...core.Value) (core.Value, error) {
arg1 := args[0]
selector := args[1].String()
err = core.ValidateType(arg1, types.HTMLDocument)
err = core.ValidateType(arg1, drivers.HTMLDocumentType)
if err != nil {
return values.None, err
}
doc, ok := arg1.(values.DHTMLDocument)
doc, err := toDocument(args[0])
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
if err != nil {
return values.None, err
}
return doc.ClickBySelectorAll(values.NewString(selector))

64
pkg/stdlib/html/document.go
View File

@ -5,13 +5,14 @@ import (
"time"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/drivers/cdp"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
type DocumentLoadParams struct {
Dynamic values.Boolean
Driver string
Timeout time.Duration
}
@ -56,17 +57,7 @@ func Document(ctx context.Context, args ...core.Value) (core.Value, error) {
ctx, cancel := context.WithTimeout(ctx, params.Timeout)
defer cancel()
if params.Dynamic {
drv, err := drivers.DynamicFrom(ctx)
if err != nil {
return values.None, err
}
return drv.GetDocument(ctx, url)
}
drv, err := drivers.StaticFrom(ctx)
drv, err := drivers.FromContext(ctx, params.Driver)
if err != nil {
return values.None, err
@ -77,7 +68,6 @@ func Document(ctx context.Context, args ...core.Value) (core.Value, error) {
func newDefaultDocLoadParams() DocumentLoadParams {
return DocumentLoadParams{
Dynamic: false,
Timeout: time.Second * 30,
}
}
@ -85,36 +75,48 @@ func newDefaultDocLoadParams() DocumentLoadParams {
func newDocLoadParams(arg core.Value) (DocumentLoadParams, error) {
res := newDefaultDocLoadParams()
if err := core.ValidateType(arg, types.Boolean, types.Object); err != nil {
if err := core.ValidateType(arg, types.Boolean, types.String, types.Object); err != nil {
return res, err
}
if arg.Type() == types.Boolean {
res.Dynamic = arg.(values.Boolean)
switch arg.Type() {
case types.Object:
obj := arg.(*values.Object)
return res, nil
}
driver, exists := obj.Get(values.NewString("driver"))
obj := arg.(*values.Object)
if exists {
if err := core.ValidateType(driver, types.String); err != nil {
return res, err
}
isDynamic, exists := obj.Get(values.NewString("dynamic"))
if exists {
if err := core.ValidateType(isDynamic, types.Boolean); err != nil {
return res, err
res.Driver = driver.(values.String).String()
}
res.Dynamic = isDynamic.(values.Boolean)
}
timeout, exists := obj.Get(values.NewString("timeout"))
timeout, exists := obj.Get(values.NewString("timeout"))
if exists {
if err := core.ValidateType(timeout, types.Int); err != nil {
return res, err
}
if exists {
if err := core.ValidateType(timeout, types.Int); err != nil {
return res, err
res.Timeout = time.Duration(timeout.(values.Int)) + time.Millisecond
}
res.Timeout = time.Duration(timeout.(values.Int)) + time.Millisecond
break
case types.String:
res.Driver = arg.(values.String).String()
break
case types.Boolean:
b := arg.(values.Boolean)
// fallback
if b {
res.Driver = cdp.DriverName
}
break
}
return res, nil

7
pkg/stdlib/html/element.go
View File

@ -3,6 +3,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
@ -23,14 +24,14 @@ func Element(_ context.Context, args ...core.Value) (core.Value, error) {
return el.QuerySelector(selector), nil
}
func queryArgs(args []core.Value) (values.HTMLNode, values.String, error) {
func queryArgs(args []core.Value) (drivers.HTMLNode, values.String, error) {
err := core.ValidateArgs(args, 2, 2)
if err != nil {
return nil, values.EmptyString, err
}
err = core.ValidateType(args[0], types.HTMLDocument, types.HTMLElement)
err = core.ValidateType(args[0], drivers.HTMLDocumentType, drivers.HTMLElementType)
if err != nil {
return nil, values.EmptyString, err
@ -42,5 +43,5 @@ func queryArgs(args []core.Value) (values.HTMLNode, values.String, error) {
return nil, values.EmptyString, err
}
return args[0].(values.HTMLNode), args[1].(values.String), nil
return args[0].(drivers.HTMLNode), args[1].(values.String), nil
}

2
pkg/stdlib/html/element_exists.go
View File

@ -8,7 +8,7 @@ import (
)
// ElementExists returns a boolean value indicating whether there is an element matched by selector.
// @param docOrEl (HTMLDocument|HTMLElement) - Parent document or element.
// @param docOrEl (HTMLDocument|HTMLNode) - Parent document or element.
// @param selector (String) - CSS selector.
// @returns (Boolean) - A boolean value indicating whether there is an element matched by selector.
func ElementExists(_ context.Context, args ...core.Value) (core.Value, error) {

2
pkg/stdlib/html/elements.go
View File

@ -9,7 +9,7 @@ import (
// Elements finds HTML elements by a given CSS selector.
// Returns an empty array if element not found.
// @param docOrEl (HTMLDocument|HTMLElement) - Parent document or element.
// @param docOrEl (HTMLDocument|HTMLNode) - Parent document or element.
// @param selector (String) - CSS selector.
// @returns (Array) - Returns an array of found HTML element.
func Elements(_ context.Context, args ...core.Value) (core.Value, error) {

2
pkg/stdlib/html/elements_count.go
View File

@ -9,7 +9,7 @@ import (
// ElementsCount returns a number of found HTML elements by a given CSS selector.
// Returns an empty array if element not found.
// @param docOrEl (HTMLDocument|HTMLElement) - Parent document or element.
// @param docOrEl (HTMLDocument|HTMLNode) - Parent document or element.
// @param selector (String) - CSS selector.
// @returns (Int) - A number of found HTML elements by a given CSS selector.
func ElementsCount(_ context.Context, args ...core.Value) (core.Value, error) {

16
pkg/stdlib/html/hover.go
View File

@ -3,6 +3,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
@ -20,7 +21,7 @@ func Hover(_ context.Context, args ...core.Value) (core.Value, error) {
}
// document or element
err = core.ValidateType(args[0], types.HTMLDocument, types.HTMLElement)
err = core.ValidateType(args[0], drivers.HTMLDocumentType, drivers.HTMLElementType)
if err != nil {
return values.None, err
@ -34,23 +35,14 @@ func Hover(_ context.Context, args ...core.Value) (core.Value, error) {
}
// Document with a selector
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)
}
doc := args[0].(drivers.HTMLDocument)
selector := args[1].(values.String)
return values.None, doc.HoverBySelector(selector)
}
// Element
el, ok := args[0].(values.DHTMLNode)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)
}
el := args[0].(drivers.HTMLElement)
return values.None, el.Hover()
}

13
pkg/stdlib/html/inner_html.go
View File

@ -3,6 +3,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
@ -19,16 +20,20 @@ func InnerHTML(_ context.Context, args ...core.Value) (core.Value, error) {
return values.EmptyString, err
}
err = core.ValidateType(args[0], types.HTMLDocument, types.HTMLElement)
err = core.ValidateType(args[0], drivers.HTMLDocumentType, drivers.HTMLElementType)
if err != nil {
return values.None, err
}
node := args[0].(values.HTMLNode)
el, err := resolveElement(args[0])
if err != nil {
return values.None, err
}
if len(args) == 1 {
return node.InnerHTML(), nil
return el.InnerHTML(), nil
}
err = core.ValidateType(args[1], types.String)
@ -39,5 +44,5 @@ func InnerHTML(_ context.Context, args ...core.Value) (core.Value, error) {
selector := args[1].(values.String)
return node.InnerHTMLBySelector(selector), nil
return el.InnerHTMLBySelector(selector), nil
}

12
pkg/stdlib/html/inner_html_all.go
View File

@ -3,6 +3,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
@ -19,7 +20,7 @@ func InnerHTMLAll(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
err = core.ValidateType(args[0], types.HTMLDocument, types.HTMLElement)
err = core.ValidateType(args[0], drivers.HTMLDocumentType, drivers.HTMLElementType)
if err != nil {
return values.None, err
@ -31,8 +32,13 @@ func InnerHTMLAll(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc := args[0].(values.HTMLNode)
el, err := resolveElement(args[0])
if err != nil {
return values.None, err
}
selector := args[1].(values.String)
return doc.InnerHTMLBySelectorAll(selector), nil
return el.InnerHTMLBySelectorAll(selector), nil
}

8
pkg/stdlib/html/inner_text.go
View File

@ -19,16 +19,14 @@ func InnerText(_ context.Context, args ...core.Value) (core.Value, error) {
return values.EmptyString, err
}
err = core.ValidateType(args[0], types.HTMLDocument, types.HTMLElement)
el, err := resolveElement(args[0])
if err != nil {
return values.None, err
}
node := args[0].(values.HTMLNode)
if len(args) == 1 {
return node.InnerText(), nil
return el.InnerText(), nil
}
err = core.ValidateType(args[1], types.String)
@ -39,5 +37,5 @@ func InnerText(_ context.Context, args ...core.Value) (core.Value, error) {
selector := args[1].(values.String)
return node.InnerTextBySelector(selector), nil
return el.InnerTextBySelector(selector), nil
}

12
pkg/stdlib/html/inner_text_all.go
View File

@ -3,6 +3,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
@ -19,7 +20,7 @@ func InnerTextAll(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
err = core.ValidateType(args[0], types.HTMLDocument, types.HTMLElement)
err = core.ValidateType(args[0], drivers.HTMLDocumentType, drivers.HTMLElementType)
if err != nil {
return values.None, err
@ -31,8 +32,13 @@ func InnerTextAll(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc := args[0].(values.HTMLNode)
el, err := resolveElement(args[0])
if err != nil {
return values.None, err
}
selector := args[1].(values.String)
return doc.InnerTextBySelectorAll(selector), nil
return el.InnerTextBySelectorAll(selector), nil
}

49
pkg/stdlib/html/input.go
View File

@ -3,6 +3,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
@ -22,19 +23,14 @@ func Input(_ context.Context, args ...core.Value) (core.Value, error) {
}
arg1 := args[0]
err = core.ValidateType(arg1, types.HTMLDocument, types.HTMLElement)
err = core.ValidateType(arg1, drivers.HTMLDocumentType, drivers.HTMLElementType)
if err != nil {
return values.False, err
}
switch args[0].(type) {
case values.DHTMLDocument:
doc, ok := arg1.(values.DHTMLDocument)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
}
if arg1.Type() == drivers.HTMLDocumentType {
doc := arg1.(drivers.HTMLDocument)
// selector
arg2 := args[1]
@ -59,35 +55,28 @@ func Input(_ context.Context, args ...core.Value) (core.Value, error) {
}
return doc.InputBySelector(arg2.(values.String), args[2], delay)
case values.DHTMLNode:
el, ok := arg1.(values.DHTMLNode)
}
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
}
el := arg1.(drivers.HTMLElement)
delay := values.Int(0)
delay := values.Int(0)
if len(args) == 3 {
arg3 := args[2]
if len(args) == 3 {
arg3 := args[2]
err = core.ValidateType(arg3, types.Int)
if err != nil {
return values.False, err
}
delay = arg3.(values.Int)
}
err = el.Input(args[1], delay)
err = core.ValidateType(arg3, types.Int)
if err != nil {
return values.False, err
}
return values.True, nil
default:
return values.False, core.Errors(core.ErrInvalidArgument)
delay = arg3.(values.Int)
}
err = el.Input(args[1], delay)
if err != nil {
return values.False, err
}
return values.True, nil
}

53
pkg/stdlib/html/lib.go
View File

@ -3,18 +3,14 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
"github.com/pkg/errors"
)
const defaultTimeout = 5000
var (
ErrNotDynamic = errors.New("expected dynamic document or element")
)
func NewLib() map[string]core.Function {
return map[string]core.Function{
"CLICK": Click,
@ -26,7 +22,6 @@ func NewLib() map[string]core.Function {
"ELEMENTS": Elements,
"ELEMENTS_COUNT": ElementsCount,
"HOVER": Hover,
"HTML_PARSE": Parse,
"INNER_HTML": InnerHTML,
"INNER_HTML_ALL": InnerHTMLAll,
"INNER_TEXT": InnerText,
@ -50,14 +45,12 @@ func NewLib() map[string]core.Function {
}
func ValidateDocument(ctx context.Context, value core.Value) (core.Value, error) {
err := core.ValidateType(value, types.HTMLDocument, types.String)
err := core.ValidateType(value, drivers.HTMLDocumentType, types.String)
if err != nil {
return values.None, err
}
var doc values.DHTMLDocument
var ok bool
var doc drivers.HTMLDocument
if value.Type() == types.String {
buf, err := Document(ctx, value, values.NewBoolean(true))
@ -66,14 +59,42 @@ func ValidateDocument(ctx context.Context, value core.Value) (core.Value, error)
return values.None, err
}
doc, ok = buf.(values.DHTMLDocument)
doc = buf.(drivers.HTMLDocument)
} else {
doc, ok = value.(values.DHTMLDocument)
}
if !ok {
return nil, ErrNotDynamic
doc = value.(drivers.HTMLDocument)
}
return doc, nil
}
func resolveElement(value core.Value) (drivers.HTMLElement, error) {
vt := value.Type()
if vt == drivers.HTMLDocumentType {
return value.(drivers.HTMLDocument).DocumentElement(), nil
} else if vt == drivers.HTMLElementType {
return value.(drivers.HTMLElement), nil
}
return nil, core.TypeError(value.Type(), drivers.HTMLDocumentType, drivers.HTMLElementType)
}
func toDocument(value core.Value) (drivers.HTMLDocument, error) {
err := core.ValidateType(value, drivers.HTMLDocumentType)
if err != nil {
return nil, err
}
return value.(drivers.HTMLDocument), nil
}
func toElement(value core.Value) (drivers.HTMLElement, error) {
err := core.ValidateType(value, drivers.HTMLElementType)
if err != nil {
return nil, err
}
return value.(drivers.HTMLElement), nil
}

14
pkg/stdlib/html/navigate.go
View File

@ -21,24 +21,12 @@ func Navigate(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
err = core.ValidateType(args[0], types.HTMLDocument)
doc, err := toDocument(args[0])
if err != nil {
return values.None, err
}
err = core.ValidateType(args[1], types.String)
if err != nil {
return values.None, err
}
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
}
timeout := values.NewInt(defaultTimeout)
if len(args) > 2 {

8
pkg/stdlib/html/navigate_back.go
View File

@ -22,18 +22,12 @@ func NavigateBack(_ context.Context, args ...core.Value) (core.Value, error) {
return values.False, err
}
err = core.ValidateType(args[0], types.HTMLDocument)
doc, err := toDocument(args[0])
if err != nil {
return values.None, err
}
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
}
skip := values.NewInt(1)
timeout := values.NewInt(defaultTimeout)

8
pkg/stdlib/html/navigate_forward.go
View File

@ -22,18 +22,12 @@ func NavigateForward(_ context.Context, args ...core.Value) (core.Value, error)
return values.False, err
}
err = core.ValidateType(args[0], types.HTMLDocument)
doc, err := toDocument(args[0])
if err != nil {
return values.None, err
}
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
}
skip := values.NewInt(1)
timeout := values.NewInt(defaultTimeout)

13
pkg/stdlib/html/pagination.go
View File

@ -3,6 +3,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
@ -20,10 +21,10 @@ func Pagination(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc, ok := args[0].(values.DHTMLDocument)
doc, err := toDocument(args[0])
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
if err != nil {
return values.None, err
}
err = core.ValidateType(args[1], types.String)
@ -37,16 +38,16 @@ func Pagination(_ context.Context, args ...core.Value) (core.Value, error) {
return &Paging{doc, selector}, nil
}
var PagingType = core.NewType("Paging")
var PagingType = core.NewType("paging")
type (
Paging struct {
document values.DHTMLDocument
document drivers.HTMLDocument
selector values.String
}
PagingIterator struct {
document values.DHTMLDocument
document drivers.HTMLDocument
selector values.String
pos values.Int
}

38
pkg/stdlib/html/parse.go
View File

@ -1,38 +0,0 @@
package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// Parse parses a given HTML string and returns a HTML document.
// Returned HTML document is always static.
// @param html (String) - Target HTML string.
// @returns (HTMLDocument) - Parsed HTML static document.
func Parse(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 1, 1)
if err != nil {
return values.None, err
}
err = core.ValidateType(args[0], types.String)
if err != nil {
return values.None, err
}
drv, err := drivers.StaticFrom(ctx)
if err != nil {
return values.None, err
}
str := args[0].(values.String)
return drv.ParseDocument(ctx, str)
}

5
pkg/stdlib/html/pdf.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"regexp"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
@ -53,10 +54,10 @@ func PDF(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc := val.(values.DHTMLDocument)
doc := val.(drivers.HTMLDocument)
defer doc.Close()
pdfParams := values.HTMLPDFParams{}
pdfParams := drivers.PDFParams{}
if len(args) == 2 {
arg2 := args[1]

13
pkg/stdlib/html/screenshot.go
View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
@ -28,7 +29,7 @@ func Screenshot(ctx context.Context, args ...core.Value) (core.Value, error) {
arg1 := args[0]
err = core.ValidateType(arg1, types.HTMLDocument, types.String)
err = core.ValidateType(arg1, drivers.HTMLDocumentType, types.String)
if err != nil {
return values.None, err
@ -40,16 +41,16 @@ func Screenshot(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc := val.(values.DHTMLDocument)
doc := val.(drivers.HTMLDocument)
defer doc.Close()
screenshotParams := values.HTMLScreenshotParams{
screenshotParams := drivers.ScreenshotParams{
X: 0,
Y: 0,
Width: -1,
Height: -1,
Format: values.HTMLScreenshotFormatJPEG,
Format: drivers.ScreenshotFormatJPEG,
Quality: 100,
}
@ -76,13 +77,13 @@ func Screenshot(ctx context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
if !values.IsHTMLScreenshotFormatValid(format.String()) {
if !drivers.IsScreenshotFormatValid(format.String()) {
return values.None, core.Error(
core.ErrInvalidArgument,
fmt.Sprintf("format is not valid, expected jpeg or png, but got %s", format.String()))
}
screenshotParams.Format = values.HTMLScreenshotFormat(format.String())
screenshotParams.Format = drivers.ScreenshotFormat(format.String())
}
x, found := params.Get("x")

10
pkg/stdlib/html/scroll_bottom.go
View File

@ -3,9 +3,9 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// ScrollTop scrolls the document's window to its bottom.
@ -17,16 +17,16 @@ func ScrollBottom(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
err = core.ValidateType(args[0], types.HTMLDocument)
err = core.ValidateType(args[0], drivers.HTMLDocumentType)
if err != nil {
return values.None, err
}
doc, ok := args[0].(values.DHTMLDocument)
doc, err := toDocument(args[0])
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)
if err != nil {
return values.None, err
}
return values.None, doc.ScrollBottom()

31
pkg/stdlib/html/scroll_element.go
View File

@ -3,6 +3,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
@ -18,14 +19,13 @@ func ScrollInto(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
// document or element
err = core.ValidateType(args[0], types.HTMLDocument, types.HTMLElement)
if err != nil {
return values.None, err
}
if len(args) == 2 {
err = core.ValidateType(args[0], drivers.HTMLDocumentType)
if err != nil {
return values.None, err
}
err = core.ValidateType(args[1], types.String)
if err != nil {
@ -33,23 +33,20 @@ func ScrollInto(_ context.Context, args ...core.Value) (core.Value, error) {
}
// Document with a selector
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)
}
doc := args[0].(drivers.HTMLDocument)
selector := args[1].(values.String)
return values.None, doc.ScrollBySelector(selector)
}
// Element
el, ok := args[0].(values.DHTMLNode)
err = core.ValidateType(args[0], drivers.HTMLElementType)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)
if err != nil {
return values.None, err
}
// Element
el := args[0].(drivers.HTMLElement)
return values.None, el.ScrollIntoView()
}

10
pkg/stdlib/html/scroll_top.go
View File

@ -3,9 +3,9 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// ScrollTop scrolls the document's window to its top.
@ -17,16 +17,16 @@ func ScrollTop(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
err = core.ValidateType(args[0], types.HTMLDocument)
err = core.ValidateType(args[0], drivers.HTMLDocumentType)
if err != nil {
return values.None, err
}
doc, ok := args[0].(values.DHTMLDocument)
doc, err := toDocument(args[0])
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)
if err != nil {
return values.None, err
}
return values.None, doc.ScrollTop()

41
pkg/stdlib/html/select.go
View File

@ -3,6 +3,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
@ -21,19 +22,14 @@ func Select(_ context.Context, args ...core.Value) (core.Value, error) {
}
arg1 := args[0]
err = core.ValidateType(arg1, types.HTMLDocument, types.HTMLElement)
err = core.ValidateType(arg1, drivers.HTMLDocumentType, drivers.HTMLElementType)
if err != nil {
return values.False, err
}
switch args[0].(type) {
case values.DHTMLDocument:
doc, ok := arg1.(values.DHTMLDocument)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
}
if arg1.Type() == drivers.HTMLDocumentType {
doc := arg1.(drivers.HTMLDocument)
// selector
arg2 := args[1]
@ -51,23 +47,16 @@ func Select(_ context.Context, args ...core.Value) (core.Value, error) {
}
return doc.SelectBySelector(arg2.(values.String), arg3.(*values.Array))
case values.DHTMLNode:
el, ok := arg1.(values.DHTMLNode)
if !ok {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
}
arg2 := args[1]
err = core.ValidateType(arg2, types.Array)
if err != nil {
return values.False, err
}
return el.Select(arg2.(*values.Array))
default:
return values.False, core.Errors(core.ErrInvalidArgument)
}
el := arg1.(drivers.HTMLElement)
arg2 := args[1]
err = core.ValidateType(arg2, types.Array)
if err != nil {
return values.False, err
}
return el.Select(arg2.(*values.Array))
}

33
pkg/stdlib/html/wait_class.go
View File

@ -3,6 +3,7 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
@ -25,7 +26,8 @@ func WaitClass(_ context.Context, args ...core.Value) (core.Value, error) {
}
// document or element
err = core.ValidateType(args[0], types.HTMLDocument, types.HTMLElement)
arg1 := args[0]
err = core.ValidateType(arg1, drivers.HTMLDocumentType, drivers.HTMLElementType)
if err != nil {
return values.None, err
@ -40,9 +42,8 @@ func WaitClass(_ context.Context, args ...core.Value) (core.Value, error) {
timeout := values.NewInt(defaultTimeout)
// lets figure out what is passed as 1st argument
switch obj := args[0].(type) {
case values.DHTMLDocument:
// if a document is passed
if arg1.Type() == drivers.HTMLDocumentType {
// revalidate args with more accurate amount
err := core.ValidateArgs(args, 3, 4)
@ -57,6 +58,7 @@ func WaitClass(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc := arg1.(drivers.HTMLDocument)
selector := args[1].(values.String)
class := args[2].(values.String)
@ -70,22 +72,21 @@ func WaitClass(_ context.Context, args ...core.Value) (core.Value, error) {
timeout = args[3].(values.Int)
}
return values.None, obj.WaitForClass(selector, class, timeout)
case values.DHTMLNode:
class := args[1].(values.String)
return values.None, doc.WaitForClassBySelector(selector, class, timeout)
}
if len(args) == 3 {
err = core.ValidateType(args[2], types.Int)
el := arg1.(drivers.HTMLElement)
class := args[1].(values.String)
if err != nil {
return values.None, err
}
if len(args) == 3 {
err = core.ValidateType(args[2], types.Int)
timeout = args[2].(values.Int)
if err != nil {
return values.None, err
}
return values.None, obj.WaitForClass(class, timeout)
default:
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)
timeout = args[2].(values.Int)
}
return values.None, el.WaitForClass(class, timeout)
}

11
pkg/stdlib/html/wait_class_all.go
View File

@ -21,8 +21,7 @@ func WaitClassAll(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
// document only
err = core.ValidateType(args[0], types.HTMLDocument)
doc, err := toDocument(args[0])
if err != nil {
return values.None, err
@ -42,12 +41,6 @@ func WaitClassAll(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)
}
selector := args[1].(values.String)
class := args[2].(values.String)
timeout := values.NewInt(defaultTimeout)
@ -62,5 +55,5 @@ func WaitClassAll(_ context.Context, args ...core.Value) (core.Value, error) {
timeout = args[3].(values.Int)
}
return values.None, doc.WaitForClassAll(selector, class, timeout)
return values.None, doc.WaitForClassBySelectorAll(selector, class, timeout)
}

21
pkg/stdlib/html/wait_element.go
View File

@ -10,7 +10,7 @@ import (
// WaitElement waits for element to appear in the DOM.
// Stops the execution until it finds an element or operation times out.
// @param doc (HTMLDocument) - Dynamic HTMLDocument.
// @param doc (HTMLDocument) - Driver HTMLDocument.
// @param selector (String) - Target element's selector.
// @param timeout (Int, optional) - Optional timeout. Default 5000 ms.
func WaitElement(_ context.Context, args ...core.Value) (core.Value, error) {
@ -20,7 +20,12 @@ func WaitElement(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
arg := args[0]
doc, err := toDocument(args[0])
if err != nil {
return values.None, err
}
selector := args[1].String()
timeout := values.NewInt(defaultTimeout)
@ -34,17 +39,5 @@ func WaitElement(_ context.Context, args ...core.Value) (core.Value, error) {
timeout = args[2].(values.Int)
}
err = core.ValidateType(arg, types.HTMLDocument)
if err != nil {
return values.None, err
}
doc, ok := arg.(values.DHTMLDocument)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)
}
return values.None, doc.WaitForSelector(values.NewString(selector), timeout)
}

10
pkg/stdlib/html/wait_navigation.go
View File

@ -10,7 +10,7 @@ import (
// WaitNavigation waits for document to navigate to a new url.
// Stops the execution until the navigation ends or operation times out.
// @param doc (HTMLDocument) - Dynamic HTMLDocument.
// @param doc (HTMLDocument) - Driver HTMLDocument.
// @param timeout (Int, optional) - Optional timeout. Default 5000 ms.
func WaitNavigation(_ context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 1, 2)
@ -19,18 +19,12 @@ func WaitNavigation(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
err = core.ValidateType(args[0], types.HTMLDocument)
doc, err := toDocument(args[0])
if err != nil {
return values.None, err
}
doc, ok := args[0].(values.DHTMLDocument)
if !ok {
return values.None, core.Errors(core.ErrInvalidType, ErrNotDynamic)
}
timeout := values.NewInt(defaultTimeout)
if len(args) > 1 {

4
pkg/stdlib/types/is_html_document.go
View File

@ -3,9 +3,9 @@ package types
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// IsHTMLDocument checks whether value is a HTMLDocument value.
@ -18,5 +18,5 @@ func IsHTMLDocument(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
return isTypeof(args[0], types.HTMLDocument), nil
return isTypeof(args[0], drivers.HTMLDocumentType), nil
}

4
pkg/stdlib/types/is_html_element.go
View File

@ -3,9 +3,9 @@ package types
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// IsHTMLElement checks whether value is a HTMLElement value.
@ -18,5 +18,5 @@ func IsHTMLElement(_ context.Context, args ...core.Value) (core.Value, error) {
return values.None, err
}
return isTypeof(args[0], types.HTMLElement), nil
return isTypeof(args[0], drivers.HTMLElementType), nil
}

6
pkg/stdlib/types/to_array.go
View File

@ -13,14 +13,12 @@ import (
// Boolean values, numbers and strings are converted to an array containing the original value as its single element
// Arrays keep their original value
// Objects / HTML nodes are converted to an array containing their attribute values as array elements.
func ToArray(_ context.Context, args ...core.Value) (core.Value, error) {
func ToArray(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 1, 1)
if err != nil {
return values.None, err
}
arg := args[0]
return values.ToArray(arg), nil
return values.ToArray(ctx, args[0])
}