ferret/pkg/stdlib/html/document.go

package html

import (
	"context"
	"strings"
	"time"

	"github.com/MontFerret/ferret/pkg/drivers"
	"github.com/MontFerret/ferret/pkg/drivers/cdp"
	"github.com/MontFerret/ferret/pkg/runtime/core"
	"github.com/MontFerret/ferret/pkg/runtime/values"
	"github.com/MontFerret/ferret/pkg/runtime/values/types"
)

type PageLoadParams struct {
	drivers.Params
	Driver  string
	Timeout time.Duration
}

// DOCUMENT opens an HTML page by a given url.
// By default, loads a page by http call - resulted page does not support any interactions.
// @param {Object} [params] - An object containing the following properties :
// @param {String} [params.driver] - Driver name to use.
// @param {Int} [params.timeout=60000] - Page load timeout.
// @param {String} [params.userAgent] - Custom user agent.
// @param {Boolean} [params.keepCookies=False] - Boolean value indicating whether to use cookies from previous sessions i.e. not to open a page in the Incognito mode.
// @param {HTTPCookies} [params.cookies] - Set of HTTP cookies to use during page loading.
// @param {HTTPHeaders} [params.headers] - Set of HTTP headers to use during page loading.
// @param {Object} [params.viewport] - Viewport params.
// @param {Int} [params.viewport.height] - Viewport height.
// @param {Int} [params.viewport.width] - Viewport width.
// @param {Float} [params.viewport.scaleFactor] - Viewport scale factor.
// @param {Boolean} [params.viewport.mobile] - Value that indicates whether to emulate mobile device.
// @param {Boolean} [params.viewport.landscape] - Value that indicates whether to render a page in landscape position.
// @return {HTMLPage} - Loaded HTML page.
func Open(ctx context.Context, args ...core.Value) (core.Value, error) {
	err := core.ValidateArgs(args, 1, 2)

	if err != nil {
		return values.None, err
	}

	err = core.ValidateType(args[0], types.String)

	if err != nil {
		return values.None, err
	}

	url := args[0].(values.String)

	var params PageLoadParams

	if len(args) == 1 {
		params = newDefaultDocLoadParams(url)
	} else {
		p, err := newPageLoadParams(url, args[1])

		if err != nil {
			return values.None, err
		}

		params = p
	}

	ctx, cancel := context.WithTimeout(ctx, params.Timeout)
	defer cancel()

	drv, err := drivers.FromContext(ctx, params.Driver)

	if err != nil {
		return values.None, err
	}

	return drv.Open(ctx, params.Params)
}

func newDefaultDocLoadParams(url values.String) PageLoadParams {
	return PageLoadParams{
		Params: drivers.Params{
			URL: url.String(),
		},
		Timeout: drivers.DefaultPageLoadTimeout * time.Millisecond,
	}
}

func newPageLoadParams(url values.String, arg core.Value) (PageLoadParams, error) {
	res := newDefaultDocLoadParams(url)

	if err := core.ValidateType(arg, types.Boolean, types.String, types.Object); err != nil {
		return res, err
	}

	switch arg.Type() {
	case types.Object:
		obj := arg.(*values.Object)

		driver, exists := obj.Get(values.NewString("driver"))

		if exists {
			if err := core.ValidateType(driver, types.String); err != nil {
				return res, err
			}

			res.Driver = driver.(values.String).String()
		}

		timeout, exists := obj.Get(values.NewString("timeout"))

		if exists {
			if err := core.ValidateType(timeout, types.Int); err != nil {
				return res, err
			}

			res.Timeout = time.Duration(timeout.(values.Int)) * time.Millisecond
		}

		userAgent, exists := obj.Get(values.NewString("userAgent"))

		if exists {
			if err := core.ValidateType(userAgent, types.String); err != nil {
				return res, err
			}

			res.UserAgent = userAgent.String()
		}

		keepCookies, exists := obj.Get(values.NewString("keepCookies"))

		if exists {
			if err := core.ValidateType(keepCookies, types.Boolean); err != nil {
				return res, err
			}

			res.KeepCookies = bool(keepCookies.(values.Boolean))
		}

		cookies, exists := obj.Get(values.NewString("cookies"))

		if exists {
			if err := core.ValidateType(cookies, types.Array, types.Object); err != nil {
				return res, err
			}

			switch c := cookies.(type) {
			case *values.Array:
				cookies, err := parseCookieArray(c)

				if err != nil {
					return res, err
				}

				res.Cookies = cookies
			case *values.Object:
				cookies, err := parseCookieObject(c)

				if err != nil {
					return res, err
				}

				res.Cookies = cookies
			default:
				res.Cookies = make(drivers.HTTPCookies)
			}
		}

		headers, exists := obj.Get(values.NewString("headers"))

		if exists {
			if err := core.ValidateType(headers, types.Object); err != nil {
				return res, err
			}

			header := parseHeader(headers.(*values.Object))
			res.Headers = header
		}

		viewport, exists := obj.Get(values.NewString("viewport"))

		if exists {
			viewport, err := parseViewport(viewport)

			if err != nil {
				return res, err
			}

			res.Viewport = viewport
		}
	case types.String:
		res.Driver = arg.(values.String).String()
	case types.Boolean:
		b := arg.(values.Boolean)

		// fallback
		if b {
			res.Driver = cdp.DriverName
		}
	}

	return res, nil
}

func parseCookieObject(obj *values.Object) (drivers.HTTPCookies, error) {
	var err error
	res := make(drivers.HTTPCookies)

	obj.ForEach(func(value core.Value, _ string) bool {
		cookie, e := parseCookie(value)

		if e != nil {
			err = e

			return false
		}

		res[cookie.Name] = cookie

		return true
	})

	return res, err
}

func parseCookieArray(arr *values.Array) (drivers.HTTPCookies, error) {
	var err error
	res := make(drivers.HTTPCookies)

	arr.ForEach(func(value core.Value, _ int) bool {
		cookie, e := parseCookie(value)

		if e != nil {
			err = e

			return false
		}

		res[cookie.Name] = cookie

		return true
	})

	return res, err
}

func parseCookie(value core.Value) (drivers.HTTPCookie, error) {
	err := core.ValidateType(value, types.Object, drivers.HTTPCookieType)

	if err != nil {
		return drivers.HTTPCookie{}, err
	}

	if value.Type() == drivers.HTTPCookieType {
		return value.(drivers.HTTPCookie), nil
	}

	co := value.(*values.Object)

	cookie := drivers.HTTPCookie{
		Name:   co.MustGet("name").String(),
		Value:  co.MustGet("value").String(),
		Path:   co.MustGet("path").String(),
		Domain: co.MustGet("domain").String(),
	}

	maxAge, exists := co.Get("maxAge")

	if exists {
		if err = core.ValidateType(maxAge, types.Int); err != nil {
			return drivers.HTTPCookie{}, err
		}

		cookie.MaxAge = int(maxAge.(values.Int))
	}

	expires, exists := co.Get("expires")

	if exists {
		if err = core.ValidateType(maxAge, types.DateTime, types.String); err != nil {
			return drivers.HTTPCookie{}, err
		}

		if expires.Type() == types.DateTime {
			cookie.Expires = expires.(values.DateTime).Unwrap().(time.Time)
		} else {
			t, err := time.Parse(expires.String(), values.DefaultTimeLayout)

			if err != nil {
				return drivers.HTTPCookie{}, err
			}

			cookie.Expires = t
		}
	}

	sameSite, exists := co.Get("sameSite")

	if exists {
		sameSite := strings.ToLower(sameSite.String())

		switch sameSite {
		case "lax":
			cookie.SameSite = drivers.SameSiteLaxMode
		case "strict":
			cookie.SameSite = drivers.SameSiteStrictMode
		default:
			cookie.SameSite = drivers.SameSiteDefaultMode
		}
	}

	httpOnly, exists := co.Get("httpOnly")

	if exists {
		if err = core.ValidateType(httpOnly, types.Boolean); err != nil {
			return drivers.HTTPCookie{}, err
		}

		cookie.HTTPOnly = bool(httpOnly.(values.Boolean))
	}

	secure, exists := co.Get("secure")

	if exists {
		if err = core.ValidateType(secure, types.Boolean); err != nil {
			return drivers.HTTPCookie{}, err
		}

		cookie.Secure = bool(secure.(values.Boolean))
	}

	return cookie, err
}

func parseHeader(headers *values.Object) drivers.HTTPHeaders {
	res := make(drivers.HTTPHeaders)

	headers.ForEach(func(value core.Value, key string) bool {
		res.Set(key, value.String())

		return true
	})

	return res
}

func parseViewport(value core.Value) (*drivers.Viewport, error) {
	if err := core.ValidateType(value, types.Object); err != nil {
		return nil, err
	}

	res := &drivers.Viewport{}

	viewport := value.(*values.Object)

	width, exists := viewport.Get(values.NewString("width"))

	if exists {
		if err := core.ValidateType(width, types.Int); err != nil {
			return nil, err
		}

		res.Width = int(values.ToInt(width))
	}

	height, exists := viewport.Get(values.NewString("height"))

	if exists {
		if err := core.ValidateType(height, types.Int); err != nil {
			return nil, err
		}

		res.Height = int(values.ToInt(height))
	}

	mobile, exists := viewport.Get(values.NewString("mobile"))

	if exists {
		res.Mobile = bool(values.ToBoolean(mobile))
	}

	landscape, exists := viewport.Get(values.NewString("landscape"))

	if exists {
		res.Landscape = bool(values.ToBoolean(landscape))
	}

	scaleFactor, exists := viewport.Get(values.NewString("scaleFactor"))

	if exists {
		res.ScaleFactor = float64(values.ToFloat(scaleFactor))
	}

	return res, nil
}