mirror of
https://github.com/MontFerret/ferret.git
synced 2025-12-01 22:19:32 +02:00
Hello world
This commit is contained in:
Tim Voronov
51
pkg/stdlib/html/driver/browser/cdp.go
Normal file
51
pkg/stdlib/html/driver/browser/cdp.go
Normal file
@@ -0,0 +1,51 @@
|
||||
package browser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
"github.com/mafredri/cdp/devtool"
|
||||
"github.com/mafredri/cdp/rpcc"
|
||||
)
|
||||
|
||||
type (
|
||||
CdpDriver struct {
|
||||
address string
|
||||
}
|
||||
|
||||
CdpConnection struct {
|
||||
target *devtool.Target
|
||||
core *rpcc.Conn
|
||||
}
|
||||
)
|
||||
|
||||
func NewDriver(conn string) *CdpDriver {
|
||||
return &CdpDriver{
|
||||
address: conn,
|
||||
}
|
||||
}
|
||||
|
||||
func (drv *CdpDriver) GetDocument(ctx context.Context, url string) (values.HtmlNode, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, DefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
devt := devtool.New(drv.address)
|
||||
|
||||
target, err := devt.CreateURL(ctx, url)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
conn, err := rpcc.DialContext(ctx, target.WebSocketDebuggerURL)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return NewHtmlDocument(ctx, conn, url)
|
||||
}
|
||||
|
||||
func (drv *CdpDriver) Close() error {
|
||||
// TODO: Do we need this method?
|
||||
return nil
|
||||
}
|
||||
110
pkg/stdlib/html/driver/browser/document.go
Normal file
110
pkg/stdlib/html/driver/browser/document.go
Normal file
@@ -0,0 +1,110 @@
|
||||
package browser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
"github.com/mafredri/cdp"
|
||||
"github.com/mafredri/cdp/protocol/dom"
|
||||
"github.com/mafredri/cdp/rpcc"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type HtmlDocument struct {
|
||||
*HtmlElement
|
||||
conn *rpcc.Conn
|
||||
client *cdp.Client
|
||||
url string
|
||||
}
|
||||
|
||||
func NewHtmlDocument(
|
||||
ctx context.Context,
|
||||
conn *rpcc.Conn,
|
||||
url string,
|
||||
) (*HtmlDocument, error) {
|
||||
if conn == nil {
|
||||
return nil, core.Error(core.ErrMissedArgument, "connection")
|
||||
}
|
||||
|
||||
if url == "" {
|
||||
return nil, core.Error(core.ErrMissedArgument, "url")
|
||||
}
|
||||
|
||||
client := cdp.NewClient(conn)
|
||||
|
||||
err := RunBatch(
|
||||
func() error {
|
||||
return client.Page.Enable(ctx)
|
||||
},
|
||||
|
||||
func() error {
|
||||
return client.DOM.Enable(ctx)
|
||||
},
|
||||
|
||||
func() error {
|
||||
return client.Runtime.Enable(ctx)
|
||||
},
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
loadEventFired, err := client.Page.LoadEventFired(ctx)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = loadEventFired.Recv()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
loadEventFired.Close()
|
||||
|
||||
args := dom.NewGetDocumentArgs()
|
||||
args.Depth = PointerInt(-1) // lets load the entire document
|
||||
|
||||
d, err := client.DOM.GetDocument(ctx, args)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &HtmlDocument{
|
||||
&HtmlElement{client, d.Root.NodeID, d.Root, nil},
|
||||
conn,
|
||||
client,
|
||||
url,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (doc *HtmlDocument) Close() error {
|
||||
doc.client.Page.Close(context.Background())
|
||||
|
||||
return doc.conn.Close()
|
||||
}
|
||||
|
||||
func (doc *HtmlDocument) Type() core.Type {
|
||||
return core.HtmlDocumentType
|
||||
}
|
||||
|
||||
func (doc *HtmlDocument) String() string {
|
||||
return doc.url
|
||||
}
|
||||
|
||||
func (doc *HtmlDocument) Compare(other core.Value) int {
|
||||
switch other.Type() {
|
||||
case core.HtmlDocumentType:
|
||||
other := other.(*HtmlDocument)
|
||||
|
||||
return strings.Compare(doc.url, other.url)
|
||||
default:
|
||||
if other.Type() > core.HtmlDocumentType {
|
||||
return -1
|
||||
}
|
||||
|
||||
return 1
|
||||
}
|
||||
}
|
||||
309
pkg/stdlib/html/driver/browser/element.go
Normal file
309
pkg/stdlib/html/driver/browser/element.go
Normal file
@@ -0,0 +1,309 @@
|
||||
package browser
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha512"
|
||||
"encoding/json"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
"github.com/MontFerret/ferret/pkg/stdlib/html/driver/common"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/mafredri/cdp"
|
||||
"github.com/mafredri/cdp/protocol/dom"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
const DefaultTimeout = time.Second * 30
|
||||
|
||||
type HtmlElement struct {
|
||||
client *cdp.Client
|
||||
id dom.NodeID
|
||||
node dom.Node
|
||||
attributes *values.Object
|
||||
}
|
||||
|
||||
func NewHtmlElement(
|
||||
client *cdp.Client,
|
||||
id dom.NodeID,
|
||||
) (*HtmlElement, error) {
|
||||
if client == nil {
|
||||
return nil, core.Error(core.ErrMissedArgument, "client")
|
||||
}
|
||||
|
||||
ctx, cancelFn := context.WithTimeout(context.Background(), DefaultTimeout)
|
||||
|
||||
defer cancelFn()
|
||||
|
||||
node, err := client.DOM.DescribeNode(
|
||||
ctx,
|
||||
dom.
|
||||
NewDescribeNodeArgs().
|
||||
SetNodeID(id).
|
||||
SetDepth(-1),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Println("ERROR:", err)
|
||||
return nil, core.Error(err, strconv.Itoa(int(id)))
|
||||
}
|
||||
|
||||
return &HtmlElement{client, id, node.Node, nil}, nil
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Close() error {
|
||||
// el.client = nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Type() core.Type {
|
||||
return core.HtmlElementType
|
||||
}
|
||||
|
||||
func (el *HtmlElement) MarshalJSON() ([]byte, error) {
|
||||
ctx, cancelFn := context.WithTimeout(context.Background(), DefaultTimeout)
|
||||
|
||||
defer cancelFn()
|
||||
|
||||
args := dom.NewGetOuterHTMLArgs()
|
||||
args.NodeID = &el.node.NodeID
|
||||
|
||||
reply, err := el.client.DOM.GetOuterHTML(ctx, args)
|
||||
|
||||
if err != nil {
|
||||
return nil, core.Error(err, strconv.Itoa(int(el.node.NodeID)))
|
||||
}
|
||||
|
||||
return json.Marshal(reply.OuterHTML)
|
||||
}
|
||||
|
||||
func (el *HtmlElement) String() string {
|
||||
return *el.node.Value
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Compare(other core.Value) int {
|
||||
switch other.Type() {
|
||||
case core.HtmlDocumentType:
|
||||
other := other.(*HtmlElement)
|
||||
|
||||
id := int(el.node.NodeID)
|
||||
otherId := int(other.node.NodeID)
|
||||
|
||||
if id == otherId {
|
||||
return 0
|
||||
}
|
||||
|
||||
if id > otherId {
|
||||
return 1
|
||||
}
|
||||
|
||||
return -1
|
||||
default:
|
||||
if other.Type() > core.HtmlElementType {
|
||||
return -1
|
||||
}
|
||||
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Unwrap() interface{} {
|
||||
return el.node
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Hash() int {
|
||||
h := sha512.New()
|
||||
|
||||
out, err := h.Write([]byte(*el.node.Value))
|
||||
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Value() core.Value {
|
||||
return values.None
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Length() values.Int {
|
||||
if el.node.ChildNodeCount == nil {
|
||||
return values.ZeroInt
|
||||
}
|
||||
|
||||
return values.NewInt(*el.node.ChildNodeCount)
|
||||
}
|
||||
|
||||
func (el *HtmlElement) NodeType() values.Int {
|
||||
return values.NewInt(el.node.NodeType)
|
||||
}
|
||||
|
||||
func (el *HtmlElement) NodeName() values.String {
|
||||
return values.NewString(el.node.NodeName)
|
||||
}
|
||||
|
||||
func (el *HtmlElement) GetAttributes() core.Value {
|
||||
if el.attributes == nil {
|
||||
el.attributes = el.parseAttrs()
|
||||
}
|
||||
|
||||
return el.attributes
|
||||
}
|
||||
|
||||
func (el *HtmlElement) GetAttribute(name values.String) core.Value {
|
||||
if el.attributes == nil {
|
||||
el.attributes = el.parseAttrs()
|
||||
}
|
||||
|
||||
val, found := el.attributes.Get(name)
|
||||
|
||||
if !found {
|
||||
return values.None
|
||||
}
|
||||
|
||||
return val
|
||||
}
|
||||
|
||||
func (el *HtmlElement) GetChildNodes() core.Value {
|
||||
arr := values.NewArray(len(el.node.Children))
|
||||
|
||||
for idx := range el.node.Children {
|
||||
el := el.GetChildNode(values.NewInt(idx))
|
||||
|
||||
if el != values.None {
|
||||
arr.Push(el)
|
||||
}
|
||||
}
|
||||
|
||||
return arr
|
||||
}
|
||||
|
||||
func (el *HtmlElement) GetChildNode(idx values.Int) core.Value {
|
||||
if el.Length() < idx {
|
||||
return values.None
|
||||
}
|
||||
|
||||
childNode := el.node.Children[idx]
|
||||
|
||||
return &HtmlElement{el.client, childNode.NodeID, childNode, nil}
|
||||
}
|
||||
|
||||
func (el *HtmlElement) QuerySelector(selector values.String) core.Value {
|
||||
ctx := context.Background()
|
||||
|
||||
selectorArgs := dom.NewQuerySelectorArgs(el.id, selector.String())
|
||||
found, err := el.client.DOM.QuerySelector(ctx, selectorArgs)
|
||||
|
||||
if err != nil {
|
||||
el.logErr(err, selector.String())
|
||||
return values.None
|
||||
}
|
||||
|
||||
res, err := NewHtmlElement(el.client, found.NodeID)
|
||||
|
||||
if err != nil {
|
||||
return values.None
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func (el *HtmlElement) QuerySelectorAll(selector values.String) core.Value {
|
||||
ctx := context.Background()
|
||||
|
||||
selectorArgs := dom.NewQuerySelectorAllArgs(el.id, selector.String())
|
||||
res, err := el.client.DOM.QuerySelectorAll(ctx, selectorArgs)
|
||||
|
||||
if err != nil {
|
||||
el.logErr(err, selector.String())
|
||||
return values.None
|
||||
}
|
||||
|
||||
arr := values.NewArray(len(res.NodeIDs))
|
||||
|
||||
for _, id := range res.NodeIDs {
|
||||
childEl, err := NewHtmlElement(el.client, id)
|
||||
|
||||
if err != nil {
|
||||
return values.None
|
||||
}
|
||||
|
||||
arr.Push(childEl)
|
||||
}
|
||||
|
||||
return arr
|
||||
}
|
||||
|
||||
func (el *HtmlElement) InnerText() values.String {
|
||||
h := el.InnerHtml()
|
||||
|
||||
if h == values.EmptyString {
|
||||
return h
|
||||
}
|
||||
|
||||
buff := bytes.NewBuffer([]byte(h))
|
||||
|
||||
parsed, err := goquery.NewDocumentFromReader(buff)
|
||||
|
||||
if err != nil {
|
||||
return values.EmptyString
|
||||
}
|
||||
|
||||
return values.NewString(parsed.Text())
|
||||
}
|
||||
|
||||
func (el *HtmlElement) InnerHtml() values.String {
|
||||
ctx, cancelFn := el.createCtx()
|
||||
|
||||
defer cancelFn()
|
||||
|
||||
res, err := el.client.DOM.GetOuterHTML(ctx, dom.NewGetOuterHTMLArgs().SetNodeID(el.id))
|
||||
|
||||
if err != nil {
|
||||
el.logErr(err)
|
||||
|
||||
return values.EmptyString
|
||||
}
|
||||
|
||||
return values.NewString(res.OuterHTML)
|
||||
}
|
||||
|
||||
func (el *HtmlElement) createCtx() (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.Background(), DefaultTimeout)
|
||||
}
|
||||
|
||||
func (el *HtmlElement) parseAttrs() *values.Object {
|
||||
var attr values.String
|
||||
|
||||
res := values.NewObject()
|
||||
|
||||
for _, el := range el.node.Attributes {
|
||||
str := values.NewString(el)
|
||||
|
||||
if common.IsAttribute(el) {
|
||||
attr = str
|
||||
res.Set(str, values.EmptyString)
|
||||
} else {
|
||||
current, ok := res.Get(attr)
|
||||
|
||||
if ok {
|
||||
res.Set(attr, current.(values.String).Concat(values.SpaceString).Concat(str))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func (el *HtmlElement) logErr(values ...interface{}) {
|
||||
args := make([]interface{}, 0, len(values)+1)
|
||||
args = append(args, "ERROR:")
|
||||
args = append(args, values...)
|
||||
args = append(args, "id:", el.node.NodeID)
|
||||
|
||||
log.Println(args...)
|
||||
}
|
||||
19
pkg/stdlib/html/driver/browser/helpers.go
Normal file
19
pkg/stdlib/html/driver/browser/helpers.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package browser
|
||||
|
||||
import "golang.org/x/sync/errgroup"
|
||||
|
||||
func PointerInt(input int) *int {
|
||||
return &input
|
||||
}
|
||||
|
||||
type BatchFunc = func() error
|
||||
|
||||
func RunBatch(funcs ...BatchFunc) error {
|
||||
eg := errgroup.Group{}
|
||||
|
||||
for _, f := range funcs {
|
||||
eg.Go(f)
|
||||
}
|
||||
|
||||
return eg.Wait()
|
||||
}
|
||||
153
pkg/stdlib/html/driver/common/attrs.go
Normal file
153
pkg/stdlib/html/driver/common/attrs.go
Normal file
@@ -0,0 +1,153 @@
|
||||
package common
|
||||
|
||||
var Attributes = []string{
|
||||
"abbr",
|
||||
"accept",
|
||||
"accept-charset",
|
||||
"accesskey",
|
||||
"action",
|
||||
"allowfullscreen",
|
||||
"allowpaymentrequest",
|
||||
"allowusermedia",
|
||||
"alt",
|
||||
"as",
|
||||
"async",
|
||||
"autocomplete",
|
||||
"autofocus",
|
||||
"autoplay",
|
||||
"challenge",
|
||||
"charset",
|
||||
"checked",
|
||||
"cite",
|
||||
"class",
|
||||
"color",
|
||||
"cols",
|
||||
"colspan",
|
||||
"command",
|
||||
"content",
|
||||
"contenteditable",
|
||||
"contextmenu",
|
||||
"controls",
|
||||
"coords",
|
||||
"crossorigin",
|
||||
"data",
|
||||
"datetime",
|
||||
"default",
|
||||
"defer",
|
||||
"dir",
|
||||
"dirname",
|
||||
"disabled",
|
||||
"download",
|
||||
"draggable",
|
||||
"dropzone",
|
||||
"enctype",
|
||||
"for",
|
||||
"form",
|
||||
"formaction",
|
||||
"formenctype",
|
||||
"formmethod",
|
||||
"formnovalidate",
|
||||
"formtarget",
|
||||
"headers",
|
||||
"height",
|
||||
"hidden",
|
||||
"high",
|
||||
"href",
|
||||
"hreflang",
|
||||
"http-equiv",
|
||||
"icon",
|
||||
"id",
|
||||
"inputmode",
|
||||
"integrity",
|
||||
"is",
|
||||
"ismap",
|
||||
"itemid",
|
||||
"itemprop",
|
||||
"itemref",
|
||||
"itemscope",
|
||||
"itemtype",
|
||||
"keytype",
|
||||
"kind",
|
||||
"label",
|
||||
"lang",
|
||||
"list",
|
||||
"loop",
|
||||
"low",
|
||||
"manifest",
|
||||
"max",
|
||||
"maxlength",
|
||||
"media",
|
||||
"mediagroup",
|
||||
"method",
|
||||
"min",
|
||||
"minlength",
|
||||
"multiple",
|
||||
"muted",
|
||||
"name",
|
||||
"nomodule",
|
||||
"nonce",
|
||||
"novalidate",
|
||||
"open",
|
||||
"optimum",
|
||||
"pattern",
|
||||
"ping",
|
||||
"placeholder",
|
||||
"playsinline",
|
||||
"poster",
|
||||
"preload",
|
||||
"radiogroup",
|
||||
"readonly",
|
||||
"referrerpolicy",
|
||||
"rel",
|
||||
"required",
|
||||
"reversed",
|
||||
"rows",
|
||||
"rowspan",
|
||||
"sandbox",
|
||||
"spellcheck",
|
||||
"scope",
|
||||
"scoped",
|
||||
"seamless",
|
||||
"selected",
|
||||
"shape",
|
||||
"size",
|
||||
"sizes",
|
||||
"sortable",
|
||||
"sorted",
|
||||
"slot",
|
||||
"span",
|
||||
"spellcheck",
|
||||
"src",
|
||||
"srcdoc",
|
||||
"srclang",
|
||||
"srcset",
|
||||
"start",
|
||||
"step",
|
||||
"style",
|
||||
"tabindex",
|
||||
"target",
|
||||
"title",
|
||||
"translate",
|
||||
"type",
|
||||
"typemustmatch",
|
||||
"updateviacache",
|
||||
"usemap",
|
||||
"value",
|
||||
"width",
|
||||
"workertype",
|
||||
"wrap",
|
||||
}
|
||||
|
||||
var attrMap = make(map[string]bool)
|
||||
|
||||
func init() {
|
||||
for _, attr := range Attributes {
|
||||
attrMap[attr] = true
|
||||
}
|
||||
}
|
||||
|
||||
func IsAttribute(name string) bool {
|
||||
_, yes := attrMap[name]
|
||||
|
||||
return yes
|
||||
}
|
||||
20
pkg/stdlib/html/driver/common/types.go
Normal file
20
pkg/stdlib/html/driver/common/types.go
Normal file
@@ -0,0 +1,20 @@
|
||||
package common
|
||||
|
||||
import "golang.org/x/net/html"
|
||||
|
||||
func ToHtmlType(nt html.NodeType) int {
|
||||
switch nt {
|
||||
case html.DocumentNode:
|
||||
return 9
|
||||
case html.ElementNode:
|
||||
return 1
|
||||
case html.TextNode:
|
||||
return 3
|
||||
case html.CommentNode:
|
||||
return 8
|
||||
case html.DoctypeNode:
|
||||
return 10
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
40
pkg/stdlib/html/driver/driver.go
Normal file
40
pkg/stdlib/html/driver/driver.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package driver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
"github.com/MontFerret/ferret/pkg/stdlib/html/driver/browser"
|
||||
"github.com/MontFerret/ferret/pkg/stdlib/html/driver/http"
|
||||
)
|
||||
|
||||
const Cdp = "cdp"
|
||||
const Http = "http"
|
||||
|
||||
type Driver interface {
|
||||
GetDocument(ctx context.Context, url string) (values.HtmlNode, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
func ToContext(ctx context.Context, name string, drv Driver) context.Context {
|
||||
return context.WithValue(ctx, name, drv)
|
||||
}
|
||||
|
||||
func FromContext(ctx context.Context, name string) Driver {
|
||||
val := ctx.Value(name)
|
||||
|
||||
drv, ok := val.(Driver)
|
||||
|
||||
if ok {
|
||||
return drv
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func WithCdpDriver(ctx context.Context, addr string) context.Context {
|
||||
return context.WithValue(ctx, Cdp, browser.NewDriver(addr))
|
||||
}
|
||||
|
||||
func WithHttpDriver(ctx context.Context, opts ...http.Option) context.Context {
|
||||
return context.WithValue(ctx, Http, http.NewDriver(opts...))
|
||||
}
|
||||
50
pkg/stdlib/html/driver/http/document.go
Normal file
50
pkg/stdlib/html/driver/http/document.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
type HtmlDocument struct {
|
||||
*HtmlElement
|
||||
url string
|
||||
}
|
||||
|
||||
func NewHtmlDocument(
|
||||
url string,
|
||||
node *goquery.Document,
|
||||
) (*HtmlDocument, error) {
|
||||
if url == "" {
|
||||
return nil, core.Error(core.ErrMissedArgument, "document url")
|
||||
}
|
||||
|
||||
if node == nil {
|
||||
return nil, core.Error(core.ErrMissedArgument, "document root selection")
|
||||
}
|
||||
|
||||
el, err := NewHtmlElement(node.Selection)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &HtmlDocument{el, url}, nil
|
||||
}
|
||||
|
||||
func (el *HtmlDocument) Type() core.Type {
|
||||
return core.HtmlDocumentType
|
||||
}
|
||||
|
||||
func (el *HtmlDocument) Compare(other core.Value) int {
|
||||
switch other.Type() {
|
||||
case core.HtmlDocumentType:
|
||||
// TODO: complete the comparison
|
||||
return -1
|
||||
default:
|
||||
if other.Type() > core.HtmlDocumentType {
|
||||
return -1
|
||||
}
|
||||
|
||||
return 1
|
||||
}
|
||||
}
|
||||
238
pkg/stdlib/html/driver/http/document_test.go
Normal file
238
pkg/stdlib/html/driver/http/document_test.go
Normal file
@@ -0,0 +1,238 @@
|
||||
package http_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/MontFerret/ferret/pkg/stdlib/html/driver/http"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDocument(t *testing.T) {
|
||||
doc := `
|
||||
<html lang="en"><head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
||||
<meta name="description" content="">
|
||||
<meta name="author" content="">
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<div class="collapse bg-dark" id="navbarHeader">
|
||||
<div class="container">
|
||||
<div class="row">
|
||||
<div class="col-sm-8 col-md-7 py-4">
|
||||
<h4 class="text-white">About</h4>
|
||||
<p class="text-muted">Add some information about the album below, the author, or any other background context. Make it a few sentences long so folks can pick up some informative tidbits. Then, link them off to some social networking sites or contact information.</p>
|
||||
</div>
|
||||
<div class="col-sm-4 offset-md-1 py-4">
|
||||
<h4 class="text-white">Contact</h4>
|
||||
<ul class="list-unstyled">
|
||||
<li><a href="#" class="text-white">Follow on Twitter</a></li>
|
||||
<li><a href="#" class="text-white">Like on Facebook</a></li>
|
||||
<li><a href="#" class="text-white">Email me</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="navbar navbar-dark bg-dark shadow-sm">
|
||||
<div class="container d-flex justify-content-between">
|
||||
<a href="#" class="navbar-brand d-flex align-items-center">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="mr-2"><path d="M23 19a2 2 0 0 1-2 2H3a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h4l2-3h6l2 3h4a2 2 0 0 1 2 2z"></path><circle cx="12" cy="13" r="4"></circle></svg>
|
||||
<strong>Album</strong>
|
||||
</a>
|
||||
<button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarHeader" aria-controls="navbarHeader" aria-expanded="false" aria-label="Toggle navigation">
|
||||
<span class="navbar-toggler-icon"></span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main role="main">
|
||||
|
||||
<section class="jumbotron text-center">
|
||||
<div class="container">
|
||||
<h1 class="jumbotron-heading">Album example</h1>
|
||||
<p class="lead text-muted">Something short and leading about the collection below—its contents, the creator, etc. Make it short and sweet, but not too short so folks don't simply skip over it entirely.</p>
|
||||
<p>
|
||||
<a href="#" class="btn btn-primary my-2">Main call to action</a>
|
||||
<a href="#" class="btn btn-secondary my-2">Secondary action</a>
|
||||
</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div class="album py-5 bg-light">
|
||||
<div class="container">
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" style="height: 225px; width: 100%; display: block;" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea5071fe%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea5071fe%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea5071fe%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea5071fe%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true" style="height: 225px; width: 100%; display: block;">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea507200%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea507200%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true" style="height: 225px; width: 100%; display: block;">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea507200%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea507200%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true" style="height: 225px; width: 100%; display: block;">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea507201%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea507201%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true" style="height: 225px; width: 100%; display: block;">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea507202%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea507202%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true" style="height: 225px; width: 100%; display: block;">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea507203%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea507203%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true" style="height: 225px; width: 100%; display: block;">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea507203%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea507203%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true" style="height: 225px; width: 100%; display: block;">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea507203%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea507203%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true" style="height: 225px; width: 100%; display: block;">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
<footer class="text-muted">
|
||||
<div class="container">
|
||||
<p class="float-right">
|
||||
<a href="#">Back to top</a>
|
||||
</p>
|
||||
<p>Album example is © Bootstrap, but please download and customize it for yourself!</p>
|
||||
<p>New to Bootstrap? <a href="../../">Visit the homepage</a> or read our <a href="../../getting-started/">getting started guide</a>.</p>
|
||||
</div>
|
||||
</footer>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="348" height="225" viewBox="0 0 348 225" preserveAspectRatio="none" style="display: none; visibility: hidden; position: absolute; top: -100%; left: -100%;"><defs><style type="text/css"></style></defs><text x="0" y="17" style="font-weight:bold;font-size:17pt;font-family:Arial, Helvetica, Open Sans, sans-serif">Thumbnail</text></svg></body></html>
|
||||
`
|
||||
Convey(".NodeType", t, func() {
|
||||
Convey("Should serialize a boolean value", func() {
|
||||
buff := bytes.NewBuffer([]byte(doc))
|
||||
|
||||
buff.Write([]byte(doc))
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(buff)
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
el, err := http.NewHtmlElement(doc.Selection)
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(el.NodeType(), ShouldEqual, 9)
|
||||
})
|
||||
})
|
||||
}
|
||||
233
pkg/stdlib/html/driver/http/element.go
Normal file
233
pkg/stdlib/html/driver/http/element.go
Normal file
@@ -0,0 +1,233 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"crypto/sha512"
|
||||
"encoding/json"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/core"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
"github.com/MontFerret/ferret/pkg/stdlib/html/driver/common"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
type HtmlElement struct {
|
||||
selection *goquery.Selection
|
||||
attrs *values.Object
|
||||
children *values.Array
|
||||
}
|
||||
|
||||
func NewHtmlElement(node *goquery.Selection) (*HtmlElement, error) {
|
||||
if node == nil {
|
||||
return nil, core.Error(core.ErrMissedArgument, "element selection")
|
||||
}
|
||||
|
||||
return &HtmlElement{node, nil, nil}, nil
|
||||
}
|
||||
|
||||
func (el *HtmlElement) MarshalJSON() ([]byte, error) {
|
||||
html, err := el.selection.Html()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return json.Marshal(html)
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Type() core.Type {
|
||||
return core.HtmlElementType
|
||||
}
|
||||
|
||||
func (el *HtmlElement) String() string {
|
||||
return el.selection.Text()
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Compare(other core.Value) int {
|
||||
switch other.Type() {
|
||||
case core.HtmlElementType:
|
||||
// TODO: complete the comparison
|
||||
return -1
|
||||
default:
|
||||
if other.Type() > core.HtmlElementType {
|
||||
return -1
|
||||
}
|
||||
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Unwrap() interface{} {
|
||||
return el.selection
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Hash() int {
|
||||
h := sha512.New()
|
||||
|
||||
str, err := el.selection.Html()
|
||||
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
out, err := h.Write([]byte(str))
|
||||
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func (el *HtmlElement) NodeType() values.Int {
|
||||
nodes := el.selection.Nodes
|
||||
|
||||
if len(nodes) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
return values.NewInt(common.ToHtmlType(nodes[0].Type))
|
||||
}
|
||||
|
||||
func (el *HtmlElement) NodeName() values.String {
|
||||
return values.NewString(goquery.NodeName(el.selection))
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Length() values.Int {
|
||||
if el.children == nil {
|
||||
el.children = el.parseChildren()
|
||||
}
|
||||
|
||||
return el.children.Length()
|
||||
}
|
||||
|
||||
func (el *HtmlElement) Value() core.Value {
|
||||
val, ok := el.selection.Attr("value")
|
||||
|
||||
if ok {
|
||||
return values.NewString(val)
|
||||
}
|
||||
|
||||
return values.EmptyString
|
||||
}
|
||||
|
||||
func (el *HtmlElement) InnerText() values.String {
|
||||
return values.NewString(el.selection.Text())
|
||||
}
|
||||
|
||||
func (el *HtmlElement) InnerHtml() values.String {
|
||||
h, err := el.selection.Html()
|
||||
|
||||
if err != nil {
|
||||
return values.EmptyString
|
||||
}
|
||||
|
||||
return values.NewString(h)
|
||||
}
|
||||
|
||||
func (el *HtmlElement) GetAttributes() core.Value {
|
||||
if el.attrs == nil {
|
||||
el.attrs = el.parseAttrs()
|
||||
}
|
||||
|
||||
return el.attrs
|
||||
}
|
||||
|
||||
func (el *HtmlElement) GetAttribute(name values.String) core.Value {
|
||||
v, ok := el.selection.Attr(name.String())
|
||||
|
||||
if ok {
|
||||
return values.NewString(v)
|
||||
}
|
||||
|
||||
return values.None
|
||||
}
|
||||
|
||||
func (el *HtmlElement) GetChildNodes() core.Value {
|
||||
if el.children == nil {
|
||||
el.children = el.parseChildren()
|
||||
}
|
||||
|
||||
return el.children
|
||||
}
|
||||
|
||||
func (el *HtmlElement) GetChildNode(idx values.Int) core.Value {
|
||||
if el.children == nil {
|
||||
el.children = el.parseChildren()
|
||||
}
|
||||
|
||||
return el.children.Get(idx)
|
||||
}
|
||||
|
||||
func (el *HtmlElement) QuerySelector(selector values.String) core.Value {
|
||||
selection := el.selection.Find(selector.String())
|
||||
|
||||
if selection == nil {
|
||||
return values.None
|
||||
}
|
||||
|
||||
res, err := NewHtmlElement(selection)
|
||||
|
||||
if err != nil {
|
||||
return values.None
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func (el *HtmlElement) QuerySelectorAll(selector values.String) core.Value {
|
||||
selection := el.selection.Find(selector.String())
|
||||
|
||||
if selection == nil {
|
||||
return values.None
|
||||
}
|
||||
|
||||
arr := values.NewArray(selection.Length())
|
||||
|
||||
selection.Each(func(i int, selection *goquery.Selection) {
|
||||
el, err := NewHtmlElement(selection)
|
||||
|
||||
if err == nil {
|
||||
arr.Push(el)
|
||||
}
|
||||
})
|
||||
|
||||
return arr
|
||||
}
|
||||
|
||||
func (el *HtmlElement) parseAttrs() *values.Object {
|
||||
obj := values.NewObject()
|
||||
|
||||
for _, name := range common.Attributes {
|
||||
val, ok := el.selection.Attr(name)
|
||||
|
||||
if ok {
|
||||
obj.Set(values.NewString(name), values.NewString(val))
|
||||
}
|
||||
}
|
||||
|
||||
return obj
|
||||
}
|
||||
|
||||
func (el *HtmlElement) parseChildren() *values.Array {
|
||||
children := el.selection.Children()
|
||||
|
||||
arr := values.NewArray(10)
|
||||
|
||||
children.Each(func(i int, selection *goquery.Selection) {
|
||||
//name := goquery.NodeName(selection)
|
||||
//if name != "#text" && name != "#comment" {
|
||||
// child, err := NewHtmlElement(selection)
|
||||
//
|
||||
// if err == nil {
|
||||
// arr.Push(child)
|
||||
// }
|
||||
//}
|
||||
|
||||
child, err := NewHtmlElement(selection)
|
||||
|
||||
if err == nil {
|
||||
arr.Push(child)
|
||||
}
|
||||
})
|
||||
|
||||
return arr
|
||||
}
|
||||
398
pkg/stdlib/html/driver/http/element_test.go
Normal file
398
pkg/stdlib/html/driver/http/element_test.go
Normal file
@@ -0,0 +1,398 @@
|
||||
package http_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/MontFerret/ferret/pkg/stdlib/html/driver/http"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestElement(t *testing.T) {
|
||||
doc := `
|
||||
<html lang="en"><head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
||||
<meta name="description" content="">
|
||||
<meta name="author" content="">
|
||||
<link rel="icon" href="../../../../favicon.ico">
|
||||
|
||||
<title>Album example for Bootstrap</title>
|
||||
|
||||
<!-- Bootstrap core CSS -->
|
||||
<link href="../../dist/css/bootstrap.min.css" rel="stylesheet">
|
||||
|
||||
<!-- Custom styles for this template -->
|
||||
<link href="album.css" rel="stylesheet">
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<div class="collapse bg-dark" id="navbarHeader">
|
||||
<div class="container">
|
||||
<div class="row">
|
||||
<div class="col-sm-8 col-md-7 py-4">
|
||||
<h4 class="text-white">About</h4>
|
||||
<p class="text-muted">Add some information about the album below, the author, or any other background context. Make it a few sentences long so folks can pick up some informative tidbits. Then, link them off to some social networking sites or contact information.</p>
|
||||
</div>
|
||||
<div class="col-sm-4 offset-md-1 py-4">
|
||||
<h4 class="text-white">Contact</h4>
|
||||
<ul class="list-unstyled">
|
||||
<li><a href="#" class="text-white">Follow on Twitter</a></li>
|
||||
<li><a href="#" class="text-white">Like on Facebook</a></li>
|
||||
<li><a href="#" class="text-white">Email me</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="navbar navbar-dark bg-dark shadow-sm">
|
||||
<div class="container d-flex justify-content-between">
|
||||
<a href="#" class="navbar-brand d-flex align-items-center">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="mr-2"><path d="M23 19a2 2 0 0 1-2 2H3a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h4l2-3h6l2 3h4a2 2 0 0 1 2 2z"></path><circle cx="12" cy="13" r="4"></circle></svg>
|
||||
<strong>Album</strong>
|
||||
</a>
|
||||
<button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarHeader" aria-controls="navbarHeader" aria-expanded="false" aria-label="Toggle navigation">
|
||||
<span class="navbar-toggler-icon"></span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main role="main">
|
||||
|
||||
<section class="jumbotron text-center">
|
||||
<div class="container">
|
||||
<h1 class="jumbotron-heading">Album example</h1>
|
||||
<p class="lead text-muted">Something short and leading about the collection below—its contents, the creator, etc. Make it short and sweet, but not too short so folks don't simply skip over it entirely.</p>
|
||||
<p>
|
||||
<a href="#" class="btn btn-primary my-2">Main call to action</a>
|
||||
<a href="#" class="btn btn-secondary my-2">Secondary action</a>
|
||||
</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div class="album py-5 bg-light">
|
||||
<div class="container">
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" style="height: 225px; width: 100%; display: block;" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea5071fe%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea5071fe%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea5071fe%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea5071fe%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true" style="height: 225px; width: 100%; display: block;">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea507200%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea507200%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true" style="height: 225px; width: 100%; display: block;">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea507200%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea507200%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true" style="height: 225px; width: 100%; display: block;">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea507201%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea507201%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true" style="height: 225px; width: 100%; display: block;">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea507202%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea507202%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true" style="height: 225px; width: 100%; display: block;">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea507203%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea507203%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true" style="height: 225px; width: 100%; display: block;">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea507203%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea507203%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true" style="height: 225px; width: 100%; display: block;">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<div class="card mb-4 shadow-sm">
|
||||
<img class="card-img-top" data-src="holder.js/100px225?theme=thumb&bg=55595c&fg=eceeef&text=Thumbnail" alt="Thumbnail [100%x225]" src="data:image/svg+xml;charset=UTF-8,%3Csvg%20width%3D%22348%22%20height%3D%22225%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20348%20225%22%20preserveAspectRatio%3D%22none%22%3E%3Cdefs%3E%3Cstyle%20type%3D%22text%2Fcss%22%3E%23holder_165ea507203%20text%20%7B%20fill%3A%23eceeef%3Bfont-weight%3Abold%3Bfont-family%3AArial%2C%20Helvetica%2C%20Open%20Sans%2C%20sans-serif%2C%20monospace%3Bfont-size%3A17pt%20%7D%20%3C%2Fstyle%3E%3C%2Fdefs%3E%3Cg%20id%3D%22holder_165ea507203%22%3E%3Crect%20width%3D%22348%22%20height%3D%22225%22%20fill%3D%22%2355595c%22%3E%3C%2Frect%3E%3Cg%3E%3Ctext%20x%3D%22116.71875%22%20y%3D%22120.15%22%3EThumbnail%3C%2Ftext%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E" data-holder-rendered="true" style="height: 225px; width: 100%; display: block;">
|
||||
<div class="card-body">
|
||||
<p class="card-text">This is a wider card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.</p>
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">View</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary">Edit</button>
|
||||
</div>
|
||||
<small class="text-muted">9 mins</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</main>
|
||||
|
||||
<footer class="text-muted">
|
||||
<div class="container">
|
||||
<p class="float-right">
|
||||
<a href="#">Back to top</a>
|
||||
</p>
|
||||
<p>Album example is © Bootstrap, but please download and customize it for yourself!</p>
|
||||
<p>New to Bootstrap? <a href="../../">Visit the homepage</a> or read our <a href="../../getting-started/">getting started guide</a>.</p>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
<!-- Bootstrap core JavaScript
|
||||
================================================== -->
|
||||
<!-- Placed at the end of the document so the pages load faster -->
|
||||
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
|
||||
<script>window.jQuery || document.write('<script src="../../assets/js/vendor/jquery-slim.min.js"><\/script>')</script>
|
||||
<script src="../../assets/js/vendor/popper.min.js"></script>
|
||||
<script src="../../dist/js/bootstrap.min.js"></script>
|
||||
<script src="../../assets/js/vendor/holder.min.js"></script>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="348" height="225" viewBox="0 0 348 225" preserveAspectRatio="none" style="display: none; visibility: hidden; position: absolute; top: -100%; left: -100%;"><defs><style type="text/css"></style></defs><text x="0" y="17" style="font-weight:bold;font-size:17pt;font-family:Arial, Helvetica, Open Sans, sans-serif">Thumbnail</text></svg></body></html>
|
||||
`
|
||||
|
||||
Convey(".NodeType", t, func() {
|
||||
buff := bytes.NewBuffer([]byte(doc))
|
||||
|
||||
buff.Write([]byte(doc))
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(buff)
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
el, err := http.NewHtmlElement(doc.Find("body"))
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(el.NodeType(), ShouldEqual, 1)
|
||||
})
|
||||
|
||||
Convey(".NodeName", t, func() {
|
||||
buff := bytes.NewBuffer([]byte(doc))
|
||||
|
||||
buff.Write([]byte(doc))
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(buff)
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
el, err := http.NewHtmlElement(doc.Find("body"))
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(el.NodeName(), ShouldEqual, "body")
|
||||
})
|
||||
|
||||
Convey(".Length", t, func() {
|
||||
buff := bytes.NewBuffer([]byte(`
|
||||
<html>
|
||||
<head></head>
|
||||
<body>
|
||||
<span></span>
|
||||
<span></span>
|
||||
<span></span>
|
||||
<span></span>
|
||||
</body>
|
||||
</html>
|
||||
`))
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(buff)
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
el, err := http.NewHtmlElement(doc.Find("body"))
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(el.Length(), ShouldEqual, 4)
|
||||
})
|
||||
|
||||
Convey(".Value", t, func() {
|
||||
buff := bytes.NewBuffer([]byte(`
|
||||
<html>
|
||||
<head></head>
|
||||
<body>
|
||||
<input id="q" value="find" />
|
||||
<span></span>
|
||||
<span></span>
|
||||
<span></span>
|
||||
<span></span>
|
||||
</body>
|
||||
</html>
|
||||
`))
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(buff)
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
el, err := http.NewHtmlElement(doc.Find("#q"))
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
v := el.Value()
|
||||
|
||||
So(v, ShouldEqual, "find")
|
||||
})
|
||||
|
||||
Convey(".InnerText", t, func() {
|
||||
buff := bytes.NewBuffer([]byte(`
|
||||
<html>
|
||||
<head></head>
|
||||
<body>
|
||||
<h2>Ferret</h2>
|
||||
<span></span>
|
||||
<span></span>
|
||||
<span></span>
|
||||
<span></span>
|
||||
</body>
|
||||
</html>
|
||||
`))
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(buff)
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
el, err := http.NewHtmlElement(doc.Find("h2"))
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
v := el.InnerText()
|
||||
|
||||
So(v, ShouldEqual, "Ferret")
|
||||
})
|
||||
|
||||
Convey(".InnerHtml", t, func() {
|
||||
buff := bytes.NewBuffer([]byte(`
|
||||
<html>
|
||||
<head></head>
|
||||
<body>
|
||||
<div id="content"><h2>Ferret</h2></div>
|
||||
<span></span>
|
||||
<span></span>
|
||||
<span></span>
|
||||
<span></span>
|
||||
</body>
|
||||
</html>
|
||||
`))
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(buff)
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
el, err := http.NewHtmlElement(doc.Find("#content"))
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
v := el.InnerHtml()
|
||||
|
||||
So(v, ShouldEqual, "<h2>Ferret</h2>")
|
||||
})
|
||||
|
||||
Convey(".QuerySelector", t, func() {
|
||||
buff := bytes.NewBuffer([]byte(doc))
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(buff)
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
el, err := http.NewHtmlElement(doc.Find("body .card-img-top:nth-child(1)"))
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
v := el.NodeName()
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(v, ShouldEqual, "img")
|
||||
})
|
||||
}
|
||||
77
pkg/stdlib/html/driver/http/http.go
Normal file
77
pkg/stdlib/html/driver/http/http.go
Normal file
@@ -0,0 +1,77 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"github.com/MontFerret/ferret/pkg/runtime/values"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/corpix/uarand"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sethgrid/pester"
|
||||
httpx "net/http"
|
||||
)
|
||||
|
||||
type HttpDriver struct {
|
||||
client *pester.Client
|
||||
}
|
||||
|
||||
func NewDriver(setters ...Option) *HttpDriver {
|
||||
client := pester.New()
|
||||
client.Concurrency = 3
|
||||
client.MaxRetries = 5
|
||||
client.Backoff = pester.ExponentialBackoff
|
||||
|
||||
for _, setter := range setters {
|
||||
setter(client)
|
||||
}
|
||||
|
||||
return &HttpDriver{client}
|
||||
}
|
||||
|
||||
func (d *HttpDriver) GetDocument(ctx context.Context, url string) (values.HtmlNode, error) {
|
||||
req, err := httpx.NewRequest(httpx.MethodGet, url, nil)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9,ru;q=0.8")
|
||||
req.Header.Set("Cache-Control", "no-cache")
|
||||
req.Header.Set("Pragma", "no-cache")
|
||||
req.Header.Set("User-Agent", uarand.GetRandom())
|
||||
|
||||
resp, err := d.client.Do(req)
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to retrieve a document %s", url)
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to parse a document %s", url)
|
||||
}
|
||||
|
||||
return NewHtmlDocument(url, doc)
|
||||
}
|
||||
|
||||
func (d *HttpDriver) ParseDocument(ctx context.Context, str string) (values.HtmlNode, error) {
|
||||
buf := bytes.NewBuffer([]byte(str))
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(buf)
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to parse a document")
|
||||
}
|
||||
|
||||
return NewHtmlDocument("#string", doc)
|
||||
}
|
||||
|
||||
func (d *HttpDriver) Close() error {
|
||||
d.client = nil
|
||||
|
||||
return nil
|
||||
}
|
||||
37
pkg/stdlib/html/driver/http/options.go
Normal file
37
pkg/stdlib/html/driver/http/options.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package http
|
||||
|
||||
import "github.com/sethgrid/pester"
|
||||
|
||||
type (
|
||||
Option func(opts *pester.Client)
|
||||
)
|
||||
|
||||
func WithDefaultBackoff() Option {
|
||||
return func(opts *pester.Client) {
|
||||
opts.Backoff = pester.DefaultBackoff
|
||||
}
|
||||
}
|
||||
|
||||
func WithLinearBackoff() Option {
|
||||
return func(opts *pester.Client) {
|
||||
opts.Backoff = pester.LinearBackoff
|
||||
}
|
||||
}
|
||||
|
||||
func WithExponentialBackoff() Option {
|
||||
return func(opts *pester.Client) {
|
||||
opts.Backoff = pester.ExponentialBackoff
|
||||
}
|
||||
}
|
||||
|
||||
func WithMaxRetries(value int) Option {
|
||||
return func(opts *pester.Client) {
|
||||
opts.MaxRetries = value
|
||||
}
|
||||
}
|
||||
|
||||
func WithConcurrency(value int) Option {
|
||||
return func(opts *pester.Client) {
|
||||
opts.Concurrency = value
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user