1
0
mirror of https://github.com/MontFerret/ferret.git synced 2025-12-01 22:19:32 +02:00

Merge branch 'master' into cleanup/fix-naming-convention

This commit is contained in:
David Landry
2018-10-05 17:42:28 -04:00
committed by GitHub
42 changed files with 571 additions and 162 deletions

View File

@@ -2,7 +2,6 @@ package dynamic
import (
"context"
"crypto/sha512"
"fmt"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/logging"
@@ -17,10 +16,14 @@ import (
"github.com/mafredri/cdp/rpcc"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"hash/fnv"
"sync"
"time"
)
const BlankPageUrl = "about:blank"
type HTMLDocument struct {
sync.Mutex
logger *zerolog.Logger
@@ -78,10 +81,12 @@ func LoadHTMLDocument(
return nil, err
}
err = waitForLoadEvent(ctx, client)
if url != BlankPageUrl {
err = waitForLoadEvent(ctx, client)
if err != nil {
return nil, err
if err != nil {
return nil, err
}
}
root, innerHTML, err := getRootElement(client)
@@ -201,19 +206,17 @@ func (doc *HTMLDocument) Unwrap() interface{} {
return doc.element
}
func (doc *HTMLDocument) Hash() int {
func (doc *HTMLDocument) Hash() uint64 {
doc.Lock()
defer doc.Unlock()
h := sha512.New()
h := fnv.New64a()
out, err := h.Write([]byte(doc.url))
h.Write([]byte(doc.Type().String()))
h.Write([]byte(":"))
h.Write([]byte(doc.url))
if err != nil {
return 0
}
return out
return h.Sum64()
}
func (doc *HTMLDocument) Clone() core.Value {
@@ -637,6 +640,10 @@ func (doc *HTMLDocument) WaitForNavigation(timeout values.Int) error {
}
func (doc *HTMLDocument) Navigate(url values.String) error {
if url == "" {
url = BlankPageUrl
}
ctx := context.Background()
repl, err := doc.client.Page.Navigate(ctx, page.NewNavigateArgs(url.String()))
@@ -648,5 +655,5 @@ func (doc *HTMLDocument) Navigate(url values.String) error {
return errors.New(*repl.ErrorText)
}
return waitForLoadEvent(ctx, doc.client)
return doc.WaitForNavigation(5000)
}

View File

@@ -38,6 +38,10 @@ func (drv *Driver) GetDocument(ctx context.Context, url string) (values.HtmlNode
ctx, cancel := context.WithTimeout(ctx, DefaultTimeout)
defer cancel()
if url == "" {
url = BlankPageUrl
}
// Create a new target belonging to the browser context, similar
// to opening a new tab in an incognito window.
createTargetArgs := target.NewCreateTargetArgs(url).SetBrowserContextID(drv.contextID)

View File

@@ -3,7 +3,6 @@ package dynamic
import (
"bytes"
"context"
"crypto/sha512"
"encoding/json"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
@@ -14,6 +13,7 @@ import (
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/protocol/dom"
"github.com/rs/zerolog"
"hash/fnv"
"strconv"
"sync"
"time"
@@ -188,24 +188,18 @@ func (el *HTMLElement) Unwrap() interface{} {
return el
}
func (el *HTMLElement) Hash() int {
func (el *HTMLElement) Hash() uint64 {
el.Lock()
defer el.Unlock()
h := sha512.New()
h := fnv.New64a()
out, err := h.Write([]byte(el.innerHTML))
h.Write([]byte(el.Type().String()))
h.Write([]byte(":"))
h.Write([]byte(el.innerHTML))
if err != nil {
el.logger.Error().
Timestamp().
Err(err).
Msg("failed to calculate hash value")
return 0
}
return out
return h.Sum64()
}
func (el *HTMLElement) Value() core.Value {
@@ -297,7 +291,7 @@ func (el *HTMLElement) GetChildNode(idx values.Int) core.Value {
func (el *HTMLElement) QuerySelector(selector values.String) core.Value {
if !el.IsConnected() {
return values.NewArray(0)
return values.None
}
ctx := context.Background()

View File

@@ -1,12 +1,12 @@
package static
import (
"crypto/sha512"
"encoding/json"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/stdlib/html/driver/common"
"github.com/PuerkitoBio/goquery"
"hash/fnv"
)
type HTMLElement struct {
@@ -53,22 +53,21 @@ func (el *HTMLElement) Unwrap() interface{} {
return el.selection
}
func (el *HTMLElement) Hash() int {
h := sha512.New()
func (el *HTMLElement) Hash() uint64 {
str, err := el.selection.Html()
if err != nil {
return 0
}
out, err := h.Write([]byte(str))
h := fnv.New64a()
if err != nil {
return 0
}
h.Write([]byte(el.Type().String()))
h.Write([]byte(":"))
h.Write([]byte(str))
return out
return h.Sum64()
}
func (el *HTMLElement) Clone() core.Value {