go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2025-11-06 08:39:09 +02:00
Code Issues Releases Activity

Fixed headers (#348)

Browse Source 
* Fixed headers

* Added e2e for static pages
This commit is contained in:
Tim Voronov
2019-08-04 17:25:47 -04:00
committed by GitHub
parent 5293561a89
commit 11bf8c365a
15 changed files with 107 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/drivers/cdp/page.go
View File

@@ -166,14 +166,14 @@ func LoadHTMLPage(
}
}
if params.Header != nil {
j, err := json.Marshal(params.Header)
if params.Headers != nil {
j, err := json.Marshal(params.Headers)
if err != nil {
return nil, err
}
for k := range params.Header {
for k := range params.Headers {
logger.
Debug().
Timestamp().

38
pkg/drivers/header.go → pkg/drivers/headers.go
View File

@@ -15,14 +15,18 @@ import (
"github.com/MontFerret/ferret/pkg/runtime/values/types"
)
// HTTPHeader HTTP header object
type HTTPHeader map[string][]string
// HTTPHeaders HTTP header object
type HTTPHeaders map[string][]string
func (h HTTPHeader) Type() core.Type {
func NewHTTPHeaders(values map[string][]string) HTTPHeaders {
return HTTPHeaders(values)
}
func (h HTTPHeaders) Type() core.Type {
return HTTPHeaderType
}
func (h HTTPHeader) String() string {
func (h HTTPHeaders) String() string {
var buf bytes.Buffer
for k := range h {
@@ -32,12 +36,12 @@ func (h HTTPHeader) String() string {
return buf.String()
}
func (h HTTPHeader) Compare(other core.Value) int64 {
func (h HTTPHeaders) Compare(other core.Value) int64 {
if other.Type() != HTTPHeaderType {
return Compare(HTTPHeaderType, other.Type())
}
oh := other.(HTTPHeader)
oh := other.(HTTPHeaders)
if len(h) > len(oh) {
return 1
@@ -56,11 +60,11 @@ func (h HTTPHeader) Compare(other core.Value) int64 {
return 0
}
func (h HTTPHeader) Unwrap() interface{} {
func (h HTTPHeaders) Unwrap() interface{} {
return h
}
func (h HTTPHeader) Hash() uint64 {
func (h HTTPHeaders) Hash() uint64 {
hash := fnv.New64a()
hash.Write([]byte(h.Type().String()))
@@ -96,12 +100,18 @@ func (h HTTPHeader) Hash() uint64 {
return hash.Sum64()
}
func (h HTTPHeader) Copy() core.Value {
func (h HTTPHeaders) Copy() core.Value {
return *(&h)
}
func (h HTTPHeader) MarshalJSON() ([]byte, error) {
out, err := json.Marshal(map[string][]string(h))
func (h HTTPHeaders) MarshalJSON() ([]byte, error) {
headers := map[string]string{}
for key, val := range h {
headers[key] = strings.Join(val, ", ")
}
out, err := json.Marshal(headers)
if err != nil {
return nil, err
@@ -110,15 +120,15 @@ func (h HTTPHeader) MarshalJSON() ([]byte, error) {
return out, err
}
func (h HTTPHeader) Set(key, value string) {
func (h HTTPHeaders) Set(key, value string) {
textproto.MIMEHeader(h).Set(key, value)
}
func (h HTTPHeader) Get(key string) string {
func (h HTTPHeaders) Get(key string) string {
return textproto.MIMEHeader(h).Get(key)
}
func (h HTTPHeader) GetIn(_ context.Context, path []core.Value) (core.Value, error) {
func (h HTTPHeaders) GetIn(_ context.Context, path []core.Value) (core.Value, error) {
if len(path) == 0 {
return values.None, nil
}

13
pkg/drivers/header_test.go → pkg/drivers/headers_test.go
View File

@@ -1,25 +1,26 @@
package drivers_test
import (
"github.com/MontFerret/ferret/pkg/drivers"
"testing"
. "github.com/smartystreets/goconvey/convey"
"github.com/MontFerret/ferret/pkg/drivers"
)
func TestHTTPHeader(t *testing.T) {
Convey("HTTPHeader", t, func() {
Convey("HTTPHeaders", t, func() {
Convey(".MarshalJSON", func() {
Convey("Should serialize header values", func() {
headers := make(drivers.HTTPHeader)
headers := make(drivers.HTTPHeaders)
headers["content-encoding"] = []string{"gzip"}
headers["content-type"] = []string{"text/html", "charset=utf-8"}
headers["Content-Encoding"] = []string{"gzip"}
headers["Content-Type"] = []string{"text/html", "charset=utf-8"}
out, err := headers.MarshalJSON()
So(err, ShouldBeNil)
So(string(out), ShouldEqual, `{"content-encoding":["gzip"],"content-type":["text/html","charset=utf-8"]}`)
So(string(out), ShouldEqual, `{"Content-Encoding":"gzip","Content-Type":"text/html, charset=utf-8"}`)
})
})
})

6
pkg/drivers/http/driver.go
View File

@@ -76,9 +76,9 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
req.Header.Set("Cache-Control", "no-cache")
req.Header.Set("Pragma", "no-cache")
if params.Header != nil {
for k := range params.Header {
req.Header.Add(k, params.Header.Get(k))
if params.Headers != nil {
for k := range params.Headers {
req.Header.Add(k, params.Headers.Get(k))
logger.
Debug().

2
pkg/drivers/params.go
View File

@@ -14,7 +14,7 @@ type (
UserAgent string
KeepCookies bool
Cookies []HTTPCookie
Header HTTPHeader
Headers HTTPHeaders
Viewport *Viewport
}
)

2
pkg/drivers/type.go
View File

@@ -3,7 +3,7 @@ package drivers
import "github.com/MontFerret/ferret/pkg/runtime/core"
var (
HTTPHeaderType = core.NewType("HTTPHeader")
HTTPHeaderType = core.NewType("HTTPHeaders")
HTTPCookieType = core.NewType("HTTPCookie")
HTMLElementType = core.NewType("HTMLElement")
HTMLDocumentType = core.NewType("HTMLDocument")

16
pkg/stdlib/html/document.go
View File

@@ -27,7 +27,7 @@ type PageLoadParams struct {
// keepCookies (Boolean) - Optional, boolean value indicating whether to use cookies from previous sessions.
// i.e. not to open a page in the Incognito mode.
// cookies (HTTPCookie) - Optional, set of HTTP cookies.
// header (HTTPHeader) - Optional, HTTP headers.
// headers (HTTPHeaders) - Optional, HTTP headers.
// viewport (Viewport) - Optional, viewport params.
// @returns (HTMLPage) - Returns loaded HTML page.
func Open(ctx context.Context, args ...core.Value) (core.Value, error) {
@@ -147,15 +147,15 @@ func newPageLoadParams(url values.String, arg core.Value) (PageLoadParams, error
res.Cookies = cookies
}
header, exists := obj.Get(values.NewString("header"))
headers, exists := obj.Get(values.NewString("headers"))
if exists {
if err := core.ValidateType(header, types.Object); err != nil {
if err := core.ValidateType(headers, types.Object); err != nil {
return res, err
}
header := parseHeader(header.(*values.Object))
res.Header = header
header := parseHeader(headers.(*values.Object))
res.Headers = header
}
viewport, exists := obj.Get(values.NewString("viewport"))
@@ -292,10 +292,10 @@ func parseCookie(value core.Value) (drivers.HTTPCookie, error) {
return cookie, err
}
func parseHeader(header *values.Object) drivers.HTTPHeader {
res := make(drivers.HTTPHeader)
func parseHeader(headers *values.Object) drivers.HTTPHeaders {
res := make(drivers.HTTPHeaders)
header.ForEach(func(value core.Value, key string) bool {
headers.ForEach(func(value core.Value, key string) bool {
res.Set(key, value.String())
return true