Fixed headers (#348)

* Fixed headers

* Added e2e for static pages

CHANGELOG.md

@@ -3,6 +3,7 @@
 ### 0.8.2
 #### Fixed
 - Scrolling position is not centered.
+- Unable to set custom headers. [#348](https://github.com/MontFerret/ferret/pull/348)
 - Unable to set custom logger fields.
 
 ### 0.8.1

e2e/tests/dynamic/doc/headers/set.fql

@@ -0,0 +1,20 @@
+LET url = @static + "/api/ts"
+LET page = DOCUMENT(url, {
+    driver: "cdp",
+    headers: {
+         "Access-Control-Allow-Origin": "*",
+         "X-Request-Id": "foobar"
+     }
+})
+
+LET el = ELEMENT(page, "#headers")
+LET actual = JSON_PARSE(el.innerText)
+LET expected = {
+    "Access-Control-Allow-Origin": ["*"],
+    "X-Request-Id": ["foobar"]
+}
+
+RETURN EXPECT(expected, {
+    "Access-Control-Allow-Origin": actual["Access-Control-Allow-Origin"],
+    "X-Request-Id": actual["X-Request-Id"]
+})

e2e/tests/static/doc/headers/set.fql

@@ -0,0 +1,20 @@
+LET url = @static + "/api/ts"
+LET page = DOCUMENT(url, {
+    driver: "http",
+    headers: {
+         "Access-Control-Allow-Origin": "*",
+         "X-Request-Id": "foobar"
+     }
+})
+
+LET el = ELEMENT(page, "#headers")
+LET actual = JSON_PARSE(el.innerText)
+LET expected = {
+    "Access-Control-Allow-Origin": ["*"],
+    "X-Request-Id": ["foobar"]
+}
+
+RETURN EXPECT(expected, {
+    "Access-Control-Allow-Origin": actual["Access-Control-Allow-Origin"],
+    "X-Request-Id": actual["X-Request-Id"]
+})

examples/google-search.fql

@@ -0,0 +1,19 @@
+LET google = DOCUMENT("https://www.google.com/", {
+    driver: "cdp",
+    userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36"
+})
+
+INPUT(google, 'input[name="q"]', @criteria)
+
+CLICK(google, 'input[name="btnK"]')
+
+WAIT_NAVIGATION(google)
+
+FOR result IN ELEMENTS(google, '.g')
+    // filter out extra elements like videos and 'People also ask'
+    FILTER TRIM(result.attributes.class) == 'g'
+    RETURN {
+        title: INNER_TEXT(result, 'h3'),
+        description: INNER_TEXT(result, '.st'),
+        url: INNER_TEXT(result, 'cite')
+    }

pkg/drivers/cdp/page.go

@@ -166,14 +166,14 @@ func LoadHTMLPage(
 		}
 	}
 
-	if params.Header != nil {
+	if params.Headers != nil {
-		j, err := json.Marshal(params.Header)
+		j, err := json.Marshal(params.Headers)
 
 		if err != nil {
 			return nil, err
 		}
 
-		for k := range params.Header {
+		for k := range params.Headers {
 			logger.
 				Debug().
 				Timestamp().

pkg/drivers/headers.go

@@ -15,14 +15,18 @@ import (
 	"github.com/MontFerret/ferret/pkg/runtime/values/types"
 )
 
-// HTTPHeader HTTP header object
+// HTTPHeaders HTTP header object
-type HTTPHeader map[string][]string
+type HTTPHeaders map[string][]string
 
-func (h HTTPHeader) Type() core.Type {
+func NewHTTPHeaders(values map[string][]string) HTTPHeaders {
+	return HTTPHeaders(values)
+}
+
+func (h HTTPHeaders) Type() core.Type {
 	return HTTPHeaderType
 }
 
-func (h HTTPHeader) String() string {
+func (h HTTPHeaders) String() string {
 	var buf bytes.Buffer
 
 	for k := range h {
@@ -32,12 +36,12 @@ func (h HTTPHeader) String() string {
 	return buf.String()
 }
 
-func (h HTTPHeader) Compare(other core.Value) int64 {
+func (h HTTPHeaders) Compare(other core.Value) int64 {
 	if other.Type() != HTTPHeaderType {
 		return Compare(HTTPHeaderType, other.Type())
 	}
 
-	oh := other.(HTTPHeader)
+	oh := other.(HTTPHeaders)
 
 	if len(h) > len(oh) {
 		return 1
@@ -56,11 +60,11 @@ func (h HTTPHeader) Compare(other core.Value) int64 {
 	return 0
 }
 
-func (h HTTPHeader) Unwrap() interface{} {
+func (h HTTPHeaders) Unwrap() interface{} {
 	return h
 }
 
-func (h HTTPHeader) Hash() uint64 {
+func (h HTTPHeaders) Hash() uint64 {
 	hash := fnv.New64a()
 
 	hash.Write([]byte(h.Type().String()))
@@ -96,12 +100,18 @@ func (h HTTPHeader) Hash() uint64 {
 	return hash.Sum64()
 }
 
-func (h HTTPHeader) Copy() core.Value {
+func (h HTTPHeaders) Copy() core.Value {
 	return *(&h)
 }
 
-func (h HTTPHeader) MarshalJSON() ([]byte, error) {
+func (h HTTPHeaders) MarshalJSON() ([]byte, error) {
-	out, err := json.Marshal(map[string][]string(h))
+	headers := map[string]string{}
+
+	for key, val := range h {
+		headers[key] = strings.Join(val, ", ")
+	}
+
+	out, err := json.Marshal(headers)
 
 	if err != nil {
 		return nil, err
@@ -110,15 +120,15 @@ func (h HTTPHeader) MarshalJSON() ([]byte, error) {
 	return out, err
 }
 
-func (h HTTPHeader) Set(key, value string) {
+func (h HTTPHeaders) Set(key, value string) {
 	textproto.MIMEHeader(h).Set(key, value)
 }
 
-func (h HTTPHeader) Get(key string) string {
+func (h HTTPHeaders) Get(key string) string {
 	return textproto.MIMEHeader(h).Get(key)
 }
 
-func (h HTTPHeader) GetIn(_ context.Context, path []core.Value) (core.Value, error) {
+func (h HTTPHeaders) GetIn(_ context.Context, path []core.Value) (core.Value, error) {
 	if len(path) == 0 {
 		return values.None, nil
 	}

pkg/drivers/headers_test.go

@@ -1,25 +1,26 @@
 package drivers_test
 
 import (
-	"github.com/MontFerret/ferret/pkg/drivers"
 	"testing"
 
 	. "github.com/smartystreets/goconvey/convey"
+
+	"github.com/MontFerret/ferret/pkg/drivers"
 )
 
 func TestHTTPHeader(t *testing.T) {
-	Convey("HTTPHeader", t, func() {
+	Convey("HTTPHeaders", t, func() {
 		Convey(".MarshalJSON", func() {
 			Convey("Should serialize header values", func() {
-				headers := make(drivers.HTTPHeader)
+				headers := make(drivers.HTTPHeaders)
 
-				headers["content-encoding"] = []string{"gzip"}
+				headers["Content-Encoding"] = []string{"gzip"}
-				headers["content-type"] = []string{"text/html", "charset=utf-8"}
+				headers["Content-Type"] = []string{"text/html", "charset=utf-8"}
 
 				out, err := headers.MarshalJSON()
 
 				So(err, ShouldBeNil)
-				So(string(out), ShouldEqual, `{"content-encoding":["gzip"],"content-type":["text/html","charset=utf-8"]}`)
+				So(string(out), ShouldEqual, `{"Content-Encoding":"gzip","Content-Type":"text/html, charset=utf-8"}`)
 			})
 		})
 	})

pkg/drivers/http/driver.go

@@ -76,9 +76,9 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
 	req.Header.Set("Cache-Control", "no-cache")
 	req.Header.Set("Pragma", "no-cache")
 
-	if params.Header != nil {
+	if params.Headers != nil {
-		for k := range params.Header {
+		for k := range params.Headers {
-			req.Header.Add(k, params.Header.Get(k))
+			req.Header.Add(k, params.Headers.Get(k))
 
 			logger.
 				Debug().

pkg/drivers/params.go

@@ -14,7 +14,7 @@ type (
 		UserAgent   string
 		KeepCookies bool
 		Cookies     []HTTPCookie
-		Header      HTTPHeader
+		Headers     HTTPHeaders
 		Viewport    *Viewport
 	}
 )

pkg/drivers/type.go

@@ -3,7 +3,7 @@ package drivers
 import "github.com/MontFerret/ferret/pkg/runtime/core"
 
 var (
-	HTTPHeaderType   = core.NewType("HTTPHeader")
+	HTTPHeaderType   = core.NewType("HTTPHeaders")
 	HTTPCookieType   = core.NewType("HTTPCookie")
 	HTMLElementType  = core.NewType("HTMLElement")
 	HTMLDocumentType = core.NewType("HTMLDocument")

pkg/stdlib/html/document.go

@@ -27,7 +27,7 @@ type PageLoadParams struct {
 //      keepCookies (Boolean) - Optional, boolean value indicating whether to use cookies from previous sessions.
 //      	i.e. not to open a page in the Incognito mode.
 //      cookies (HTTPCookie) - Optional, set of HTTP cookies.
-//      header (HTTPHeader) - Optional, HTTP headers.
+//      headers (HTTPHeaders) - Optional, HTTP headers.
 //      viewport (Viewport) - Optional, viewport params.
 // @returns (HTMLPage) - Returns loaded HTML page.
 func Open(ctx context.Context, args ...core.Value) (core.Value, error) {
@@ -147,15 +147,15 @@ func newPageLoadParams(url values.String, arg core.Value) (PageLoadParams, error
 			res.Cookies = cookies
 		}
 
-		header, exists := obj.Get(values.NewString("header"))
+		headers, exists := obj.Get(values.NewString("headers"))
 
 		if exists {
-			if err := core.ValidateType(header, types.Object); err != nil {
+			if err := core.ValidateType(headers, types.Object); err != nil {
 				return res, err
 			}
 
-			header := parseHeader(header.(*values.Object))
+			header := parseHeader(headers.(*values.Object))
-			res.Header = header
+			res.Headers = header
 		}
 
 		viewport, exists := obj.Get(values.NewString("viewport"))
@@ -292,10 +292,10 @@ func parseCookie(value core.Value) (drivers.HTTPCookie, error) {
 	return cookie, err
 }
 
-func parseHeader(header *values.Object) drivers.HTTPHeader {
+func parseHeader(headers *values.Object) drivers.HTTPHeaders {
-	res := make(drivers.HTTPHeader)
+	res := make(drivers.HTTPHeaders)
 
-	header.ForEach(func(value core.Value, key string) bool {
+	headers.ForEach(func(value core.Value, key string) bool {
 		res.Set(key, value.String())
 
 		return true