Feature/#382 response cdp (#450)

* wip

* Added support of response information to CDP driver

* Fixed response look up

e2e/tests/dynamic/response.fql

@@ -0,0 +1,4 @@
+LET url = @dynamic
+LET doc = DOCUMENT(url, true)
+
+RETURN EXPECT(doc.response.status, "OK")

pkg/drivers/cdp/network/events.go

@@ -3,5 +3,6 @@ package network
 import "github.com/MontFerret/ferret/pkg/drivers/cdp/events"
 
 var (
-	eventFrameLoad = events.New("frame_load")
+	eventFrameLoad   = events.New("frame_load")
+	responseReceived = events.New("response_received")
 )

pkg/drivers/cdp/network/manager.go

@@ -3,6 +3,7 @@ package network
 import (
 	"context"
 	"encoding/json"
+	"io"
 	"regexp"
 	"sync"
 
@@ -15,6 +16,7 @@ import (
 
 	"github.com/MontFerret/ferret/pkg/drivers"
 	"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
+	"github.com/MontFerret/ferret/pkg/drivers/common"
 	"github.com/MontFerret/ferret/pkg/runtime/core"
 	"github.com/MontFerret/ferret/pkg/runtime/values"
 )
@@ -25,13 +27,14 @@ type (
 	FrameLoadedListener = func(ctx context.Context, frame page.Frame)
 
 	Manager struct {
-		mu        sync.Mutex
-		logger    *zerolog.Logger
-		client    *cdp.Client
-		headers   drivers.HTTPHeaders
-		eventLoop *events.Loop
-		cancel    context.CancelFunc
-		listeners []FrameLoadedListener
+		mu                 sync.Mutex
+		logger             *zerolog.Logger
+		client             *cdp.Client
+		headers            drivers.HTTPHeaders
+		eventLoop          *events.Loop
+		cancel             context.CancelFunc
+		responseListenerID events.ListenerID
+		response           *sync.Map
 	}
 )
 
@@ -48,6 +51,17 @@ func New(
 	m.headers = make(drivers.HTTPHeaders)
 	m.eventLoop = eventLoop
 	m.cancel = cancel
+	m.response = new(sync.Map)
+
+	var err error
+
+	closers := make([]io.Closer, 0, 10)
+
+	defer func() {
+		if err != nil {
+			common.CloseAll(logger, closers, "failed to close a DOM event stream")
+		}
+	}()
 
 	frameNavigatedStream, err := m.client.Page.FrameNavigated(ctx)
 
@@ -55,10 +69,22 @@ func New(
 		return nil, err
 	}
 
+	responseReceivedStream, err := m.client.Network.ResponseReceived(ctx)
+
+	if err != nil {
+		return nil, err
+	}
+
 	m.eventLoop.AddSource(events.NewSource(eventFrameLoad, frameNavigatedStream, func(stream rpcc.Stream) (interface{}, error) {
 		return stream.(page.FrameNavigatedClient).Recv()
 	}))
 
+	m.eventLoop.AddSource(events.NewSource(responseReceived, responseReceivedStream, func(stream rpcc.Stream) (interface{}, error) {
+		return stream.(network.ResponseReceivedClient).Recv()
+	}))
+
+	m.responseListenerID = m.eventLoop.AddListener(responseReceived, m.onResponse)
+
 	return m, nil
 }
 
@@ -170,6 +196,16 @@ func (m *Manager) SetHeaders(ctx context.Context, headers drivers.HTTPHeaders) e
 	return nil
 }
 
+func (m *Manager) GetResponse(_ context.Context, frameID page.FrameID) (drivers.HTTPResponse, error) {
+	value, found := m.response.Load(frameID)
+
+	if !found {
+		return drivers.HTTPResponse{}, core.ErrNotFound
+	}
+
+	return value.(drivers.HTTPResponse), nil
+}
+
 func (m *Manager) Navigate(ctx context.Context, url values.String) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
@@ -338,3 +374,41 @@ func (m *Manager) AddFrameLoadedListener(listener FrameLoadedListener) events.Li
 func (m *Manager) RemoveFrameLoadedListener(id events.ListenerID) {
 	m.eventLoop.RemoveListener(eventFrameLoad, id)
 }
+
+func (m *Manager) onResponse(_ context.Context, message interface{}) (out bool) {
+	out = true
+	msg, ok := message.(*network.ResponseReceivedReply)
+
+	if !ok {
+		return
+	}
+
+	// we are interested in documents only
+	if msg.Type != network.ResourceTypeDocument {
+		return
+	}
+
+	response := drivers.HTTPResponse{
+		StatusCode: msg.Response.Status,
+		Status:     msg.Response.StatusText,
+		Headers:    make(drivers.HTTPHeaders),
+	}
+
+	deserialized := make(map[string]string)
+
+	if len(msg.Response.Headers) > 0 {
+		err := json.Unmarshal(msg.Response.Headers, &deserialized)
+
+		if err != nil {
+			m.logger.Error().Err(err).Msg("failed to deserialize response headers")
+		}
+	}
+
+	for key, value := range deserialized {
+		response.Headers.Set(key, value)
+	}
+
+	m.response.Store(*msg.FrameID, response)
+
+	return
+}

pkg/drivers/cdp/page.go

@@ -369,8 +369,14 @@ func (p *HTMLPage) DeleteCookies(ctx context.Context, cookies drivers.HTTPCookie
 	return p.network.DeleteCookies(ctx, p.getCurrentDocument().GetURL().String(), cookies)
 }
 
-func (p *HTMLPage) GetResponse(_ context.Context) (*drivers.HTTPResponse, error) {
-	return nil, core.ErrNotSupported
+func (p *HTMLPage) GetResponse(ctx context.Context) (drivers.HTTPResponse, error) {
+	doc := p.getCurrentDocument()
+
+	if doc == nil {
+		return drivers.HTTPResponse{}, nil
+	}
+
+	return p.network.GetResponse(ctx, doc.Frame().Frame.ID)
 }
 
 func (p *HTMLPage) PrintToPDF(ctx context.Context, params drivers.PDFParams) (values.Binary, error) {

pkg/drivers/common/getter.go

@@ -24,12 +24,12 @@ func GetInPage(ctx context.Context, page drivers.HTMLPage, path []core.Value) (c
 		switch segment {
 		case "response":
 			resp, err := page.GetResponse(ctx)
+
 			if err != nil {
 				return nil, errors.Wrap(err, "get response")
 			}
 
 			return resp.GetIn(ctx, path[1:])
-
 		case "mainFrame", "document":
 			return GetInDocument(ctx, page.GetMainFrame(), path[1:])
 		case "frames":

pkg/drivers/http/driver.go

@@ -174,7 +174,7 @@ func (drv *Driver) Open(ctx context.Context, params drivers.Params) (drivers.HTM
 		Headers:    drivers.HTTPHeaders(resp.Header),
 	}
 
-	return NewHTMLPage(doc, params.URL, &r, cookies)
+	return NewHTMLPage(doc, params.URL, r, cookies)
 }
 
 func (drv *Driver) Parse(_ context.Context, params drivers.ParseParams) (drivers.HTMLPage, error) {
@@ -186,7 +186,7 @@ func (drv *Driver) Parse(_ context.Context, params drivers.ParseParams) (drivers
 		return nil, errors.Wrap(err, "failed to parse a document")
 	}
 
-	return NewHTMLPage(doc, "#blank", nil, nil)
+	return NewHTMLPage(doc, "#blank", drivers.HTTPResponse{}, nil)
 }
 
 func (drv *Driver) Close() error {

pkg/drivers/http/page.go

@@ -16,13 +16,13 @@ type HTMLPage struct {
 	document *HTMLDocument
 	cookies  drivers.HTTPCookies
 	frames   *values.Array
-	response *drivers.HTTPResponse
+	response drivers.HTTPResponse
 }
 
 func NewHTMLPage(
 	qdoc *goquery.Document,
 	url string,
-	response *drivers.HTTPResponse,
+	response drivers.HTTPResponse,
 	cookies drivers.HTTPCookies,
 ) (*HTMLPage, error) {
 	doc, err := NewRootHTMLDocument(qdoc, url)
@@ -178,7 +178,7 @@ func (p *HTMLPage) GetCookies(_ context.Context) (drivers.HTTPCookies, error) {
 	return res, nil
 }
 
-func (p *HTMLPage) GetResponse(_ context.Context) (*drivers.HTTPResponse, error) {
+func (p *HTMLPage) GetResponse(_ context.Context) (drivers.HTTPResponse, error) {
 	return p.response, nil
 }
 

pkg/drivers/value.go

@@ -198,7 +198,7 @@ type (
 
 		DeleteCookies(ctx context.Context, cookies HTTPCookies) error
 
-		GetResponse(ctx context.Context) (*HTTPResponse, error)
+		GetResponse(ctx context.Context) (HTTPResponse, error)
 
 		PrintToPDF(ctx context.Context, params PDFParams) (values.Binary, error)