Feature/425 iframe navigation (#535)

* Updated navigation logic

* Fixed goroutine deadlock

* Fixed closing chan

* Added support of waiting for individual frame navigation

* Updated EventLoop API in order to avoid double closing of event sources

* Fixed attr retrieval

* Removed redundant println

* Updated DOM Readiness check

.travis.yml

@@ -1,80 +0,0 @@
-language: go
-
-sudo: required
-
-os:
-- linux
-
-go:
-- "1.13.x"
-- stable
-
-env:
-  - ANTLR_VERSION=4.8
-
-services:
-  - docker
-
-addons:
-  apt:
-    packages:
-    - openjdk-9-jre-headless
-
-install:
-- go get github.com/mgechev/revive
-- sudo curl -o /usr/local/lib/antlr-${ANTLR_VERSION}-complete.jar https://www.antlr.org/download/antlr-${ANTLR_VERSION}-complete.jar
-- export CLASSPATH=".:/usr/local/lib/antlr-${ANTLR_VERSION}-complete.jar:$CLASSPATH"
-- mkdir $HOME/travis-bin
-- echo -e '#!/bin/bash\njava -jar /usr/local/lib/antlr-${ANTLR_VERSION}-complete.jar "$@"' > $HOME/travis-bin/antlr
-- echo -e '#!/bin/bash\njava org.antlr.v4.gui.TestRig "$@"' > $HOME/travis-bin/grun
-- chmod +x $HOME/travis-bin/*
-- export PATH=$PATH:$HOME/travis-bin
-- export GO111MODULE=on
-- git reset --hard
-
-stages:
-- install
-- lint
-- compile
-- test
-- e2e
-- bench
-
-jobs:
-  include:
-  - stage: install
-    go: stable
-    script:
-      - make install
-  - stage: lint
-    go: stable
-    script:
-    - make vet
-    - make lint
-    - make fmt
-    - git diff
-    - if [[ $(git diff) != '' ]]; then echo 'Invalid formatting!' >&2; exit 1; fi
-  - stage: compile
-    go: stable
-    script:
-    - make generate
-    - make compile
-  - stage: test
-    script:
-    - make cover
-  - stage: e2e
-    go: stable
-    before_script:
-      - curl https://raw.githubusercontent.com/MontFerret/lab/master/install.sh -o install.sh
-      - sudo sh install.sh
-      - docker run -d -p 9222:9222 -e CHROME_OPTS='--disable-dev-shm-usage --full-memory-crash-report' alpeware/chrome-headless-stable:ver-83.0.4103.61
-      - docker ps
-    script:
-      - make compile
-      - make e2e
-    after_script:
-      - docker stop $(docker ps -q)
-  - stage: bench
-    go: stable
-    script:
-    - make bench

Makefile

@@ -18,7 +18,7 @@ install:
 	go get
 
 compile:
-	go build -v -o ${DIR_BIN}/ferret \
+	go build -race -v -o ${DIR_BIN}/ferret \
 	-ldflags "-X main.version=${VERSION}" \
 	./main.go
 
@@ -30,7 +30,7 @@ cover:
 	curl -s https://codecov.io/bash | bash
 
 e2e:
-	lab --timeout=120 --concurrency=2 --wait=http://127.0.0.1:9222/json/version --runtime=bin://./bin/ferret --files=file://./e2e/tests --dir=./e2e/pages/dynamic:8080@dynamic --dir=./e2e/pages/static:8081@static
+	lab --timeout=120 --times=5 --concurrency=1 --wait=http://127.0.0.1:9222/json/version --runtime=bin://./bin/ferret --files=file://./e2e/tests --cdn=./e2e/pages/dynamic:8080@dynamic --cdn=./e2e/pages/static:8081@static
 
 bench:
 	go test -run=XXX -bench=. ${DIR_PKG}/...

cli/exec.go

@@ -41,6 +41,7 @@ func Exec(query string, opts Options) {
 
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, os.Interrupt)
+	signal.Notify(c, os.Kill)
 
 	go func() {
 		for {

e2e/pages/dynamic/components/pages/iframes/index.js

@@ -3,6 +3,24 @@ import { parse } from '../../../utils/qs.js';
 const e = React.createElement;
 
 export default class IFramePage extends React.Component {
+    constructor(props) {
+        super(props);
+
+        this.state = {
+            url: '/'
+        }
+    }
+
+    handleUrlInput(evt) {
+        this.setState({
+            url: evt.target.value
+        })
+    }
+
+    handleReload() {
+        window.location.href = this.state.url;
+    }
+
     render() {
         const search = parse(this.props.location.search);
 
@@ -12,7 +30,30 @@ export default class IFramePage extends React.Component {
             redirect = search.src;
         }
 
+        let navGroup;
+
+        if (window.top !== window) {
+            navGroup = [
+                e("div", { className: "form-group row" }, [
+                    e("input", {
+                        id: "url_input",
+                        type: "text",
+                        className: "form-control",
+                        onChange: this.handleUrlInput.bind(this)
+                    }),
+                    e("button", {
+                        id: "submit",
+                        className: "btn btn-primary",
+                        onClick: this.handleReload.bind(this)
+                    }, [
+                        "Navigate"
+                    ]),
+                ]),
+            ];
+        }
+
         return e("div", { id: "iframe" }, [
+            navGroup,
             e("iframe", {
                 name: 'nested',
                 style: {

e2e/tests/dynamic/doc/click/click.fql

@@ -0,0 +1,8 @@
+LET url = @lab.cdn.dynamic + "/#/events"
+LET page = DOCUMENT(url, { driver: "cdp" })
+
+CLICK(page, "#wait-class-random-btn")
+
+WAIT_CLASS(page, "#wait-class-random-content", "alert-success")
+
+RETURN ""

e2e/tests/dynamic/doc/click/click_by_selector.fql

@@ -0,0 +1,8 @@
+LET url = @lab.cdn.dynamic + "/#/events"
+LET page = DOCUMENT(url, true)
+
+CLICK(page, "#wait-class-random button")
+
+WAIT_CLASS(page, "#wait-class-random-content", "alert-success", 10000)
+
+RETURN ""

e2e/tests/dynamic/doc/click/click_by_selector_with_count.fql

@@ -0,0 +1,16 @@
+LET url = @lab.cdn.dynamic + "/#/forms"
+LET page = DOCUMENT(url, true)
+
+WAIT_ELEMENT(page, "form")
+
+LET input = ELEMENT(page, "#text_input")
+
+INPUT(input, "Foo")
+
+CLICK(page, "#text_input", 2)
+
+INPUT(input, "Bar")
+
+WAIT(100)
+
+RETURN T::EQ(input.value, "Bar")

e2e/tests/dynamic/doc/click/click_with_count.fql

@@ -0,0 +1,16 @@
+LET url = @lab.cdn.dynamic + "/#/forms"
+LET page = DOCUMENT(url, true)
+
+WAIT_ELEMENT(page, "form")
+
+LET input = ELEMENT(page, "#text_input")
+
+INPUT(input, "Foo")
+
+CLICK(page, "#text_input", 2)
+
+INPUT(input, "Bar")
+
+WAIT(100)
+
+RETURN T::EQ(input.value, "Bar")

e2e/tests/dynamic/doc/iframes/lookup.fql

@@ -1,5 +1,5 @@
 LET url = @lab.cdn.dynamic + "?redirect=/iframe"
-LET page = DOCUMENT(url, { driver: 'cdp' })
+LET page = DOCUMENT(url, { driver: 'cdp', timeout: 5000 })
 
 LET frames = (
     FOR frame IN page.frames

e2e/tests/dynamic/doc/wait/frame_navigation.fql

@@ -0,0 +1,14 @@
+LET url = @lab.cdn.dynamic + "?redirect=/iframe&src=/iframe"
+LET page = DOCUMENT(url, { driver: 'cdp' })
+LET original = FIRST(FRAMES(page, "name", "nested"))
+
+INPUT(original, "#url_input", "https://getbootstrap.com/")
+CLICK(original, "#submit")
+
+WAIT_NAVIGATION(page, {
+    frame: original
+})
+
+LET current = FIRST(FRAMES(page, "name", "nested"))
+
+RETURN T::EQ(current.URL, "https://getbootstrap.com/")

examples/navigate_back.fql

@@ -2,6 +2,6 @@ LET origin = "https://github.com/"
 LET doc = DOCUMENT(origin, { driver: "cdp" })
 
 NAVIGATE(doc, "https://github.com/features", 10000)
-NAVIGATE_BACK(doc, 10000)
+NAVIGATE_BACK(doc)
 
 RETURN doc.url == origin

examples/navigate_forward.fql

@@ -2,8 +2,8 @@ LET origin = "https://github.com/"
 LET target = "https://github.com/features"
 LET doc = DOCUMENT(origin, { driver: "cdp" })
 
-NAVIGATE(doc, target, 10000)
-NAVIGATE_BACK(doc, 10000)
-NAVIGATE_FORWARD(doc, 10000)
+NAVIGATE(doc, target)
+NAVIGATE_BACK(doc)
+NAVIGATE_FORWARD(doc)
 
 RETURN doc.url == target

examples/navigate_forward_2.fql

@@ -1,9 +1,9 @@
 LET doc = DOCUMENT("https://github.com/", { driver: "cdp" })
 
-NAVIGATE(doc, "https://github.com/features", 10000)
-NAVIGATE(doc, "https://github.com/enterprise", 10000)
-NAVIGATE(doc, "https://github.com/marketplace", 10000)
-NAVIGATE_BACK(doc, 3, 10000)
-NAVIGATE_FORWARD(doc, 2, 10000)
+NAVIGATE(doc, "https://github.com/features")
+NAVIGATE(doc, "https://github.com/enterprise")
+NAVIGATE(doc, "https://github.com/marketplace")
+NAVIGATE_BACK(doc, 3)
+NAVIGATE_FORWARD(doc, 2)
 
 RETURN doc.url == "https://github.com/enterprise"

pkg/drivers/cdp/dom/element.go

@@ -416,7 +416,8 @@ func (el *HTMLElement) GetAttribute(ctx context.Context, name values.String) (co
 		return values.None, err
 	}
 
-	result := values.EmptyString
+	var result core.Value
+	result = values.None
 	targetName := strings.ToLower(name.String())
 
 	traverseAttrs(repl.Attributes, func(name, value string) bool {

pkg/drivers/cdp/dom/frame.go

@@ -0,0 +1,129 @@
+package dom
+
+import (
+	"github.com/mafredri/cdp/protocol/page"
+	"sync"
+)
+
+type (
+	Frame struct {
+		tree  page.FrameTree
+		node  *HTMLDocument
+		ready bool
+	}
+
+	AtomicFrameID struct {
+		mu    sync.Mutex
+		value page.FrameID
+	}
+
+	AtomicFrameCollection struct {
+		mu    sync.Mutex
+		value map[page.FrameID]Frame
+	}
+)
+
+func NewAtomicFrameID() *AtomicFrameID {
+	return &AtomicFrameID{}
+}
+
+func (id *AtomicFrameID) Get() page.FrameID {
+	id.mu.Lock()
+	defer id.mu.Unlock()
+
+	return id.value
+}
+
+func (id *AtomicFrameID) Set(value page.FrameID) {
+	id.mu.Lock()
+	defer id.mu.Unlock()
+
+	id.value = value
+}
+
+func (id *AtomicFrameID) Reset() {
+	id.mu.Lock()
+	defer id.mu.Unlock()
+
+	id.value = ""
+}
+
+func (id *AtomicFrameID) IsEmpty() bool {
+	id.mu.Lock()
+	defer id.mu.Unlock()
+
+	return id.value == ""
+}
+
+func NewAtomicFrameCollection() *AtomicFrameCollection {
+	return &AtomicFrameCollection{
+		value: make(map[page.FrameID]Frame),
+	}
+}
+
+func (fc *AtomicFrameCollection) Length() int {
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+
+	return len(fc.value)
+}
+
+func (fc *AtomicFrameCollection) ForEach(predicate func(value Frame, key page.FrameID) bool) {
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+
+	for k, v := range fc.value {
+		if predicate(v, k) == false {
+			break
+		}
+	}
+}
+
+func (fc *AtomicFrameCollection) Has(key page.FrameID) bool {
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+
+	_, ok := fc.value[key]
+
+	return ok
+}
+
+func (fc *AtomicFrameCollection) Get(key page.FrameID) (Frame, bool) {
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+
+	found, ok := fc.value[key]
+
+	if ok {
+		return found, ok
+	}
+
+	return Frame{}, false
+}
+
+func (fc *AtomicFrameCollection) Set(key page.FrameID, value Frame) {
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+
+	fc.value[key] = value
+}
+
+func (fc *AtomicFrameCollection) Remove(key page.FrameID) {
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+
+	delete(fc.value, key)
+}
+
+func (fc *AtomicFrameCollection) ToSlice() []Frame {
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+
+	slice := make([]Frame, 0, len(fc.value))
+
+	for _, v := range fc.value {
+		slice = append(slice, v)
+	}
+
+	return slice
+}

pkg/drivers/cdp/dom/manager.go

@@ -2,21 +2,18 @@ package dom
 
 import (
 	"context"
+	"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
 	"github.com/MontFerret/ferret/pkg/drivers/cdp/input"
 	"github.com/MontFerret/ferret/pkg/drivers/common"
 	"github.com/MontFerret/ferret/pkg/runtime/core"
 	"github.com/MontFerret/ferret/pkg/runtime/values"
+	"github.com/mafredri/cdp"
+	"github.com/mafredri/cdp/protocol/dom"
 	"github.com/mafredri/cdp/protocol/page"
+	"github.com/mafredri/cdp/rpcc"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"io"
-	"sync"
-
-	"github.com/mafredri/cdp"
-	"github.com/mafredri/cdp/protocol/dom"
-	"github.com/mafredri/cdp/rpcc"
-
-	"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
 )
 
 var (
@@ -38,21 +35,14 @@ type (
 
 	ChildNodeRemovedListener func(ctx context.Context, nodeID, previousNodeID dom.NodeID)
 
-	Frame struct {
-		tree  page.FrameTree
-		node  *HTMLDocument
-		ready bool
-	}
-
 	Manager struct {
-		mu        sync.Mutex
 		logger    *zerolog.Logger
 		client    *cdp.Client
 		events    *events.Loop
 		mouse     *input.Mouse
 		keyboard  *input.Keyboard
-		mainFrame page.FrameID
-		frames    map[page.FrameID]Frame
+		mainFrame *AtomicFrameID
+		frames    *AtomicFrameCollection
 		cancel    context.CancelFunc
 	}
 )
@@ -154,39 +144,33 @@ func New(
 	manager.events = eventLoop
 	manager.mouse = mouse
 	manager.keyboard = keyboard
-	manager.frames = make(map[page.FrameID]Frame)
+	manager.mainFrame = NewAtomicFrameID()
+	manager.frames = NewAtomicFrameCollection()
 	manager.cancel = cancel
 
-	eventLoop.Start()
+	eventLoop.Run(ctx)
 
 	return manager, nil
 }
 
 func (m *Manager) Close() error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	errs := make([]error, 0, len(m.frames)+1)
+	errs := make([]error, 0, m.frames.Length()+1)
 
 	if m.cancel != nil {
 		m.cancel()
 		m.cancel = nil
-
-		err := m.events.Stop().Close()
-
-		if err != nil {
-			errs = append(errs, err)
-		}
 	}
 
-	for _, f := range m.frames {
+	m.frames.ForEach(func(f Frame, key page.FrameID) bool {
 		// if initialized
 		if f.node != nil {
 			if err := f.node.Close(); err != nil {
 				errs = append(errs, err)
 			}
 		}
-	}
+
+		return true
+	})
 
 	if len(errs) > 0 {
 		return core.Errors(errs...)
@@ -196,14 +180,13 @@ func (m *Manager) Close() error {
 }
 
 func (m *Manager) GetMainFrame() *HTMLDocument {
-	m.mu.Lock()
-	defer m.mu.Unlock()
+	mainFrameID := m.mainFrame.Get()
 
-	if m.mainFrame == "" {
+	if mainFrameID == "" {
 		return nil
 	}
 
-	mainFrame, exists := m.frames[m.mainFrame]
+	mainFrame, exists := m.frames.Get(mainFrameID)
 
 	if exists {
 		return mainFrame.node
@@ -213,46 +196,33 @@ func (m *Manager) GetMainFrame() *HTMLDocument {
 }
 
 func (m *Manager) SetMainFrame(doc *HTMLDocument) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
+	mainFrameID := m.mainFrame.Get()
 
-	if m.mainFrame != "" {
-		if err := m.removeFrameRecursivelyInternal(m.mainFrame); err != nil {
+	if mainFrameID != "" {
+		if err := m.removeFrameRecursivelyInternal(mainFrameID); err != nil {
 			m.logger.Error().Err(err).Msg("failed to close previous main frame")
 		}
 	}
 
-	m.mainFrame = doc.frameTree.Frame.ID
+	m.mainFrame.Set(doc.frameTree.Frame.ID)
 
 	m.addPreloadedFrame(doc)
 }
 
 func (m *Manager) AddFrame(frame page.FrameTree) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
 	m.addFrameInternal(frame)
 }
 
 func (m *Manager) RemoveFrame(frameID page.FrameID) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
 	return m.removeFrameInternal(frameID)
 }
 
 func (m *Manager) RemoveFrameRecursively(frameID page.FrameID) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
 	return m.removeFrameRecursivelyInternal(frameID)
 }
 
 func (m *Manager) RemoveFramesByParentID(parentFrameID page.FrameID) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	frame, found := m.frames[parentFrameID]
+	frame, found := m.frames.Get(parentFrameID)
 
 	if !found {
 		return errors.New("frame not found")
@@ -268,17 +238,11 @@ func (m *Manager) RemoveFramesByParentID(parentFrameID page.FrameID) error {
 }
 
 func (m *Manager) GetFrameNode(ctx context.Context, frameID page.FrameID) (*HTMLDocument, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
 	return m.getFrameInternal(ctx, frameID)
 }
 
 func (m *Manager) GetFrameTree(_ context.Context, frameID page.FrameID) (page.FrameTree, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	frame, found := m.frames[frameID]
+	frame, found := m.frames.Get(frameID)
 
 	if !found {
 		return page.FrameTree{}, core.ErrNotFound
@@ -288,12 +252,9 @@ func (m *Manager) GetFrameTree(_ context.Context, frameID page.FrameID) (page.Fr
 }
 
 func (m *Manager) GetFrameNodes(ctx context.Context) (*values.Array, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
+	arr := values.NewArray(m.frames.Length())
 
-	arr := values.NewArray(len(m.frames))
-
-	for _, f := range m.frames {
+	for _, f := range m.frames.ToSlice() {
 		doc, err := m.getFrameInternal(ctx, f.tree.Frame.ID)
 
 		if err != nil {
@@ -346,29 +307,11 @@ func (m *Manager) RemoveChildNodeRemovedListener(listenerID events.ListenerID) {
 	m.events.RemoveListener(eventChildNodeRemoved, listenerID)
 }
 
-func (m *Manager) WaitForDOMReady(ctx context.Context) error {
-	onContentReady, err := m.client.Page.DOMContentEventFired(ctx)
-
-	if err != nil {
-		return err
-	}
-
-	defer func() {
-		if err := onContentReady.Close(); err != nil {
-			m.logger.Error().Err(err).Msg("failed to close DOM content ready stream event")
-		}
-	}()
-
-	_, err = onContentReady.Recv()
-
-	return err
-}
-
 func (m *Manager) addFrameInternal(frame page.FrameTree) {
-	m.frames[frame.Frame.ID] = Frame{
+	m.frames.Set(frame.Frame.ID, Frame{
 		tree: frame,
 		node: nil,
-	}
+	})
 
 	for _, child := range frame.ChildFrames {
 		m.addFrameInternal(child)
@@ -376,10 +319,10 @@ func (m *Manager) addFrameInternal(frame page.FrameTree) {
 }
 
 func (m *Manager) addPreloadedFrame(doc *HTMLDocument) {
-	m.frames[doc.frameTree.Frame.ID] = Frame{
+	m.frames.Set(doc.frameTree.Frame.ID, Frame{
 		tree: doc.frameTree,
 		node: doc,
-	}
+	})
 
 	for _, child := range doc.frameTree.ChildFrames {
 		m.addFrameInternal(child)
@@ -387,7 +330,7 @@ func (m *Manager) addPreloadedFrame(doc *HTMLDocument) {
 }
 
 func (m *Manager) getFrameInternal(ctx context.Context, frameID page.FrameID) (*HTMLDocument, error) {
-	frame, found := m.frames[frameID]
+	frame, found := m.frames.Get(frameID)
 
 	if !found {
 		return nil, core.ErrNotFound
@@ -402,7 +345,7 @@ func (m *Manager) getFrameInternal(ctx context.Context, frameID page.FrameID) (*
 	node, execID, err := resolveFrame(ctx, m.client, frameID)
 
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to resolve frame node")
+		return nil, errors.Wrapf(err, "failed to resolve frame node: %s", frameID)
 	}
 
 	doc, err := LoadHTMLDocument(
@@ -427,16 +370,18 @@ func (m *Manager) getFrameInternal(ctx context.Context, frameID page.FrameID) (*
 }
 
 func (m *Manager) removeFrameInternal(frameID page.FrameID) error {
-	current, exists := m.frames[frameID]
+	current, exists := m.frames.Get(frameID)
 
 	if !exists {
 		return core.Error(core.ErrNotFound, "frame")
 	}
 
-	delete(m.frames, frameID)
+	m.frames.Remove(frameID)
 
-	if frameID == m.mainFrame {
-		m.mainFrame = ""
+	mainFrameID := m.mainFrame.Get()
+
+	if frameID == mainFrameID {
+		m.mainFrame.Reset()
 	}
 
 	if current.node == nil {
@@ -447,7 +392,7 @@ func (m *Manager) removeFrameInternal(frameID page.FrameID) error {
 }
 
 func (m *Manager) removeFrameRecursivelyInternal(frameID page.FrameID) error {
-	parent, exists := m.frames[frameID]
+	parent, exists := m.frames.Get(frameID)
 
 	if !exists {
 		return core.Error(core.ErrNotFound, "frame")

pkg/drivers/cdp/eval/context.go

@@ -24,6 +24,16 @@ type ExecutionContext struct {
 	contextID runtime.ExecutionContextID
 }
 
+func NewExecutionContextFrom(ctx context.Context, client *cdp.Client, frame page.Frame) (*ExecutionContext, error) {
+	world, err := client.Page.CreateIsolatedWorld(ctx, page.NewCreateIsolatedWorldArgs(frame.ID))
+
+	if err != nil {
+		return nil, err
+	}
+
+	return NewExecutionContext(client, frame, world.ExecutionContextID), nil
+}
+
 func NewExecutionContext(client *cdp.Client, frame page.Frame, contextID runtime.ExecutionContextID) *ExecutionContext {
 	ec := new(ExecutionContext)
 	ec.client = client

pkg/drivers/cdp/events/loop.go

@@ -3,12 +3,9 @@ package events
 import (
 	"context"
 	"math/rand"
-	"sync"
 )
 
 type Loop struct {
-	mu        sync.Mutex
-	cancel    context.CancelFunc
 	sources   *SourceCollection
 	listeners *ListenerCollection
 }
@@ -21,47 +18,8 @@ func NewLoop() *Loop {
 	return loop
 }
 
-func (loop *Loop) Start() *Loop {
-	loop.mu.Lock()
-	defer loop.mu.Unlock()
-
-	if loop.cancel != nil {
-		return loop
-	}
-
-	loopCtx, cancel := context.WithCancel(context.Background())
-
-	loop.cancel = cancel
-
-	go loop.run(loopCtx)
-
-	return loop
-}
-
-func (loop *Loop) Stop() *Loop {
-	loop.mu.Lock()
-	defer loop.mu.Unlock()
-
-	if loop.cancel == nil {
-		return loop
-	}
-
-	loop.cancel()
-	loop.cancel = nil
-
-	return loop
-}
-
-func (loop *Loop) Close() error {
-	loop.mu.Lock()
-	defer loop.mu.Unlock()
-
-	if loop.cancel != nil {
-		loop.cancel()
-		loop.cancel = nil
-	}
-
-	return loop.sources.Close()
+func (loop *Loop) Run(ctx context.Context) {
+	go loop.run(ctx)
 }
 
 func (loop *Loop) AddSource(source Source) {
@@ -125,7 +83,6 @@ func (loop *Loop) run(ctx context.Context) {
 			source = noop
 		}
 
-		// commands have higher priority
 		select {
 		case <-ctx.Done():
 			return

pkg/drivers/cdp/events/loop_test.go

@@ -180,8 +180,10 @@ func TestLoop(t *testing.T) {
 
 			loop.AddSource(src)
 
-			loop.Start()
-			defer loop.Stop()
+			ctx, cancel := context.WithCancel(context.Background())
+
+			loop.Run(ctx)
+			defer cancel()
 
 			onLoad.EmitDefault()
 
@@ -219,8 +221,10 @@ func TestLoop(t *testing.T) {
 					counter.Increase()
 				}))
 
-				loop.Start()
-				defer loop.Stop()
+				ctx, cancel := context.WithCancel(context.Background())
+
+				loop.Run(ctx)
+				defer cancel()
 
 				onLoad.EmitDefault()
 
@@ -252,8 +256,10 @@ func TestLoop(t *testing.T) {
 				counter.Increase()
 			}))
 
-			loop.Start()
-			defer loop.Stop()
+			ctx, cancel := context.WithCancel(context.Background())
+
+			loop.Run(ctx)
+			defer cancel()
 
 			onLoad := &TestLoadEventFiredClient{NewTestEventStream()}
 
@@ -282,8 +288,10 @@ func TestLoop(t *testing.T) {
 			loop := events.NewLoop()
 			counter := NewCounter()
 
-			loop.Start()
-			defer loop.Stop()
+			ctx, cancel := context.WithCancel(context.Background())
+
+			loop.Run(ctx)
+			defer cancel()
 
 			loop.AddListener(TestEvent, events.Always(func(ctx context.Context, message interface{}) {
 				counter.Increase()
@@ -342,8 +350,10 @@ func TestLoop(t *testing.T) {
 			return onLoad.Recv()
 		}))
 
-		loop.Start()
-		defer loop.Stop()
+		ctx, cancel := context.WithCancel(context.Background())
+
+		loop.Run(ctx)
+		defer cancel()
 
 		time.Sleep(time.Duration(100) * time.Millisecond)
 
@@ -365,8 +375,10 @@ func BenchmarkLoop_AddListenerSync(b *testing.B) {
 
 func BenchmarkLoop_AddListenerAsync(b *testing.B) {
 	loop := events.NewLoop()
-	loop.Start()
-	defer loop.Stop()
+	ctx, cancel := context.WithCancel(context.Background())
+
+	loop.Run(ctx)
+	defer cancel()
 
 	for n := 0; n < b.N; n++ {
 		loop.AddListener(TestEvent, events.Always(func(ctx context.Context, message interface{}) {}))
@@ -375,8 +387,10 @@ func BenchmarkLoop_AddListenerAsync(b *testing.B) {
 
 func BenchmarkLoop_AddListenerAsync2(b *testing.B) {
 	loop := events.NewLoop()
-	loop.Start()
-	defer loop.Stop()
+	ctx, cancel := context.WithCancel(context.Background())
+
+	loop.Run(ctx)
+	defer cancel()
 
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
@@ -417,8 +431,10 @@ func BenchmarkLoop_Start(b *testing.B) {
 		return onLoad.Recv()
 	}))
 
-	loop.Start()
-	defer loop.Stop()
+	ctx, cancel := context.WithCancel(context.Background())
+
+	loop.Run(ctx)
+	defer cancel()
 
 	for n := 0; n < b.N; n++ {
 		onLoad.Emit(&page.LoadEventFiredReply{})

pkg/drivers/cdp/events/sources.go

@@ -1,6 +1,7 @@
 package events
 
 import (
+	"errors"
 	"github.com/MontFerret/ferret/pkg/runtime/core"
 	"sync"
 )
@@ -21,6 +22,10 @@ func (sc *SourceCollection) Close() error {
 	sc.mu.Lock()
 	defer sc.mu.Unlock()
 
+	if sc.values == nil {
+		return errors.New("sources are already closed")
+	}
+
 	errs := make([]error, 0, len(sc.values))
 
 	for _, e := range sc.values {
@@ -29,6 +34,8 @@ func (sc *SourceCollection) Close() error {
 		}
 	}
 
+	sc.values = nil
+
 	if len(errs) > 0 {
 		return core.Errors(errs...)
 	}

pkg/drivers/cdp/network/manager.go

@@ -3,16 +3,17 @@ package network
 import (
 	"context"
 	"encoding/json"
-	"io"
-	"regexp"
-	"sync"
-
+	"github.com/MontFerret/ferret/pkg/drivers/cdp/eval"
+	"github.com/MontFerret/ferret/pkg/drivers/cdp/templates"
 	"github.com/mafredri/cdp"
 	"github.com/mafredri/cdp/protocol/network"
 	"github.com/mafredri/cdp/protocol/page"
 	"github.com/mafredri/cdp/rpcc"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+	"io"
+	"regexp"
+	"sync"
 
 	"github.com/MontFerret/ferret/pkg/drivers"
 	"github.com/MontFerret/ferret/pkg/drivers/cdp/events"
@@ -84,7 +85,7 @@ func New(
 
 	m.responseListenerID = m.eventLoop.AddListener(responseReceived, m.onResponse)
 
-	m.eventLoop.Start()
+	m.eventLoop.Run(ctx)
 
 	return m, nil
 }
@@ -96,8 +97,6 @@ func (m *Manager) Close() error {
 	if m.cancel != nil {
 		m.cancel()
 		m.cancel = nil
-
-		return m.eventLoop.Stop().Close()
 	}
 
 	return nil
@@ -326,10 +325,6 @@ func (m *Manager) WaitForNavigation(ctx context.Context, pattern *regexp.Regexp)
 func (m *Manager) WaitForFrameNavigation(ctx context.Context, frameID page.FrameID, urlPattern *regexp.Regexp) error {
 	onEvent := make(chan struct{})
 
-	defer func() {
-		close(onEvent)
-	}()
-
 	m.eventLoop.AddListener(eventFrameLoad, func(_ context.Context, message interface{}) bool {
 		repl := message.(*page.FrameNavigatedReply)
 
@@ -348,7 +343,26 @@ func (m *Manager) WaitForFrameNavigation(ctx context.Context, frameID page.Frame
 
 		if matched {
 			if ctx.Err() == nil {
+				ec, err := eval.NewExecutionContextFrom(ctx, m.client, repl.Frame)
+
+				if err != nil {
+					close(onEvent)
+					return false
+				}
+
+				_, err = events.NewEvalWaitTask(
+					ec,
+					templates.DOMReady(),
+					events.DefaultPolling,
+				).Run(ctx)
+
+				if err != nil {
+					close(onEvent)
+					return false
+				}
+
 				onEvent <- struct{}{}
+				close(onEvent)
 			}
 		}
 

pkg/drivers/cdp/page.go

@@ -288,7 +288,17 @@ func (p *HTMLPage) Close() error {
 			Msg("failed to close browser page")
 	}
 
-	return p.conn.Close()
+	err = p.conn.Close()
+
+	if err != nil {
+		p.logger.Warn().
+			Timestamp().
+			Str("url", doc.GetURL().String()).
+			Err(err).
+			Msg("failed to close connection")
+	}
+
+	return err
 }
 
 func (p *HTMLPage) IsClosed() values.Boolean {
@@ -511,16 +521,10 @@ func (p *HTMLPage) NavigateForward(ctx context.Context, skip values.Int) (values
 }
 
 func (p *HTMLPage) WaitForNavigation(ctx context.Context, targetURL values.String) error {
-	var pattern *regexp.Regexp
+	pattern, err := p.urlToRegexp(targetURL)
 
-	if targetURL != "" {
-		r, err := regexp.Compile(targetURL.String())
-
-		if err != nil {
-			return errors.Wrap(err, "invalid URL pattern")
-		}
-
-		pattern = r
+	if err != nil {
+		return err
 	}
 
 	if err := p.network.WaitForNavigation(ctx, pattern); err != nil {
@@ -530,11 +534,56 @@ func (p *HTMLPage) WaitForNavigation(ctx context.Context, targetURL values.Strin
 	return p.reloadMainFrame(ctx)
 }
 
-func (p *HTMLPage) reloadMainFrame(ctx context.Context) error {
-	if err := p.dom.WaitForDOMReady(ctx); err != nil {
+func (p *HTMLPage) WaitForFrameNavigation(ctx context.Context, frame drivers.HTMLDocument, targetURL values.String) error {
+	current := p.dom.GetMainFrame()
+	doc, ok := frame.(*dom.HTMLDocument)
+
+	if !ok {
+		return errors.New("invalid frame type")
+	}
+
+	pattern, err := p.urlToRegexp(targetURL)
+
+	if err != nil {
 		return err
 	}
 
+	frameID := doc.Frame().Frame.ID
+	isMain := current.Frame().Frame.ID == frameID
+
+	// if it's the current document
+	if isMain {
+		err = p.network.WaitForNavigation(ctx, pattern)
+	} else {
+		err = p.network.WaitForFrameNavigation(ctx, frameID, pattern)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	//if isMain {
+	//
+	//}
+
+	return p.reloadMainFrame(ctx)
+}
+
+func (p *HTMLPage) urlToRegexp(targetURL values.String) (*regexp.Regexp, error) {
+	if targetURL == "" {
+		return nil, nil
+	}
+
+	r, err := regexp.Compile(targetURL.String())
+
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid URL pattern")
+	}
+
+	return r, nil
+}
+
+func (p *HTMLPage) reloadMainFrame(ctx context.Context) error {
 	prev := p.dom.GetMainFrame()
 
 	next, err := dom.LoadRootHTMLDocument(

pkg/drivers/cdp/templates/dom_ready.go

@@ -0,0 +1,13 @@
+package templates
+
+const domReadyTemplate = `
+if (document.readyState === 'complete') {
+	return true;
+}
+
+return null;
+`
+
+func DOMReady() string {
+	return domReadyTemplate
+}

pkg/drivers/http/page.go

@@ -202,6 +202,10 @@ func (p *HTMLPage) WaitForNavigation(_ context.Context, _ values.String) error {
 	return core.ErrNotSupported
 }
 
+func (p *HTMLPage) WaitForFrameNavigation(_ context.Context, _ drivers.HTMLDocument, _ values.String) error {
+	return core.ErrNotSupported
+}
+
 func (p *HTMLPage) Navigate(_ context.Context, _ values.String) error {
 	return core.ErrNotSupported
 }

pkg/drivers/value.go

@@ -206,6 +206,8 @@ type (
 
 		WaitForNavigation(ctx context.Context, targetURL values.String) error
 
+		WaitForFrameNavigation(ctx context.Context, frame HTMLDocument, targetURL values.String) error
+
 		Navigate(ctx context.Context, url values.String) error
 
 		NavigateBack(ctx context.Context, skip values.Int) (values.Boolean, error)

pkg/runtime/values/array.go

@@ -136,7 +136,19 @@ func (t *Array) ForEach(predicate ArrayPredicate) {
 	}
 }
 
-func (t *Array) Find(predicate ArrayPredicate) (core.Value, Boolean) {
+func (t *Array) Find(predicate ArrayPredicate) (*Array, Boolean) {
+	result := NewArray(len(t.items))
+
+	for idx, val := range t.items {
+		if predicate(val, idx) {
+			result.Push(val)
+		}
+	}
+
+	return result, result.Length() > 0
+}
+
+func (t *Array) FindOne(predicate ArrayPredicate) (core.Value, Boolean) {
 	for idx, val := range t.items {
 		if predicate(val, idx) {
 			return val, True

pkg/stdlib/collections/include.go

@@ -29,7 +29,7 @@ func Includes(ctx context.Context, args ...core.Value) (core.Value, error) {
 
 		break
 	case *values.Array:
-		_, result = v.Find(func(value core.Value, _ int) bool {
+		_, result = v.FindOne(func(value core.Value, _ int) bool {
 			return needle.Compare(value) == 0
 		})
 

pkg/stdlib/html/find_frames.go

@@ -0,0 +1,58 @@
+package html
+
+import (
+	"context"
+	"github.com/MontFerret/ferret/pkg/drivers"
+	"github.com/MontFerret/ferret/pkg/runtime/core"
+	"github.com/MontFerret/ferret/pkg/runtime/values"
+)
+
+// FRAMES finds HTML frames by a given property selector.
+// Returns an empty array if frames not found.
+// @param page (HTMLPage) - HTML page.
+// @param prop (String) - Property selector.
+// @param value (Any) - Property value.
+// @returns (Array) - Returns an array of found HTML frames.
+func Frames(ctx context.Context, args ...core.Value) (core.Value, error) {
+	err := core.ValidateArgs(args, 3, 3)
+
+	if err != nil {
+		return values.None, err
+	}
+
+	page, err := drivers.ToPage(args[0])
+
+	if err != nil {
+		return values.None, err
+	}
+
+	frames, err := page.GetFrames(ctx)
+
+	if err != nil {
+		return values.None, err
+	}
+
+	propName := values.ToString(args[1])
+	propValue := args[2]
+
+	result, _ := frames.Find(func(value core.Value, idx int) bool {
+		doc, e := drivers.ToDocument(value)
+
+		if e != nil {
+			err = e
+			return false
+		}
+
+		currentPropValue, e := doc.GetIn(ctx, []core.Value{propName})
+
+		if e != nil {
+			err = e
+
+			return false
+		}
+
+		return currentPropValue.Compare(propValue) == 0
+	})
+
+	return result, err
+}

pkg/stdlib/html/lib.go

@@ -28,6 +28,7 @@ func RegisterLib(ns core.Namespace) error {
 			"ELEMENT_EXISTS":    ElementExists,
 			"ELEMENTS":          Elements,
 			"ELEMENTS_COUNT":    ElementsCount,
+			"FRAMES":            Frames,
 			"FOCUS":             Focus,
 			"HOVER":             Hover,
 			"INNER_HTML":        GetInnerHTML,

pkg/stdlib/html/wait_navigation.go

@@ -13,6 +13,7 @@ import (
 type WaitNavigationParams struct {
 	TargetURL values.String
 	Timeout   values.Int
+	Frame     drivers.HTMLDocument
 }
 
 // WAIT_NAVIGATION waits for a given page to navigate to a new url.
@@ -49,7 +50,11 @@ func WaitNavigation(ctx context.Context, args ...core.Value) (core.Value, error)
 	ctx, fn := waitTimeout(ctx, params.Timeout)
 	defer fn()
 
-	return values.None, doc.WaitForNavigation(ctx, params.TargetURL)
+	if params.Frame == nil {
+		return values.None, doc.WaitForNavigation(ctx, params.TargetURL)
+	}
+
+	return values.None, doc.WaitForFrameNavigation(ctx, params.Frame, params.TargetURL)
 }
 
 func parseWaitNavigationParams(arg core.Value) (WaitNavigationParams, error) {
@@ -62,7 +67,6 @@ func parseWaitNavigationParams(arg core.Value) (WaitNavigationParams, error) {
 
 	if arg.Type() == types.Int {
 		params.Timeout = arg.(values.Int)
-
 	} else {
 		obj := arg.(*values.Object)
 
@@ -85,6 +89,16 @@ func parseWaitNavigationParams(arg core.Value) (WaitNavigationParams, error) {
 
 			params.TargetURL = v.(values.String)
 		}
+
+		if v, exists := obj.Get("frame"); exists {
+			doc, err := drivers.ToDocument(v)
+
+			if err != nil {
+				return params, errors.Wrap(err, "navigation parameters: frame")
+			}
+
+			params.Frame = doc
+		}
 	}
 
 	return params, nil