Added optional timeout to NAVIGATE

examples/crawler.fql

@@ -6,7 +6,8 @@ LET links = (
         RETURN article.attributes.href
 )
 FOR link IN links
-    NAVIGATE(doc, link)
+    // The Verge has pretty heavy pages, so let's increase the navigation wait time
+    NAVIGATE(doc, link, 10000)
     WAIT_ELEMENT(doc, '.c-entry-content', 5000)
     LET texter = ELEMENT(doc, '.c-entry-content')
     RETURN texter.innerText

pkg/stdlib/html/driver/dynamic/document.go

@@ -150,32 +150,8 @@ func NewHTMLDocument(
 		doc.url = values.NewString(*root.BaseURL)
 	}
 
-	broker.AddEventListener("load", func(_ interface{}) {
+	broker.AddEventListener("load", doc.onLoad)
-		doc.Lock()
+	broker.AddEventListener("error", doc.onError)
-		defer doc.Unlock()
-
-		updated, innerHTML, err := getRootElement(client)
-
-		if err != nil {
-			doc.logger.Error().
-				Timestamp().
-				Err(err).
-				Msg("failed to get root node after page load")
-
-			return
-		}
-
-		// close the prev element
-		doc.element.Close()
-
-		// create a new root element wrapper
-		doc.element = NewHTMLElement(doc.logger, client, broker, updated.NodeID, updated, innerHTML)
-		doc.url = ""
-
-		if updated.BaseURL != nil {
-			doc.url = values.NewString(*updated.BaseURL)
-		}
-	})
 
 	return doc
 }
@@ -735,7 +711,7 @@ func (doc *HTMLDocument) WaitForNavigation(timeout values.Int) error {
 	}
 }
 
-func (doc *HTMLDocument) Navigate(url values.String) error {
+func (doc *HTMLDocument) Navigate(url values.String, timeout values.Int) error {
 	if url == "" {
 		url = BlankPageURL
 	}
@@ -751,5 +727,52 @@ func (doc *HTMLDocument) Navigate(url values.String) error {
 		return errors.New(*repl.ErrorText)
 	}
 
-	return doc.WaitForNavigation(5000)
+	return doc.WaitForNavigation(timeout)
+}
+
+func (doc *HTMLDocument) onLoad(_ interface{}) {
+	doc.Lock()
+	defer doc.Unlock()
+
+	updated, innerHTML, err := getRootElement(doc.client)
+
+	if err != nil {
+		doc.logger.Error().
+			Timestamp().
+			Err(err).
+			Msg("failed to get root node after page load")
+
+		return
+	}
+
+	// close the prev element
+	doc.element.Close()
+
+	// create a new root element wrapper
+	doc.element = NewHTMLElement(
+		doc.logger,
+		doc.client,
+		doc.events,
+		updated.NodeID,
+		updated,
+		innerHTML,
+	)
+	doc.url = ""
+
+	if updated.BaseURL != nil {
+		doc.url = values.NewString(*updated.BaseURL)
+	}
+}
+
+func (doc *HTMLDocument) onError(val interface{}) {
+	err, ok := val.(error)
+
+	if !ok {
+		return
+	}
+
+	doc.logger.Error().
+		Timestamp().
+		Err(err).
+		Msg("unexpected error")
 }

pkg/stdlib/html/navigate.go

@@ -13,9 +13,10 @@ import (
  * Which means there is no need in WAIT_NAVIGATION function.
  * @param doc (Document) - Target document.
  * @param url (String) - Target url to navigate.
+ * @param timeout (Int, optional) - Optional timeout. Default is 5000.
  */
 func Navigate(_ context.Context, args ...core.Value) (core.Value, error) {
-	err := core.ValidateArgs(args, 2, 2)
+	err := core.ValidateArgs(args, 2, 3)
 
 	if err != nil {
 		return values.None, err
@@ -39,5 +40,17 @@ func Navigate(_ context.Context, args ...core.Value) (core.Value, error) {
 		return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
 	}
 
-	return values.None, doc.Navigate(args[1].(values.String))
+	timeout := values.NewInt(defaultTimeout)
+
+	if len(args) > 2 {
+		err = core.ValidateType(args[2], core.IntType)
+
+		if err != nil {
+			return values.None, err
+		}
+
+		timeout = args[2].(values.Int)
+	}
+
+	return values.None, doc.Navigate(args[1].(values.String), timeout)
 }