go/ferret
go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2025-02-13 13:58:32 +02:00
Code Issues Releases Activity

Added optional timeout to NAVIGATE

Browse Source
This commit is contained in:
Tim Voronov 2018-10-07 20:15:41 -04:00
parent fa5a10925b
commit fa466af940
3 changed files with 68 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
examples/crawler.fql
View File

@ -6,7 +6,8 @@ LET links = (
RETURN article.attributes.href
)
FOR link IN links
NAVIGATE(doc, link)
// The Verge has pretty heavy pages, so let's increase the navigation wait time
NAVIGATE(doc, link, 10000)
WAIT_ELEMENT(doc, '.c-entry-content', 5000)
LET texter = ELEMENT(doc, '.c-entry-content')
RETURN texter.innerText

79
pkg/stdlib/html/driver/dynamic/document.go
View File

@ -150,32 +150,8 @@ func NewHTMLDocument(
doc.url = values.NewString(*root.BaseURL)
}
broker.AddEventListener("load", func(_ interface{}) {
doc.Lock()
defer doc.Unlock()
updated, innerHTML, err := getRootElement(client)
if err != nil {
doc.logger.Error().
Timestamp().
Err(err).
Msg("failed to get root node after page load")
return
}
// close the prev element
doc.element.Close()
// create a new root element wrapper
doc.element = NewHTMLElement(doc.logger, client, broker, updated.NodeID, updated, innerHTML)
doc.url = ""
if updated.BaseURL != nil {
doc.url = values.NewString(*updated.BaseURL)
}
})
broker.AddEventListener("load", doc.onLoad)
broker.AddEventListener("error", doc.onError)
return doc
}
@ -735,7 +711,7 @@ func (doc *HTMLDocument) WaitForNavigation(timeout values.Int) error {
}
}
func (doc *HTMLDocument) Navigate(url values.String) error {
func (doc *HTMLDocument) Navigate(url values.String, timeout values.Int) error {
if url == "" {
url = BlankPageURL
}
@ -751,5 +727,52 @@ func (doc *HTMLDocument) Navigate(url values.String) error {
return errors.New(*repl.ErrorText)
}
return doc.WaitForNavigation(5000)
return doc.WaitForNavigation(timeout)
}
func (doc *HTMLDocument) onLoad(_ interface{}) {
doc.Lock()
defer doc.Unlock()
updated, innerHTML, err := getRootElement(doc.client)
if err != nil {
doc.logger.Error().
Timestamp().
Err(err).
Msg("failed to get root node after page load")
return
}
// close the prev element
doc.element.Close()
// create a new root element wrapper
doc.element = NewHTMLElement(
doc.logger,
doc.client,
doc.events,
updated.NodeID,
updated,
innerHTML,
)
doc.url = ""
if updated.BaseURL != nil {
doc.url = values.NewString(*updated.BaseURL)
}
}
func (doc *HTMLDocument) onError(val interface{}) {
err, ok := val.(error)
if !ok {
return
}
doc.logger.Error().
Timestamp().
Err(err).
Msg("unexpected error")
}

17
pkg/stdlib/html/navigate.go
View File

@ -13,9 +13,10 @@ import (
* Which means there is no need in WAIT_NAVIGATION function.
* @param doc (Document) - Target document.
* @param url (String) - Target url to navigate.
* @param timeout (Int, optional) - Optional timeout. Default is 5000.
*/
func Navigate(_ context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 2, 2)
err := core.ValidateArgs(args, 2, 3)
if err != nil {
return values.None, err
@ -39,5 +40,17 @@ func Navigate(_ context.Context, args ...core.Value) (core.Value, error) {
return values.False, core.Errors(core.ErrInvalidType, ErrNotDynamic)
}
return values.None, doc.Navigate(args[1].(values.String))
timeout := values.NewInt(defaultTimeout)
if len(args) > 2 {
err = core.ValidateType(args[2], core.IntType)
if err != nil {
return values.None, err
}
timeout = args[2].(values.Int)
}
return values.None, doc.Navigate(args[1].(values.String), timeout)
}