1
0
mirror of https://github.com/MontFerret/ferret.git synced 2024-12-14 11:23:02 +02:00
ferret/examples/crawler.fql
Tim Voronov 957490efec
Feature/#19 proxy (#72)
* #19 Some work on proxy

* Fixed linter issue
2018-10-07 21:23:36 -04:00

13 lines
519 B
Plaintext

LET doc = DOCUMENT('https://www.theverge.com/tech', true)
WAIT_ELEMENT(doc, '.c-compact-river__entry', 5000)
LET articles = ELEMENTS(doc, '.c-entry-box--compact__image-wrapper')
LET links = (
FOR article IN articles
RETURN article.attributes.href
)
FOR link IN links
// The Verge has pretty heavy pages, so let's increase the navigation wait time
NAVIGATE(doc, link, 20000)
WAIT_ELEMENT(doc, '.c-entry-content', 5000)
LET texter = ELEMENT(doc, '.c-entry-content')
RETURN texter.innerText