go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2025-07-05 00:49:00 +02:00
Code Issues Releases Activity

Bugfix/e2e tests (#648)

Browse Source 
* Fixed logger level

* Fixed WAITFOR EVENT parser

* Added tracing to Network Manager

* Updated logging

* Swtitched to value type of logger

* Added tracing

* Increased websocket maximum buffer size

* Ignore unimportant error message

* Added support of new CDP API for layouts

* Switched to value type of logger

* Added log level

* Fixed early context cancellation

* Updated example of 'click' action

* Switched to val for elements lookup

* Fixed unit tests

* Refactored 'eval' module

* Fixed SetStyle eval expression

* Fixed style deletion

* Updated logic of setting multiple styles
This commit is contained in:
Tim Voronov
2021-09-02 11:09:48 -04:00
committed by GitHub
parent 25c97b86b8
commit e6dd5689b4
59 changed files with 2772 additions and 1507 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
pkg/stdlib/html/pagination.go
View File

@ -2,11 +2,12 @@ package html
import (
"context"
"github.com/MontFerret/ferret/pkg/drivers"
"github.com/MontFerret/ferret/pkg/runtime/core"
"github.com/MontFerret/ferret/pkg/runtime/logging"
"github.com/MontFerret/ferret/pkg/runtime/values"
"github.com/MontFerret/ferret/pkg/runtime/values/types"
"github.com/rs/zerolog"
)
// PAGINATION creates an iterator that goes through pages using CSS selector.
@ -14,7 +15,7 @@ import (
// That allows you to keep scraping logic inside FOR loop.
// @param {HTMLPage | HTMLDocument | HTMLElement} node - Target html node.
// @param {String} selector - CSS selector for a pagination on the page.
func Pagination(_ context.Context, args ...core.Value) (core.Value, error) {
func Pagination(ctx context.Context, args ...core.Value) (core.Value, error) {
err := core.ValidateArgs(args, 2, 2)
if err != nil {
@ -35,18 +36,25 @@ func Pagination(_ context.Context, args ...core.Value) (core.Value, error) {
selector := args[1].(values.String)
return &Paging{page, selector}, nil
logger := logging.
WithName(logging.FromContext(ctx).With(), "stdlib_html_pagination").
Str("selector", selector.String()).
Logger()
return &Paging{logger, page, selector}, nil
}
var PagingType = core.NewType("paging")
type (
Paging struct {
logger zerolog.Logger
page drivers.HTMLPage
selector values.String
}
PagingIterator struct {
logger zerolog.Logger
page drivers.HTMLPage
selector values.String
pos values.Int
@ -82,32 +90,46 @@ func (p *Paging) Copy() core.Value {
}
func (p *Paging) Iterate(_ context.Context) (core.Iterator, error) {
return &PagingIterator{p.page, p.selector, -1}, nil
return &PagingIterator{p.logger, p.page, p.selector, -1}, nil
}
func (i *PagingIterator) Next(ctx context.Context) (core.Value, core.Value, error) {
i.pos++
i.logger.Trace().Int("position", int(i.pos)).Msg("starting to advance iteration")
if i.pos == 0 {
i.logger.Trace().Msg("starting point of pagination. nothing to do. exit")
return values.ZeroInt, values.ZeroInt, nil
}
i.logger.Trace().Msg("checking if an element exists...")
exists, err := i.page.GetMainFrame().ExistsBySelector(ctx, i.selector)
if err != nil {
i.logger.Trace().Err(err).Msg("failed to check")
return values.None, values.None, err
}
if !exists {
i.logger.Trace().Bool("exists", bool(exists)).Msg("element does not exist. exit")
return values.None, values.None, core.ErrNoMoreData
}
i.logger.Trace().Bool("exists", bool(exists)).Msg("element exists. clicking...")
err = i.page.GetMainFrame().GetElement().ClickBySelector(ctx, i.selector, 1)
if err != nil {
i.logger.Trace().Err(err).Msg("failed to click. exit")
return values.None, values.None, err
}
i.logger.Trace().Msg("successfully clicked on element. iteration has succeeded")
// terminate
return i.pos, i.pos, nil
}