mirror of
https://github.com/MontFerret/ferret.git
synced 2024-12-16 11:37:36 +02:00
d7b923e4c3
* Refactored Virtual DOM structure * Added new E2E tests * Updated E2E Test Runner
156 lines
3.1 KiB
Go
156 lines
3.1 KiB
Go
package http
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"net/http"
|
|
"net/url"
|
|
|
|
"github.com/MontFerret/ferret/pkg/drivers"
|
|
"github.com/MontFerret/ferret/pkg/drivers/common"
|
|
"github.com/MontFerret/ferret/pkg/runtime/logging"
|
|
"github.com/MontFerret/ferret/pkg/runtime/values"
|
|
"github.com/PuerkitoBio/goquery"
|
|
"github.com/pkg/errors"
|
|
"github.com/sethgrid/pester"
|
|
)
|
|
|
|
const DriverName = "http"
|
|
|
|
type Driver struct {
|
|
client *pester.Client
|
|
options *Options
|
|
}
|
|
|
|
func NewDriver(opts ...Option) *Driver {
|
|
drv := new(Driver)
|
|
drv.options = newOptions(opts)
|
|
|
|
if drv.options.proxy == "" {
|
|
drv.client = pester.New()
|
|
} else {
|
|
client, err := newClientWithProxy(drv.options)
|
|
|
|
if err != nil {
|
|
drv.client = pester.New()
|
|
} else {
|
|
drv.client = pester.NewExtendedClient(client)
|
|
}
|
|
}
|
|
|
|
drv.client.Concurrency = drv.options.concurrency
|
|
drv.client.MaxRetries = drv.options.maxRetries
|
|
drv.client.Backoff = drv.options.backoff
|
|
|
|
return drv
|
|
}
|
|
|
|
func newClientWithProxy(options *Options) (*http.Client, error) {
|
|
proxyURL, err := url.Parse(options.proxy)
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
proxy := http.ProxyURL(proxyURL)
|
|
tr := &http.Transport{Proxy: proxy}
|
|
|
|
return &http.Client{Transport: tr}, nil
|
|
}
|
|
|
|
func (drv *Driver) Name() string {
|
|
return DriverName
|
|
}
|
|
|
|
func (drv *Driver) Open(ctx context.Context, params drivers.OpenPageParams) (drivers.HTMLPage, error) {
|
|
req, err := http.NewRequest(http.MethodGet, params.URL, nil)
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
logger := logging.FromContext(ctx)
|
|
|
|
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8")
|
|
req.Header.Set("Accept-Language", "en-US,en;q=0.9,ru;q=0.8")
|
|
req.Header.Set("Cache-Control", "no-cache")
|
|
req.Header.Set("Pragma", "no-cache")
|
|
|
|
if params.Header != nil {
|
|
for k := range params.Header {
|
|
req.Header.Add(k, params.Header.Get(k))
|
|
|
|
logger.
|
|
Debug().
|
|
Timestamp().
|
|
Str("header", k).
|
|
Msg("set header")
|
|
}
|
|
}
|
|
|
|
if params.Cookies != nil {
|
|
for _, c := range params.Cookies {
|
|
req.AddCookie(&http.Cookie{
|
|
Name: c.Name,
|
|
Value: c.Value,
|
|
})
|
|
|
|
logger.
|
|
Debug().
|
|
Timestamp().
|
|
Str("cookie", c.Name).
|
|
Msg("set cookie")
|
|
}
|
|
}
|
|
|
|
req = req.WithContext(ctx)
|
|
|
|
var ua string
|
|
|
|
if params.UserAgent != "" {
|
|
ua = common.GetUserAgent(params.UserAgent)
|
|
} else {
|
|
ua = common.GetUserAgent(drv.options.userAgent)
|
|
}
|
|
|
|
logger.
|
|
Debug().
|
|
Timestamp().
|
|
Str("user-agent", ua).
|
|
Msg("using User-Agent")
|
|
|
|
resp, err := drv.client.Do(req)
|
|
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "failed to retrieve a document %s", params.URL)
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
|
|
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
|
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "failed to parse a document %s", params.URL)
|
|
}
|
|
|
|
return NewHTMLPage(doc, params.URL, params.Cookies)
|
|
}
|
|
|
|
func (drv *Driver) Parse(_ context.Context, str values.String) (drivers.HTMLPage, error) {
|
|
buf := bytes.NewBuffer([]byte(str))
|
|
|
|
doc, err := goquery.NewDocumentFromReader(buf)
|
|
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "failed to parse a document")
|
|
}
|
|
|
|
return NewHTMLPage(doc, "#blank", nil)
|
|
}
|
|
|
|
func (drv *Driver) Close() error {
|
|
drv.client = nil
|
|
|
|
return nil
|
|
}
|