2018-12-21 23:14:41 -05:00
|
|
|
package http
|
2018-09-18 16:42:38 -04:00
|
|
|
|
2018-10-07 21:23:36 -04:00
|
|
|
import (
|
2019-10-13 19:41:53 +02:00
|
|
|
stdhttp "net/http"
|
|
|
|
|
2021-03-26 12:01:00 -04:00
|
|
|
"github.com/gobwas/glob"
|
2018-10-07 21:23:36 -04:00
|
|
|
"github.com/sethgrid/pester"
|
2021-03-26 12:01:00 -04:00
|
|
|
|
|
|
|
"github.com/MontFerret/ferret/pkg/drivers"
|
|
|
|
)
|
|
|
|
|
|
|
|
var (
|
2021-08-01 15:00:20 -04:00
|
|
|
DefaultConcurrency = 1
|
2021-03-26 12:01:00 -04:00
|
|
|
DefaultMaxRetries = 5
|
2018-10-07 21:23:36 -04:00
|
|
|
)
|
2018-09-18 16:42:38 -04:00
|
|
|
|
|
|
|
type (
|
2018-12-21 23:14:41 -05:00
|
|
|
Option func(opts *Options)
|
|
|
|
|
2021-03-05 08:08:28 -05:00
|
|
|
compiledStatusCodeFilter struct {
|
|
|
|
URL glob.Glob
|
|
|
|
Code int
|
|
|
|
}
|
|
|
|
|
2018-10-07 21:23:36 -04:00
|
|
|
Options struct {
|
2021-03-26 12:01:00 -04:00
|
|
|
*drivers.Options
|
2021-03-05 08:08:28 -05:00
|
|
|
Backoff pester.BackoffStrategy
|
|
|
|
MaxRetries int
|
|
|
|
Concurrency int
|
|
|
|
HTTPCodesFilter []compiledStatusCodeFilter
|
|
|
|
HTTPTransport *stdhttp.Transport
|
2018-10-07 21:23:36 -04:00
|
|
|
}
|
2018-09-18 16:42:38 -04:00
|
|
|
)
|
|
|
|
|
2021-03-26 12:01:00 -04:00
|
|
|
func NewOptions(setters []Option) *Options {
|
2018-11-30 19:30:55 -05:00
|
|
|
opts := new(Options)
|
2021-03-26 12:01:00 -04:00
|
|
|
opts.Options = new(drivers.Options)
|
2019-09-05 11:49:21 -04:00
|
|
|
opts.Name = DriverName
|
|
|
|
opts.Backoff = pester.ExponentialBackoff
|
2021-03-26 12:01:00 -04:00
|
|
|
opts.Concurrency = DefaultConcurrency
|
|
|
|
opts.MaxRetries = DefaultMaxRetries
|
2021-03-05 08:08:28 -05:00
|
|
|
opts.HTTPCodesFilter = make([]compiledStatusCodeFilter, 0, 5)
|
2018-11-30 19:30:55 -05:00
|
|
|
|
|
|
|
for _, setter := range setters {
|
|
|
|
setter(opts)
|
|
|
|
}
|
|
|
|
|
|
|
|
return opts
|
|
|
|
}
|
|
|
|
|
2018-09-18 16:42:38 -04:00
|
|
|
func WithDefaultBackoff() Option {
|
2018-10-07 21:23:36 -04:00
|
|
|
return func(opts *Options) {
|
2019-09-05 11:49:21 -04:00
|
|
|
opts.Backoff = pester.DefaultBackoff
|
2018-09-18 16:42:38 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithLinearBackoff() Option {
|
2018-10-07 21:23:36 -04:00
|
|
|
return func(opts *Options) {
|
2019-09-05 11:49:21 -04:00
|
|
|
opts.Backoff = pester.LinearBackoff
|
2018-09-18 16:42:38 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithExponentialBackoff() Option {
|
2018-10-07 21:23:36 -04:00
|
|
|
return func(opts *Options) {
|
2019-09-05 11:49:21 -04:00
|
|
|
opts.Backoff = pester.ExponentialBackoff
|
2018-09-18 16:42:38 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithMaxRetries(value int) Option {
|
2018-10-07 21:23:36 -04:00
|
|
|
return func(opts *Options) {
|
2019-09-05 11:49:21 -04:00
|
|
|
opts.MaxRetries = value
|
2018-09-18 16:42:38 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithConcurrency(value int) Option {
|
2018-10-07 21:23:36 -04:00
|
|
|
return func(opts *Options) {
|
2019-09-05 11:49:21 -04:00
|
|
|
opts.Concurrency = value
|
2018-10-07 21:23:36 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithProxy(address string) Option {
|
|
|
|
return func(opts *Options) {
|
2021-03-26 12:01:00 -04:00
|
|
|
drivers.WithProxy(address)(opts.Options)
|
2018-09-18 16:42:38 -04:00
|
|
|
}
|
|
|
|
}
|
2018-10-07 22:18:57 -04:00
|
|
|
|
|
|
|
func WithUserAgent(value string) Option {
|
|
|
|
return func(opts *Options) {
|
2021-03-26 12:01:00 -04:00
|
|
|
drivers.WithUserAgent(value)(opts.Options)
|
2019-09-05 11:49:21 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithCustomName(name string) Option {
|
|
|
|
return func(opts *Options) {
|
2021-03-26 12:01:00 -04:00
|
|
|
drivers.WithCustomName(name)(opts.Options)
|
2019-09-05 11:49:21 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithHeader(name string, value []string) Option {
|
|
|
|
return func(opts *Options) {
|
2021-03-26 12:01:00 -04:00
|
|
|
drivers.WithHeader(name, value)(opts.Options)
|
2019-09-05 11:49:21 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-03-26 12:01:00 -04:00
|
|
|
func WithHeaders(headers *drivers.HTTPHeaders) Option {
|
2019-09-05 11:49:21 -04:00
|
|
|
return func(opts *Options) {
|
2021-03-26 12:01:00 -04:00
|
|
|
drivers.WithHeaders(headers)(opts.Options)
|
2019-09-05 11:49:21 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithCookie(cookie drivers.HTTPCookie) Option {
|
|
|
|
return func(opts *Options) {
|
2021-03-26 12:01:00 -04:00
|
|
|
drivers.WithCookie(cookie)(opts.Options)
|
2019-09-05 11:49:21 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithCookies(cookies []drivers.HTTPCookie) Option {
|
|
|
|
return func(opts *Options) {
|
2021-03-26 12:01:00 -04:00
|
|
|
drivers.WithCookies(cookies)(opts.Options)
|
2018-10-07 22:18:57 -04:00
|
|
|
}
|
|
|
|
}
|
2019-10-13 19:41:53 +02:00
|
|
|
|
|
|
|
func WithAllowedHTTPCode(httpCode int) Option {
|
|
|
|
return func(opts *Options) {
|
2021-03-05 08:08:28 -05:00
|
|
|
opts.HTTPCodesFilter = append(opts.HTTPCodesFilter, compiledStatusCodeFilter{
|
|
|
|
Code: httpCode,
|
|
|
|
})
|
2019-10-13 19:41:53 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithAllowedHTTPCodes(httpCodes []int) Option {
|
|
|
|
return func(opts *Options) {
|
2019-10-16 00:07:01 +02:00
|
|
|
for _, code := range httpCodes {
|
2021-03-05 08:08:28 -05:00
|
|
|
opts.HTTPCodesFilter = append(opts.HTTPCodesFilter, compiledStatusCodeFilter{
|
|
|
|
Code: code,
|
|
|
|
})
|
2019-10-16 00:07:01 +02:00
|
|
|
}
|
2019-10-13 19:41:53 +02:00
|
|
|
}
|
|
|
|
}
|
2021-01-22 23:00:01 +03:00
|
|
|
|
|
|
|
func WithCustomTransport(transport *stdhttp.Transport) Option {
|
|
|
|
return func(opts *Options) {
|
|
|
|
opts.HTTPTransport = transport
|
|
|
|
}
|
|
|
|
}
|