2018-12-22 06:14:41 +02:00
|
|
|
package http
|
2018-09-18 22:42:38 +02:00
|
|
|
|
2018-10-08 03:23:36 +02:00
|
|
|
import (
|
2019-10-13 19:41:53 +02:00
|
|
|
stdhttp "net/http"
|
|
|
|
|
2019-09-05 17:49:21 +02:00
|
|
|
"github.com/MontFerret/ferret/pkg/drivers"
|
2018-10-08 03:23:36 +02:00
|
|
|
"github.com/sethgrid/pester"
|
|
|
|
)
|
2018-09-18 22:42:38 +02:00
|
|
|
|
|
|
|
type (
|
2018-12-22 06:14:41 +02:00
|
|
|
Option func(opts *Options)
|
|
|
|
|
2018-10-08 03:23:36 +02:00
|
|
|
Options struct {
|
2019-10-13 19:46:13 +02:00
|
|
|
Name string
|
|
|
|
Backoff pester.BackoffStrategy
|
|
|
|
MaxRetries int
|
|
|
|
Concurrency int
|
|
|
|
Proxy string
|
|
|
|
UserAgent string
|
|
|
|
Headers drivers.HTTPHeaders
|
|
|
|
Cookies drivers.HTTPCookies
|
2019-10-16 00:07:01 +02:00
|
|
|
AllowedHTTPCodes map[int]struct{}
|
2018-10-08 03:23:36 +02:00
|
|
|
}
|
2018-09-18 22:42:38 +02:00
|
|
|
)
|
|
|
|
|
2018-12-01 02:30:55 +02:00
|
|
|
func newOptions(setters []Option) *Options {
|
|
|
|
opts := new(Options)
|
2019-09-05 17:49:21 +02:00
|
|
|
opts.Name = DriverName
|
|
|
|
opts.Backoff = pester.ExponentialBackoff
|
|
|
|
opts.Concurrency = 3
|
|
|
|
opts.MaxRetries = 5
|
2019-10-16 00:07:01 +02:00
|
|
|
opts.AllowedHTTPCodes = map[int]struct{}{stdhttp.StatusOK: struct{}{}}
|
2018-12-01 02:30:55 +02:00
|
|
|
|
|
|
|
for _, setter := range setters {
|
|
|
|
setter(opts)
|
|
|
|
}
|
|
|
|
|
|
|
|
return opts
|
|
|
|
}
|
|
|
|
|
2018-09-18 22:42:38 +02:00
|
|
|
func WithDefaultBackoff() Option {
|
2018-10-08 03:23:36 +02:00
|
|
|
return func(opts *Options) {
|
2019-09-05 17:49:21 +02:00
|
|
|
opts.Backoff = pester.DefaultBackoff
|
2018-09-18 22:42:38 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithLinearBackoff() Option {
|
2018-10-08 03:23:36 +02:00
|
|
|
return func(opts *Options) {
|
2019-09-05 17:49:21 +02:00
|
|
|
opts.Backoff = pester.LinearBackoff
|
2018-09-18 22:42:38 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithExponentialBackoff() Option {
|
2018-10-08 03:23:36 +02:00
|
|
|
return func(opts *Options) {
|
2019-09-05 17:49:21 +02:00
|
|
|
opts.Backoff = pester.ExponentialBackoff
|
2018-09-18 22:42:38 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithMaxRetries(value int) Option {
|
2018-10-08 03:23:36 +02:00
|
|
|
return func(opts *Options) {
|
2019-09-05 17:49:21 +02:00
|
|
|
opts.MaxRetries = value
|
2018-09-18 22:42:38 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithConcurrency(value int) Option {
|
2018-10-08 03:23:36 +02:00
|
|
|
return func(opts *Options) {
|
2019-09-05 17:49:21 +02:00
|
|
|
opts.Concurrency = value
|
2018-10-08 03:23:36 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithProxy(address string) Option {
|
|
|
|
return func(opts *Options) {
|
2019-09-05 17:49:21 +02:00
|
|
|
opts.Proxy = address
|
2018-09-18 22:42:38 +02:00
|
|
|
}
|
|
|
|
}
|
2018-10-08 04:18:57 +02:00
|
|
|
|
|
|
|
func WithUserAgent(value string) Option {
|
|
|
|
return func(opts *Options) {
|
2019-09-05 17:49:21 +02:00
|
|
|
opts.UserAgent = value
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithCustomName(name string) Option {
|
|
|
|
return func(opts *Options) {
|
|
|
|
opts.Name = name
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithHeader(name string, value []string) Option {
|
|
|
|
return func(opts *Options) {
|
|
|
|
if opts.Headers == nil {
|
|
|
|
opts.Headers = make(drivers.HTTPHeaders)
|
|
|
|
}
|
|
|
|
|
|
|
|
opts.Headers[name] = value
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithHeaders(headers drivers.HTTPHeaders) Option {
|
|
|
|
return func(opts *Options) {
|
|
|
|
if opts.Headers == nil {
|
|
|
|
opts.Headers = make(drivers.HTTPHeaders)
|
|
|
|
}
|
|
|
|
|
|
|
|
for k, v := range headers {
|
|
|
|
opts.Headers[k] = v
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithCookie(cookie drivers.HTTPCookie) Option {
|
|
|
|
return func(opts *Options) {
|
|
|
|
if opts.Cookies == nil {
|
|
|
|
opts.Cookies = make(drivers.HTTPCookies)
|
|
|
|
}
|
|
|
|
|
|
|
|
opts.Cookies[cookie.Name] = cookie
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithCookies(cookies []drivers.HTTPCookie) Option {
|
|
|
|
return func(opts *Options) {
|
|
|
|
if opts.Cookies == nil {
|
|
|
|
opts.Cookies = make(drivers.HTTPCookies)
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, c := range cookies {
|
|
|
|
opts.Cookies[c.Name] = c
|
|
|
|
}
|
2018-10-08 04:18:57 +02:00
|
|
|
}
|
|
|
|
}
|
2019-10-13 19:41:53 +02:00
|
|
|
|
|
|
|
func WithAllowedHTTPCode(httpCode int) Option {
|
|
|
|
return func(opts *Options) {
|
2019-10-16 00:07:01 +02:00
|
|
|
opts.AllowedHTTPCodes[httpCode] = struct{}{}
|
2019-10-13 19:41:53 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithAllowedHTTPCodes(httpCodes []int) Option {
|
|
|
|
return func(opts *Options) {
|
2019-10-16 00:07:01 +02:00
|
|
|
for _, code := range httpCodes {
|
|
|
|
opts.AllowedHTTPCodes[code] = struct{}{}
|
|
|
|
}
|
2019-10-13 19:41:53 +02:00
|
|
|
}
|
|
|
|
}
|