go/ferret
go/ferret
1
0
Fork 0
mirror of https://github.com/MontFerret/ferret.git synced 2025-04-13 11:50:30 +02:00
Code Issues Releases Activity

Bugfix/#638 http driver multiple requsets (#642)

Browse Source 
* Set DefaultConcurrency to 1

* Added unit test
This commit is contained in:
Tim Voronov 2021-08-01 15:00:20 -04:00 committed by GitHub
parent bc5dd5feed
commit 25c97b86b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 38 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -10,6 +10,7 @@ require (
github.com/corpix/uarand v0.1.1
github.com/gobwas/glob v0.2.3
github.com/gorilla/css v1.0.0
github.com/jarcoal/httpmock v1.0.8
github.com/mafredri/cdp v0.32.0
github.com/pkg/errors v0.9.1
github.com/rs/zerolog v1.23.0

2
go.sum
View File

@ -35,6 +35,8 @@ github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k=
github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=

46
pkg/drivers/http/driver_test.go
View File

@ -2,8 +2,8 @@ package http
import (
"bytes"
"context"
"crypto/tls"
"github.com/MontFerret/ferret/pkg/drivers"
"io"
"io/ioutil"
"net/http"
@ -11,9 +11,11 @@ import (
"testing"
"unsafe"
"github.com/jarcoal/httpmock"
. "github.com/smartystreets/goconvey/convey"
"golang.org/x/text/encoding/charmap"
"github.com/smartystreets/goconvey/convey"
"github.com/MontFerret/ferret/pkg/drivers"
)
func Test_newHTTPClientWithTransport(t *testing.T) {
@ -49,7 +51,7 @@ func Test_newHTTPClientWithTransport(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
convey.Convey(tt.name, t, func() {
Convey(tt.name, t, func() {
var (
transport *http.Transport
client = newHTTPClient(tt.args.options)
@ -68,7 +70,7 @@ func Test_newHTTPClientWithTransport(t *testing.T) {
verify := transport.TLSClientConfig.InsecureSkipVerify
convey.So(verify, convey.ShouldBeTrue)
So(verify, ShouldBeTrue)
})
})
}
@ -76,7 +78,7 @@ func Test_newHTTPClientWithTransport(t *testing.T) {
func Test_newHTTPClient(t *testing.T) {
convey.Convey("pester.New()", t, func() {
Convey("pester.New()", t, func() {
var (
client = newHTTPClient(&Options{
Options: &drivers.Options{
@ -91,10 +93,10 @@ func Test_newHTTPClient(t *testing.T) {
rField = reflect.NewAt(rField.Type(), unsafe.Pointer(rField.UnsafeAddr())).Elem()
hc := rField.Interface().(*http.Client)
convey.So(hc, convey.ShouldBeNil)
So(hc, ShouldBeNil)
})
convey.Convey("pester.NewExtend()", t, func() {
Convey("pester.NewExtend()", t, func() {
var (
client = newHTTPClient(&Options{
Options: &drivers.Options{
@ -109,7 +111,7 @@ func Test_newHTTPClient(t *testing.T) {
rField = reflect.NewAt(rField.Type(), unsafe.Pointer(rField.UnsafeAddr())).Elem()
hc := rField.Interface().(*http.Client)
convey.So(hc, convey.ShouldNotBeNil)
So(hc, ShouldNotBeNil)
})
}
@ -139,7 +141,7 @@ func TestDriver_convertToUTF8(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
drv := &Driver{}
convey.Convey(tt.name, t, func() {
Convey(tt.name, t, func() {
data, err := ioutil.ReadAll(bytes.NewBufferString(tt.args.inputData))
if err != nil {
@ -157,15 +159,35 @@ func TestDriver_convertToUTF8(t *testing.T) {
encodedData = encodedData[:nDst]
gotData, err := drv.convertToUTF8(bytes.NewReader(encodedData), tt.args.srcCharset)
convey.So(err, convey.ShouldBeNil)
So(err, ShouldBeNil)
outData, err := ioutil.ReadAll(gotData)
convey.So(err, convey.ShouldBeNil)
So(err, ShouldBeNil)
convey.So(string(outData), convey.ShouldEqual, tt.expected)
So(string(outData), ShouldEqual, tt.expected)
})
})
}
}
func TestDriver_Concurrency(t *testing.T) {
Convey("Should make only 1 request", t, func() {
httpmock.Activate()
defer httpmock.DeactivateAndReset()
httpmock.RegisterResponder("GET", "http://localhost:1111",
httpmock.NewStringResponder(200, `<!DOCTYPE html><html><head></head><body></body></html>`))
drv := NewDriver()
page, err := drv.Open(context.Background(), drivers.Params{
URL: "http://localhost:1111",
})
So(err, ShouldBeNil)
So(page, ShouldNotBeNil)
So(httpmock.GetTotalCallCount(), ShouldEqual, 1)
})
}

2
pkg/drivers/http/options.go
View File

@ -10,7 +10,7 @@ import (
)
var (
DefaultConcurrency = 3
DefaultConcurrency = 1
DefaultMaxRetries = 5
)