From 25c97b86b85d70624e54066638a23f91ed3322f2 Mon Sep 17 00:00:00 2001 From: Tim Voronov Date: Sun, 1 Aug 2021 15:00:20 -0400 Subject: [PATCH] Bugfix/#638 http driver multiple requsets (#642) * Set DefaultConcurrency to 1 * Added unit test --- go.mod | 1 + go.sum | 2 ++ pkg/drivers/http/driver_test.go | 46 ++++++++++++++++++++++++--------- pkg/drivers/http/options.go | 2 +- 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index e99a2c5f..72e3f64a 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/corpix/uarand v0.1.1 github.com/gobwas/glob v0.2.3 github.com/gorilla/css v1.0.0 + github.com/jarcoal/httpmock v1.0.8 github.com/mafredri/cdp v0.32.0 github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.23.0 diff --git a/go.sum b/go.sum index c040deeb..642e76e0 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k= +github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= diff --git a/pkg/drivers/http/driver_test.go b/pkg/drivers/http/driver_test.go index ea55c96f..ce546f0a 100644 --- a/pkg/drivers/http/driver_test.go +++ b/pkg/drivers/http/driver_test.go @@ -2,8 +2,8 @@ package http import ( "bytes" + "context" "crypto/tls" - "github.com/MontFerret/ferret/pkg/drivers" "io" "io/ioutil" "net/http" @@ -11,9 +11,11 @@ import ( "testing" "unsafe" + "github.com/jarcoal/httpmock" + . "github.com/smartystreets/goconvey/convey" "golang.org/x/text/encoding/charmap" - "github.com/smartystreets/goconvey/convey" + "github.com/MontFerret/ferret/pkg/drivers" ) func Test_newHTTPClientWithTransport(t *testing.T) { @@ -49,7 +51,7 @@ func Test_newHTTPClientWithTransport(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - convey.Convey(tt.name, t, func() { + Convey(tt.name, t, func() { var ( transport *http.Transport client = newHTTPClient(tt.args.options) @@ -68,7 +70,7 @@ func Test_newHTTPClientWithTransport(t *testing.T) { verify := transport.TLSClientConfig.InsecureSkipVerify - convey.So(verify, convey.ShouldBeTrue) + So(verify, ShouldBeTrue) }) }) } @@ -76,7 +78,7 @@ func Test_newHTTPClientWithTransport(t *testing.T) { func Test_newHTTPClient(t *testing.T) { - convey.Convey("pester.New()", t, func() { + Convey("pester.New()", t, func() { var ( client = newHTTPClient(&Options{ Options: &drivers.Options{ @@ -91,10 +93,10 @@ func Test_newHTTPClient(t *testing.T) { rField = reflect.NewAt(rField.Type(), unsafe.Pointer(rField.UnsafeAddr())).Elem() hc := rField.Interface().(*http.Client) - convey.So(hc, convey.ShouldBeNil) + So(hc, ShouldBeNil) }) - convey.Convey("pester.NewExtend()", t, func() { + Convey("pester.NewExtend()", t, func() { var ( client = newHTTPClient(&Options{ Options: &drivers.Options{ @@ -109,7 +111,7 @@ func Test_newHTTPClient(t *testing.T) { rField = reflect.NewAt(rField.Type(), unsafe.Pointer(rField.UnsafeAddr())).Elem() hc := rField.Interface().(*http.Client) - convey.So(hc, convey.ShouldNotBeNil) + So(hc, ShouldNotBeNil) }) } @@ -139,7 +141,7 @@ func TestDriver_convertToUTF8(t *testing.T) { t.Run(tt.name, func(t *testing.T) { drv := &Driver{} - convey.Convey(tt.name, t, func() { + Convey(tt.name, t, func() { data, err := ioutil.ReadAll(bytes.NewBufferString(tt.args.inputData)) if err != nil { @@ -157,15 +159,35 @@ func TestDriver_convertToUTF8(t *testing.T) { encodedData = encodedData[:nDst] gotData, err := drv.convertToUTF8(bytes.NewReader(encodedData), tt.args.srcCharset) - convey.So(err, convey.ShouldBeNil) + So(err, ShouldBeNil) outData, err := ioutil.ReadAll(gotData) - convey.So(err, convey.ShouldBeNil) + So(err, ShouldBeNil) - convey.So(string(outData), convey.ShouldEqual, tt.expected) + So(string(outData), ShouldEqual, tt.expected) }) }) } } + +func TestDriver_Concurrency(t *testing.T) { + Convey("Should make only 1 request", t, func() { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + httpmock.RegisterResponder("GET", "http://localhost:1111", + httpmock.NewStringResponder(200, ``)) + + drv := NewDriver() + + page, err := drv.Open(context.Background(), drivers.Params{ + URL: "http://localhost:1111", + }) + + So(err, ShouldBeNil) + So(page, ShouldNotBeNil) + So(httpmock.GetTotalCallCount(), ShouldEqual, 1) + }) +} diff --git a/pkg/drivers/http/options.go b/pkg/drivers/http/options.go index 5adc81b5..c7f084f6 100644 --- a/pkg/drivers/http/options.go +++ b/pkg/drivers/http/options.go @@ -10,7 +10,7 @@ import ( ) var ( - DefaultConcurrency = 3 + DefaultConcurrency = 1 DefaultMaxRetries = 5 )