diff --git a/config.go b/config.go index 1c3bdeb1..6a842724 100644 --- a/config.go +++ b/config.go @@ -202,6 +202,7 @@ type config struct { FreeMemoryInterval int DownloadBufferSize int + GZipBufferSize int BufferPoolCalibrationThreshold int } @@ -329,6 +330,7 @@ func init() { intEnvConfig(&conf.FreeMemoryInterval, "IMGPROXY_FREE_MEMORY_INTERVAL") intEnvConfig(&conf.DownloadBufferSize, "IMGPROXY_DOWNLOAD_BUFFER_SIZE") + intEnvConfig(&conf.GZipBufferSize, "IMGPROXY_GZIP_BUFFER_SIZE") intEnvConfig(&conf.BufferPoolCalibrationThreshold, "IMGPROXY_BUFFER_POOL_CALIBRATION_THRESHOLD") if len(conf.Keys) != len(conf.Salts) { @@ -453,6 +455,12 @@ func init() { logFatal("Download buffer size can't be greater than %d", ^uint32(0)) } + if conf.GZipBufferSize < 0 { + logFatal("GZip buffer size should be greater than or equal to 0") + } else if conf.GZipBufferSize > int(^uint32(0)) { + logFatal("GZip buffer size can't be greater than %d", ^uint32(0)) + } + if conf.BufferPoolCalibrationThreshold < 64 { logFatal("Buffer pool calibration threshold should be greater than or equal to 64") } diff --git a/docs/memory_usage_tweaks.md b/docs/memory_usage_tweaks.md index af4464ea..8da2f0b9 100644 --- a/docs/memory_usage_tweaks.md +++ b/docs/memory_usage_tweaks.md @@ -8,6 +8,10 @@ There are some imgproxy options that can help you to optimize memory usage and d imgproxy uses memory buffers to download source images. While these buffers are empty at the start by default, they can grow to a required size when imgproxy downloads an image. Allocating new memory to grow the buffers can cause memory fragmentation. Allocating required memory at the start can eliminate much of memory fragmentation since buffers won't grow. Setting `IMGPROXY_DOWNLOAD_BUFFER_SIZE` will tell imgproxy to initialize download buffers with _at least_ the specified size. It's recommended to use the estimated 95 percentile of your image sizes as the initial download buffers size. +### `IMGPROXY_GZIP_BUFFER_SIZE` + +The same as `IMGPROXY_DOWNLOAD_BUFFER_SIZE` but for GZip buffers. If you use GZip compression of the resulting images, you can reduce memory fragmentation by using the estimated maximum size of the GZipped resulting image as the initial size of GZip buffers. + ### `IMGPROXY_FREE_MEMORY_INTERVAL` Working with a large amount of data can cause allocating some memory that is not used most of the time. That's why imgproxy enforces Go's garbage collector to free as much memory as possible and return it to the OS. The default interval of this action is 10 seconds, but you can change it by setting `IMGPROXY_FREE_MEMORY_INTERVAL`. Decreasing the interval can smooth the memory usage graph but it can also slow down imgproxy a little. Increasing has the opposite effect. diff --git a/etag.go b/etag.go index bcbfd860..ea228c82 100644 --- a/etag.go +++ b/etag.go @@ -9,70 +9,26 @@ import ( "sync" ) -type etagPool struct { - mutex sync.Mutex - top *etagPoolEntry -} - -type etagPoolEntry struct { +type eTagCalc struct { hash hash.Hash enc *json.Encoder - next *etagPoolEntry - b []byte } -func newEtagPool(n int) *etagPool { - pool := new(etagPool) +var eTagCalcPool = sync.Pool{ + New: func() interface{} { + h := sha256.New() - for i := 0; i < n; i++ { - pool.grow() - } + enc := json.NewEncoder(h) + enc.SetEscapeHTML(false) + enc.SetIndent("", "") - return pool + return &eTagCalc{h, enc} + }, } -func (p *etagPool) grow() { - h := sha256.New() - - enc := json.NewEncoder(h) - enc.SetEscapeHTML(false) - enc.SetIndent("", "") - - p.top = &etagPoolEntry{ - hash: h, - enc: enc, - b: make([]byte, 64), - next: p.top, - } -} - -func (p *etagPool) Get() *etagPoolEntry { - p.mutex.Lock() - defer p.mutex.Unlock() - - if p.top == nil { - p.grow() - } - - entry := p.top - p.top = p.top.next - - return entry -} - -func (p *etagPool) Put(e *etagPoolEntry) { - p.mutex.Lock() - defer p.mutex.Unlock() - - e.next = p.top - p.top = e -} - -var eTagCalcPool *etagPool - -func calcETag(ctx context.Context) ([]byte, context.CancelFunc) { - c := eTagCalcPool.Get() - cancel := func() { eTagCalcPool.Put(c) } +func calcETag(ctx context.Context) string { + c := eTagCalcPool.Get().(*eTagCalc) + defer eTagCalcPool.Put(c) c.hash.Reset() c.hash.Write(getImageData(ctx).Bytes()) @@ -84,7 +40,5 @@ func calcETag(ctx context.Context) ([]byte, context.CancelFunc) { c.enc.Encode(conf) c.enc.Encode(getProcessingOptions(ctx)) - hex.Encode(c.b, c.hash.Sum(nil)) - - return c.b, cancel + return hex.EncodeToString(c.hash.Sum(nil)) } diff --git a/go.mod b/go.mod index 4fe24535..20c4f0c9 100644 --- a/go.mod +++ b/go.mod @@ -19,8 +19,6 @@ require ( github.com/googleapis/gax-go v0.0.0-20181219185031-c8a15bac9b9f // indirect github.com/honeybadger-io/honeybadger-go v0.4.0 github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 // indirect - github.com/klauspost/compress v1.4.1 // indirect - github.com/klauspost/cpuid v1.2.0 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/mat/besticon v3.9.0+incompatible github.com/matoous/go-nanoid v0.0.0-20181114085210-eab626deece6 @@ -33,9 +31,8 @@ require ( github.com/prometheus/procfs v0.0.0-20190104112138-b1a0a9a36d74 // indirect github.com/shirou/gopsutil v2.18.12+incompatible // indirect github.com/stretchr/testify v1.3.0 - github.com/valyala/fasthttp v1.2.0 golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b - golang.org/x/net v0.0.0-20190110200230-915654e7eabc // indirect + golang.org/x/net v0.0.0-20190110200230-915654e7eabc golang.org/x/oauth2 v0.0.0-20190110195249-fd3eaa146cbb // indirect golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb // indirect diff --git a/go.sum b/go.sum index e17e3eff..89ba0dc7 100644 --- a/go.sum +++ b/go.sum @@ -60,12 +60,6 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 h1:PJPDf8OUfOK1bb/NeTKd4f1QXZItOX389VN3B6qC8ro= github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -115,11 +109,6 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.2.0 h1:dzZJf2IuMiclVjdw0kkT+f9u4YdrapbNyGAN47E/qnk= -github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= go.opencensus.io v0.18.0 h1:Mk5rgZcggtbvtAun5aJzAtjKKN/t0R3jJPlWILlv938= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -130,7 +119,6 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/log.go b/log.go index 0c8aea1f..baf619d9 100644 --- a/log.go +++ b/log.go @@ -4,8 +4,7 @@ import ( "fmt" "log" "log/syslog" - - "github.com/valyala/fasthttp" + "net/http" ) const ( @@ -18,13 +17,13 @@ const ( logFatalSyslogFmt = "FATAL %s" ) -func logRequest(reqID string, rctx *fasthttp.RequestCtx) { - path := rctx.RequestURI() +func logRequest(reqID string, r *http.Request) { + path := r.URL.RequestURI() - log.Printf(logRequestFmt, reqID, rctx.Method(), path) + log.Printf(logRequestFmt, reqID, r.Method, path) if syslogWriter != nil { - syslogWriter.Notice(fmt.Sprintf(logRequestSyslogFmt, reqID, rctx.Method(), path)) + syslogWriter.Notice(fmt.Sprintf(logRequestSyslogFmt, reqID, r.Method, path)) } } diff --git a/newrelic.go b/newrelic.go index 017f4706..33c75d70 100644 --- a/newrelic.go +++ b/newrelic.go @@ -38,8 +38,8 @@ func initNewrelic() { newRelicEnabled = true } -func startNewRelicTransaction(ctx context.Context, r *http.Request) (context.Context, context.CancelFunc) { - txn := newRelicApp.StartTransaction("request", nil, r) +func startNewRelicTransaction(ctx context.Context, rw http.ResponseWriter, r *http.Request) (context.Context, context.CancelFunc) { + txn := newRelicApp.StartTransaction("request", rw, r) cancel := func() { txn.End() } return context.WithValue(ctx, newRelicTransactionCtxKey, txn), cancel } diff --git a/processing_options.go b/processing_options.go index 7458e233..f43d6410 100644 --- a/processing_options.go +++ b/processing_options.go @@ -11,12 +11,11 @@ import ( "encoding/base64" "errors" "fmt" + "net/http" "net/url" "regexp" "strconv" "strings" - - "github.com/valyala/fasthttp" ) type urlOptions map[string][]string @@ -234,7 +233,7 @@ func decodeBase64URL(parts []string) (string, string, error) { return "", "", errInvalidURLEncoding } - fullURL := fmt.Sprintf("%s%s", conf.BaseURL, imageURL) + fullURL := fmt.Sprintf("%s%s", conf.BaseURL, string(imageURL)) if _, err := url.ParseRequestURI(fullURL); err != nil { return "", "", errInvalidImageURL @@ -259,7 +258,7 @@ func decodePlainURL(parts []string) (string, string, error) { if unescaped, err := url.PathUnescape(urlParts[0]); err == nil { fullURL := fmt.Sprintf("%s%s", conf.BaseURL, unescaped) if _, err := url.ParseRequestURI(fullURL); err == nil { - return fmt.Sprintf("%s%s", conf.BaseURL, unescaped), format, nil + return fullURL, format, nil } } @@ -848,8 +847,11 @@ func parsePathBasic(parts []string, headers *processingHeaders) (string, *proces return url, po, nil } -func parsePath(ctx context.Context, rctx *fasthttp.RequestCtx) (context.Context, error) { - path := string(rctx.Request.URI().PathOriginal()) +func parsePath(ctx context.Context, r *http.Request) (context.Context, error) { + path := r.URL.RawPath + if len(path) == 0 { + path = r.URL.Path + } parts := strings.Split(strings.TrimPrefix(path, "/"), "/") if len(parts) < 3 { @@ -863,10 +865,10 @@ func parsePath(ctx context.Context, rctx *fasthttp.RequestCtx) (context.Context, } headers := &processingHeaders{ - Accept: string(rctx.Request.Header.Peek("Accept")), - Width: string(rctx.Request.Header.Peek("Width")), - ViewportWidth: string(rctx.Request.Header.Peek("Viewport-Width")), - DPR: string(rctx.Request.Header.Peek("DPR")), + Accept: r.Header.Get("Accept"), + Width: r.Header.Get("Width"), + ViewportWidth: r.Header.Get("Viewport-Width"), + DPR: r.Header.Get("DPR"), } var imageURL string diff --git a/processing_options_test.go b/processing_options_test.go index 756b52ee..97c6020e 100644 --- a/processing_options_test.go +++ b/processing_options_test.go @@ -4,26 +4,20 @@ import ( "context" "encoding/base64" "fmt" + "net/http" "net/url" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/valyala/fasthttp" ) type ProcessingOptionsTestSuite struct{ MainTestSuite } -func (s *ProcessingOptionsTestSuite) getRequest(url string) *fasthttp.RequestCtx { - ctx := fasthttp.RequestCtx{ - Request: fasthttp.Request{}, - Response: fasthttp.Response{}, - } - - ctx.Request.SetRequestURI(url) - - return &ctx +func (s *ProcessingOptionsTestSuite) getRequest(url string) *http.Request { + req, _ := http.NewRequest("GET", url, nil) + return req } func (s *ProcessingOptionsTestSuite) TestParseBase64URL() { @@ -411,7 +405,7 @@ func (s *ProcessingOptionsTestSuite) TestParsePathWebpDetection() { conf.EnableWebpDetection = true req := s.getRequest("http://example.com/unsafe/plain/http://images.dev/lorem/ipsum.jpg") - req.Request.Header.Set("Accept", "image/webp") + req.Header.Set("Accept", "image/webp") ctx, err := parsePath(context.Background(), req) require.Nil(s.T(), err) @@ -424,7 +418,7 @@ func (s *ProcessingOptionsTestSuite) TestParsePathWebpDetectionRedefine() { conf.EnableWebpDetection = true req := s.getRequest("http://example.com/unsafe/plain/http://images.dev/lorem/ipsum.jpg@png") - req.Request.Header.Set("Accept", "image/webp") + req.Header.Set("Accept", "image/webp") ctx, err := parsePath(context.Background(), req) require.Nil(s.T(), err) @@ -437,7 +431,7 @@ func (s *ProcessingOptionsTestSuite) TestParsePathWebpEnforce() { conf.EnforceWebp = true req := s.getRequest("http://example.com/unsafe/plain/http://images.dev/lorem/ipsum.jpg@png") - req.Request.Header.Set("Accept", "image/webp") + req.Header.Set("Accept", "image/webp") ctx, err := parsePath(context.Background(), req) require.Nil(s.T(), err) @@ -450,7 +444,7 @@ func (s *ProcessingOptionsTestSuite) TestParsePathWidthHeader() { conf.EnableClientHints = true req := s.getRequest("http://example.com/unsafe/plain/http://images.dev/lorem/ipsum.jpg@png") - req.Request.Header.Set("Width", "100") + req.Header.Set("Width", "100") ctx, err := parsePath(context.Background(), req) require.Nil(s.T(), err) @@ -461,7 +455,7 @@ func (s *ProcessingOptionsTestSuite) TestParsePathWidthHeader() { func (s *ProcessingOptionsTestSuite) TestParsePathWidthHeaderDisabled() { req := s.getRequest("http://example.com/unsafe/plain/http://images.dev/lorem/ipsum.jpg@png") - req.Request.Header.Set("Width", "100") + req.Header.Set("Width", "100") ctx, err := parsePath(context.Background(), req) require.Nil(s.T(), err) @@ -474,7 +468,7 @@ func (s *ProcessingOptionsTestSuite) TestParsePathWidthHeaderRedefine() { conf.EnableClientHints = true req := s.getRequest("http://example.com/unsafe/width:150/plain/http://images.dev/lorem/ipsum.jpg@png") - req.Request.Header.Set("Width", "100") + req.Header.Set("Width", "100") ctx, err := parsePath(context.Background(), req) require.Nil(s.T(), err) @@ -487,7 +481,7 @@ func (s *ProcessingOptionsTestSuite) TestParsePathViewportWidthHeader() { conf.EnableClientHints = true req := s.getRequest("http://example.com/unsafe/plain/http://images.dev/lorem/ipsum.jpg@png") - req.Request.Header.Set("Viewport-Width", "100") + req.Header.Set("Viewport-Width", "100") ctx, err := parsePath(context.Background(), req) require.Nil(s.T(), err) @@ -498,7 +492,7 @@ func (s *ProcessingOptionsTestSuite) TestParsePathViewportWidthHeader() { func (s *ProcessingOptionsTestSuite) TestParsePathViewportWidthHeaderDisabled() { req := s.getRequest("http://example.com/unsafe/plain/http://images.dev/lorem/ipsum.jpg@png") - req.Request.Header.Set("Viewport-Width", "100") + req.Header.Set("Viewport-Width", "100") ctx, err := parsePath(context.Background(), req) require.Nil(s.T(), err) @@ -511,7 +505,7 @@ func (s *ProcessingOptionsTestSuite) TestParsePathViewportWidthHeaderRedefine() conf.EnableClientHints = true req := s.getRequest("http://example.com/unsafe/width:150/plain/http://images.dev/lorem/ipsum.jpg@png") - req.Request.Header.Set("Viewport-Width", "100") + req.Header.Set("Viewport-Width", "100") ctx, err := parsePath(context.Background(), req) require.Nil(s.T(), err) @@ -524,7 +518,7 @@ func (s *ProcessingOptionsTestSuite) TestParsePathDprHeader() { conf.EnableClientHints = true req := s.getRequest("http://example.com/unsafe/plain/http://images.dev/lorem/ipsum.jpg@png") - req.Request.Header.Set("DPR", "2") + req.Header.Set("DPR", "2") ctx, err := parsePath(context.Background(), req) require.Nil(s.T(), err) @@ -535,7 +529,7 @@ func (s *ProcessingOptionsTestSuite) TestParsePathDprHeader() { func (s *ProcessingOptionsTestSuite) TestParsePathDprHeaderDisabled() { req := s.getRequest("http://example.com/unsafe/plain/http://images.dev/lorem/ipsum.jpg@png") - req.Request.Header.Set("DPR", "2") + req.Header.Set("DPR", "2") ctx, err := parsePath(context.Background(), req) require.Nil(s.T(), err) diff --git a/server.go b/server.go index 4aff69a3..644977a7 100644 --- a/server.go +++ b/server.go @@ -1,22 +1,24 @@ package main import ( - "bytes" "context" "crypto/subtle" "fmt" + "net" "net/http" "net/url" "path/filepath" "regexp" + "strconv" "strings" "time" nanoid "github.com/matoous/go-nanoid" - "github.com/valyala/fasthttp" + "golang.org/x/net/netutil" ) const ( + healthPath = "/health" contextDispositionFilenameFallback = "image" xRequestIDHeader = "X-Request-ID" ) @@ -40,16 +42,15 @@ var ( authHeaderMust []byte - healthPath = []byte("/health") - imgproxyIsRunningMsg = []byte("imgproxy is running") errInvalidMethod = newError(422, "Invalid request method", "Method doesn't allowed") errInvalidSecret = newError(403, "Invalid secret", "Forbidden") - responseGzipPool *gzipPool - requestIDRe = regexp.MustCompile(`^[A-Za-z0-9_\-]+$`) + + responseGzipBufPool *bufPool + responseGzipPool *gzipPool ) type httpHandler struct { @@ -60,43 +61,45 @@ func newHTTPHandler() *httpHandler { return &httpHandler{make(chan struct{}, conf.Concurrency)} } -func startServer() *fasthttp.Server { - handler := newHTTPHandler() - - server := &fasthttp.Server{ - Name: "imgproxy", - Handler: handler.ServeHTTP, - Concurrency: conf.MaxClients, - ReadTimeout: time.Duration(conf.ReadTimeout) * time.Second, +func startServer() *http.Server { + l, err := net.Listen("tcp", conf.Bind) + if err != nil { + logFatal(err.Error()) + } + s := &http.Server{ + Handler: newHTTPHandler(), + ReadTimeout: time.Duration(conf.ReadTimeout) * time.Second, + MaxHeaderBytes: 1 << 20, } if conf.GZipCompression > 0 { + responseGzipBufPool = newBufPool("gzip", conf.Concurrency, conf.GZipBufferSize) responseGzipPool = newGzipPool(conf.Concurrency) } - if conf.ETagEnabled { - eTagCalcPool = newEtagPool(conf.Concurrency) - } - go func() { logNotice("Starting server at %s", conf.Bind) - if err := server.ListenAndServe(conf.Bind); err != nil { + if err := s.Serve(netutil.LimitListener(l, conf.MaxClients)); err != nil && err != http.ErrServerClosed { logFatal(err.Error()) } }() - return server + return s } -func shutdownServer(s *fasthttp.Server) { +func shutdownServer(s *http.Server) { logNotice("Shutting down the server...") - s.Shutdown() + + ctx, close := context.WithTimeout(context.Background(), 5*time.Second) + defer close() + + s.Shutdown(ctx) } -func writeCORS(rctx *fasthttp.RequestCtx) { +func writeCORS(rw http.ResponseWriter) { if len(conf.AllowOrigin) > 0 { - rctx.Response.Header.Set("Access-Control-Allow-Origin", conf.AllowOrigin) - rctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, OPTIONS") + rw.Header().Set("Access-Control-Allow-Origin", conf.AllowOrigin) + rw.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS") } } @@ -114,35 +117,42 @@ func contentDisposition(imageURL string, imgtype imageType) string { return fmt.Sprintf(contentDispositionsFmt[imgtype], strings.TrimSuffix(filename, filepath.Ext(filename))) } -func respondWithImage(ctx context.Context, reqID string, rctx *fasthttp.RequestCtx, data []byte) { +func respondWithImage(ctx context.Context, reqID string, r *http.Request, rw http.ResponseWriter, data []byte) { po := getProcessingOptions(ctx) - rctx.SetStatusCode(200) + rw.Header().Set("Expires", time.Now().Add(time.Second*time.Duration(conf.TTL)).Format(http.TimeFormat)) + rw.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d, public", conf.TTL)) + rw.Header().Set("Content-Type", mimes[po.Format]) + rw.Header().Set("Content-Disposition", contentDisposition(getImageURL(ctx), po.Format)) - rctx.Response.Header.Set("Expires", time.Now().Add(time.Second*time.Duration(conf.TTL)).Format(http.TimeFormat)) - rctx.Response.Header.Set("Cache-Control", fmt.Sprintf("max-age=%d, public", conf.TTL)) - rctx.Response.Header.Set("Content-Type", mimes[po.Format]) - rctx.Response.Header.Set("Content-Disposition", contentDisposition(getImageURL(ctx), po.Format)) + addVaryHeader(rw) - addVaryHeader(rctx) + if conf.GZipCompression > 0 && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + buf := responseGzipBufPool.Get(0) + defer responseGzipBufPool.Put(buf) - if conf.GZipCompression > 0 && rctx.Request.Header.HasAcceptEncoding("gzip") { - gz := responseGzipPool.Get(rctx) + gz := responseGzipPool.Get(buf) defer responseGzipPool.Put(gz) gz.Write(data) gz.Close() - rctx.Response.Header.Set("Content-Encoding", "gzip") + rw.Header().Set("Content-Encoding", "gzip") + rw.Header().Set("Content-Length", strconv.Itoa(buf.Len())) + + rw.WriteHeader(200) + rw.Write(buf.Bytes()) } else { - rctx.SetBody(data) + rw.Header().Set("Content-Length", strconv.Itoa(len(data))) + rw.WriteHeader(200) + rw.Write(data) } logResponse(reqID, 200, fmt.Sprintf("Processed in %s: %s; %+v", getTimerSince(ctx), getImageURL(ctx), po)) } -func addVaryHeader(rctx *fasthttp.RequestCtx) { - vary := make([]string, 0, 5) +func addVaryHeader(rw http.ResponseWriter) { + vary := make([]string, 0) if conf.EnableWebpDetection || conf.EnforceWebp { vary = append(vary, "Accept") @@ -157,37 +167,35 @@ func addVaryHeader(rctx *fasthttp.RequestCtx) { } if len(vary) > 0 { - rctx.Response.Header.Set("Vary", strings.Join(vary, ", ")) + rw.Header().Set("Vary", strings.Join(vary, ", ")) } } -func respondWithError(reqID string, rctx *fasthttp.RequestCtx, err *imgproxyError) { +func respondWithError(reqID string, rw http.ResponseWriter, err *imgproxyError) { logResponse(reqID, err.StatusCode, err.Message) - rctx.SetStatusCode(err.StatusCode) - rctx.SetBodyString(err.PublicMessage) + rw.WriteHeader(err.StatusCode) + rw.Write([]byte(err.PublicMessage)) } -func respondWithOptions(reqID string, rctx *fasthttp.RequestCtx) { +func respondWithOptions(reqID string, rw http.ResponseWriter) { logResponse(reqID, 200, "Respond with options") - rctx.SetStatusCode(200) + rw.WriteHeader(200) } -func respondWithNotModified(reqID string, rctx *fasthttp.RequestCtx) { - logResponse(reqID, 304, "Not modified") - rctx.SetStatusCode(304) +func respondWithNotModified(reqID string, rw http.ResponseWriter) { + logResponse(reqID, 200, "Not modified") + rw.WriteHeader(304) } -func generateRequestID(rctx *fasthttp.RequestCtx) (reqID string) { - reqIDb := rctx.Request.Header.Peek(xRequestIDHeader) +func generateRequestID(rw http.ResponseWriter, r *http.Request) (reqID string) { + reqID = r.Header.Get(xRequestIDHeader) - if len(reqIDb) > 0 && requestIDRe.Match(reqIDb) { - reqID = string(reqIDb) - } else { + if len(reqID) == 0 || !requestIDRe.MatchString(reqID) { reqID, _ = nanoid.Nanoid() } - rctx.Response.Header.Set(xRequestIDHeader, reqID) + rw.Header().Set(xRequestIDHeader, reqID) return } @@ -200,46 +208,17 @@ func prepareAuthHeaderMust() []byte { return authHeaderMust } -func checkSecret(rctx *fasthttp.RequestCtx) bool { +func checkSecret(r *http.Request) bool { if len(conf.Secret) == 0 { return true } return subtle.ConstantTimeCompare( - rctx.Request.Header.Peek("Authorization"), + []byte(r.Header.Get("Authorization")), prepareAuthHeaderMust(), ) == 1 } -func requestCtxToRequest(rctx *fasthttp.RequestCtx) *http.Request { - if r, ok := rctx.UserValue("httpRequest").(*http.Request); ok { - return r - } - - reqURL, _ := url.Parse(rctx.Request.URI().String()) - - r := &http.Request{ - Method: http.MethodGet, // Only GET is supported - URL: reqURL, - Proto: "HTTP/1.0", - ProtoMajor: 1, - ProtoMinor: 0, - Header: make(http.Header), - Body: http.NoBody, - Host: reqURL.Host, - RequestURI: reqURL.RequestURI(), - RemoteAddr: rctx.RemoteAddr().String(), - } - - rctx.Request.Header.VisitAll(func(key, value []byte) { - r.Header.Add(string(key), string(value)) - }) - - rctx.SetUserValue("httpRequest", r) - - return r -} - func (h *httpHandler) lock() { h.sem <- struct{}{} } @@ -248,18 +227,20 @@ func (h *httpHandler) unlock() { <-h.sem } -func (h *httpHandler) ServeHTTP(rctx *fasthttp.RequestCtx) { - reqID := generateRequestID(rctx) +func (h *httpHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { + rw.Header().Set("Server", "imgproxy") + + reqID := generateRequestID(rw, r) defer func() { if rerr := recover(); rerr != nil { if err, ok := rerr.(error); ok { - reportError(err, requestCtxToRequest(rctx)) + reportError(err, r) if ierr, ok := err.(*imgproxyError); ok { - respondWithError(reqID, rctx, ierr) + respondWithError(reqID, rw, ierr) } else { - respondWithError(reqID, rctx, newUnexpectedError(err, 4)) + respondWithError(reqID, rw, newUnexpectedError(err, 4)) } } else { panic(rerr) @@ -267,26 +248,26 @@ func (h *httpHandler) ServeHTTP(rctx *fasthttp.RequestCtx) { } }() - logRequest(reqID, rctx) + logRequest(reqID, r) - writeCORS(rctx) + writeCORS(rw) - if rctx.Request.Header.IsOptions() { - respondWithOptions(reqID, rctx) + if r.Method == http.MethodOptions { + respondWithOptions(reqID, rw) return } - if !rctx.Request.Header.IsGet() { + if r.Method != http.MethodGet { panic(errInvalidMethod) } - if bytes.Equal(rctx.RequestURI(), healthPath) { - rctx.SetStatusCode(200) - rctx.SetBody(imgproxyIsRunningMsg) + if r.URL.RequestURI() == healthPath { + rw.WriteHeader(200) + rw.Write(imgproxyIsRunningMsg) return } - if !checkSecret(rctx) { + if !checkSecret(r) { panic(errInvalidSecret) } @@ -294,7 +275,7 @@ func (h *httpHandler) ServeHTTP(rctx *fasthttp.RequestCtx) { if newRelicEnabled { var newRelicCancel context.CancelFunc - ctx, newRelicCancel = startNewRelicTransaction(ctx, requestCtxToRequest(rctx)) + ctx, newRelicCancel = startNewRelicTransaction(ctx, rw, r) defer newRelicCancel() } @@ -309,7 +290,7 @@ func (h *httpHandler) ServeHTTP(rctx *fasthttp.RequestCtx) { ctx, timeoutCancel := startTimer(ctx, time.Duration(conf.WriteTimeout)*time.Second) defer timeoutCancel() - ctx, err := parsePath(ctx, rctx) + ctx, err := parsePath(ctx, r) if err != nil { panic(err) } @@ -329,13 +310,11 @@ func (h *httpHandler) ServeHTTP(rctx *fasthttp.RequestCtx) { checkTimeout(ctx) if conf.ETagEnabled { - eTag, etagcancel := calcETag(ctx) - defer etagcancel() + eTag := calcETag(ctx) + rw.Header().Set("ETag", eTag) - rctx.Response.Header.SetBytesV("ETag", eTag) - - if bytes.Equal(eTag, rctx.Request.Header.Peek("If-None-Match")) { - respondWithNotModified(reqID, rctx) + if eTag == r.Header.Get("If-None-Match") { + respondWithNotModified(reqID, rw) return } } @@ -356,5 +335,5 @@ func (h *httpHandler) ServeHTTP(rctx *fasthttp.RequestCtx) { checkTimeout(ctx) - respondWithImage(ctx, reqID, rctx, imageData) + respondWithImage(ctx, reqID, r, rw, imageData) } diff --git a/vendor/github.com/klauspost/compress/LICENSE b/vendor/github.com/klauspost/compress/LICENSE deleted file mode 100644 index 74487567..00000000 --- a/vendor/github.com/klauspost/compress/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/klauspost/compress/flate/copy.go b/vendor/github.com/klauspost/compress/flate/copy.go deleted file mode 100644 index a3200a8f..00000000 --- a/vendor/github.com/klauspost/compress/flate/copy.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -// forwardCopy is like the built-in copy function except that it always goes -// forward from the start, even if the dst and src overlap. -// It is equivalent to: -// for i := 0; i < n; i++ { -// mem[dst+i] = mem[src+i] -// } -func forwardCopy(mem []byte, dst, src, n int) { - if dst <= src { - copy(mem[dst:dst+n], mem[src:src+n]) - return - } - for { - if dst >= src+n { - copy(mem[dst:dst+n], mem[src:src+n]) - return - } - // There is some forward overlap. The destination - // will be filled with a repeated pattern of mem[src:src+k]. - // We copy one instance of the pattern here, then repeat. - // Each time around this loop k will double. - k := dst - src - copy(mem[dst:dst+k], mem[src:src+k]) - n -= k - dst += k - } -} diff --git a/vendor/github.com/klauspost/compress/flate/crc32_amd64.go b/vendor/github.com/klauspost/compress/flate/crc32_amd64.go deleted file mode 100644 index 8298d309..00000000 --- a/vendor/github.com/klauspost/compress/flate/crc32_amd64.go +++ /dev/null @@ -1,42 +0,0 @@ -//+build !noasm -//+build !appengine -//+build !gccgo - -// Copyright 2015, Klaus Post, see LICENSE for details. - -package flate - -import ( - "github.com/klauspost/cpuid" -) - -// crc32sse returns a hash for the first 4 bytes of the slice -// len(a) must be >= 4. -//go:noescape -func crc32sse(a []byte) uint32 - -// crc32sseAll calculates hashes for each 4-byte set in a. -// dst must be east len(a) - 4 in size. -// The size is not checked by the assembly. -//go:noescape -func crc32sseAll(a []byte, dst []uint32) - -// matchLenSSE4 returns the number of matching bytes in a and b -// up to length 'max'. Both slices must be at least 'max' -// bytes in size. -// -// TODO: drop the "SSE4" name, since it doesn't use any SSE instructions. -// -//go:noescape -func matchLenSSE4(a, b []byte, max int) int - -// histogram accumulates a histogram of b in h. -// h must be at least 256 entries in length, -// and must be cleared before calling this function. -//go:noescape -func histogram(b []byte, h []int32) - -// Detect SSE 4.2 feature. -func init() { - useSSE42 = cpuid.CPU.SSE42() -} diff --git a/vendor/github.com/klauspost/compress/flate/crc32_amd64.s b/vendor/github.com/klauspost/compress/flate/crc32_amd64.s deleted file mode 100644 index a7994372..00000000 --- a/vendor/github.com/klauspost/compress/flate/crc32_amd64.s +++ /dev/null @@ -1,214 +0,0 @@ -//+build !noasm -//+build !appengine -//+build !gccgo - -// Copyright 2015, Klaus Post, see LICENSE for details. - -// func crc32sse(a []byte) uint32 -TEXT ·crc32sse(SB), 4, $0 - MOVQ a+0(FP), R10 - XORQ BX, BX - - // CRC32 dword (R10), EBX - BYTE $0xF2; BYTE $0x41; BYTE $0x0f - BYTE $0x38; BYTE $0xf1; BYTE $0x1a - - MOVL BX, ret+24(FP) - RET - -// func crc32sseAll(a []byte, dst []uint32) -TEXT ·crc32sseAll(SB), 4, $0 - MOVQ a+0(FP), R8 // R8: src - MOVQ a_len+8(FP), R10 // input length - MOVQ dst+24(FP), R9 // R9: dst - SUBQ $4, R10 - JS end - JZ one_crc - MOVQ R10, R13 - SHRQ $2, R10 // len/4 - ANDQ $3, R13 // len&3 - XORQ BX, BX - ADDQ $1, R13 - TESTQ R10, R10 - JZ rem_loop - -crc_loop: - MOVQ (R8), R11 - XORQ BX, BX - XORQ DX, DX - XORQ DI, DI - MOVQ R11, R12 - SHRQ $8, R11 - MOVQ R12, AX - MOVQ R11, CX - SHRQ $16, R12 - SHRQ $16, R11 - MOVQ R12, SI - - // CRC32 EAX, EBX - BYTE $0xF2; BYTE $0x0f - BYTE $0x38; BYTE $0xf1; BYTE $0xd8 - - // CRC32 ECX, EDX - BYTE $0xF2; BYTE $0x0f - BYTE $0x38; BYTE $0xf1; BYTE $0xd1 - - // CRC32 ESI, EDI - BYTE $0xF2; BYTE $0x0f - BYTE $0x38; BYTE $0xf1; BYTE $0xfe - MOVL BX, (R9) - MOVL DX, 4(R9) - MOVL DI, 8(R9) - - XORQ BX, BX - MOVL R11, AX - - // CRC32 EAX, EBX - BYTE $0xF2; BYTE $0x0f - BYTE $0x38; BYTE $0xf1; BYTE $0xd8 - MOVL BX, 12(R9) - - ADDQ $16, R9 - ADDQ $4, R8 - XORQ BX, BX - SUBQ $1, R10 - JNZ crc_loop - -rem_loop: - MOVL (R8), AX - - // CRC32 EAX, EBX - BYTE $0xF2; BYTE $0x0f - BYTE $0x38; BYTE $0xf1; BYTE $0xd8 - - MOVL BX, (R9) - ADDQ $4, R9 - ADDQ $1, R8 - XORQ BX, BX - SUBQ $1, R13 - JNZ rem_loop - -end: - RET - -one_crc: - MOVQ $1, R13 - XORQ BX, BX - JMP rem_loop - -// func matchLenSSE4(a, b []byte, max int) int -TEXT ·matchLenSSE4(SB), 4, $0 - MOVQ a_base+0(FP), SI - MOVQ b_base+24(FP), DI - MOVQ DI, DX - MOVQ max+48(FP), CX - -cmp8: - // As long as we are 8 or more bytes before the end of max, we can load and - // compare 8 bytes at a time. If those 8 bytes are equal, repeat. - CMPQ CX, $8 - JLT cmp1 - MOVQ (SI), AX - MOVQ (DI), BX - CMPQ AX, BX - JNE bsf - ADDQ $8, SI - ADDQ $8, DI - SUBQ $8, CX - JMP cmp8 - -bsf: - // If those 8 bytes were not equal, XOR the two 8 byte values, and return - // the index of the first byte that differs. The BSF instruction finds the - // least significant 1 bit, the amd64 architecture is little-endian, and - // the shift by 3 converts a bit index to a byte index. - XORQ AX, BX - BSFQ BX, BX - SHRQ $3, BX - ADDQ BX, DI - - // Subtract off &b[0] to convert from &b[ret] to ret, and return. - SUBQ DX, DI - MOVQ DI, ret+56(FP) - RET - -cmp1: - // In the slices' tail, compare 1 byte at a time. - CMPQ CX, $0 - JEQ matchLenEnd - MOVB (SI), AX - MOVB (DI), BX - CMPB AX, BX - JNE matchLenEnd - ADDQ $1, SI - ADDQ $1, DI - SUBQ $1, CX - JMP cmp1 - -matchLenEnd: - // Subtract off &b[0] to convert from &b[ret] to ret, and return. - SUBQ DX, DI - MOVQ DI, ret+56(FP) - RET - -// func histogram(b []byte, h []int32) -TEXT ·histogram(SB), 4, $0 - MOVQ b+0(FP), SI // SI: &b - MOVQ b_len+8(FP), R9 // R9: len(b) - MOVQ h+24(FP), DI // DI: Histogram - MOVQ R9, R8 - SHRQ $3, R8 - JZ hist1 - XORQ R11, R11 - -loop_hist8: - MOVQ (SI), R10 - - MOVB R10, R11 - INCL (DI)(R11*4) - SHRQ $8, R10 - - MOVB R10, R11 - INCL (DI)(R11*4) - SHRQ $8, R10 - - MOVB R10, R11 - INCL (DI)(R11*4) - SHRQ $8, R10 - - MOVB R10, R11 - INCL (DI)(R11*4) - SHRQ $8, R10 - - MOVB R10, R11 - INCL (DI)(R11*4) - SHRQ $8, R10 - - MOVB R10, R11 - INCL (DI)(R11*4) - SHRQ $8, R10 - - MOVB R10, R11 - INCL (DI)(R11*4) - SHRQ $8, R10 - - INCL (DI)(R10*4) - - ADDQ $8, SI - DECQ R8 - JNZ loop_hist8 - -hist1: - ANDQ $7, R9 - JZ end_hist - XORQ R10, R10 - -loop_hist1: - MOVB (SI), R10 - INCL (DI)(R10*4) - INCQ SI - DECQ R9 - JNZ loop_hist1 - -end_hist: - RET diff --git a/vendor/github.com/klauspost/compress/flate/crc32_noasm.go b/vendor/github.com/klauspost/compress/flate/crc32_noasm.go deleted file mode 100644 index dcf43bd5..00000000 --- a/vendor/github.com/klauspost/compress/flate/crc32_noasm.go +++ /dev/null @@ -1,35 +0,0 @@ -//+build !amd64 noasm appengine gccgo - -// Copyright 2015, Klaus Post, see LICENSE for details. - -package flate - -func init() { - useSSE42 = false -} - -// crc32sse should never be called. -func crc32sse(a []byte) uint32 { - panic("no assembler") -} - -// crc32sseAll should never be called. -func crc32sseAll(a []byte, dst []uint32) { - panic("no assembler") -} - -// matchLenSSE4 should never be called. -func matchLenSSE4(a, b []byte, max int) int { - panic("no assembler") - return 0 -} - -// histogram accumulates a histogram of b in h. -// -// len(h) must be >= 256, and h's elements must be all zeroes. -func histogram(b []byte, h []int32) { - h = h[:256] - for _, t := range b { - h[t]++ - } -} diff --git a/vendor/github.com/klauspost/compress/flate/deflate.go b/vendor/github.com/klauspost/compress/flate/deflate.go deleted file mode 100644 index 9e6e7ff0..00000000 --- a/vendor/github.com/klauspost/compress/flate/deflate.go +++ /dev/null @@ -1,1353 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Copyright (c) 2015 Klaus Post -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import ( - "fmt" - "io" - "math" -) - -const ( - NoCompression = 0 - BestSpeed = 1 - BestCompression = 9 - DefaultCompression = -1 - - // HuffmanOnly disables Lempel-Ziv match searching and only performs Huffman - // entropy encoding. This mode is useful in compressing data that has - // already been compressed with an LZ style algorithm (e.g. Snappy or LZ4) - // that lacks an entropy encoder. Compression gains are achieved when - // certain bytes in the input stream occur more frequently than others. - // - // Note that HuffmanOnly produces a compressed output that is - // RFC 1951 compliant. That is, any valid DEFLATE decompressor will - // continue to be able to decompress this output. - HuffmanOnly = -2 - ConstantCompression = HuffmanOnly // compatibility alias. - - logWindowSize = 15 - windowSize = 1 << logWindowSize - windowMask = windowSize - 1 - logMaxOffsetSize = 15 // Standard DEFLATE - minMatchLength = 4 // The smallest match that the compressor looks for - maxMatchLength = 258 // The longest match for the compressor - minOffsetSize = 1 // The shortest offset that makes any sense - - // The maximum number of tokens we put into a single flat block, just too - // stop things from getting too large. - maxFlateBlockTokens = 1 << 14 - maxStoreBlockSize = 65535 - hashBits = 17 // After 17 performance degrades - hashSize = 1 << hashBits - hashMask = (1 << hashBits) - 1 - hashShift = (hashBits + minMatchLength - 1) / minMatchLength - maxHashOffset = 1 << 24 - - skipNever = math.MaxInt32 -) - -var useSSE42 bool - -type compressionLevel struct { - good, lazy, nice, chain, fastSkipHashing, level int -} - -// Compression levels have been rebalanced from zlib deflate defaults -// to give a bigger spread in speed and compression. -// See https://blog.klauspost.com/rebalancing-deflate-compression-levels/ -var levels = []compressionLevel{ - {}, // 0 - // Level 1-4 uses specialized algorithm - values not used - {0, 0, 0, 0, 0, 1}, - {0, 0, 0, 0, 0, 2}, - {0, 0, 0, 0, 0, 3}, - {0, 0, 0, 0, 0, 4}, - // For levels 5-6 we don't bother trying with lazy matches. - // Lazy matching is at least 30% slower, with 1.5% increase. - {6, 0, 12, 8, 12, 5}, - {8, 0, 24, 16, 16, 6}, - // Levels 7-9 use increasingly more lazy matching - // and increasingly stringent conditions for "good enough". - {8, 8, 24, 16, skipNever, 7}, - {10, 16, 24, 64, skipNever, 8}, - {32, 258, 258, 4096, skipNever, 9}, -} - -type compressor struct { - compressionLevel - - w *huffmanBitWriter - bulkHasher func([]byte, []uint32) - - // compression algorithm - fill func(*compressor, []byte) int // copy data to window - step func(*compressor) // process window - sync bool // requesting flush - - // Input hash chains - // hashHead[hashValue] contains the largest inputIndex with the specified hash value - // If hashHead[hashValue] is within the current window, then - // hashPrev[hashHead[hashValue] & windowMask] contains the previous index - // with the same hash value. - chainHead int - hashHead [hashSize]uint32 - hashPrev [windowSize]uint32 - hashOffset int - - // input window: unprocessed data is window[index:windowEnd] - index int - window []byte - windowEnd int - blockStart int // window index where current tokens start - byteAvailable bool // if true, still need to process window[index-1]. - - // queued output tokens - tokens tokens - - // deflate state - length int - offset int - hash uint32 - maxInsertIndex int - err error - ii uint16 // position of last match, intended to overflow to reset. - - snap snappyEnc - hashMatch [maxMatchLength + minMatchLength]uint32 -} - -func (d *compressor) fillDeflate(b []byte) int { - if d.index >= 2*windowSize-(minMatchLength+maxMatchLength) { - // shift the window by windowSize - copy(d.window[:], d.window[windowSize:2*windowSize]) - d.index -= windowSize - d.windowEnd -= windowSize - if d.blockStart >= windowSize { - d.blockStart -= windowSize - } else { - d.blockStart = math.MaxInt32 - } - d.hashOffset += windowSize - if d.hashOffset > maxHashOffset { - delta := d.hashOffset - 1 - d.hashOffset -= delta - d.chainHead -= delta - // Iterate over slices instead of arrays to avoid copying - // the entire table onto the stack (Issue #18625). - for i, v := range d.hashPrev[:] { - if int(v) > delta { - d.hashPrev[i] = uint32(int(v) - delta) - } else { - d.hashPrev[i] = 0 - } - } - for i, v := range d.hashHead[:] { - if int(v) > delta { - d.hashHead[i] = uint32(int(v) - delta) - } else { - d.hashHead[i] = 0 - } - } - } - } - n := copy(d.window[d.windowEnd:], b) - d.windowEnd += n - return n -} - -func (d *compressor) writeBlock(tok tokens, index int, eof bool) error { - if index > 0 || eof { - var window []byte - if d.blockStart <= index { - window = d.window[d.blockStart:index] - } - d.blockStart = index - d.w.writeBlock(tok.tokens[:tok.n], eof, window) - return d.w.err - } - return nil -} - -// writeBlockSkip writes the current block and uses the number of tokens -// to determine if the block should be stored on no matches, or -// only huffman encoded. -func (d *compressor) writeBlockSkip(tok tokens, index int, eof bool) error { - if index > 0 || eof { - if d.blockStart <= index { - window := d.window[d.blockStart:index] - // If we removed less than a 64th of all literals - // we huffman compress the block. - if int(tok.n) > len(window)-int(tok.n>>6) { - d.w.writeBlockHuff(eof, window) - } else { - // Write a dynamic huffman block. - d.w.writeBlockDynamic(tok.tokens[:tok.n], eof, window) - } - } else { - d.w.writeBlock(tok.tokens[:tok.n], eof, nil) - } - d.blockStart = index - return d.w.err - } - return nil -} - -// fillWindow will fill the current window with the supplied -// dictionary and calculate all hashes. -// This is much faster than doing a full encode. -// Should only be used after a start/reset. -func (d *compressor) fillWindow(b []byte) { - // Do not fill window if we are in store-only mode, - // use constant or Snappy compression. - switch d.compressionLevel.level { - case 0, 1, 2: - return - } - // If we are given too much, cut it. - if len(b) > windowSize { - b = b[len(b)-windowSize:] - } - // Add all to window. - n := copy(d.window[d.windowEnd:], b) - - // Calculate 256 hashes at the time (more L1 cache hits) - loops := (n + 256 - minMatchLength) / 256 - for j := 0; j < loops; j++ { - startindex := j * 256 - end := startindex + 256 + minMatchLength - 1 - if end > n { - end = n - } - tocheck := d.window[startindex:end] - dstSize := len(tocheck) - minMatchLength + 1 - - if dstSize <= 0 { - continue - } - - dst := d.hashMatch[:dstSize] - d.bulkHasher(tocheck, dst) - var newH uint32 - for i, val := range dst { - di := i + startindex - newH = val & hashMask - // Get previous value with the same hash. - // Our chain should point to the previous value. - d.hashPrev[di&windowMask] = d.hashHead[newH] - // Set the head of the hash chain to us. - d.hashHead[newH] = uint32(di + d.hashOffset) - } - d.hash = newH - } - // Update window information. - d.windowEnd += n - d.index = n -} - -// Try to find a match starting at index whose length is greater than prevSize. -// We only look at chainCount possibilities before giving up. -// pos = d.index, prevHead = d.chainHead-d.hashOffset, prevLength=minMatchLength-1, lookahead -func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) { - minMatchLook := maxMatchLength - if lookahead < minMatchLook { - minMatchLook = lookahead - } - - win := d.window[0 : pos+minMatchLook] - - // We quit when we get a match that's at least nice long - nice := len(win) - pos - if d.nice < nice { - nice = d.nice - } - - // If we've got a match that's good enough, only look in 1/4 the chain. - tries := d.chain - length = prevLength - if length >= d.good { - tries >>= 2 - } - - wEnd := win[pos+length] - wPos := win[pos:] - minIndex := pos - windowSize - - for i := prevHead; tries > 0; tries-- { - if wEnd == win[i+length] { - n := matchLen(win[i:], wPos, minMatchLook) - - if n > length && (n > minMatchLength || pos-i <= 4096) { - length = n - offset = pos - i - ok = true - if n >= nice { - // The match is good enough that we don't try to find a better one. - break - } - wEnd = win[pos+n] - } - } - if i == minIndex { - // hashPrev[i & windowMask] has already been overwritten, so stop now. - break - } - i = int(d.hashPrev[i&windowMask]) - d.hashOffset - if i < minIndex || i < 0 { - break - } - } - return -} - -// Try to find a match starting at index whose length is greater than prevSize. -// We only look at chainCount possibilities before giving up. -// pos = d.index, prevHead = d.chainHead-d.hashOffset, prevLength=minMatchLength-1, lookahead -func (d *compressor) findMatchSSE(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) { - minMatchLook := maxMatchLength - if lookahead < minMatchLook { - minMatchLook = lookahead - } - - win := d.window[0 : pos+minMatchLook] - - // We quit when we get a match that's at least nice long - nice := len(win) - pos - if d.nice < nice { - nice = d.nice - } - - // If we've got a match that's good enough, only look in 1/4 the chain. - tries := d.chain - length = prevLength - if length >= d.good { - tries >>= 2 - } - - wEnd := win[pos+length] - wPos := win[pos:] - minIndex := pos - windowSize - - for i := prevHead; tries > 0; tries-- { - if wEnd == win[i+length] { - n := matchLenSSE4(win[i:], wPos, minMatchLook) - - if n > length && (n > minMatchLength || pos-i <= 4096) { - length = n - offset = pos - i - ok = true - if n >= nice { - // The match is good enough that we don't try to find a better one. - break - } - wEnd = win[pos+n] - } - } - if i == minIndex { - // hashPrev[i & windowMask] has already been overwritten, so stop now. - break - } - i = int(d.hashPrev[i&windowMask]) - d.hashOffset - if i < minIndex || i < 0 { - break - } - } - return -} - -func (d *compressor) writeStoredBlock(buf []byte) error { - if d.w.writeStoredHeader(len(buf), false); d.w.err != nil { - return d.w.err - } - d.w.writeBytes(buf) - return d.w.err -} - -const hashmul = 0x1e35a7bd - -// hash4 returns a hash representation of the first 4 bytes -// of the supplied slice. -// The caller must ensure that len(b) >= 4. -func hash4(b []byte) uint32 { - return ((uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24) * hashmul) >> (32 - hashBits) -} - -// bulkHash4 will compute hashes using the same -// algorithm as hash4 -func bulkHash4(b []byte, dst []uint32) { - if len(b) < minMatchLength { - return - } - hb := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 - dst[0] = (hb * hashmul) >> (32 - hashBits) - end := len(b) - minMatchLength + 1 - for i := 1; i < end; i++ { - hb = (hb << 8) | uint32(b[i+3]) - dst[i] = (hb * hashmul) >> (32 - hashBits) - } -} - -// matchLen returns the number of matching bytes in a and b -// up to length 'max'. Both slices must be at least 'max' -// bytes in size. -func matchLen(a, b []byte, max int) int { - a = a[:max] - b = b[:len(a)] - for i, av := range a { - if b[i] != av { - return i - } - } - return max -} - -func (d *compressor) initDeflate() { - d.window = make([]byte, 2*windowSize) - d.hashOffset = 1 - d.length = minMatchLength - 1 - d.offset = 0 - d.byteAvailable = false - d.index = 0 - d.hash = 0 - d.chainHead = -1 - d.bulkHasher = bulkHash4 - if useSSE42 { - d.bulkHasher = crc32sseAll - } -} - -// Assumes that d.fastSkipHashing != skipNever, -// otherwise use deflateLazy -func (d *compressor) deflate() { - - // Sanity enables additional runtime tests. - // It's intended to be used during development - // to supplement the currently ad-hoc unit tests. - const sanity = false - - if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync { - return - } - - d.maxInsertIndex = d.windowEnd - (minMatchLength - 1) - if d.index < d.maxInsertIndex { - d.hash = hash4(d.window[d.index : d.index+minMatchLength]) - } - - for { - if sanity && d.index > d.windowEnd { - panic("index > windowEnd") - } - lookahead := d.windowEnd - d.index - if lookahead < minMatchLength+maxMatchLength { - if !d.sync { - return - } - if sanity && d.index > d.windowEnd { - panic("index > windowEnd") - } - if lookahead == 0 { - if d.tokens.n > 0 { - if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - return - } - } - if d.index < d.maxInsertIndex { - // Update the hash - d.hash = hash4(d.window[d.index : d.index+minMatchLength]) - ch := d.hashHead[d.hash&hashMask] - d.chainHead = int(ch) - d.hashPrev[d.index&windowMask] = ch - d.hashHead[d.hash&hashMask] = uint32(d.index + d.hashOffset) - } - d.length = minMatchLength - 1 - d.offset = 0 - minIndex := d.index - windowSize - if minIndex < 0 { - minIndex = 0 - } - - if d.chainHead-d.hashOffset >= minIndex && lookahead > minMatchLength-1 { - if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok { - d.length = newLength - d.offset = newOffset - } - } - if d.length >= minMatchLength { - d.ii = 0 - // There was a match at the previous step, and the current match is - // not better. Output the previous match. - // "d.length-3" should NOT be "d.length-minMatchLength", since the format always assume 3 - d.tokens.tokens[d.tokens.n] = matchToken(uint32(d.length-3), uint32(d.offset-minOffsetSize)) - d.tokens.n++ - // Insert in the hash table all strings up to the end of the match. - // index and index-1 are already inserted. If there is not enough - // lookahead, the last two strings are not inserted into the hash - // table. - if d.length <= d.fastSkipHashing { - var newIndex int - newIndex = d.index + d.length - // Calculate missing hashes - end := newIndex - if end > d.maxInsertIndex { - end = d.maxInsertIndex - } - end += minMatchLength - 1 - startindex := d.index + 1 - if startindex > d.maxInsertIndex { - startindex = d.maxInsertIndex - } - tocheck := d.window[startindex:end] - dstSize := len(tocheck) - minMatchLength + 1 - if dstSize > 0 { - dst := d.hashMatch[:dstSize] - bulkHash4(tocheck, dst) - var newH uint32 - for i, val := range dst { - di := i + startindex - newH = val & hashMask - // Get previous value with the same hash. - // Our chain should point to the previous value. - d.hashPrev[di&windowMask] = d.hashHead[newH] - // Set the head of the hash chain to us. - d.hashHead[newH] = uint32(di + d.hashOffset) - } - d.hash = newH - } - d.index = newIndex - } else { - // For matches this long, we don't bother inserting each individual - // item into the table. - d.index += d.length - if d.index < d.maxInsertIndex { - d.hash = hash4(d.window[d.index : d.index+minMatchLength]) - } - } - if d.tokens.n == maxFlateBlockTokens { - // The block includes the current character - if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - } else { - d.ii++ - end := d.index + int(d.ii>>uint(d.fastSkipHashing)) + 1 - if end > d.windowEnd { - end = d.windowEnd - } - for i := d.index; i < end; i++ { - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[i])) - d.tokens.n++ - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlockSkip(d.tokens, i+1, false); d.err != nil { - return - } - d.tokens.n = 0 - } - } - d.index = end - } - } -} - -// deflateLazy is the same as deflate, but with d.fastSkipHashing == skipNever, -// meaning it always has lazy matching on. -func (d *compressor) deflateLazy() { - // Sanity enables additional runtime tests. - // It's intended to be used during development - // to supplement the currently ad-hoc unit tests. - const sanity = false - - if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync { - return - } - - d.maxInsertIndex = d.windowEnd - (minMatchLength - 1) - if d.index < d.maxInsertIndex { - d.hash = hash4(d.window[d.index : d.index+minMatchLength]) - } - - for { - if sanity && d.index > d.windowEnd { - panic("index > windowEnd") - } - lookahead := d.windowEnd - d.index - if lookahead < minMatchLength+maxMatchLength { - if !d.sync { - return - } - if sanity && d.index > d.windowEnd { - panic("index > windowEnd") - } - if lookahead == 0 { - // Flush current output block if any. - if d.byteAvailable { - // There is still one pending token that needs to be flushed - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) - d.tokens.n++ - d.byteAvailable = false - } - if d.tokens.n > 0 { - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - return - } - } - if d.index < d.maxInsertIndex { - // Update the hash - d.hash = hash4(d.window[d.index : d.index+minMatchLength]) - ch := d.hashHead[d.hash&hashMask] - d.chainHead = int(ch) - d.hashPrev[d.index&windowMask] = ch - d.hashHead[d.hash&hashMask] = uint32(d.index + d.hashOffset) - } - prevLength := d.length - prevOffset := d.offset - d.length = minMatchLength - 1 - d.offset = 0 - minIndex := d.index - windowSize - if minIndex < 0 { - minIndex = 0 - } - - if d.chainHead-d.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy { - if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok { - d.length = newLength - d.offset = newOffset - } - } - if prevLength >= minMatchLength && d.length <= prevLength { - // There was a match at the previous step, and the current match is - // not better. Output the previous match. - d.tokens.tokens[d.tokens.n] = matchToken(uint32(prevLength-3), uint32(prevOffset-minOffsetSize)) - d.tokens.n++ - - // Insert in the hash table all strings up to the end of the match. - // index and index-1 are already inserted. If there is not enough - // lookahead, the last two strings are not inserted into the hash - // table. - var newIndex int - newIndex = d.index + prevLength - 1 - // Calculate missing hashes - end := newIndex - if end > d.maxInsertIndex { - end = d.maxInsertIndex - } - end += minMatchLength - 1 - startindex := d.index + 1 - if startindex > d.maxInsertIndex { - startindex = d.maxInsertIndex - } - tocheck := d.window[startindex:end] - dstSize := len(tocheck) - minMatchLength + 1 - if dstSize > 0 { - dst := d.hashMatch[:dstSize] - bulkHash4(tocheck, dst) - var newH uint32 - for i, val := range dst { - di := i + startindex - newH = val & hashMask - // Get previous value with the same hash. - // Our chain should point to the previous value. - d.hashPrev[di&windowMask] = d.hashHead[newH] - // Set the head of the hash chain to us. - d.hashHead[newH] = uint32(di + d.hashOffset) - } - d.hash = newH - } - - d.index = newIndex - d.byteAvailable = false - d.length = minMatchLength - 1 - if d.tokens.n == maxFlateBlockTokens { - // The block includes the current character - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - } else { - // Reset, if we got a match this run. - if d.length >= minMatchLength { - d.ii = 0 - } - // We have a byte waiting. Emit it. - if d.byteAvailable { - d.ii++ - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) - d.tokens.n++ - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - d.index++ - - // If we have a long run of no matches, skip additional bytes - // Resets when d.ii overflows after 64KB. - if d.ii > 31 { - n := int(d.ii >> 5) - for j := 0; j < n; j++ { - if d.index >= d.windowEnd-1 { - break - } - - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) - d.tokens.n++ - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - d.index++ - } - // Flush last byte - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) - d.tokens.n++ - d.byteAvailable = false - // d.length = minMatchLength - 1 // not needed, since d.ii is reset above, so it should never be > minMatchLength - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - } - } else { - d.index++ - d.byteAvailable = true - } - } - } -} - -// Assumes that d.fastSkipHashing != skipNever, -// otherwise use deflateLazySSE -func (d *compressor) deflateSSE() { - - // Sanity enables additional runtime tests. - // It's intended to be used during development - // to supplement the currently ad-hoc unit tests. - const sanity = false - - if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync { - return - } - - d.maxInsertIndex = d.windowEnd - (minMatchLength - 1) - if d.index < d.maxInsertIndex { - d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask - } - - for { - if sanity && d.index > d.windowEnd { - panic("index > windowEnd") - } - lookahead := d.windowEnd - d.index - if lookahead < minMatchLength+maxMatchLength { - if !d.sync { - return - } - if sanity && d.index > d.windowEnd { - panic("index > windowEnd") - } - if lookahead == 0 { - if d.tokens.n > 0 { - if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - return - } - } - if d.index < d.maxInsertIndex { - // Update the hash - d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask - ch := d.hashHead[d.hash] - d.chainHead = int(ch) - d.hashPrev[d.index&windowMask] = ch - d.hashHead[d.hash] = uint32(d.index + d.hashOffset) - } - d.length = minMatchLength - 1 - d.offset = 0 - minIndex := d.index - windowSize - if minIndex < 0 { - minIndex = 0 - } - - if d.chainHead-d.hashOffset >= minIndex && lookahead > minMatchLength-1 { - if newLength, newOffset, ok := d.findMatchSSE(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok { - d.length = newLength - d.offset = newOffset - } - } - if d.length >= minMatchLength { - d.ii = 0 - // There was a match at the previous step, and the current match is - // not better. Output the previous match. - // "d.length-3" should NOT be "d.length-minMatchLength", since the format always assume 3 - d.tokens.tokens[d.tokens.n] = matchToken(uint32(d.length-3), uint32(d.offset-minOffsetSize)) - d.tokens.n++ - // Insert in the hash table all strings up to the end of the match. - // index and index-1 are already inserted. If there is not enough - // lookahead, the last two strings are not inserted into the hash - // table. - if d.length <= d.fastSkipHashing { - var newIndex int - newIndex = d.index + d.length - // Calculate missing hashes - end := newIndex - if end > d.maxInsertIndex { - end = d.maxInsertIndex - } - end += minMatchLength - 1 - startindex := d.index + 1 - if startindex > d.maxInsertIndex { - startindex = d.maxInsertIndex - } - tocheck := d.window[startindex:end] - dstSize := len(tocheck) - minMatchLength + 1 - if dstSize > 0 { - dst := d.hashMatch[:dstSize] - - crc32sseAll(tocheck, dst) - var newH uint32 - for i, val := range dst { - di := i + startindex - newH = val & hashMask - // Get previous value with the same hash. - // Our chain should point to the previous value. - d.hashPrev[di&windowMask] = d.hashHead[newH] - // Set the head of the hash chain to us. - d.hashHead[newH] = uint32(di + d.hashOffset) - } - d.hash = newH - } - d.index = newIndex - } else { - // For matches this long, we don't bother inserting each individual - // item into the table. - d.index += d.length - if d.index < d.maxInsertIndex { - d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask - } - } - if d.tokens.n == maxFlateBlockTokens { - // The block includes the current character - if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - } else { - d.ii++ - end := d.index + int(d.ii>>5) + 1 - if end > d.windowEnd { - end = d.windowEnd - } - for i := d.index; i < end; i++ { - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[i])) - d.tokens.n++ - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlockSkip(d.tokens, i+1, false); d.err != nil { - return - } - d.tokens.n = 0 - } - } - d.index = end - } - } -} - -// deflateLazy is the same as deflate, but with d.fastSkipHashing == skipNever, -// meaning it always has lazy matching on. -func (d *compressor) deflateLazySSE() { - // Sanity enables additional runtime tests. - // It's intended to be used during development - // to supplement the currently ad-hoc unit tests. - const sanity = false - - if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync { - return - } - - d.maxInsertIndex = d.windowEnd - (minMatchLength - 1) - if d.index < d.maxInsertIndex { - d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask - } - - for { - if sanity && d.index > d.windowEnd { - panic("index > windowEnd") - } - lookahead := d.windowEnd - d.index - if lookahead < minMatchLength+maxMatchLength { - if !d.sync { - return - } - if sanity && d.index > d.windowEnd { - panic("index > windowEnd") - } - if lookahead == 0 { - // Flush current output block if any. - if d.byteAvailable { - // There is still one pending token that needs to be flushed - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) - d.tokens.n++ - d.byteAvailable = false - } - if d.tokens.n > 0 { - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - return - } - } - if d.index < d.maxInsertIndex { - // Update the hash - d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask - ch := d.hashHead[d.hash] - d.chainHead = int(ch) - d.hashPrev[d.index&windowMask] = ch - d.hashHead[d.hash] = uint32(d.index + d.hashOffset) - } - prevLength := d.length - prevOffset := d.offset - d.length = minMatchLength - 1 - d.offset = 0 - minIndex := d.index - windowSize - if minIndex < 0 { - minIndex = 0 - } - - if d.chainHead-d.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy { - if newLength, newOffset, ok := d.findMatchSSE(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok { - d.length = newLength - d.offset = newOffset - } - } - if prevLength >= minMatchLength && d.length <= prevLength { - // There was a match at the previous step, and the current match is - // not better. Output the previous match. - d.tokens.tokens[d.tokens.n] = matchToken(uint32(prevLength-3), uint32(prevOffset-minOffsetSize)) - d.tokens.n++ - - // Insert in the hash table all strings up to the end of the match. - // index and index-1 are already inserted. If there is not enough - // lookahead, the last two strings are not inserted into the hash - // table. - var newIndex int - newIndex = d.index + prevLength - 1 - // Calculate missing hashes - end := newIndex - if end > d.maxInsertIndex { - end = d.maxInsertIndex - } - end += minMatchLength - 1 - startindex := d.index + 1 - if startindex > d.maxInsertIndex { - startindex = d.maxInsertIndex - } - tocheck := d.window[startindex:end] - dstSize := len(tocheck) - minMatchLength + 1 - if dstSize > 0 { - dst := d.hashMatch[:dstSize] - crc32sseAll(tocheck, dst) - var newH uint32 - for i, val := range dst { - di := i + startindex - newH = val & hashMask - // Get previous value with the same hash. - // Our chain should point to the previous value. - d.hashPrev[di&windowMask] = d.hashHead[newH] - // Set the head of the hash chain to us. - d.hashHead[newH] = uint32(di + d.hashOffset) - } - d.hash = newH - } - - d.index = newIndex - d.byteAvailable = false - d.length = minMatchLength - 1 - if d.tokens.n == maxFlateBlockTokens { - // The block includes the current character - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - } else { - // Reset, if we got a match this run. - if d.length >= minMatchLength { - d.ii = 0 - } - // We have a byte waiting. Emit it. - if d.byteAvailable { - d.ii++ - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) - d.tokens.n++ - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - d.index++ - - // If we have a long run of no matches, skip additional bytes - // Resets when d.ii overflows after 64KB. - if d.ii > 31 { - n := int(d.ii >> 6) - for j := 0; j < n; j++ { - if d.index >= d.windowEnd-1 { - break - } - - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) - d.tokens.n++ - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - d.index++ - } - // Flush last byte - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) - d.tokens.n++ - d.byteAvailable = false - // d.length = minMatchLength - 1 // not needed, since d.ii is reset above, so it should never be > minMatchLength - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - } - } else { - d.index++ - d.byteAvailable = true - } - } - } -} - -func (d *compressor) store() { - if d.windowEnd > 0 && (d.windowEnd == maxStoreBlockSize || d.sync) { - d.err = d.writeStoredBlock(d.window[:d.windowEnd]) - d.windowEnd = 0 - } -} - -// fillWindow will fill the buffer with data for huffman-only compression. -// The number of bytes copied is returned. -func (d *compressor) fillBlock(b []byte) int { - n := copy(d.window[d.windowEnd:], b) - d.windowEnd += n - return n -} - -// storeHuff will compress and store the currently added data, -// if enough has been accumulated or we at the end of the stream. -// Any error that occurred will be in d.err -func (d *compressor) storeHuff() { - if d.windowEnd < len(d.window) && !d.sync || d.windowEnd == 0 { - return - } - d.w.writeBlockHuff(false, d.window[:d.windowEnd]) - d.err = d.w.err - d.windowEnd = 0 -} - -// storeHuff will compress and store the currently added data, -// if enough has been accumulated or we at the end of the stream. -// Any error that occurred will be in d.err -func (d *compressor) storeSnappy() { - // We only compress if we have maxStoreBlockSize. - if d.windowEnd < maxStoreBlockSize { - if !d.sync { - return - } - // Handle extremely small sizes. - if d.windowEnd < 128 { - if d.windowEnd == 0 { - return - } - if d.windowEnd <= 32 { - d.err = d.writeStoredBlock(d.window[:d.windowEnd]) - d.tokens.n = 0 - d.windowEnd = 0 - } else { - d.w.writeBlockHuff(false, d.window[:d.windowEnd]) - d.err = d.w.err - } - d.tokens.n = 0 - d.windowEnd = 0 - d.snap.Reset() - return - } - } - - d.snap.Encode(&d.tokens, d.window[:d.windowEnd]) - // If we made zero matches, store the block as is. - if int(d.tokens.n) == d.windowEnd { - d.err = d.writeStoredBlock(d.window[:d.windowEnd]) - // If we removed less than 1/16th, huffman compress the block. - } else if int(d.tokens.n) > d.windowEnd-(d.windowEnd>>4) { - d.w.writeBlockHuff(false, d.window[:d.windowEnd]) - d.err = d.w.err - } else { - d.w.writeBlockDynamic(d.tokens.tokens[:d.tokens.n], false, d.window[:d.windowEnd]) - d.err = d.w.err - } - d.tokens.n = 0 - d.windowEnd = 0 -} - -// write will add input byte to the stream. -// Unless an error occurs all bytes will be consumed. -func (d *compressor) write(b []byte) (n int, err error) { - if d.err != nil { - return 0, d.err - } - n = len(b) - for len(b) > 0 { - d.step(d) - b = b[d.fill(d, b):] - if d.err != nil { - return 0, d.err - } - } - return n, d.err -} - -func (d *compressor) syncFlush() error { - d.sync = true - if d.err != nil { - return d.err - } - d.step(d) - if d.err == nil { - d.w.writeStoredHeader(0, false) - d.w.flush() - d.err = d.w.err - } - d.sync = false - return d.err -} - -func (d *compressor) init(w io.Writer, level int) (err error) { - d.w = newHuffmanBitWriter(w) - - switch { - case level == NoCompression: - d.window = make([]byte, maxStoreBlockSize) - d.fill = (*compressor).fillBlock - d.step = (*compressor).store - case level == ConstantCompression: - d.window = make([]byte, maxStoreBlockSize) - d.fill = (*compressor).fillBlock - d.step = (*compressor).storeHuff - case level >= 1 && level <= 4: - d.snap = newSnappy(level) - d.window = make([]byte, maxStoreBlockSize) - d.fill = (*compressor).fillBlock - d.step = (*compressor).storeSnappy - case level == DefaultCompression: - level = 5 - fallthrough - case 5 <= level && level <= 9: - d.compressionLevel = levels[level] - d.initDeflate() - d.fill = (*compressor).fillDeflate - if d.fastSkipHashing == skipNever { - if useSSE42 { - d.step = (*compressor).deflateLazySSE - } else { - d.step = (*compressor).deflateLazy - } - } else { - if useSSE42 { - d.step = (*compressor).deflateSSE - } else { - d.step = (*compressor).deflate - - } - } - default: - return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level) - } - return nil -} - -// reset the state of the compressor. -func (d *compressor) reset(w io.Writer) { - d.w.reset(w) - d.sync = false - d.err = nil - // We only need to reset a few things for Snappy. - if d.snap != nil { - d.snap.Reset() - d.windowEnd = 0 - d.tokens.n = 0 - return - } - switch d.compressionLevel.chain { - case 0: - // level was NoCompression or ConstantCompresssion. - d.windowEnd = 0 - default: - d.chainHead = -1 - for i := range d.hashHead { - d.hashHead[i] = 0 - } - for i := range d.hashPrev { - d.hashPrev[i] = 0 - } - d.hashOffset = 1 - d.index, d.windowEnd = 0, 0 - d.blockStart, d.byteAvailable = 0, false - d.tokens.n = 0 - d.length = minMatchLength - 1 - d.offset = 0 - d.hash = 0 - d.ii = 0 - d.maxInsertIndex = 0 - } -} - -func (d *compressor) close() error { - if d.err != nil { - return d.err - } - d.sync = true - d.step(d) - if d.err != nil { - return d.err - } - if d.w.writeStoredHeader(0, true); d.w.err != nil { - return d.w.err - } - d.w.flush() - return d.w.err -} - -// NewWriter returns a new Writer compressing data at the given level. -// Following zlib, levels range from 1 (BestSpeed) to 9 (BestCompression); -// higher levels typically run slower but compress more. -// Level 0 (NoCompression) does not attempt any compression; it only adds the -// necessary DEFLATE framing. -// Level -1 (DefaultCompression) uses the default compression level. -// Level -2 (ConstantCompression) will use Huffman compression only, giving -// a very fast compression for all types of input, but sacrificing considerable -// compression efficiency. -// -// If level is in the range [-2, 9] then the error returned will be nil. -// Otherwise the error returned will be non-nil. -func NewWriter(w io.Writer, level int) (*Writer, error) { - var dw Writer - if err := dw.d.init(w, level); err != nil { - return nil, err - } - return &dw, nil -} - -// NewWriterDict is like NewWriter but initializes the new -// Writer with a preset dictionary. The returned Writer behaves -// as if the dictionary had been written to it without producing -// any compressed output. The compressed data written to w -// can only be decompressed by a Reader initialized with the -// same dictionary. -func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) { - dw := &dictWriter{w} - zw, err := NewWriter(dw, level) - if err != nil { - return nil, err - } - zw.d.fillWindow(dict) - zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method. - return zw, err -} - -type dictWriter struct { - w io.Writer -} - -func (w *dictWriter) Write(b []byte) (n int, err error) { - return w.w.Write(b) -} - -// A Writer takes data written to it and writes the compressed -// form of that data to an underlying writer (see NewWriter). -type Writer struct { - d compressor - dict []byte -} - -// Write writes data to w, which will eventually write the -// compressed form of data to its underlying writer. -func (w *Writer) Write(data []byte) (n int, err error) { - return w.d.write(data) -} - -// Flush flushes any pending data to the underlying writer. -// It is useful mainly in compressed network protocols, to ensure that -// a remote reader has enough data to reconstruct a packet. -// Flush does not return until the data has been written. -// Calling Flush when there is no pending data still causes the Writer -// to emit a sync marker of at least 4 bytes. -// If the underlying writer returns an error, Flush returns that error. -// -// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH. -func (w *Writer) Flush() error { - // For more about flushing: - // http://www.bolet.org/~pornin/deflate-flush.html - return w.d.syncFlush() -} - -// Close flushes and closes the writer. -func (w *Writer) Close() error { - return w.d.close() -} - -// Reset discards the writer's state and makes it equivalent to -// the result of NewWriter or NewWriterDict called with dst -// and w's level and dictionary. -func (w *Writer) Reset(dst io.Writer) { - if dw, ok := w.d.w.writer.(*dictWriter); ok { - // w was created with NewWriterDict - dw.w = dst - w.d.reset(dw) - w.d.fillWindow(w.dict) - } else { - // w was created with NewWriter - w.d.reset(dst) - } -} - -// ResetDict discards the writer's state and makes it equivalent to -// the result of NewWriter or NewWriterDict called with dst -// and w's level, but sets a specific dictionary. -func (w *Writer) ResetDict(dst io.Writer, dict []byte) { - w.dict = dict - w.d.reset(dst) - w.d.fillWindow(w.dict) -} diff --git a/vendor/github.com/klauspost/compress/flate/dict_decoder.go b/vendor/github.com/klauspost/compress/flate/dict_decoder.go deleted file mode 100644 index 71c75a06..00000000 --- a/vendor/github.com/klauspost/compress/flate/dict_decoder.go +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -// dictDecoder implements the LZ77 sliding dictionary as used in decompression. -// LZ77 decompresses data through sequences of two forms of commands: -// -// * Literal insertions: Runs of one or more symbols are inserted into the data -// stream as is. This is accomplished through the writeByte method for a -// single symbol, or combinations of writeSlice/writeMark for multiple symbols. -// Any valid stream must start with a literal insertion if no preset dictionary -// is used. -// -// * Backward copies: Runs of one or more symbols are copied from previously -// emitted data. Backward copies come as the tuple (dist, length) where dist -// determines how far back in the stream to copy from and length determines how -// many bytes to copy. Note that it is valid for the length to be greater than -// the distance. Since LZ77 uses forward copies, that situation is used to -// perform a form of run-length encoding on repeated runs of symbols. -// The writeCopy and tryWriteCopy are used to implement this command. -// -// For performance reasons, this implementation performs little to no sanity -// checks about the arguments. As such, the invariants documented for each -// method call must be respected. -type dictDecoder struct { - hist []byte // Sliding window history - - // Invariant: 0 <= rdPos <= wrPos <= len(hist) - wrPos int // Current output position in buffer - rdPos int // Have emitted hist[:rdPos] already - full bool // Has a full window length been written yet? -} - -// init initializes dictDecoder to have a sliding window dictionary of the given -// size. If a preset dict is provided, it will initialize the dictionary with -// the contents of dict. -func (dd *dictDecoder) init(size int, dict []byte) { - *dd = dictDecoder{hist: dd.hist} - - if cap(dd.hist) < size { - dd.hist = make([]byte, size) - } - dd.hist = dd.hist[:size] - - if len(dict) > len(dd.hist) { - dict = dict[len(dict)-len(dd.hist):] - } - dd.wrPos = copy(dd.hist, dict) - if dd.wrPos == len(dd.hist) { - dd.wrPos = 0 - dd.full = true - } - dd.rdPos = dd.wrPos -} - -// histSize reports the total amount of historical data in the dictionary. -func (dd *dictDecoder) histSize() int { - if dd.full { - return len(dd.hist) - } - return dd.wrPos -} - -// availRead reports the number of bytes that can be flushed by readFlush. -func (dd *dictDecoder) availRead() int { - return dd.wrPos - dd.rdPos -} - -// availWrite reports the available amount of output buffer space. -func (dd *dictDecoder) availWrite() int { - return len(dd.hist) - dd.wrPos -} - -// writeSlice returns a slice of the available buffer to write data to. -// -// This invariant will be kept: len(s) <= availWrite() -func (dd *dictDecoder) writeSlice() []byte { - return dd.hist[dd.wrPos:] -} - -// writeMark advances the writer pointer by cnt. -// -// This invariant must be kept: 0 <= cnt <= availWrite() -func (dd *dictDecoder) writeMark(cnt int) { - dd.wrPos += cnt -} - -// writeByte writes a single byte to the dictionary. -// -// This invariant must be kept: 0 < availWrite() -func (dd *dictDecoder) writeByte(c byte) { - dd.hist[dd.wrPos] = c - dd.wrPos++ -} - -// writeCopy copies a string at a given (dist, length) to the output. -// This returns the number of bytes copied and may be less than the requested -// length if the available space in the output buffer is too small. -// -// This invariant must be kept: 0 < dist <= histSize() -func (dd *dictDecoder) writeCopy(dist, length int) int { - dstBase := dd.wrPos - dstPos := dstBase - srcPos := dstPos - dist - endPos := dstPos + length - if endPos > len(dd.hist) { - endPos = len(dd.hist) - } - - // Copy non-overlapping section after destination position. - // - // This section is non-overlapping in that the copy length for this section - // is always less than or equal to the backwards distance. This can occur - // if a distance refers to data that wraps-around in the buffer. - // Thus, a backwards copy is performed here; that is, the exact bytes in - // the source prior to the copy is placed in the destination. - if srcPos < 0 { - srcPos += len(dd.hist) - dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:]) - srcPos = 0 - } - - // Copy possibly overlapping section before destination position. - // - // This section can overlap if the copy length for this section is larger - // than the backwards distance. This is allowed by LZ77 so that repeated - // strings can be succinctly represented using (dist, length) pairs. - // Thus, a forwards copy is performed here; that is, the bytes copied is - // possibly dependent on the resulting bytes in the destination as the copy - // progresses along. This is functionally equivalent to the following: - // - // for i := 0; i < endPos-dstPos; i++ { - // dd.hist[dstPos+i] = dd.hist[srcPos+i] - // } - // dstPos = endPos - // - for dstPos < endPos { - dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos]) - } - - dd.wrPos = dstPos - return dstPos - dstBase -} - -// tryWriteCopy tries to copy a string at a given (distance, length) to the -// output. This specialized version is optimized for short distances. -// -// This method is designed to be inlined for performance reasons. -// -// This invariant must be kept: 0 < dist <= histSize() -func (dd *dictDecoder) tryWriteCopy(dist, length int) int { - dstPos := dd.wrPos - endPos := dstPos + length - if dstPos < dist || endPos > len(dd.hist) { - return 0 - } - dstBase := dstPos - srcPos := dstPos - dist - - // Copy possibly overlapping section before destination position. -loop: - dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos]) - if dstPos < endPos { - goto loop // Avoid for-loop so that this function can be inlined - } - - dd.wrPos = dstPos - return dstPos - dstBase -} - -// readFlush returns a slice of the historical buffer that is ready to be -// emitted to the user. The data returned by readFlush must be fully consumed -// before calling any other dictDecoder methods. -func (dd *dictDecoder) readFlush() []byte { - toRead := dd.hist[dd.rdPos:dd.wrPos] - dd.rdPos = dd.wrPos - if dd.wrPos == len(dd.hist) { - dd.wrPos, dd.rdPos = 0, 0 - dd.full = true - } - return toRead -} diff --git a/vendor/github.com/klauspost/compress/flate/gen.go b/vendor/github.com/klauspost/compress/flate/gen.go deleted file mode 100644 index 154c89a4..00000000 --- a/vendor/github.com/klauspost/compress/flate/gen.go +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -// This program generates fixedhuff.go -// Invoke as -// -// go run gen.go -output fixedhuff.go - -package main - -import ( - "bytes" - "flag" - "fmt" - "go/format" - "io/ioutil" - "log" -) - -var filename = flag.String("output", "fixedhuff.go", "output file name") - -const maxCodeLen = 16 - -// Note: the definition of the huffmanDecoder struct is copied from -// inflate.go, as it is private to the implementation. - -// chunk & 15 is number of bits -// chunk >> 4 is value, including table link - -const ( - huffmanChunkBits = 9 - huffmanNumChunks = 1 << huffmanChunkBits - huffmanCountMask = 15 - huffmanValueShift = 4 -) - -type huffmanDecoder struct { - min int // the minimum code length - chunks [huffmanNumChunks]uint32 // chunks as described above - links [][]uint32 // overflow links - linkMask uint32 // mask the width of the link table -} - -// Initialize Huffman decoding tables from array of code lengths. -// Following this function, h is guaranteed to be initialized into a complete -// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a -// degenerate case where the tree has only a single symbol with length 1. Empty -// trees are permitted. -func (h *huffmanDecoder) init(bits []int) bool { - // Sanity enables additional runtime tests during Huffman - // table construction. It's intended to be used during - // development to supplement the currently ad-hoc unit tests. - const sanity = false - - if h.min != 0 { - *h = huffmanDecoder{} - } - - // Count number of codes of each length, - // compute min and max length. - var count [maxCodeLen]int - var min, max int - for _, n := range bits { - if n == 0 { - continue - } - if min == 0 || n < min { - min = n - } - if n > max { - max = n - } - count[n]++ - } - - // Empty tree. The decompressor.huffSym function will fail later if the tree - // is used. Technically, an empty tree is only valid for the HDIST tree and - // not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree - // is guaranteed to fail since it will attempt to use the tree to decode the - // codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is - // guaranteed to fail later since the compressed data section must be - // composed of at least one symbol (the end-of-block marker). - if max == 0 { - return true - } - - code := 0 - var nextcode [maxCodeLen]int - for i := min; i <= max; i++ { - code <<= 1 - nextcode[i] = code - code += count[i] - } - - // Check that the coding is complete (i.e., that we've - // assigned all 2-to-the-max possible bit sequences). - // Exception: To be compatible with zlib, we also need to - // accept degenerate single-code codings. See also - // TestDegenerateHuffmanCoding. - if code != 1< huffmanChunkBits { - numLinks := 1 << (uint(max) - huffmanChunkBits) - h.linkMask = uint32(numLinks - 1) - - // create link tables - link := nextcode[huffmanChunkBits+1] >> 1 - h.links = make([][]uint32, huffmanNumChunks-link) - for j := uint(link); j < huffmanNumChunks; j++ { - reverse := int(reverseByte[j>>8]) | int(reverseByte[j&0xff])<<8 - reverse >>= uint(16 - huffmanChunkBits) - off := j - uint(link) - if sanity && h.chunks[reverse] != 0 { - panic("impossible: overwriting existing chunk") - } - h.chunks[reverse] = uint32(off<>8]) | int(reverseByte[code&0xff])<<8 - reverse >>= uint(16 - n) - if n <= huffmanChunkBits { - for off := reverse; off < len(h.chunks); off += 1 << uint(n) { - // We should never need to overwrite - // an existing chunk. Also, 0 is - // never a valid chunk, because the - // lower 4 "count" bits should be - // between 1 and 15. - if sanity && h.chunks[off] != 0 { - panic("impossible: overwriting existing chunk") - } - h.chunks[off] = chunk - } - } else { - j := reverse & (huffmanNumChunks - 1) - if sanity && h.chunks[j]&huffmanCountMask != huffmanChunkBits+1 { - // Longer codes should have been - // associated with a link table above. - panic("impossible: not an indirect chunk") - } - value := h.chunks[j] >> huffmanValueShift - linktab := h.links[value] - reverse >>= huffmanChunkBits - for off := reverse; off < len(linktab); off += 1 << uint(n-huffmanChunkBits) { - if sanity && linktab[off] != 0 { - panic("impossible: overwriting existing chunk") - } - linktab[off] = chunk - } - } - } - - if sanity { - // Above we've sanity checked that we never overwrote - // an existing entry. Here we additionally check that - // we filled the tables completely. - for i, chunk := range h.chunks { - if chunk == 0 { - // As an exception, in the degenerate - // single-code case, we allow odd - // chunks to be missing. - if code == 1 && i%2 == 1 { - continue - } - panic("impossible: missing chunk") - } - } - for _, linktab := range h.links { - for _, chunk := range linktab { - if chunk == 0 { - panic("impossible: missing chunk") - } - } - } - } - - return true -} - -func main() { - flag.Parse() - - var h huffmanDecoder - var bits [288]int - initReverseByte() - for i := 0; i < 144; i++ { - bits[i] = 8 - } - for i := 144; i < 256; i++ { - bits[i] = 9 - } - for i := 256; i < 280; i++ { - bits[i] = 7 - } - for i := 280; i < 288; i++ { - bits[i] = 8 - } - h.init(bits[:]) - if h.links != nil { - log.Fatal("Unexpected links table in fixed Huffman decoder") - } - - var buf bytes.Buffer - - fmt.Fprintf(&buf, `// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file.`+"\n\n") - - fmt.Fprintln(&buf, "package flate") - fmt.Fprintln(&buf) - fmt.Fprintln(&buf, "// autogenerated by go run gen.go -output fixedhuff.go, DO NOT EDIT") - fmt.Fprintln(&buf) - fmt.Fprintln(&buf, "var fixedHuffmanDecoder = huffmanDecoder{") - fmt.Fprintf(&buf, "\t%d,\n", h.min) - fmt.Fprintln(&buf, "\t[huffmanNumChunks]uint32{") - for i := 0; i < huffmanNumChunks; i++ { - if i&7 == 0 { - fmt.Fprintf(&buf, "\t\t") - } else { - fmt.Fprintf(&buf, " ") - } - fmt.Fprintf(&buf, "0x%04x,", h.chunks[i]) - if i&7 == 7 { - fmt.Fprintln(&buf) - } - } - fmt.Fprintln(&buf, "\t},") - fmt.Fprintln(&buf, "\tnil, 0,") - fmt.Fprintln(&buf, "}") - - data, err := format.Source(buf.Bytes()) - if err != nil { - log.Fatal(err) - } - err = ioutil.WriteFile(*filename, data, 0644) - if err != nil { - log.Fatal(err) - } -} - -var reverseByte [256]byte - -func initReverseByte() { - for x := 0; x < 256; x++ { - var result byte - for i := uint(0); i < 8; i++ { - result |= byte(((x >> i) & 1) << (7 - i)) - } - reverseByte[x] = result - } -} diff --git a/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go b/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go deleted file mode 100644 index f9b2a699..00000000 --- a/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go +++ /dev/null @@ -1,701 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import ( - "io" -) - -const ( - // The largest offset code. - offsetCodeCount = 30 - - // The special code used to mark the end of a block. - endBlockMarker = 256 - - // The first length code. - lengthCodesStart = 257 - - // The number of codegen codes. - codegenCodeCount = 19 - badCode = 255 - - // bufferFlushSize indicates the buffer size - // after which bytes are flushed to the writer. - // Should preferably be a multiple of 6, since - // we accumulate 6 bytes between writes to the buffer. - bufferFlushSize = 240 - - // bufferSize is the actual output byte buffer size. - // It must have additional headroom for a flush - // which can contain up to 8 bytes. - bufferSize = bufferFlushSize + 8 -) - -// The number of extra bits needed by length code X - LENGTH_CODES_START. -var lengthExtraBits = []int8{ - /* 257 */ 0, 0, 0, - /* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, - /* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, - /* 280 */ 4, 5, 5, 5, 5, 0, -} - -// The length indicated by length code X - LENGTH_CODES_START. -var lengthBase = []uint32{ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, - 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, - 64, 80, 96, 112, 128, 160, 192, 224, 255, -} - -// offset code word extra bits. -var offsetExtraBits = []int8{ - 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, - 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, - 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, - /* extended window */ - 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, -} - -var offsetBase = []uint32{ - /* normal deflate */ - 0x000000, 0x000001, 0x000002, 0x000003, 0x000004, - 0x000006, 0x000008, 0x00000c, 0x000010, 0x000018, - 0x000020, 0x000030, 0x000040, 0x000060, 0x000080, - 0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300, - 0x000400, 0x000600, 0x000800, 0x000c00, 0x001000, - 0x001800, 0x002000, 0x003000, 0x004000, 0x006000, - - /* extended window */ - 0x008000, 0x00c000, 0x010000, 0x018000, 0x020000, - 0x030000, 0x040000, 0x060000, 0x080000, 0x0c0000, - 0x100000, 0x180000, 0x200000, 0x300000, -} - -// The odd order in which the codegen code sizes are written. -var codegenOrder = []uint32{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} - -type huffmanBitWriter struct { - // writer is the underlying writer. - // Do not use it directly; use the write method, which ensures - // that Write errors are sticky. - writer io.Writer - - // Data waiting to be written is bytes[0:nbytes] - // and then the low nbits of bits. - bits uint64 - nbits uint - bytes [bufferSize]byte - codegenFreq [codegenCodeCount]int32 - nbytes int - literalFreq []int32 - offsetFreq []int32 - codegen []uint8 - literalEncoding *huffmanEncoder - offsetEncoding *huffmanEncoder - codegenEncoding *huffmanEncoder - err error -} - -func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter { - return &huffmanBitWriter{ - writer: w, - literalFreq: make([]int32, maxNumLit), - offsetFreq: make([]int32, offsetCodeCount), - codegen: make([]uint8, maxNumLit+offsetCodeCount+1), - literalEncoding: newHuffmanEncoder(maxNumLit), - codegenEncoding: newHuffmanEncoder(codegenCodeCount), - offsetEncoding: newHuffmanEncoder(offsetCodeCount), - } -} - -func (w *huffmanBitWriter) reset(writer io.Writer) { - w.writer = writer - w.bits, w.nbits, w.nbytes, w.err = 0, 0, 0, nil - w.bytes = [bufferSize]byte{} -} - -func (w *huffmanBitWriter) flush() { - if w.err != nil { - w.nbits = 0 - return - } - n := w.nbytes - for w.nbits != 0 { - w.bytes[n] = byte(w.bits) - w.bits >>= 8 - if w.nbits > 8 { // Avoid underflow - w.nbits -= 8 - } else { - w.nbits = 0 - } - n++ - } - w.bits = 0 - w.write(w.bytes[:n]) - w.nbytes = 0 -} - -func (w *huffmanBitWriter) write(b []byte) { - if w.err != nil { - return - } - _, w.err = w.writer.Write(b) -} - -func (w *huffmanBitWriter) writeBits(b int32, nb uint) { - if w.err != nil { - return - } - w.bits |= uint64(b) << w.nbits - w.nbits += nb - if w.nbits >= 48 { - bits := w.bits - w.bits >>= 48 - w.nbits -= 48 - n := w.nbytes - bytes := w.bytes[n : n+6] - bytes[0] = byte(bits) - bytes[1] = byte(bits >> 8) - bytes[2] = byte(bits >> 16) - bytes[3] = byte(bits >> 24) - bytes[4] = byte(bits >> 32) - bytes[5] = byte(bits >> 40) - n += 6 - if n >= bufferFlushSize { - w.write(w.bytes[:n]) - n = 0 - } - w.nbytes = n - } -} - -func (w *huffmanBitWriter) writeBytes(bytes []byte) { - if w.err != nil { - return - } - n := w.nbytes - if w.nbits&7 != 0 { - w.err = InternalError("writeBytes with unfinished bits") - return - } - for w.nbits != 0 { - w.bytes[n] = byte(w.bits) - w.bits >>= 8 - w.nbits -= 8 - n++ - } - if n != 0 { - w.write(w.bytes[:n]) - } - w.nbytes = 0 - w.write(bytes) -} - -// RFC 1951 3.2.7 specifies a special run-length encoding for specifying -// the literal and offset lengths arrays (which are concatenated into a single -// array). This method generates that run-length encoding. -// -// The result is written into the codegen array, and the frequencies -// of each code is written into the codegenFreq array. -// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional -// information. Code badCode is an end marker -// -// numLiterals The number of literals in literalEncoding -// numOffsets The number of offsets in offsetEncoding -// litenc, offenc The literal and offset encoder to use -func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litEnc, offEnc *huffmanEncoder) { - for i := range w.codegenFreq { - w.codegenFreq[i] = 0 - } - // Note that we are using codegen both as a temporary variable for holding - // a copy of the frequencies, and as the place where we put the result. - // This is fine because the output is always shorter than the input used - // so far. - codegen := w.codegen // cache - // Copy the concatenated code sizes to codegen. Put a marker at the end. - cgnl := codegen[:numLiterals] - for i := range cgnl { - cgnl[i] = uint8(litEnc.codes[i].len) - } - - cgnl = codegen[numLiterals : numLiterals+numOffsets] - for i := range cgnl { - cgnl[i] = uint8(offEnc.codes[i].len) - } - codegen[numLiterals+numOffsets] = badCode - - size := codegen[0] - count := 1 - outIndex := 0 - for inIndex := 1; size != badCode; inIndex++ { - // INVARIANT: We have seen "count" copies of size that have not yet - // had output generated for them. - nextSize := codegen[inIndex] - if nextSize == size { - count++ - continue - } - // We need to generate codegen indicating "count" of size. - if size != 0 { - codegen[outIndex] = size - outIndex++ - w.codegenFreq[size]++ - count-- - for count >= 3 { - n := 6 - if n > count { - n = count - } - codegen[outIndex] = 16 - outIndex++ - codegen[outIndex] = uint8(n - 3) - outIndex++ - w.codegenFreq[16]++ - count -= n - } - } else { - for count >= 11 { - n := 138 - if n > count { - n = count - } - codegen[outIndex] = 18 - outIndex++ - codegen[outIndex] = uint8(n - 11) - outIndex++ - w.codegenFreq[18]++ - count -= n - } - if count >= 3 { - // count >= 3 && count <= 10 - codegen[outIndex] = 17 - outIndex++ - codegen[outIndex] = uint8(count - 3) - outIndex++ - w.codegenFreq[17]++ - count = 0 - } - } - count-- - for ; count >= 0; count-- { - codegen[outIndex] = size - outIndex++ - w.codegenFreq[size]++ - } - // Set up invariant for next time through the loop. - size = nextSize - count = 1 - } - // Marker indicating the end of the codegen. - codegen[outIndex] = badCode -} - -// dynamicSize returns the size of dynamically encoded data in bits. -func (w *huffmanBitWriter) dynamicSize(litEnc, offEnc *huffmanEncoder, extraBits int) (size, numCodegens int) { - numCodegens = len(w.codegenFreq) - for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 { - numCodegens-- - } - header := 3 + 5 + 5 + 4 + (3 * numCodegens) + - w.codegenEncoding.bitLength(w.codegenFreq[:]) + - int(w.codegenFreq[16])*2 + - int(w.codegenFreq[17])*3 + - int(w.codegenFreq[18])*7 - size = header + - litEnc.bitLength(w.literalFreq) + - offEnc.bitLength(w.offsetFreq) + - extraBits - - return size, numCodegens -} - -// fixedSize returns the size of dynamically encoded data in bits. -func (w *huffmanBitWriter) fixedSize(extraBits int) int { - return 3 + - fixedLiteralEncoding.bitLength(w.literalFreq) + - fixedOffsetEncoding.bitLength(w.offsetFreq) + - extraBits -} - -// storedSize calculates the stored size, including header. -// The function returns the size in bits and whether the block -// fits inside a single block. -func (w *huffmanBitWriter) storedSize(in []byte) (int, bool) { - if in == nil { - return 0, false - } - if len(in) <= maxStoreBlockSize { - return (len(in) + 5) * 8, true - } - return 0, false -} - -func (w *huffmanBitWriter) writeCode(c hcode) { - if w.err != nil { - return - } - w.bits |= uint64(c.code) << w.nbits - w.nbits += uint(c.len) - if w.nbits >= 48 { - bits := w.bits - w.bits >>= 48 - w.nbits -= 48 - n := w.nbytes - bytes := w.bytes[n : n+6] - bytes[0] = byte(bits) - bytes[1] = byte(bits >> 8) - bytes[2] = byte(bits >> 16) - bytes[3] = byte(bits >> 24) - bytes[4] = byte(bits >> 32) - bytes[5] = byte(bits >> 40) - n += 6 - if n >= bufferFlushSize { - w.write(w.bytes[:n]) - n = 0 - } - w.nbytes = n - } -} - -// Write the header of a dynamic Huffman block to the output stream. -// -// numLiterals The number of literals specified in codegen -// numOffsets The number of offsets specified in codegen -// numCodegens The number of codegens used in codegen -func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) { - if w.err != nil { - return - } - var firstBits int32 = 4 - if isEof { - firstBits = 5 - } - w.writeBits(firstBits, 3) - w.writeBits(int32(numLiterals-257), 5) - w.writeBits(int32(numOffsets-1), 5) - w.writeBits(int32(numCodegens-4), 4) - - for i := 0; i < numCodegens; i++ { - value := uint(w.codegenEncoding.codes[codegenOrder[i]].len) - w.writeBits(int32(value), 3) - } - - i := 0 - for { - var codeWord int = int(w.codegen[i]) - i++ - if codeWord == badCode { - break - } - w.writeCode(w.codegenEncoding.codes[uint32(codeWord)]) - - switch codeWord { - case 16: - w.writeBits(int32(w.codegen[i]), 2) - i++ - break - case 17: - w.writeBits(int32(w.codegen[i]), 3) - i++ - break - case 18: - w.writeBits(int32(w.codegen[i]), 7) - i++ - break - } - } -} - -func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) { - if w.err != nil { - return - } - var flag int32 - if isEof { - flag = 1 - } - w.writeBits(flag, 3) - w.flush() - w.writeBits(int32(length), 16) - w.writeBits(int32(^uint16(length)), 16) -} - -func (w *huffmanBitWriter) writeFixedHeader(isEof bool) { - if w.err != nil { - return - } - // Indicate that we are a fixed Huffman block - var value int32 = 2 - if isEof { - value = 3 - } - w.writeBits(value, 3) -} - -// writeBlock will write a block of tokens with the smallest encoding. -// The original input can be supplied, and if the huffman encoded data -// is larger than the original bytes, the data will be written as a -// stored block. -// If the input is nil, the tokens will always be Huffman encoded. -func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) { - if w.err != nil { - return - } - - tokens = append(tokens, endBlockMarker) - numLiterals, numOffsets := w.indexTokens(tokens) - - var extraBits int - storedSize, storable := w.storedSize(input) - if storable { - // We only bother calculating the costs of the extra bits required by - // the length of offset fields (which will be the same for both fixed - // and dynamic encoding), if we need to compare those two encodings - // against stored encoding. - for lengthCode := lengthCodesStart + 8; lengthCode < numLiterals; lengthCode++ { - // First eight length codes have extra size = 0. - extraBits += int(w.literalFreq[lengthCode]) * int(lengthExtraBits[lengthCode-lengthCodesStart]) - } - for offsetCode := 4; offsetCode < numOffsets; offsetCode++ { - // First four offset codes have extra size = 0. - extraBits += int(w.offsetFreq[offsetCode]) * int(offsetExtraBits[offsetCode]) - } - } - - // Figure out smallest code. - // Fixed Huffman baseline. - var literalEncoding = fixedLiteralEncoding - var offsetEncoding = fixedOffsetEncoding - var size = w.fixedSize(extraBits) - - // Dynamic Huffman? - var numCodegens int - - // Generate codegen and codegenFrequencies, which indicates how to encode - // the literalEncoding and the offsetEncoding. - w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding) - w.codegenEncoding.generate(w.codegenFreq[:], 7) - dynamicSize, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits) - - if dynamicSize < size { - size = dynamicSize - literalEncoding = w.literalEncoding - offsetEncoding = w.offsetEncoding - } - - // Stored bytes? - if storable && storedSize < size { - w.writeStoredHeader(len(input), eof) - w.writeBytes(input) - return - } - - // Huffman. - if literalEncoding == fixedLiteralEncoding { - w.writeFixedHeader(eof) - } else { - w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) - } - - // Write the tokens. - w.writeTokens(tokens, literalEncoding.codes, offsetEncoding.codes) -} - -// writeBlockDynamic encodes a block using a dynamic Huffman table. -// This should be used if the symbols used have a disproportionate -// histogram distribution. -// If input is supplied and the compression savings are below 1/16th of the -// input size the block is stored. -func (w *huffmanBitWriter) writeBlockDynamic(tokens []token, eof bool, input []byte) { - if w.err != nil { - return - } - - tokens = append(tokens, endBlockMarker) - numLiterals, numOffsets := w.indexTokens(tokens) - - // Generate codegen and codegenFrequencies, which indicates how to encode - // the literalEncoding and the offsetEncoding. - w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding) - w.codegenEncoding.generate(w.codegenFreq[:], 7) - size, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, 0) - - // Store bytes, if we don't get a reasonable improvement. - if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) { - w.writeStoredHeader(len(input), eof) - w.writeBytes(input) - return - } - - // Write Huffman table. - w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) - - // Write the tokens. - w.writeTokens(tokens, w.literalEncoding.codes, w.offsetEncoding.codes) -} - -// indexTokens indexes a slice of tokens, and updates -// literalFreq and offsetFreq, and generates literalEncoding -// and offsetEncoding. -// The number of literal and offset tokens is returned. -func (w *huffmanBitWriter) indexTokens(tokens []token) (numLiterals, numOffsets int) { - for i := range w.literalFreq { - w.literalFreq[i] = 0 - } - for i := range w.offsetFreq { - w.offsetFreq[i] = 0 - } - - for _, t := range tokens { - if t < matchType { - w.literalFreq[t.literal()]++ - continue - } - length := t.length() - offset := t.offset() - w.literalFreq[lengthCodesStart+lengthCode(length)]++ - w.offsetFreq[offsetCode(offset)]++ - } - - // get the number of literals - numLiterals = len(w.literalFreq) - for w.literalFreq[numLiterals-1] == 0 { - numLiterals-- - } - // get the number of offsets - numOffsets = len(w.offsetFreq) - for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 { - numOffsets-- - } - if numOffsets == 0 { - // We haven't found a single match. If we want to go with the dynamic encoding, - // we should count at least one offset to be sure that the offset huffman tree could be encoded. - w.offsetFreq[0] = 1 - numOffsets = 1 - } - w.literalEncoding.generate(w.literalFreq, 15) - w.offsetEncoding.generate(w.offsetFreq, 15) - return -} - -// writeTokens writes a slice of tokens to the output. -// codes for literal and offset encoding must be supplied. -func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) { - if w.err != nil { - return - } - for _, t := range tokens { - if t < matchType { - w.writeCode(leCodes[t.literal()]) - continue - } - // Write the length - length := t.length() - lengthCode := lengthCode(length) - w.writeCode(leCodes[lengthCode+lengthCodesStart]) - extraLengthBits := uint(lengthExtraBits[lengthCode]) - if extraLengthBits > 0 { - extraLength := int32(length - lengthBase[lengthCode]) - w.writeBits(extraLength, extraLengthBits) - } - // Write the offset - offset := t.offset() - offsetCode := offsetCode(offset) - w.writeCode(oeCodes[offsetCode]) - extraOffsetBits := uint(offsetExtraBits[offsetCode]) - if extraOffsetBits > 0 { - extraOffset := int32(offset - offsetBase[offsetCode]) - w.writeBits(extraOffset, extraOffsetBits) - } - } -} - -// huffOffset is a static offset encoder used for huffman only encoding. -// It can be reused since we will not be encoding offset values. -var huffOffset *huffmanEncoder - -func init() { - w := newHuffmanBitWriter(nil) - w.offsetFreq[0] = 1 - huffOffset = newHuffmanEncoder(offsetCodeCount) - huffOffset.generate(w.offsetFreq, 15) -} - -// writeBlockHuff encodes a block of bytes as either -// Huffman encoded literals or uncompressed bytes if the -// results only gains very little from compression. -func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte) { - if w.err != nil { - return - } - - // Clear histogram - for i := range w.literalFreq { - w.literalFreq[i] = 0 - } - - // Add everything as literals - histogram(input, w.literalFreq) - - w.literalFreq[endBlockMarker] = 1 - - const numLiterals = endBlockMarker + 1 - const numOffsets = 1 - - w.literalEncoding.generate(w.literalFreq, 15) - - // Figure out smallest code. - // Always use dynamic Huffman or Store - var numCodegens int - - // Generate codegen and codegenFrequencies, which indicates how to encode - // the literalEncoding and the offsetEncoding. - w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, huffOffset) - w.codegenEncoding.generate(w.codegenFreq[:], 7) - size, numCodegens := w.dynamicSize(w.literalEncoding, huffOffset, 0) - - // Store bytes, if we don't get a reasonable improvement. - if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) { - w.writeStoredHeader(len(input), eof) - w.writeBytes(input) - return - } - - // Huffman. - w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) - encoding := w.literalEncoding.codes[:257] - n := w.nbytes - for _, t := range input { - // Bitwriting inlined, ~30% speedup - c := encoding[t] - w.bits |= uint64(c.code) << w.nbits - w.nbits += uint(c.len) - if w.nbits < 48 { - continue - } - // Store 6 bytes - bits := w.bits - w.bits >>= 48 - w.nbits -= 48 - bytes := w.bytes[n : n+6] - bytes[0] = byte(bits) - bytes[1] = byte(bits >> 8) - bytes[2] = byte(bits >> 16) - bytes[3] = byte(bits >> 24) - bytes[4] = byte(bits >> 32) - bytes[5] = byte(bits >> 40) - n += 6 - if n < bufferFlushSize { - continue - } - w.write(w.bytes[:n]) - if w.err != nil { - return // Return early in the event of write failures - } - n = 0 - } - w.nbytes = n - w.writeCode(encoding[endBlockMarker]) -} diff --git a/vendor/github.com/klauspost/compress/flate/huffman_code.go b/vendor/github.com/klauspost/compress/flate/huffman_code.go deleted file mode 100644 index bdcbd823..00000000 --- a/vendor/github.com/klauspost/compress/flate/huffman_code.go +++ /dev/null @@ -1,344 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import ( - "math" - "sort" -) - -// hcode is a huffman code with a bit code and bit length. -type hcode struct { - code, len uint16 -} - -type huffmanEncoder struct { - codes []hcode - freqcache []literalNode - bitCount [17]int32 - lns byLiteral // stored to avoid repeated allocation in generate - lfs byFreq // stored to avoid repeated allocation in generate -} - -type literalNode struct { - literal uint16 - freq int32 -} - -// A levelInfo describes the state of the constructed tree for a given depth. -type levelInfo struct { - // Our level. for better printing - level int32 - - // The frequency of the last node at this level - lastFreq int32 - - // The frequency of the next character to add to this level - nextCharFreq int32 - - // The frequency of the next pair (from level below) to add to this level. - // Only valid if the "needed" value of the next lower level is 0. - nextPairFreq int32 - - // The number of chains remaining to generate for this level before moving - // up to the next level - needed int32 -} - -// set sets the code and length of an hcode. -func (h *hcode) set(code uint16, length uint16) { - h.len = length - h.code = code -} - -func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} } - -func newHuffmanEncoder(size int) *huffmanEncoder { - return &huffmanEncoder{codes: make([]hcode, size)} -} - -// Generates a HuffmanCode corresponding to the fixed literal table -func generateFixedLiteralEncoding() *huffmanEncoder { - h := newHuffmanEncoder(maxNumLit) - codes := h.codes - var ch uint16 - for ch = 0; ch < maxNumLit; ch++ { - var bits uint16 - var size uint16 - switch { - case ch < 144: - // size 8, 000110000 .. 10111111 - bits = ch + 48 - size = 8 - break - case ch < 256: - // size 9, 110010000 .. 111111111 - bits = ch + 400 - 144 - size = 9 - break - case ch < 280: - // size 7, 0000000 .. 0010111 - bits = ch - 256 - size = 7 - break - default: - // size 8, 11000000 .. 11000111 - bits = ch + 192 - 280 - size = 8 - } - codes[ch] = hcode{code: reverseBits(bits, byte(size)), len: size} - } - return h -} - -func generateFixedOffsetEncoding() *huffmanEncoder { - h := newHuffmanEncoder(30) - codes := h.codes - for ch := range codes { - codes[ch] = hcode{code: reverseBits(uint16(ch), 5), len: 5} - } - return h -} - -var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding() -var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding() - -func (h *huffmanEncoder) bitLength(freq []int32) int { - var total int - for i, f := range freq { - if f != 0 { - total += int(f) * int(h.codes[i].len) - } - } - return total -} - -const maxBitsLimit = 16 - -// Return the number of literals assigned to each bit size in the Huffman encoding -// -// This method is only called when list.length >= 3 -// The cases of 0, 1, and 2 literals are handled by special case code. -// -// list An array of the literals with non-zero frequencies -// and their associated frequencies. The array is in order of increasing -// frequency, and has as its last element a special element with frequency -// MaxInt32 -// maxBits The maximum number of bits that should be used to encode any literal. -// Must be less than 16. -// return An integer array in which array[i] indicates the number of literals -// that should be encoded in i bits. -func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 { - if maxBits >= maxBitsLimit { - panic("flate: maxBits too large") - } - n := int32(len(list)) - list = list[0 : n+1] - list[n] = maxNode() - - // The tree can't have greater depth than n - 1, no matter what. This - // saves a little bit of work in some small cases - if maxBits > n-1 { - maxBits = n - 1 - } - - // Create information about each of the levels. - // A bogus "Level 0" whose sole purpose is so that - // level1.prev.needed==0. This makes level1.nextPairFreq - // be a legitimate value that never gets chosen. - var levels [maxBitsLimit]levelInfo - // leafCounts[i] counts the number of literals at the left - // of ancestors of the rightmost node at level i. - // leafCounts[i][j] is the number of literals at the left - // of the level j ancestor. - var leafCounts [maxBitsLimit][maxBitsLimit]int32 - - for level := int32(1); level <= maxBits; level++ { - // For every level, the first two items are the first two characters. - // We initialize the levels as if we had already figured this out. - levels[level] = levelInfo{ - level: level, - lastFreq: list[1].freq, - nextCharFreq: list[2].freq, - nextPairFreq: list[0].freq + list[1].freq, - } - leafCounts[level][level] = 2 - if level == 1 { - levels[level].nextPairFreq = math.MaxInt32 - } - } - - // We need a total of 2*n - 2 items at top level and have already generated 2. - levels[maxBits].needed = 2*n - 4 - - level := maxBits - for { - l := &levels[level] - if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 { - // We've run out of both leafs and pairs. - // End all calculations for this level. - // To make sure we never come back to this level or any lower level, - // set nextPairFreq impossibly large. - l.needed = 0 - levels[level+1].nextPairFreq = math.MaxInt32 - level++ - continue - } - - prevFreq := l.lastFreq - if l.nextCharFreq < l.nextPairFreq { - // The next item on this row is a leaf node. - n := leafCounts[level][level] + 1 - l.lastFreq = l.nextCharFreq - // Lower leafCounts are the same of the previous node. - leafCounts[level][level] = n - l.nextCharFreq = list[n].freq - } else { - // The next item on this row is a pair from the previous row. - // nextPairFreq isn't valid until we generate two - // more values in the level below - l.lastFreq = l.nextPairFreq - // Take leaf counts from the lower level, except counts[level] remains the same. - copy(leafCounts[level][:level], leafCounts[level-1][:level]) - levels[l.level-1].needed = 2 - } - - if l.needed--; l.needed == 0 { - // We've done everything we need to do for this level. - // Continue calculating one level up. Fill in nextPairFreq - // of that level with the sum of the two nodes we've just calculated on - // this level. - if l.level == maxBits { - // All done! - break - } - levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq - level++ - } else { - // If we stole from below, move down temporarily to replenish it. - for levels[level-1].needed > 0 { - level-- - } - } - } - - // Somethings is wrong if at the end, the top level is null or hasn't used - // all of the leaves. - if leafCounts[maxBits][maxBits] != n { - panic("leafCounts[maxBits][maxBits] != n") - } - - bitCount := h.bitCount[:maxBits+1] - bits := 1 - counts := &leafCounts[maxBits] - for level := maxBits; level > 0; level-- { - // chain.leafCount gives the number of literals requiring at least "bits" - // bits to encode. - bitCount[bits] = counts[level] - counts[level-1] - bits++ - } - return bitCount -} - -// Look at the leaves and assign them a bit count and an encoding as specified -// in RFC 1951 3.2.2 -func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) { - code := uint16(0) - for n, bits := range bitCount { - code <<= 1 - if n == 0 || bits == 0 { - continue - } - // The literals list[len(list)-bits] .. list[len(list)-bits] - // are encoded using "bits" bits, and get the values - // code, code + 1, .... The code values are - // assigned in literal order (not frequency order). - chunk := list[len(list)-int(bits):] - - h.lns.sort(chunk) - for _, node := range chunk { - h.codes[node.literal] = hcode{code: reverseBits(code, uint8(n)), len: uint16(n)} - code++ - } - list = list[0 : len(list)-int(bits)] - } -} - -// Update this Huffman Code object to be the minimum code for the specified frequency count. -// -// freq An array of frequencies, in which frequency[i] gives the frequency of literal i. -// maxBits The maximum number of bits to use for any literal. -func (h *huffmanEncoder) generate(freq []int32, maxBits int32) { - if h.freqcache == nil { - // Allocate a reusable buffer with the longest possible frequency table. - // Possible lengths are codegenCodeCount, offsetCodeCount and maxNumLit. - // The largest of these is maxNumLit, so we allocate for that case. - h.freqcache = make([]literalNode, maxNumLit+1) - } - list := h.freqcache[:len(freq)+1] - // Number of non-zero literals - count := 0 - // Set list to be the set of all non-zero literals and their frequencies - for i, f := range freq { - if f != 0 { - list[count] = literalNode{uint16(i), f} - count++ - } else { - list[count] = literalNode{} - h.codes[i].len = 0 - } - } - list[len(freq)] = literalNode{} - - list = list[:count] - if count <= 2 { - // Handle the small cases here, because they are awkward for the general case code. With - // two or fewer literals, everything has bit length 1. - for i, node := range list { - // "list" is in order of increasing literal value. - h.codes[node.literal].set(uint16(i), 1) - } - return - } - h.lfs.sort(list) - - // Get the number of literals for each bit count - bitCount := h.bitCounts(list, maxBits) - // And do the assignment - h.assignEncodingAndSize(bitCount, list) -} - -type byLiteral []literalNode - -func (s *byLiteral) sort(a []literalNode) { - *s = byLiteral(a) - sort.Sort(s) -} - -func (s byLiteral) Len() int { return len(s) } - -func (s byLiteral) Less(i, j int) bool { - return s[i].literal < s[j].literal -} - -func (s byLiteral) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -type byFreq []literalNode - -func (s *byFreq) sort(a []literalNode) { - *s = byFreq(a) - sort.Sort(s) -} - -func (s byFreq) Len() int { return len(s) } - -func (s byFreq) Less(i, j int) bool { - if s[i].freq == s[j].freq { - return s[i].literal < s[j].literal - } - return s[i].freq < s[j].freq -} - -func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/vendor/github.com/klauspost/compress/flate/inflate.go b/vendor/github.com/klauspost/compress/flate/inflate.go deleted file mode 100644 index 800d0ce9..00000000 --- a/vendor/github.com/klauspost/compress/flate/inflate.go +++ /dev/null @@ -1,880 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package flate implements the DEFLATE compressed data format, described in -// RFC 1951. The gzip and zlib packages implement access to DEFLATE-based file -// formats. -package flate - -import ( - "bufio" - "io" - "math/bits" - "strconv" - "sync" -) - -const ( - maxCodeLen = 16 // max length of Huffman code - maxCodeLenMask = 15 // mask for max length of Huffman code - // The next three numbers come from the RFC section 3.2.7, with the - // additional proviso in section 3.2.5 which implies that distance codes - // 30 and 31 should never occur in compressed data. - maxNumLit = 286 - maxNumDist = 30 - numCodes = 19 // number of codes in Huffman meta-code -) - -// Initialize the fixedHuffmanDecoder only once upon first use. -var fixedOnce sync.Once -var fixedHuffmanDecoder huffmanDecoder - -// A CorruptInputError reports the presence of corrupt input at a given offset. -type CorruptInputError int64 - -func (e CorruptInputError) Error() string { - return "flate: corrupt input before offset " + strconv.FormatInt(int64(e), 10) -} - -// An InternalError reports an error in the flate code itself. -type InternalError string - -func (e InternalError) Error() string { return "flate: internal error: " + string(e) } - -// A ReadError reports an error encountered while reading input. -// -// Deprecated: No longer returned. -type ReadError struct { - Offset int64 // byte offset where error occurred - Err error // error returned by underlying Read -} - -func (e *ReadError) Error() string { - return "flate: read error at offset " + strconv.FormatInt(e.Offset, 10) + ": " + e.Err.Error() -} - -// A WriteError reports an error encountered while writing output. -// -// Deprecated: No longer returned. -type WriteError struct { - Offset int64 // byte offset where error occurred - Err error // error returned by underlying Write -} - -func (e *WriteError) Error() string { - return "flate: write error at offset " + strconv.FormatInt(e.Offset, 10) + ": " + e.Err.Error() -} - -// Resetter resets a ReadCloser returned by NewReader or NewReaderDict to -// to switch to a new underlying Reader. This permits reusing a ReadCloser -// instead of allocating a new one. -type Resetter interface { - // Reset discards any buffered data and resets the Resetter as if it was - // newly initialized with the given reader. - Reset(r io.Reader, dict []byte) error -} - -// The data structure for decoding Huffman tables is based on that of -// zlib. There is a lookup table of a fixed bit width (huffmanChunkBits), -// For codes smaller than the table width, there are multiple entries -// (each combination of trailing bits has the same value). For codes -// larger than the table width, the table contains a link to an overflow -// table. The width of each entry in the link table is the maximum code -// size minus the chunk width. -// -// Note that you can do a lookup in the table even without all bits -// filled. Since the extra bits are zero, and the DEFLATE Huffman codes -// have the property that shorter codes come before longer ones, the -// bit length estimate in the result is a lower bound on the actual -// number of bits. -// -// See the following: -// http://www.gzip.org/algorithm.txt - -// chunk & 15 is number of bits -// chunk >> 4 is value, including table link - -const ( - huffmanChunkBits = 9 - huffmanNumChunks = 1 << huffmanChunkBits - huffmanCountMask = 15 - huffmanValueShift = 4 -) - -type huffmanDecoder struct { - min int // the minimum code length - chunks *[huffmanNumChunks]uint32 // chunks as described above - links [][]uint32 // overflow links - linkMask uint32 // mask the width of the link table -} - -// Initialize Huffman decoding tables from array of code lengths. -// Following this function, h is guaranteed to be initialized into a complete -// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a -// degenerate case where the tree has only a single symbol with length 1. Empty -// trees are permitted. -func (h *huffmanDecoder) init(lengths []int) bool { - // Sanity enables additional runtime tests during Huffman - // table construction. It's intended to be used during - // development to supplement the currently ad-hoc unit tests. - const sanity = false - - if h.chunks == nil { - h.chunks = &[huffmanNumChunks]uint32{} - } - if h.min != 0 { - *h = huffmanDecoder{chunks: h.chunks, links: h.links} - } - - // Count number of codes of each length, - // compute min and max length. - var count [maxCodeLen]int - var min, max int - for _, n := range lengths { - if n == 0 { - continue - } - if min == 0 || n < min { - min = n - } - if n > max { - max = n - } - count[n&maxCodeLenMask]++ - } - - // Empty tree. The decompressor.huffSym function will fail later if the tree - // is used. Technically, an empty tree is only valid for the HDIST tree and - // not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree - // is guaranteed to fail since it will attempt to use the tree to decode the - // codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is - // guaranteed to fail later since the compressed data section must be - // composed of at least one symbol (the end-of-block marker). - if max == 0 { - return true - } - - code := 0 - var nextcode [maxCodeLen]int - for i := min; i <= max; i++ { - code <<= 1 - nextcode[i&maxCodeLenMask] = code - code += count[i&maxCodeLenMask] - } - - // Check that the coding is complete (i.e., that we've - // assigned all 2-to-the-max possible bit sequences). - // Exception: To be compatible with zlib, we also need to - // accept degenerate single-code codings. See also - // TestDegenerateHuffmanCoding. - if code != 1< huffmanChunkBits { - numLinks := 1 << (uint(max) - huffmanChunkBits) - h.linkMask = uint32(numLinks - 1) - - // create link tables - link := nextcode[huffmanChunkBits+1] >> 1 - if cap(h.links) < huffmanNumChunks-link { - h.links = make([][]uint32, huffmanNumChunks-link) - } else { - h.links = h.links[:huffmanNumChunks-link] - } - for j := uint(link); j < huffmanNumChunks; j++ { - reverse := int(bits.Reverse16(uint16(j))) - reverse >>= uint(16 - huffmanChunkBits) - off := j - uint(link) - if sanity && h.chunks[reverse] != 0 { - panic("impossible: overwriting existing chunk") - } - h.chunks[reverse] = uint32(off<>= uint(16 - n) - if n <= huffmanChunkBits { - for off := reverse; off < len(h.chunks); off += 1 << uint(n) { - // We should never need to overwrite - // an existing chunk. Also, 0 is - // never a valid chunk, because the - // lower 4 "count" bits should be - // between 1 and 15. - if sanity && h.chunks[off] != 0 { - panic("impossible: overwriting existing chunk") - } - h.chunks[off] = chunk - } - } else { - j := reverse & (huffmanNumChunks - 1) - if sanity && h.chunks[j]&huffmanCountMask != huffmanChunkBits+1 { - // Longer codes should have been - // associated with a link table above. - panic("impossible: not an indirect chunk") - } - value := h.chunks[j] >> huffmanValueShift - linktab := h.links[value] - reverse >>= huffmanChunkBits - for off := reverse; off < len(linktab); off += 1 << uint(n-huffmanChunkBits) { - if sanity && linktab[off] != 0 { - panic("impossible: overwriting existing chunk") - } - linktab[off] = chunk - } - } - } - - if sanity { - // Above we've sanity checked that we never overwrote - // an existing entry. Here we additionally check that - // we filled the tables completely. - for i, chunk := range h.chunks { - if chunk == 0 { - // As an exception, in the degenerate - // single-code case, we allow odd - // chunks to be missing. - if code == 1 && i%2 == 1 { - continue - } - panic("impossible: missing chunk") - } - } - for _, linktab := range h.links { - for _, chunk := range linktab { - if chunk == 0 { - panic("impossible: missing chunk") - } - } - } - } - - return true -} - -// The actual read interface needed by NewReader. -// If the passed in io.Reader does not also have ReadByte, -// the NewReader will introduce its own buffering. -type Reader interface { - io.Reader - io.ByteReader -} - -// Decompress state. -type decompressor struct { - // Input source. - r Reader - roffset int64 - - // Input bits, in top of b. - b uint32 - nb uint - - // Huffman decoders for literal/length, distance. - h1, h2 huffmanDecoder - - // Length arrays used to define Huffman codes. - bits *[maxNumLit + maxNumDist]int - codebits *[numCodes]int - - // Output history, buffer. - dict dictDecoder - - // Temporary buffer (avoids repeated allocation). - buf [4]byte - - // Next step in the decompression, - // and decompression state. - step func(*decompressor) - stepState int - final bool - err error - toRead []byte - hl, hd *huffmanDecoder - copyLen int - copyDist int -} - -func (f *decompressor) nextBlock() { - for f.nb < 1+2 { - if f.err = f.moreBits(); f.err != nil { - return - } - } - f.final = f.b&1 == 1 - f.b >>= 1 - typ := f.b & 3 - f.b >>= 2 - f.nb -= 1 + 2 - switch typ { - case 0: - f.dataBlock() - case 1: - // compressed, fixed Huffman tables - f.hl = &fixedHuffmanDecoder - f.hd = nil - f.huffmanBlock() - case 2: - // compressed, dynamic Huffman tables - if f.err = f.readHuffman(); f.err != nil { - break - } - f.hl = &f.h1 - f.hd = &f.h2 - f.huffmanBlock() - default: - // 3 is reserved. - f.err = CorruptInputError(f.roffset) - } -} - -func (f *decompressor) Read(b []byte) (int, error) { - for { - if len(f.toRead) > 0 { - n := copy(b, f.toRead) - f.toRead = f.toRead[n:] - if len(f.toRead) == 0 { - return n, f.err - } - return n, nil - } - if f.err != nil { - return 0, f.err - } - f.step(f) - if f.err != nil && len(f.toRead) == 0 { - f.toRead = f.dict.readFlush() // Flush what's left in case of error - } - } -} - -// Support the io.WriteTo interface for io.Copy and friends. -func (f *decompressor) WriteTo(w io.Writer) (int64, error) { - total := int64(0) - flushed := false - for { - if len(f.toRead) > 0 { - n, err := w.Write(f.toRead) - total += int64(n) - if err != nil { - f.err = err - return total, err - } - if n != len(f.toRead) { - return total, io.ErrShortWrite - } - f.toRead = f.toRead[:0] - } - if f.err != nil && flushed { - if f.err == io.EOF { - return total, nil - } - return total, f.err - } - if f.err == nil { - f.step(f) - } - if len(f.toRead) == 0 && f.err != nil && !flushed { - f.toRead = f.dict.readFlush() // Flush what's left in case of error - flushed = true - } - } -} - -func (f *decompressor) Close() error { - if f.err == io.EOF { - return nil - } - return f.err -} - -// RFC 1951 section 3.2.7. -// Compression with dynamic Huffman codes - -var codeOrder = [...]int{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} - -func (f *decompressor) readHuffman() error { - // HLIT[5], HDIST[5], HCLEN[4]. - for f.nb < 5+5+4 { - if err := f.moreBits(); err != nil { - return err - } - } - nlit := int(f.b&0x1F) + 257 - if nlit > maxNumLit { - return CorruptInputError(f.roffset) - } - f.b >>= 5 - ndist := int(f.b&0x1F) + 1 - if ndist > maxNumDist { - return CorruptInputError(f.roffset) - } - f.b >>= 5 - nclen := int(f.b&0xF) + 4 - // numCodes is 19, so nclen is always valid. - f.b >>= 4 - f.nb -= 5 + 5 + 4 - - // (HCLEN+4)*3 bits: code lengths in the magic codeOrder order. - for i := 0; i < nclen; i++ { - for f.nb < 3 { - if err := f.moreBits(); err != nil { - return err - } - } - f.codebits[codeOrder[i]] = int(f.b & 0x7) - f.b >>= 3 - f.nb -= 3 - } - for i := nclen; i < len(codeOrder); i++ { - f.codebits[codeOrder[i]] = 0 - } - if !f.h1.init(f.codebits[0:]) { - return CorruptInputError(f.roffset) - } - - // HLIT + 257 code lengths, HDIST + 1 code lengths, - // using the code length Huffman code. - for i, n := 0, nlit+ndist; i < n; { - x, err := f.huffSym(&f.h1) - if err != nil { - return err - } - if x < 16 { - // Actual length. - f.bits[i] = x - i++ - continue - } - // Repeat previous length or zero. - var rep int - var nb uint - var b int - switch x { - default: - return InternalError("unexpected length code") - case 16: - rep = 3 - nb = 2 - if i == 0 { - return CorruptInputError(f.roffset) - } - b = f.bits[i-1] - case 17: - rep = 3 - nb = 3 - b = 0 - case 18: - rep = 11 - nb = 7 - b = 0 - } - for f.nb < nb { - if err := f.moreBits(); err != nil { - return err - } - } - rep += int(f.b & uint32(1<>= nb - f.nb -= nb - if i+rep > n { - return CorruptInputError(f.roffset) - } - for j := 0; j < rep; j++ { - f.bits[i] = b - i++ - } - } - - if !f.h1.init(f.bits[0:nlit]) || !f.h2.init(f.bits[nlit:nlit+ndist]) { - return CorruptInputError(f.roffset) - } - - // As an optimization, we can initialize the min bits to read at a time - // for the HLIT tree to the length of the EOB marker since we know that - // every block must terminate with one. This preserves the property that - // we never read any extra bytes after the end of the DEFLATE stream. - if f.h1.min < f.bits[endBlockMarker] { - f.h1.min = f.bits[endBlockMarker] - } - - return nil -} - -// Decode a single Huffman block from f. -// hl and hd are the Huffman states for the lit/length values -// and the distance values, respectively. If hd == nil, using the -// fixed distance encoding associated with fixed Huffman blocks. -func (f *decompressor) huffmanBlock() { - const ( - stateInit = iota // Zero value must be stateInit - stateDict - ) - - switch f.stepState { - case stateInit: - goto readLiteral - case stateDict: - goto copyHistory - } - -readLiteral: - // Read literal and/or (length, distance) according to RFC section 3.2.3. - { - v, err := f.huffSym(f.hl) - if err != nil { - f.err = err - return - } - var n uint // number of bits extra - var length int - switch { - case v < 256: - f.dict.writeByte(byte(v)) - if f.dict.availWrite() == 0 { - f.toRead = f.dict.readFlush() - f.step = (*decompressor).huffmanBlock - f.stepState = stateInit - return - } - goto readLiteral - case v == 256: - f.finishBlock() - return - // otherwise, reference to older data - case v < 265: - length = v - (257 - 3) - n = 0 - case v < 269: - length = v*2 - (265*2 - 11) - n = 1 - case v < 273: - length = v*4 - (269*4 - 19) - n = 2 - case v < 277: - length = v*8 - (273*8 - 35) - n = 3 - case v < 281: - length = v*16 - (277*16 - 67) - n = 4 - case v < 285: - length = v*32 - (281*32 - 131) - n = 5 - case v < maxNumLit: - length = 258 - n = 0 - default: - f.err = CorruptInputError(f.roffset) - return - } - if n > 0 { - for f.nb < n { - if err = f.moreBits(); err != nil { - f.err = err - return - } - } - length += int(f.b & uint32(1<>= n - f.nb -= n - } - - var dist int - if f.hd == nil { - for f.nb < 5 { - if err = f.moreBits(); err != nil { - f.err = err - return - } - } - dist = int(bits.Reverse8(uint8(f.b & 0x1F << 3))) - f.b >>= 5 - f.nb -= 5 - } else { - if dist, err = f.huffSym(f.hd); err != nil { - f.err = err - return - } - } - - switch { - case dist < 4: - dist++ - case dist < maxNumDist: - nb := uint(dist-2) >> 1 - // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << nb - for f.nb < nb { - if err = f.moreBits(); err != nil { - f.err = err - return - } - } - extra |= int(f.b & uint32(1<>= nb - f.nb -= nb - dist = 1<<(nb+1) + 1 + extra - default: - f.err = CorruptInputError(f.roffset) - return - } - - // No check on length; encoding can be prescient. - if dist > f.dict.histSize() { - f.err = CorruptInputError(f.roffset) - return - } - - f.copyLen, f.copyDist = length, dist - goto copyHistory - } - -copyHistory: - // Perform a backwards copy according to RFC section 3.2.3. - { - cnt := f.dict.tryWriteCopy(f.copyDist, f.copyLen) - if cnt == 0 { - cnt = f.dict.writeCopy(f.copyDist, f.copyLen) - } - f.copyLen -= cnt - - if f.dict.availWrite() == 0 || f.copyLen > 0 { - f.toRead = f.dict.readFlush() - f.step = (*decompressor).huffmanBlock // We need to continue this work - f.stepState = stateDict - return - } - goto readLiteral - } -} - -// Copy a single uncompressed data block from input to output. -func (f *decompressor) dataBlock() { - // Uncompressed. - // Discard current half-byte. - f.nb = 0 - f.b = 0 - - // Length then ones-complement of length. - nr, err := io.ReadFull(f.r, f.buf[0:4]) - f.roffset += int64(nr) - if err != nil { - f.err = noEOF(err) - return - } - n := int(f.buf[0]) | int(f.buf[1])<<8 - nn := int(f.buf[2]) | int(f.buf[3])<<8 - if uint16(nn) != uint16(^n) { - f.err = CorruptInputError(f.roffset) - return - } - - if n == 0 { - f.toRead = f.dict.readFlush() - f.finishBlock() - return - } - - f.copyLen = n - f.copyData() -} - -// copyData copies f.copyLen bytes from the underlying reader into f.hist. -// It pauses for reads when f.hist is full. -func (f *decompressor) copyData() { - buf := f.dict.writeSlice() - if len(buf) > f.copyLen { - buf = buf[:f.copyLen] - } - - cnt, err := io.ReadFull(f.r, buf) - f.roffset += int64(cnt) - f.copyLen -= cnt - f.dict.writeMark(cnt) - if err != nil { - f.err = noEOF(err) - return - } - - if f.dict.availWrite() == 0 || f.copyLen > 0 { - f.toRead = f.dict.readFlush() - f.step = (*decompressor).copyData - return - } - f.finishBlock() -} - -func (f *decompressor) finishBlock() { - if f.final { - if f.dict.availRead() > 0 { - f.toRead = f.dict.readFlush() - } - f.err = io.EOF - } - f.step = (*decompressor).nextBlock -} - -// noEOF returns err, unless err == io.EOF, in which case it returns io.ErrUnexpectedEOF. -func noEOF(e error) error { - if e == io.EOF { - return io.ErrUnexpectedEOF - } - return e -} - -func (f *decompressor) moreBits() error { - c, err := f.r.ReadByte() - if err != nil { - return noEOF(err) - } - f.roffset++ - f.b |= uint32(c) << f.nb - f.nb += 8 - return nil -} - -// Read the next Huffman-encoded symbol from f according to h. -func (f *decompressor) huffSym(h *huffmanDecoder) (int, error) { - // Since a huffmanDecoder can be empty or be composed of a degenerate tree - // with single element, huffSym must error on these two edge cases. In both - // cases, the chunks slice will be 0 for the invalid sequence, leading it - // satisfy the n == 0 check below. - n := uint(h.min) - // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, - // but is smart enough to keep local variables in registers, so use nb and b, - // inline call to moreBits and reassign b,nb back to f on return. - nb, b := f.nb, f.b - for { - for nb < n { - c, err := f.r.ReadByte() - if err != nil { - f.b = b - f.nb = nb - return 0, noEOF(err) - } - f.roffset++ - b |= uint32(c) << (nb & 31) - nb += 8 - } - chunk := h.chunks[b&(huffmanNumChunks-1)] - n = uint(chunk & huffmanCountMask) - if n > huffmanChunkBits { - chunk = h.links[chunk>>huffmanValueShift][(b>>huffmanChunkBits)&h.linkMask] - n = uint(chunk & huffmanCountMask) - } - if n <= nb { - if n == 0 { - f.b = b - f.nb = nb - f.err = CorruptInputError(f.roffset) - return 0, f.err - } - f.b = b >> (n & 31) - f.nb = nb - n - return int(chunk >> huffmanValueShift), nil - } - } -} - -func makeReader(r io.Reader) Reader { - if rr, ok := r.(Reader); ok { - return rr - } - return bufio.NewReader(r) -} - -func fixedHuffmanDecoderInit() { - fixedOnce.Do(func() { - // These come from the RFC section 3.2.6. - var bits [288]int - for i := 0; i < 144; i++ { - bits[i] = 8 - } - for i := 144; i < 256; i++ { - bits[i] = 9 - } - for i := 256; i < 280; i++ { - bits[i] = 7 - } - for i := 280; i < 288; i++ { - bits[i] = 8 - } - fixedHuffmanDecoder.init(bits[:]) - }) -} - -func (f *decompressor) Reset(r io.Reader, dict []byte) error { - *f = decompressor{ - r: makeReader(r), - bits: f.bits, - codebits: f.codebits, - h1: f.h1, - h2: f.h2, - dict: f.dict, - step: (*decompressor).nextBlock, - } - f.dict.init(maxMatchOffset, dict) - return nil -} - -// NewReader returns a new ReadCloser that can be used -// to read the uncompressed version of r. -// If r does not also implement io.ByteReader, -// the decompressor may read more data than necessary from r. -// It is the caller's responsibility to call Close on the ReadCloser -// when finished reading. -// -// The ReadCloser returned by NewReader also implements Resetter. -func NewReader(r io.Reader) io.ReadCloser { - fixedHuffmanDecoderInit() - - var f decompressor - f.r = makeReader(r) - f.bits = new([maxNumLit + maxNumDist]int) - f.codebits = new([numCodes]int) - f.step = (*decompressor).nextBlock - f.dict.init(maxMatchOffset, nil) - return &f -} - -// NewReaderDict is like NewReader but initializes the reader -// with a preset dictionary. The returned Reader behaves as if -// the uncompressed data stream started with the given dictionary, -// which has already been read. NewReaderDict is typically used -// to read data compressed by NewWriterDict. -// -// The ReadCloser returned by NewReader also implements Resetter. -func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser { - fixedHuffmanDecoderInit() - - var f decompressor - f.r = makeReader(r) - f.bits = new([maxNumLit + maxNumDist]int) - f.codebits = new([numCodes]int) - f.step = (*decompressor).nextBlock - f.dict.init(maxMatchOffset, dict) - return &f -} diff --git a/vendor/github.com/klauspost/compress/flate/reverse_bits.go b/vendor/github.com/klauspost/compress/flate/reverse_bits.go deleted file mode 100644 index c1a02720..00000000 --- a/vendor/github.com/klauspost/compress/flate/reverse_bits.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -var reverseByte = [256]byte{ - 0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, - 0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0, - 0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8, - 0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8, - 0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4, - 0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4, - 0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec, - 0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc, - 0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2, - 0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2, - 0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea, - 0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa, - 0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6, - 0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6, - 0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee, - 0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe, - 0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1, - 0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1, - 0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9, - 0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9, - 0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5, - 0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5, - 0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed, - 0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd, - 0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3, - 0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3, - 0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb, - 0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb, - 0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7, - 0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7, - 0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef, - 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff, -} - -func reverseUint16(v uint16) uint16 { - return uint16(reverseByte[v>>8]) | uint16(reverseByte[v&0xFF])<<8 -} - -func reverseBits(number uint16, bitLength byte) uint16 { - return reverseUint16(number << uint8(16-bitLength)) -} diff --git a/vendor/github.com/klauspost/compress/flate/snappy.go b/vendor/github.com/klauspost/compress/flate/snappy.go deleted file mode 100644 index d853320a..00000000 --- a/vendor/github.com/klauspost/compress/flate/snappy.go +++ /dev/null @@ -1,900 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Modified for deflate by Klaus Post (c) 2015. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -// emitLiteral writes a literal chunk and returns the number of bytes written. -func emitLiteral(dst *tokens, lit []byte) { - ol := int(dst.n) - for i, v := range lit { - dst.tokens[(i+ol)&maxStoreBlockSize] = token(v) - } - dst.n += uint16(len(lit)) -} - -// emitCopy writes a copy chunk and returns the number of bytes written. -func emitCopy(dst *tokens, offset, length int) { - dst.tokens[dst.n] = matchToken(uint32(length-3), uint32(offset-minOffsetSize)) - dst.n++ -} - -type snappyEnc interface { - Encode(dst *tokens, src []byte) - Reset() -} - -func newSnappy(level int) snappyEnc { - switch level { - case 1: - return &snappyL1{} - case 2: - return &snappyL2{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}} - case 3: - return &snappyL3{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}} - case 4: - return &snappyL4{snappyL3{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}}} - default: - panic("invalid level specified") - } -} - -const ( - tableBits = 14 // Bits used in the table - tableSize = 1 << tableBits // Size of the table - tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. - tableShift = 32 - tableBits // Right-shift to get the tableBits most significant bits of a uint32. - baseMatchOffset = 1 // The smallest match offset - baseMatchLength = 3 // The smallest match length per the RFC section 3.2.5 - maxMatchOffset = 1 << 15 // The largest match offset -) - -func load32(b []byte, i int) uint32 { - b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line. - return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 -} - -func load64(b []byte, i int) uint64 { - b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line. - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 -} - -func hash(u uint32) uint32 { - return (u * 0x1e35a7bd) >> tableShift -} - -// snappyL1 encapsulates level 1 compression -type snappyL1 struct{} - -func (e *snappyL1) Reset() {} - -func (e *snappyL1) Encode(dst *tokens, src []byte) { - const ( - inputMargin = 16 - 1 - minNonLiteralBlockSize = 1 + 1 + inputMargin - ) - - // This check isn't in the Snappy implementation, but there, the caller - // instead of the callee handles this case. - if len(src) < minNonLiteralBlockSize { - // We do not fill the token table. - // This will be picked up by caller. - dst.n = uint16(len(src)) - return - } - - // Initialize the hash table. - // - // The table element type is uint16, as s < sLimit and sLimit < len(src) - // and len(src) <= maxStoreBlockSize and maxStoreBlockSize == 65535. - var table [tableSize]uint16 - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := len(src) - inputMargin - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := 0 - - // The encoded form must start with a literal, as there are no previous - // bytes to copy, so we start looking for hash matches at s == 1. - s := 1 - nextHash := hash(load32(src, s)) - - for { - // Copied from the C++ snappy implementation: - // - // Heuristic match skipping: If 32 bytes are scanned with no matches - // found, start looking only at every other byte. If 32 more bytes are - // scanned (or skipped), look at every third byte, etc.. When a match - // is found, immediately go back to looking at every byte. This is a - // small loss (~5% performance, ~0.1% density) for compressible data - // due to more bookkeeping, but for non-compressible data (such as - // JPEG) it's a huge win since the compressor quickly "realizes" the - // data is incompressible and doesn't bother looking for matches - // everywhere. - // - // The "skip" variable keeps track of how many bytes there are since - // the last match; dividing it by 32 (ie. right-shifting by five) gives - // the number of bytes to move ahead for each iteration. - skip := 32 - - nextS := s - candidate := 0 - for { - s = nextS - bytesBetweenHashLookups := skip >> 5 - nextS = s + bytesBetweenHashLookups - skip += bytesBetweenHashLookups - if nextS > sLimit { - goto emitRemainder - } - candidate = int(table[nextHash&tableMask]) - table[nextHash&tableMask] = uint16(s) - nextHash = hash(load32(src, nextS)) - if s-candidate <= maxMatchOffset && load32(src, s) == load32(src, candidate) { - break - } - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - emitLiteral(dst, src[nextEmit:s]) - - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - base := s - - // Extend the 4-byte match as long as possible. - // - // This is an inlined version of Snappy's: - // s = extendMatch(src, candidate+4, s+4) - s += 4 - s1 := base + maxMatchLength - if s1 > len(src) { - s1 = len(src) - } - a := src[s:s1] - b := src[candidate+4:] - b = b[:len(a)] - l := len(a) - for i := range a { - if a[i] != b[i] { - l = i - break - } - } - s += l - - // matchToken is flate's equivalent of Snappy's emitCopy. - dst.tokens[dst.n] = matchToken(uint32(s-base-baseMatchLength), uint32(base-candidate-baseMatchOffset)) - dst.n++ - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-1 and at s. If - // another emitCopy is not our next move, also calculate nextHash - // at s+1. At least on GOARCH=amd64, these three hash calculations - // are faster as one load64 call (with some shifts) instead of - // three load32 calls. - x := load64(src, s-1) - prevHash := hash(uint32(x >> 0)) - table[prevHash&tableMask] = uint16(s - 1) - currHash := hash(uint32(x >> 8)) - candidate = int(table[currHash&tableMask]) - table[currHash&tableMask] = uint16(s) - if s-candidate > maxMatchOffset || uint32(x>>8) != load32(src, candidate) { - nextHash = hash(uint32(x >> 16)) - s++ - break - } - } - } - -emitRemainder: - if nextEmit < len(src) { - emitLiteral(dst, src[nextEmit:]) - } -} - -type tableEntry struct { - val uint32 - offset int32 -} - -func load3232(b []byte, i int32) uint32 { - b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line. - return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 -} - -func load6432(b []byte, i int32) uint64 { - b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line. - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 -} - -// snappyGen maintains the table for matches, -// and the previous byte block for level 2. -// This is the generic implementation. -type snappyGen struct { - prev []byte - cur int32 -} - -// snappyGen maintains the table for matches, -// and the previous byte block for level 2. -// This is the generic implementation. -type snappyL2 struct { - snappyGen - table [tableSize]tableEntry -} - -// EncodeL2 uses a similar algorithm to level 1, but is capable -// of matching across blocks giving better compression at a small slowdown. -func (e *snappyL2) Encode(dst *tokens, src []byte) { - const ( - inputMargin = 8 - 1 - minNonLiteralBlockSize = 1 + 1 + inputMargin - ) - - // Protect against e.cur wraparound. - if e.cur > 1<<30 { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } - e.cur = maxStoreBlockSize - } - - // This check isn't in the Snappy implementation, but there, the caller - // instead of the callee handles this case. - if len(src) < minNonLiteralBlockSize { - // We do not fill the token table. - // This will be picked up by caller. - dst.n = uint16(len(src)) - e.cur += maxStoreBlockSize - e.prev = e.prev[:0] - return - } - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := int32(len(src) - inputMargin) - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := int32(0) - s := int32(0) - cv := load3232(src, s) - nextHash := hash(cv) - - for { - // Copied from the C++ snappy implementation: - // - // Heuristic match skipping: If 32 bytes are scanned with no matches - // found, start looking only at every other byte. If 32 more bytes are - // scanned (or skipped), look at every third byte, etc.. When a match - // is found, immediately go back to looking at every byte. This is a - // small loss (~5% performance, ~0.1% density) for compressible data - // due to more bookkeeping, but for non-compressible data (such as - // JPEG) it's a huge win since the compressor quickly "realizes" the - // data is incompressible and doesn't bother looking for matches - // everywhere. - // - // The "skip" variable keeps track of how many bytes there are since - // the last match; dividing it by 32 (ie. right-shifting by five) gives - // the number of bytes to move ahead for each iteration. - skip := int32(32) - - nextS := s - var candidate tableEntry - for { - s = nextS - bytesBetweenHashLookups := skip >> 5 - nextS = s + bytesBetweenHashLookups - skip += bytesBetweenHashLookups - if nextS > sLimit { - goto emitRemainder - } - candidate = e.table[nextHash&tableMask] - now := load3232(src, nextS) - e.table[nextHash&tableMask] = tableEntry{offset: s + e.cur, val: cv} - nextHash = hash(now) - - offset := s - (candidate.offset - e.cur) - if offset > maxMatchOffset || cv != candidate.val { - // Out of range or not matched. - cv = now - continue - } - break - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - emitLiteral(dst, src[nextEmit:s]) - - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - - // Extend the 4-byte match as long as possible. - // - s += 4 - t := candidate.offset - e.cur + 4 - l := e.matchlen(s, t, src) - - // matchToken is flate's equivalent of Snappy's emitCopy. (length,offset) - dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset)) - dst.n++ - s += l - nextEmit = s - if s >= sLimit { - t += l - // Index first pair after match end. - if int(t+4) < len(src) && t > 0 { - cv := load3232(src, t) - e.table[hash(cv)&tableMask] = tableEntry{offset: t + e.cur, val: cv} - } - goto emitRemainder - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-1 and at s. If - // another emitCopy is not our next move, also calculate nextHash - // at s+1. At least on GOARCH=amd64, these three hash calculations - // are faster as one load64 call (with some shifts) instead of - // three load32 calls. - x := load6432(src, s-1) - prevHash := hash(uint32(x)) - e.table[prevHash&tableMask] = tableEntry{offset: e.cur + s - 1, val: uint32(x)} - x >>= 8 - currHash := hash(uint32(x)) - candidate = e.table[currHash&tableMask] - e.table[currHash&tableMask] = tableEntry{offset: e.cur + s, val: uint32(x)} - - offset := s - (candidate.offset - e.cur) - if offset > maxMatchOffset || uint32(x) != candidate.val { - cv = uint32(x >> 8) - nextHash = hash(cv) - s++ - break - } - } - } - -emitRemainder: - if int(nextEmit) < len(src) { - emitLiteral(dst, src[nextEmit:]) - } - e.cur += int32(len(src)) - e.prev = e.prev[:len(src)] - copy(e.prev, src) -} - -type tableEntryPrev struct { - Cur tableEntry - Prev tableEntry -} - -// snappyL3 -type snappyL3 struct { - snappyGen - table [tableSize]tableEntryPrev -} - -// Encode uses a similar algorithm to level 2, will check up to two candidates. -func (e *snappyL3) Encode(dst *tokens, src []byte) { - const ( - inputMargin = 8 - 1 - minNonLiteralBlockSize = 1 + 1 + inputMargin - ) - - // Protect against e.cur wraparound. - if e.cur > 1<<30 { - for i := range e.table[:] { - e.table[i] = tableEntryPrev{} - } - e.snappyGen = snappyGen{cur: maxStoreBlockSize, prev: e.prev[:0]} - } - - // This check isn't in the Snappy implementation, but there, the caller - // instead of the callee handles this case. - if len(src) < minNonLiteralBlockSize { - // We do not fill the token table. - // This will be picked up by caller. - dst.n = uint16(len(src)) - e.cur += maxStoreBlockSize - e.prev = e.prev[:0] - return - } - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := int32(len(src) - inputMargin) - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := int32(0) - s := int32(0) - cv := load3232(src, s) - nextHash := hash(cv) - - for { - // Copied from the C++ snappy implementation: - // - // Heuristic match skipping: If 32 bytes are scanned with no matches - // found, start looking only at every other byte. If 32 more bytes are - // scanned (or skipped), look at every third byte, etc.. When a match - // is found, immediately go back to looking at every byte. This is a - // small loss (~5% performance, ~0.1% density) for compressible data - // due to more bookkeeping, but for non-compressible data (such as - // JPEG) it's a huge win since the compressor quickly "realizes" the - // data is incompressible and doesn't bother looking for matches - // everywhere. - // - // The "skip" variable keeps track of how many bytes there are since - // the last match; dividing it by 32 (ie. right-shifting by five) gives - // the number of bytes to move ahead for each iteration. - skip := int32(32) - - nextS := s - var candidate tableEntry - for { - s = nextS - bytesBetweenHashLookups := skip >> 5 - nextS = s + bytesBetweenHashLookups - skip += bytesBetweenHashLookups - if nextS > sLimit { - goto emitRemainder - } - candidates := e.table[nextHash&tableMask] - now := load3232(src, nextS) - e.table[nextHash&tableMask] = tableEntryPrev{Prev: candidates.Cur, Cur: tableEntry{offset: s + e.cur, val: cv}} - nextHash = hash(now) - - // Check both candidates - candidate = candidates.Cur - if cv == candidate.val { - offset := s - (candidate.offset - e.cur) - if offset <= maxMatchOffset { - break - } - } else { - // We only check if value mismatches. - // Offset will always be invalid in other cases. - candidate = candidates.Prev - if cv == candidate.val { - offset := s - (candidate.offset - e.cur) - if offset <= maxMatchOffset { - break - } - } - } - cv = now - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - emitLiteral(dst, src[nextEmit:s]) - - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - - // Extend the 4-byte match as long as possible. - // - s += 4 - t := candidate.offset - e.cur + 4 - l := e.matchlen(s, t, src) - - // matchToken is flate's equivalent of Snappy's emitCopy. (length,offset) - dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset)) - dst.n++ - s += l - nextEmit = s - if s >= sLimit { - t += l - // Index first pair after match end. - if int(t+4) < len(src) && t > 0 { - cv := load3232(src, t) - nextHash = hash(cv) - e.table[nextHash&tableMask] = tableEntryPrev{ - Prev: e.table[nextHash&tableMask].Cur, - Cur: tableEntry{offset: e.cur + t, val: cv}, - } - } - goto emitRemainder - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-3 to s. If - // another emitCopy is not our next move, also calculate nextHash - // at s+1. At least on GOARCH=amd64, these three hash calculations - // are faster as one load64 call (with some shifts) instead of - // three load32 calls. - x := load6432(src, s-3) - prevHash := hash(uint32(x)) - e.table[prevHash&tableMask] = tableEntryPrev{ - Prev: e.table[prevHash&tableMask].Cur, - Cur: tableEntry{offset: e.cur + s - 3, val: uint32(x)}, - } - x >>= 8 - prevHash = hash(uint32(x)) - - e.table[prevHash&tableMask] = tableEntryPrev{ - Prev: e.table[prevHash&tableMask].Cur, - Cur: tableEntry{offset: e.cur + s - 2, val: uint32(x)}, - } - x >>= 8 - prevHash = hash(uint32(x)) - - e.table[prevHash&tableMask] = tableEntryPrev{ - Prev: e.table[prevHash&tableMask].Cur, - Cur: tableEntry{offset: e.cur + s - 1, val: uint32(x)}, - } - x >>= 8 - currHash := hash(uint32(x)) - candidates := e.table[currHash&tableMask] - cv = uint32(x) - e.table[currHash&tableMask] = tableEntryPrev{ - Prev: candidates.Cur, - Cur: tableEntry{offset: s + e.cur, val: cv}, - } - - // Check both candidates - candidate = candidates.Cur - if cv == candidate.val { - offset := s - (candidate.offset - e.cur) - if offset <= maxMatchOffset { - continue - } - } else { - // We only check if value mismatches. - // Offset will always be invalid in other cases. - candidate = candidates.Prev - if cv == candidate.val { - offset := s - (candidate.offset - e.cur) - if offset <= maxMatchOffset { - continue - } - } - } - cv = uint32(x >> 8) - nextHash = hash(cv) - s++ - break - } - } - -emitRemainder: - if int(nextEmit) < len(src) { - emitLiteral(dst, src[nextEmit:]) - } - e.cur += int32(len(src)) - e.prev = e.prev[:len(src)] - copy(e.prev, src) -} - -// snappyL4 -type snappyL4 struct { - snappyL3 -} - -// Encode uses a similar algorithm to level 3, -// but will check up to two candidates if first isn't long enough. -func (e *snappyL4) Encode(dst *tokens, src []byte) { - const ( - inputMargin = 8 - 3 - minNonLiteralBlockSize = 1 + 1 + inputMargin - matchLenGood = 12 - ) - - // Protect against e.cur wraparound. - if e.cur > 1<<30 { - for i := range e.table[:] { - e.table[i] = tableEntryPrev{} - } - e.snappyGen = snappyGen{cur: maxStoreBlockSize, prev: e.prev[:0]} - } - - // This check isn't in the Snappy implementation, but there, the caller - // instead of the callee handles this case. - if len(src) < minNonLiteralBlockSize { - // We do not fill the token table. - // This will be picked up by caller. - dst.n = uint16(len(src)) - e.cur += maxStoreBlockSize - e.prev = e.prev[:0] - return - } - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := int32(len(src) - inputMargin) - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := int32(0) - s := int32(0) - cv := load3232(src, s) - nextHash := hash(cv) - - for { - // Copied from the C++ snappy implementation: - // - // Heuristic match skipping: If 32 bytes are scanned with no matches - // found, start looking only at every other byte. If 32 more bytes are - // scanned (or skipped), look at every third byte, etc.. When a match - // is found, immediately go back to looking at every byte. This is a - // small loss (~5% performance, ~0.1% density) for compressible data - // due to more bookkeeping, but for non-compressible data (such as - // JPEG) it's a huge win since the compressor quickly "realizes" the - // data is incompressible and doesn't bother looking for matches - // everywhere. - // - // The "skip" variable keeps track of how many bytes there are since - // the last match; dividing it by 32 (ie. right-shifting by five) gives - // the number of bytes to move ahead for each iteration. - skip := int32(32) - - nextS := s - var candidate tableEntry - var candidateAlt tableEntry - for { - s = nextS - bytesBetweenHashLookups := skip >> 5 - nextS = s + bytesBetweenHashLookups - skip += bytesBetweenHashLookups - if nextS > sLimit { - goto emitRemainder - } - candidates := e.table[nextHash&tableMask] - now := load3232(src, nextS) - e.table[nextHash&tableMask] = tableEntryPrev{Prev: candidates.Cur, Cur: tableEntry{offset: s + e.cur, val: cv}} - nextHash = hash(now) - - // Check both candidates - candidate = candidates.Cur - if cv == candidate.val { - offset := s - (candidate.offset - e.cur) - if offset < maxMatchOffset { - offset = s - (candidates.Prev.offset - e.cur) - if cv == candidates.Prev.val && offset < maxMatchOffset { - candidateAlt = candidates.Prev - } - break - } - } else { - // We only check if value mismatches. - // Offset will always be invalid in other cases. - candidate = candidates.Prev - if cv == candidate.val { - offset := s - (candidate.offset - e.cur) - if offset < maxMatchOffset { - break - } - } - } - cv = now - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - emitLiteral(dst, src[nextEmit:s]) - - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - - // Extend the 4-byte match as long as possible. - // - s += 4 - t := candidate.offset - e.cur + 4 - l := e.matchlen(s, t, src) - // Try alternative candidate if match length < matchLenGood. - if l < matchLenGood-4 && candidateAlt.offset != 0 { - t2 := candidateAlt.offset - e.cur + 4 - l2 := e.matchlen(s, t2, src) - if l2 > l { - l = l2 - t = t2 - } - } - // matchToken is flate's equivalent of Snappy's emitCopy. (length,offset) - dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset)) - dst.n++ - s += l - nextEmit = s - if s >= sLimit { - t += l - // Index first pair after match end. - if int(t+4) < len(src) && t > 0 { - cv := load3232(src, t) - nextHash = hash(cv) - e.table[nextHash&tableMask] = tableEntryPrev{ - Prev: e.table[nextHash&tableMask].Cur, - Cur: tableEntry{offset: e.cur + t, val: cv}, - } - } - goto emitRemainder - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-3 to s. If - // another emitCopy is not our next move, also calculate nextHash - // at s+1. At least on GOARCH=amd64, these three hash calculations - // are faster as one load64 call (with some shifts) instead of - // three load32 calls. - x := load6432(src, s-3) - prevHash := hash(uint32(x)) - e.table[prevHash&tableMask] = tableEntryPrev{ - Prev: e.table[prevHash&tableMask].Cur, - Cur: tableEntry{offset: e.cur + s - 3, val: uint32(x)}, - } - x >>= 8 - prevHash = hash(uint32(x)) - - e.table[prevHash&tableMask] = tableEntryPrev{ - Prev: e.table[prevHash&tableMask].Cur, - Cur: tableEntry{offset: e.cur + s - 2, val: uint32(x)}, - } - x >>= 8 - prevHash = hash(uint32(x)) - - e.table[prevHash&tableMask] = tableEntryPrev{ - Prev: e.table[prevHash&tableMask].Cur, - Cur: tableEntry{offset: e.cur + s - 1, val: uint32(x)}, - } - x >>= 8 - currHash := hash(uint32(x)) - candidates := e.table[currHash&tableMask] - cv = uint32(x) - e.table[currHash&tableMask] = tableEntryPrev{ - Prev: candidates.Cur, - Cur: tableEntry{offset: s + e.cur, val: cv}, - } - - // Check both candidates - candidate = candidates.Cur - candidateAlt = tableEntry{} - if cv == candidate.val { - offset := s - (candidate.offset - e.cur) - if offset <= maxMatchOffset { - offset = s - (candidates.Prev.offset - e.cur) - if cv == candidates.Prev.val && offset <= maxMatchOffset { - candidateAlt = candidates.Prev - } - continue - } - } else { - // We only check if value mismatches. - // Offset will always be invalid in other cases. - candidate = candidates.Prev - if cv == candidate.val { - offset := s - (candidate.offset - e.cur) - if offset <= maxMatchOffset { - continue - } - } - } - cv = uint32(x >> 8) - nextHash = hash(cv) - s++ - break - } - } - -emitRemainder: - if int(nextEmit) < len(src) { - emitLiteral(dst, src[nextEmit:]) - } - e.cur += int32(len(src)) - e.prev = e.prev[:len(src)] - copy(e.prev, src) -} - -func (e *snappyGen) matchlen(s, t int32, src []byte) int32 { - s1 := int(s) + maxMatchLength - 4 - if s1 > len(src) { - s1 = len(src) - } - - // If we are inside the current block - if t >= 0 { - b := src[t:] - a := src[s:s1] - b = b[:len(a)] - // Extend the match to be as long as possible. - for i := range a { - if a[i] != b[i] { - return int32(i) - } - } - return int32(len(a)) - } - - // We found a match in the previous block. - tp := int32(len(e.prev)) + t - if tp < 0 { - return 0 - } - - // Extend the match to be as long as possible. - a := src[s:s1] - b := e.prev[tp:] - if len(b) > len(a) { - b = b[:len(a)] - } - a = a[:len(b)] - for i := range b { - if a[i] != b[i] { - return int32(i) - } - } - - // If we reached our limit, we matched everything we are - // allowed to in the previous block and we return. - n := int32(len(b)) - if int(s+n) == s1 { - return n - } - - // Continue looking for more matches in the current block. - a = src[s+n : s1] - b = src[:len(a)] - for i := range a { - if a[i] != b[i] { - return int32(i) + n - } - } - return int32(len(a)) + n -} - -// Reset the encoding table. -func (e *snappyGen) Reset() { - e.prev = e.prev[:0] - e.cur += maxMatchOffset -} diff --git a/vendor/github.com/klauspost/compress/flate/token.go b/vendor/github.com/klauspost/compress/flate/token.go deleted file mode 100644 index 4f275ea6..00000000 --- a/vendor/github.com/klauspost/compress/flate/token.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import "fmt" - -const ( - // 2 bits: type 0 = literal 1=EOF 2=Match 3=Unused - // 8 bits: xlength = length - MIN_MATCH_LENGTH - // 22 bits xoffset = offset - MIN_OFFSET_SIZE, or literal - lengthShift = 22 - offsetMask = 1< pair into a match token. -func matchToken(xlength uint32, xoffset uint32) token { - return token(matchType + xlength< maxMatchLength || xoffset > maxMatchOffset { - panic(fmt.Sprintf("Invalid match: len: %d, offset: %d\n", xlength, xoffset)) - return token(matchType) - } - return token(matchType + xlength<> lengthShift) } - -func lengthCode(len uint32) uint32 { return lengthCodes[len] } - -// Returns the offset code corresponding to a specific offset -func offsetCode(off uint32) uint32 { - if off < uint32(len(offsetCodes)) { - return offsetCodes[off] - } else if off>>7 < uint32(len(offsetCodes)) { - return offsetCodes[off>>7] + 14 - } else { - return offsetCodes[off>>14] + 28 - } -} diff --git a/vendor/github.com/klauspost/compress/gzip/gunzip.go b/vendor/github.com/klauspost/compress/gzip/gunzip.go deleted file mode 100644 index 568b5d4f..00000000 --- a/vendor/github.com/klauspost/compress/gzip/gunzip.go +++ /dev/null @@ -1,344 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package gzip implements reading and writing of gzip format compressed files, -// as specified in RFC 1952. -package gzip - -import ( - "bufio" - "encoding/binary" - "errors" - "hash/crc32" - "io" - "time" - - "github.com/klauspost/compress/flate" -) - -const ( - gzipID1 = 0x1f - gzipID2 = 0x8b - gzipDeflate = 8 - flagText = 1 << 0 - flagHdrCrc = 1 << 1 - flagExtra = 1 << 2 - flagName = 1 << 3 - flagComment = 1 << 4 -) - -var ( - // ErrChecksum is returned when reading GZIP data that has an invalid checksum. - ErrChecksum = errors.New("gzip: invalid checksum") - // ErrHeader is returned when reading GZIP data that has an invalid header. - ErrHeader = errors.New("gzip: invalid header") -) - -var le = binary.LittleEndian - -// noEOF converts io.EOF to io.ErrUnexpectedEOF. -func noEOF(err error) error { - if err == io.EOF { - return io.ErrUnexpectedEOF - } - return err -} - -// The gzip file stores a header giving metadata about the compressed file. -// That header is exposed as the fields of the Writer and Reader structs. -// -// Strings must be UTF-8 encoded and may only contain Unicode code points -// U+0001 through U+00FF, due to limitations of the GZIP file format. -type Header struct { - Comment string // comment - Extra []byte // "extra data" - ModTime time.Time // modification time - Name string // file name - OS byte // operating system type -} - -// A Reader is an io.Reader that can be read to retrieve -// uncompressed data from a gzip-format compressed file. -// -// In general, a gzip file can be a concatenation of gzip files, -// each with its own header. Reads from the Reader -// return the concatenation of the uncompressed data of each. -// Only the first header is recorded in the Reader fields. -// -// Gzip files store a length and checksum of the uncompressed data. -// The Reader will return a ErrChecksum when Read -// reaches the end of the uncompressed data if it does not -// have the expected length or checksum. Clients should treat data -// returned by Read as tentative until they receive the io.EOF -// marking the end of the data. -type Reader struct { - Header // valid after NewReader or Reader.Reset - r flate.Reader - decompressor io.ReadCloser - digest uint32 // CRC-32, IEEE polynomial (section 8) - size uint32 // Uncompressed size (section 2.3.1) - buf [512]byte - err error - multistream bool -} - -// NewReader creates a new Reader reading the given reader. -// If r does not also implement io.ByteReader, -// the decompressor may read more data than necessary from r. -// -// It is the caller's responsibility to call Close on the Reader when done. -// -// The Reader.Header fields will be valid in the Reader returned. -func NewReader(r io.Reader) (*Reader, error) { - z := new(Reader) - if err := z.Reset(r); err != nil { - return nil, err - } - return z, nil -} - -// Reset discards the Reader z's state and makes it equivalent to the -// result of its original state from NewReader, but reading from r instead. -// This permits reusing a Reader rather than allocating a new one. -func (z *Reader) Reset(r io.Reader) error { - *z = Reader{ - decompressor: z.decompressor, - multistream: true, - } - if rr, ok := r.(flate.Reader); ok { - z.r = rr - } else { - z.r = bufio.NewReader(r) - } - z.Header, z.err = z.readHeader() - return z.err -} - -// Multistream controls whether the reader supports multistream files. -// -// If enabled (the default), the Reader expects the input to be a sequence -// of individually gzipped data streams, each with its own header and -// trailer, ending at EOF. The effect is that the concatenation of a sequence -// of gzipped files is treated as equivalent to the gzip of the concatenation -// of the sequence. This is standard behavior for gzip readers. -// -// Calling Multistream(false) disables this behavior; disabling the behavior -// can be useful when reading file formats that distinguish individual gzip -// data streams or mix gzip data streams with other data streams. -// In this mode, when the Reader reaches the end of the data stream, -// Read returns io.EOF. If the underlying reader implements io.ByteReader, -// it will be left positioned just after the gzip stream. -// To start the next stream, call z.Reset(r) followed by z.Multistream(false). -// If there is no next stream, z.Reset(r) will return io.EOF. -func (z *Reader) Multistream(ok bool) { - z.multistream = ok -} - -// readString reads a NUL-terminated string from z.r. -// It treats the bytes read as being encoded as ISO 8859-1 (Latin-1) and -// will output a string encoded using UTF-8. -// This method always updates z.digest with the data read. -func (z *Reader) readString() (string, error) { - var err error - needConv := false - for i := 0; ; i++ { - if i >= len(z.buf) { - return "", ErrHeader - } - z.buf[i], err = z.r.ReadByte() - if err != nil { - return "", err - } - if z.buf[i] > 0x7f { - needConv = true - } - if z.buf[i] == 0 { - // Digest covers the NUL terminator. - z.digest = crc32.Update(z.digest, crc32.IEEETable, z.buf[:i+1]) - - // Strings are ISO 8859-1, Latin-1 (RFC 1952, section 2.3.1). - if needConv { - s := make([]rune, 0, i) - for _, v := range z.buf[:i] { - s = append(s, rune(v)) - } - return string(s), nil - } - return string(z.buf[:i]), nil - } - } -} - -// readHeader reads the GZIP header according to section 2.3.1. -// This method does not set z.err. -func (z *Reader) readHeader() (hdr Header, err error) { - if _, err = io.ReadFull(z.r, z.buf[:10]); err != nil { - // RFC 1952, section 2.2, says the following: - // A gzip file consists of a series of "members" (compressed data sets). - // - // Other than this, the specification does not clarify whether a - // "series" is defined as "one or more" or "zero or more". To err on the - // side of caution, Go interprets this to mean "zero or more". - // Thus, it is okay to return io.EOF here. - return hdr, err - } - if z.buf[0] != gzipID1 || z.buf[1] != gzipID2 || z.buf[2] != gzipDeflate { - return hdr, ErrHeader - } - flg := z.buf[3] - hdr.ModTime = time.Unix(int64(le.Uint32(z.buf[4:8])), 0) - // z.buf[8] is XFL and is currently ignored. - hdr.OS = z.buf[9] - z.digest = crc32.ChecksumIEEE(z.buf[:10]) - - if flg&flagExtra != 0 { - if _, err = io.ReadFull(z.r, z.buf[:2]); err != nil { - return hdr, noEOF(err) - } - z.digest = crc32.Update(z.digest, crc32.IEEETable, z.buf[:2]) - data := make([]byte, le.Uint16(z.buf[:2])) - if _, err = io.ReadFull(z.r, data); err != nil { - return hdr, noEOF(err) - } - z.digest = crc32.Update(z.digest, crc32.IEEETable, data) - hdr.Extra = data - } - - var s string - if flg&flagName != 0 { - if s, err = z.readString(); err != nil { - return hdr, err - } - hdr.Name = s - } - - if flg&flagComment != 0 { - if s, err = z.readString(); err != nil { - return hdr, err - } - hdr.Comment = s - } - - if flg&flagHdrCrc != 0 { - if _, err = io.ReadFull(z.r, z.buf[:2]); err != nil { - return hdr, noEOF(err) - } - digest := le.Uint16(z.buf[:2]) - if digest != uint16(z.digest) { - return hdr, ErrHeader - } - } - - z.digest = 0 - if z.decompressor == nil { - z.decompressor = flate.NewReader(z.r) - } else { - z.decompressor.(flate.Resetter).Reset(z.r, nil) - } - return hdr, nil -} - -// Read implements io.Reader, reading uncompressed bytes from its underlying Reader. -func (z *Reader) Read(p []byte) (n int, err error) { - if z.err != nil { - return 0, z.err - } - - n, z.err = z.decompressor.Read(p) - z.digest = crc32.Update(z.digest, crc32.IEEETable, p[:n]) - z.size += uint32(n) - if z.err != io.EOF { - // In the normal case we return here. - return n, z.err - } - - // Finished file; check checksum and size. - if _, err := io.ReadFull(z.r, z.buf[:8]); err != nil { - z.err = noEOF(err) - return n, z.err - } - digest := le.Uint32(z.buf[:4]) - size := le.Uint32(z.buf[4:8]) - if digest != z.digest || size != z.size { - z.err = ErrChecksum - return n, z.err - } - z.digest, z.size = 0, 0 - - // File is ok; check if there is another. - if !z.multistream { - return n, io.EOF - } - z.err = nil // Remove io.EOF - - if _, z.err = z.readHeader(); z.err != nil { - return n, z.err - } - - // Read from next file, if necessary. - if n > 0 { - return n, nil - } - return z.Read(p) -} - -// Support the io.WriteTo interface for io.Copy and friends. -func (z *Reader) WriteTo(w io.Writer) (int64, error) { - total := int64(0) - crcWriter := crc32.NewIEEE() - for { - if z.err != nil { - if z.err == io.EOF { - return total, nil - } - return total, z.err - } - - // We write both to output and digest. - mw := io.MultiWriter(w, crcWriter) - n, err := z.decompressor.(io.WriterTo).WriteTo(mw) - total += n - z.size += uint32(n) - if err != nil { - z.err = err - return total, z.err - } - - // Finished file; check checksum + size. - if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - z.err = err - return total, err - } - z.digest = crcWriter.Sum32() - digest := le.Uint32(z.buf[:4]) - size := le.Uint32(z.buf[4:8]) - if digest != z.digest || size != z.size { - z.err = ErrChecksum - return total, z.err - } - z.digest, z.size = 0, 0 - - // File is ok; check if there is another. - if !z.multistream { - return total, nil - } - crcWriter.Reset() - z.err = nil // Remove io.EOF - - if _, z.err = z.readHeader(); z.err != nil { - if z.err == io.EOF { - return total, nil - } - return total, z.err - } - } -} - -// Close closes the Reader. It does not close the underlying io.Reader. -// In order for the GZIP checksum to be verified, the reader must be -// fully consumed until the io.EOF. -func (z *Reader) Close() error { return z.decompressor.Close() } diff --git a/vendor/github.com/klauspost/compress/gzip/gzip.go b/vendor/github.com/klauspost/compress/gzip/gzip.go deleted file mode 100644 index 7da7ee74..00000000 --- a/vendor/github.com/klauspost/compress/gzip/gzip.go +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package gzip - -import ( - "errors" - "fmt" - "hash/crc32" - "io" - - "github.com/klauspost/compress/flate" -) - -// These constants are copied from the flate package, so that code that imports -// "compress/gzip" does not also have to import "compress/flate". -const ( - NoCompression = flate.NoCompression - BestSpeed = flate.BestSpeed - BestCompression = flate.BestCompression - DefaultCompression = flate.DefaultCompression - ConstantCompression = flate.ConstantCompression - HuffmanOnly = flate.HuffmanOnly -) - -// A Writer is an io.WriteCloser. -// Writes to a Writer are compressed and written to w. -type Writer struct { - Header // written at first call to Write, Flush, or Close - w io.Writer - level int - wroteHeader bool - compressor *flate.Writer - digest uint32 // CRC-32, IEEE polynomial (section 8) - size uint32 // Uncompressed size (section 2.3.1) - closed bool - buf [10]byte - err error -} - -// NewWriter returns a new Writer. -// Writes to the returned writer are compressed and written to w. -// -// It is the caller's responsibility to call Close on the WriteCloser when done. -// Writes may be buffered and not flushed until Close. -// -// Callers that wish to set the fields in Writer.Header must do so before -// the first call to Write, Flush, or Close. -func NewWriter(w io.Writer) *Writer { - z, _ := NewWriterLevel(w, DefaultCompression) - return z -} - -// NewWriterLevel is like NewWriter but specifies the compression level instead -// of assuming DefaultCompression. -// -// The compression level can be DefaultCompression, NoCompression, or any -// integer value between BestSpeed and BestCompression inclusive. The error -// returned will be nil if the level is valid. -func NewWriterLevel(w io.Writer, level int) (*Writer, error) { - if level < HuffmanOnly || level > BestCompression { - return nil, fmt.Errorf("gzip: invalid compression level: %d", level) - } - z := new(Writer) - z.init(w, level) - return z, nil -} - -func (z *Writer) init(w io.Writer, level int) { - compressor := z.compressor - if compressor != nil { - compressor.Reset(w) - } - *z = Writer{ - Header: Header{ - OS: 255, // unknown - }, - w: w, - level: level, - compressor: compressor, - } -} - -// Reset discards the Writer z's state and makes it equivalent to the -// result of its original state from NewWriter or NewWriterLevel, but -// writing to w instead. This permits reusing a Writer rather than -// allocating a new one. -func (z *Writer) Reset(w io.Writer) { - z.init(w, z.level) -} - -// writeBytes writes a length-prefixed byte slice to z.w. -func (z *Writer) writeBytes(b []byte) error { - if len(b) > 0xffff { - return errors.New("gzip.Write: Extra data is too large") - } - le.PutUint16(z.buf[:2], uint16(len(b))) - _, err := z.w.Write(z.buf[:2]) - if err != nil { - return err - } - _, err = z.w.Write(b) - return err -} - -// writeString writes a UTF-8 string s in GZIP's format to z.w. -// GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1). -func (z *Writer) writeString(s string) (err error) { - // GZIP stores Latin-1 strings; error if non-Latin-1; convert if non-ASCII. - needconv := false - for _, v := range s { - if v == 0 || v > 0xff { - return errors.New("gzip.Write: non-Latin-1 header string") - } - if v > 0x7f { - needconv = true - } - } - if needconv { - b := make([]byte, 0, len(s)) - for _, v := range s { - b = append(b, byte(v)) - } - _, err = z.w.Write(b) - } else { - _, err = io.WriteString(z.w, s) - } - if err != nil { - return err - } - // GZIP strings are NUL-terminated. - z.buf[0] = 0 - _, err = z.w.Write(z.buf[:1]) - return err -} - -// Write writes a compressed form of p to the underlying io.Writer. The -// compressed bytes are not necessarily flushed until the Writer is closed. -func (z *Writer) Write(p []byte) (int, error) { - if z.err != nil { - return 0, z.err - } - var n int - // Write the GZIP header lazily. - if !z.wroteHeader { - z.wroteHeader = true - z.buf[0] = gzipID1 - z.buf[1] = gzipID2 - z.buf[2] = gzipDeflate - z.buf[3] = 0 - if z.Extra != nil { - z.buf[3] |= 0x04 - } - if z.Name != "" { - z.buf[3] |= 0x08 - } - if z.Comment != "" { - z.buf[3] |= 0x10 - } - le.PutUint32(z.buf[4:8], uint32(z.ModTime.Unix())) - if z.level == BestCompression { - z.buf[8] = 2 - } else if z.level == BestSpeed { - z.buf[8] = 4 - } else { - z.buf[8] = 0 - } - z.buf[9] = z.OS - n, z.err = z.w.Write(z.buf[:10]) - if z.err != nil { - return n, z.err - } - if z.Extra != nil { - z.err = z.writeBytes(z.Extra) - if z.err != nil { - return n, z.err - } - } - if z.Name != "" { - z.err = z.writeString(z.Name) - if z.err != nil { - return n, z.err - } - } - if z.Comment != "" { - z.err = z.writeString(z.Comment) - if z.err != nil { - return n, z.err - } - } - if z.compressor == nil { - z.compressor, _ = flate.NewWriter(z.w, z.level) - } - } - z.size += uint32(len(p)) - z.digest = crc32.Update(z.digest, crc32.IEEETable, p) - n, z.err = z.compressor.Write(p) - return n, z.err -} - -// Flush flushes any pending compressed data to the underlying writer. -// -// It is useful mainly in compressed network protocols, to ensure that -// a remote reader has enough data to reconstruct a packet. Flush does -// not return until the data has been written. If the underlying -// writer returns an error, Flush returns that error. -// -// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH. -func (z *Writer) Flush() error { - if z.err != nil { - return z.err - } - if z.closed { - return nil - } - if !z.wroteHeader { - z.Write(nil) - if z.err != nil { - return z.err - } - } - z.err = z.compressor.Flush() - return z.err -} - -// Close closes the Writer, flushing any unwritten data to the underlying -// io.Writer, but does not close the underlying io.Writer. -func (z *Writer) Close() error { - if z.err != nil { - return z.err - } - if z.closed { - return nil - } - z.closed = true - if !z.wroteHeader { - z.Write(nil) - if z.err != nil { - return z.err - } - } - z.err = z.compressor.Close() - if z.err != nil { - return z.err - } - le.PutUint32(z.buf[:4], z.digest) - le.PutUint32(z.buf[4:8], z.size) - _, z.err = z.w.Write(z.buf[:8]) - return z.err -} diff --git a/vendor/github.com/klauspost/compress/zlib/reader.go b/vendor/github.com/klauspost/compress/zlib/reader.go deleted file mode 100644 index d9091e83..00000000 --- a/vendor/github.com/klauspost/compress/zlib/reader.go +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -/* -Package zlib implements reading and writing of zlib format compressed data, -as specified in RFC 1950. - -The implementation provides filters that uncompress during reading -and compress during writing. For example, to write compressed data -to a buffer: - - var b bytes.Buffer - w := zlib.NewWriter(&b) - w.Write([]byte("hello, world\n")) - w.Close() - -and to read that data back: - - r, err := zlib.NewReader(&b) - io.Copy(os.Stdout, r) - r.Close() -*/ -package zlib - -import ( - "bufio" - "errors" - "hash" - "hash/adler32" - "io" - - "github.com/klauspost/compress/flate" -) - -const zlibDeflate = 8 - -var ( - // ErrChecksum is returned when reading ZLIB data that has an invalid checksum. - ErrChecksum = errors.New("zlib: invalid checksum") - // ErrDictionary is returned when reading ZLIB data that has an invalid dictionary. - ErrDictionary = errors.New("zlib: invalid dictionary") - // ErrHeader is returned when reading ZLIB data that has an invalid header. - ErrHeader = errors.New("zlib: invalid header") -) - -type reader struct { - r flate.Reader - decompressor io.ReadCloser - digest hash.Hash32 - err error - scratch [4]byte -} - -// Resetter resets a ReadCloser returned by NewReader or NewReaderDict to -// to switch to a new underlying Reader. This permits reusing a ReadCloser -// instead of allocating a new one. -type Resetter interface { - // Reset discards any buffered data and resets the Resetter as if it was - // newly initialized with the given reader. - Reset(r io.Reader, dict []byte) error -} - -// NewReader creates a new ReadCloser. -// Reads from the returned ReadCloser read and decompress data from r. -// If r does not implement io.ByteReader, the decompressor may read more -// data than necessary from r. -// It is the caller's responsibility to call Close on the ReadCloser when done. -// -// The ReadCloser returned by NewReader also implements Resetter. -func NewReader(r io.Reader) (io.ReadCloser, error) { - return NewReaderDict(r, nil) -} - -// NewReaderDict is like NewReader but uses a preset dictionary. -// NewReaderDict ignores the dictionary if the compressed data does not refer to it. -// If the compressed data refers to a different dictionary, NewReaderDict returns ErrDictionary. -// -// The ReadCloser returned by NewReaderDict also implements Resetter. -func NewReaderDict(r io.Reader, dict []byte) (io.ReadCloser, error) { - z := new(reader) - err := z.Reset(r, dict) - if err != nil { - return nil, err - } - return z, nil -} - -func (z *reader) Read(p []byte) (int, error) { - if z.err != nil { - return 0, z.err - } - - var n int - n, z.err = z.decompressor.Read(p) - z.digest.Write(p[0:n]) - if z.err != io.EOF { - // In the normal case we return here. - return n, z.err - } - - // Finished file; check checksum. - if _, err := io.ReadFull(z.r, z.scratch[0:4]); err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - z.err = err - return n, z.err - } - // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). - checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3]) - if checksum != z.digest.Sum32() { - z.err = ErrChecksum - return n, z.err - } - return n, io.EOF -} - -// Calling Close does not close the wrapped io.Reader originally passed to NewReader. -// In order for the ZLIB checksum to be verified, the reader must be -// fully consumed until the io.EOF. -func (z *reader) Close() error { - if z.err != nil && z.err != io.EOF { - return z.err - } - z.err = z.decompressor.Close() - return z.err -} - -func (z *reader) Reset(r io.Reader, dict []byte) error { - *z = reader{decompressor: z.decompressor, digest: z.digest} - if fr, ok := r.(flate.Reader); ok { - z.r = fr - } else { - z.r = bufio.NewReader(r) - } - - // Read the header (RFC 1950 section 2.2.). - _, z.err = io.ReadFull(z.r, z.scratch[0:2]) - if z.err != nil { - if z.err == io.EOF { - z.err = io.ErrUnexpectedEOF - } - return z.err - } - h := uint(z.scratch[0])<<8 | uint(z.scratch[1]) - if (z.scratch[0]&0x0f != zlibDeflate) || (h%31 != 0) { - z.err = ErrHeader - return z.err - } - haveDict := z.scratch[1]&0x20 != 0 - if haveDict { - _, z.err = io.ReadFull(z.r, z.scratch[0:4]) - if z.err != nil { - if z.err == io.EOF { - z.err = io.ErrUnexpectedEOF - } - return z.err - } - checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3]) - if checksum != adler32.Checksum(dict) { - z.err = ErrDictionary - return z.err - } - } - - if z.decompressor == nil { - if haveDict { - z.decompressor = flate.NewReaderDict(z.r, dict) - } else { - z.decompressor = flate.NewReader(z.r) - } - } else { - z.decompressor.(flate.Resetter).Reset(z.r, dict) - } - - if z.digest != nil { - z.digest.Reset() - } else { - z.digest = adler32.New() - } - return nil -} diff --git a/vendor/github.com/klauspost/compress/zlib/writer.go b/vendor/github.com/klauspost/compress/zlib/writer.go deleted file mode 100644 index 605816ba..00000000 --- a/vendor/github.com/klauspost/compress/zlib/writer.go +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package zlib - -import ( - "fmt" - "hash" - "hash/adler32" - "io" - - "github.com/klauspost/compress/flate" -) - -// These constants are copied from the flate package, so that code that imports -// "compress/zlib" does not also have to import "compress/flate". -const ( - NoCompression = flate.NoCompression - BestSpeed = flate.BestSpeed - BestCompression = flate.BestCompression - DefaultCompression = flate.DefaultCompression - ConstantCompression = flate.ConstantCompression - HuffmanOnly = flate.HuffmanOnly -) - -// A Writer takes data written to it and writes the compressed -// form of that data to an underlying writer (see NewWriter). -type Writer struct { - w io.Writer - level int - dict []byte - compressor *flate.Writer - digest hash.Hash32 - err error - scratch [4]byte - wroteHeader bool -} - -// NewWriter creates a new Writer. -// Writes to the returned Writer are compressed and written to w. -// -// It is the caller's responsibility to call Close on the WriteCloser when done. -// Writes may be buffered and not flushed until Close. -func NewWriter(w io.Writer) *Writer { - z, _ := NewWriterLevelDict(w, DefaultCompression, nil) - return z -} - -// NewWriterLevel is like NewWriter but specifies the compression level instead -// of assuming DefaultCompression. -// -// The compression level can be DefaultCompression, NoCompression, HuffmanOnly -// or any integer value between BestSpeed and BestCompression inclusive. -// The error returned will be nil if the level is valid. -func NewWriterLevel(w io.Writer, level int) (*Writer, error) { - return NewWriterLevelDict(w, level, nil) -} - -// NewWriterLevelDict is like NewWriterLevel but specifies a dictionary to -// compress with. -// -// The dictionary may be nil. If not, its contents should not be modified until -// the Writer is closed. -func NewWriterLevelDict(w io.Writer, level int, dict []byte) (*Writer, error) { - if level < HuffmanOnly || level > BestCompression { - return nil, fmt.Errorf("zlib: invalid compression level: %d", level) - } - return &Writer{ - w: w, - level: level, - dict: dict, - }, nil -} - -// Reset clears the state of the Writer z such that it is equivalent to its -// initial state from NewWriterLevel or NewWriterLevelDict, but instead writing -// to w. -func (z *Writer) Reset(w io.Writer) { - z.w = w - // z.level and z.dict left unchanged. - if z.compressor != nil { - z.compressor.Reset(w) - } - if z.digest != nil { - z.digest.Reset() - } - z.err = nil - z.scratch = [4]byte{} - z.wroteHeader = false -} - -// writeHeader writes the ZLIB header. -func (z *Writer) writeHeader() (err error) { - z.wroteHeader = true - // ZLIB has a two-byte header (as documented in RFC 1950). - // The first four bits is the CINFO (compression info), which is 7 for the default deflate window size. - // The next four bits is the CM (compression method), which is 8 for deflate. - z.scratch[0] = 0x78 - // The next two bits is the FLEVEL (compression level). The four values are: - // 0=fastest, 1=fast, 2=default, 3=best. - // The next bit, FDICT, is set if a dictionary is given. - // The final five FCHECK bits form a mod-31 checksum. - switch z.level { - case -2, 0, 1: - z.scratch[1] = 0 << 6 - case 2, 3, 4, 5: - z.scratch[1] = 1 << 6 - case 6, -1: - z.scratch[1] = 2 << 6 - case 7, 8, 9: - z.scratch[1] = 3 << 6 - default: - panic("unreachable") - } - if z.dict != nil { - z.scratch[1] |= 1 << 5 - } - z.scratch[1] += uint8(31 - (uint16(z.scratch[0])<<8+uint16(z.scratch[1]))%31) - if _, err = z.w.Write(z.scratch[0:2]); err != nil { - return err - } - if z.dict != nil { - // The next four bytes are the Adler-32 checksum of the dictionary. - checksum := adler32.Checksum(z.dict) - z.scratch[0] = uint8(checksum >> 24) - z.scratch[1] = uint8(checksum >> 16) - z.scratch[2] = uint8(checksum >> 8) - z.scratch[3] = uint8(checksum >> 0) - if _, err = z.w.Write(z.scratch[0:4]); err != nil { - return err - } - } - if z.compressor == nil { - // Initialize deflater unless the Writer is being reused - // after a Reset call. - z.compressor, err = flate.NewWriterDict(z.w, z.level, z.dict) - if err != nil { - return err - } - z.digest = adler32.New() - } - return nil -} - -// Write writes a compressed form of p to the underlying io.Writer. The -// compressed bytes are not necessarily flushed until the Writer is closed or -// explicitly flushed. -func (z *Writer) Write(p []byte) (n int, err error) { - if !z.wroteHeader { - z.err = z.writeHeader() - } - if z.err != nil { - return 0, z.err - } - if len(p) == 0 { - return 0, nil - } - n, err = z.compressor.Write(p) - if err != nil { - z.err = err - return - } - z.digest.Write(p) - return -} - -// Flush flushes the Writer to its underlying io.Writer. -func (z *Writer) Flush() error { - if !z.wroteHeader { - z.err = z.writeHeader() - } - if z.err != nil { - return z.err - } - z.err = z.compressor.Flush() - return z.err -} - -// Close closes the Writer, flushing any unwritten data to the underlying -// io.Writer, but does not close the underlying io.Writer. -func (z *Writer) Close() error { - if !z.wroteHeader { - z.err = z.writeHeader() - } - if z.err != nil { - return z.err - } - z.err = z.compressor.Close() - if z.err != nil { - return z.err - } - checksum := z.digest.Sum32() - // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). - z.scratch[0] = uint8(checksum >> 24) - z.scratch[1] = uint8(checksum >> 16) - z.scratch[2] = uint8(checksum >> 8) - z.scratch[3] = uint8(checksum >> 0) - _, z.err = z.w.Write(z.scratch[0:4]) - return z.err -} diff --git a/vendor/github.com/klauspost/cpuid/.gitignore b/vendor/github.com/klauspost/cpuid/.gitignore deleted file mode 100644 index daf913b1..00000000 --- a/vendor/github.com/klauspost/cpuid/.gitignore +++ /dev/null @@ -1,24 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe -*.test -*.prof diff --git a/vendor/github.com/klauspost/cpuid/.travis.yml b/vendor/github.com/klauspost/cpuid/.travis.yml deleted file mode 100644 index 630192d5..00000000 --- a/vendor/github.com/klauspost/cpuid/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -language: go - -sudo: false - -os: - - linux - - osx -go: - - 1.8.x - - 1.9.x - - 1.10.x - - master - -script: - - go vet ./... - - go test -v ./... - - go test -race ./... - - diff <(gofmt -d .) <("") - -matrix: - allow_failures: - - go: 'master' - fast_finish: true diff --git a/vendor/github.com/klauspost/cpuid/CONTRIBUTING.txt b/vendor/github.com/klauspost/cpuid/CONTRIBUTING.txt deleted file mode 100644 index 2ef4714f..00000000 --- a/vendor/github.com/klauspost/cpuid/CONTRIBUTING.txt +++ /dev/null @@ -1,35 +0,0 @@ -Developer Certificate of Origin -Version 1.1 - -Copyright (C) 2015- Klaus Post & Contributors. -Email: klauspost@gmail.com - -Everyone is permitted to copy and distribute verbatim copies of this -license document, but changing it is not allowed. - - -Developer's Certificate of Origin 1.1 - -By making a contribution to this project, I certify that: - -(a) The contribution was created in whole or in part by me and I - have the right to submit it under the open source license - indicated in the file; or - -(b) The contribution is based upon previous work that, to the best - of my knowledge, is covered under an appropriate open source - license and I have the right under that license to submit that - work with modifications, whether created in whole or in part - by me, under the same open source license (unless I am - permitted to submit under a different license), as indicated - in the file; or - -(c) The contribution was provided directly to me by some other - person who certified (a), (b) or (c) and I have not modified - it. - -(d) I understand and agree that this project and the contribution - are public and that a record of the contribution (including all - personal information I submit with it, including my sign-off) is - maintained indefinitely and may be redistributed consistent with - this project or the open source license(s) involved. diff --git a/vendor/github.com/klauspost/cpuid/LICENSE b/vendor/github.com/klauspost/cpuid/LICENSE deleted file mode 100644 index 5cec7ee9..00000000 --- a/vendor/github.com/klauspost/cpuid/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 Klaus Post - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/vendor/github.com/klauspost/cpuid/README.md b/vendor/github.com/klauspost/cpuid/README.md deleted file mode 100644 index b2b6bee8..00000000 --- a/vendor/github.com/klauspost/cpuid/README.md +++ /dev/null @@ -1,145 +0,0 @@ -# cpuid -Package cpuid provides information about the CPU running the current program. - -CPU features are detected on startup, and kept for fast access through the life of the application. -Currently x86 / x64 (AMD64) is supported, and no external C (cgo) code is used, which should make the library very easy to use. - -You can access the CPU information by accessing the shared CPU variable of the cpuid library. - -Package home: https://github.com/klauspost/cpuid - -[![GoDoc][1]][2] [![Build Status][3]][4] - -[1]: https://godoc.org/github.com/klauspost/cpuid?status.svg -[2]: https://godoc.org/github.com/klauspost/cpuid -[3]: https://travis-ci.org/klauspost/cpuid.svg -[4]: https://travis-ci.org/klauspost/cpuid - -# features -## CPU Instructions -* **CMOV** (i686 CMOV) -* **NX** (NX (No-Execute) bit) -* **AMD3DNOW** (AMD 3DNOW) -* **AMD3DNOWEXT** (AMD 3DNowExt) -* **MMX** (standard MMX) -* **MMXEXT** (SSE integer functions or AMD MMX ext) -* **SSE** (SSE functions) -* **SSE2** (P4 SSE functions) -* **SSE3** (Prescott SSE3 functions) -* **SSSE3** (Conroe SSSE3 functions) -* **SSE4** (Penryn SSE4.1 functions) -* **SSE4A** (AMD Barcelona microarchitecture SSE4a instructions) -* **SSE42** (Nehalem SSE4.2 functions) -* **AVX** (AVX functions) -* **AVX2** (AVX2 functions) -* **FMA3** (Intel FMA 3) -* **FMA4** (Bulldozer FMA4 functions) -* **XOP** (Bulldozer XOP functions) -* **F16C** (Half-precision floating-point conversion) -* **BMI1** (Bit Manipulation Instruction Set 1) -* **BMI2** (Bit Manipulation Instruction Set 2) -* **TBM** (AMD Trailing Bit Manipulation) -* **LZCNT** (LZCNT instruction) -* **POPCNT** (POPCNT instruction) -* **AESNI** (Advanced Encryption Standard New Instructions) -* **CLMUL** (Carry-less Multiplication) -* **HTT** (Hyperthreading (enabled)) -* **HLE** (Hardware Lock Elision) -* **RTM** (Restricted Transactional Memory) -* **RDRAND** (RDRAND instruction is available) -* **RDSEED** (RDSEED instruction is available) -* **ADX** (Intel ADX (Multi-Precision Add-Carry Instruction Extensions)) -* **SHA** (Intel SHA Extensions) -* **AVX512F** (AVX-512 Foundation) -* **AVX512DQ** (AVX-512 Doubleword and Quadword Instructions) -* **AVX512IFMA** (AVX-512 Integer Fused Multiply-Add Instructions) -* **AVX512PF** (AVX-512 Prefetch Instructions) -* **AVX512ER** (AVX-512 Exponential and Reciprocal Instructions) -* **AVX512CD** (AVX-512 Conflict Detection Instructions) -* **AVX512BW** (AVX-512 Byte and Word Instructions) -* **AVX512VL** (AVX-512 Vector Length Extensions) -* **AVX512VBMI** (AVX-512 Vector Bit Manipulation Instructions) -* **MPX** (Intel MPX (Memory Protection Extensions)) -* **ERMS** (Enhanced REP MOVSB/STOSB) -* **RDTSCP** (RDTSCP Instruction) -* **CX16** (CMPXCHG16B Instruction) -* **SGX** (Software Guard Extensions, with activation details) - -## Performance -* **RDTSCP()** Returns current cycle count. Can be used for benchmarking. -* **SSE2SLOW** (SSE2 is supported, but usually not faster) -* **SSE3SLOW** (SSE3 is supported, but usually not faster) -* **ATOM** (Atom processor, some SSSE3 instructions are slower) -* **Cache line** (Probable size of a cache line). -* **L1, L2, L3 Cache size** on newer Intel/AMD CPUs. - -## Cpu Vendor/VM -* **Intel** -* **AMD** -* **VIA** -* **Transmeta** -* **NSC** -* **KVM** (Kernel-based Virtual Machine) -* **MSVM** (Microsoft Hyper-V or Windows Virtual PC) -* **VMware** -* **XenHVM** - -# installing - -```go get github.com/klauspost/cpuid``` - -# example - -```Go -package main - -import ( - "fmt" - "github.com/klauspost/cpuid" -) - -func main() { - // Print basic CPU information: - fmt.Println("Name:", cpuid.CPU.BrandName) - fmt.Println("PhysicalCores:", cpuid.CPU.PhysicalCores) - fmt.Println("ThreadsPerCore:", cpuid.CPU.ThreadsPerCore) - fmt.Println("LogicalCores:", cpuid.CPU.LogicalCores) - fmt.Println("Family", cpuid.CPU.Family, "Model:", cpuid.CPU.Model) - fmt.Println("Features:", cpuid.CPU.Features) - fmt.Println("Cacheline bytes:", cpuid.CPU.CacheLine) - fmt.Println("L1 Data Cache:", cpuid.CPU.Cache.L1D, "bytes") - fmt.Println("L1 Instruction Cache:", cpuid.CPU.Cache.L1D, "bytes") - fmt.Println("L2 Cache:", cpuid.CPU.Cache.L2, "bytes") - fmt.Println("L3 Cache:", cpuid.CPU.Cache.L3, "bytes") - - // Test if we have a specific feature: - if cpuid.CPU.SSE() { - fmt.Println("We have Streaming SIMD Extensions") - } -} -``` - -Sample output: -``` ->go run main.go -Name: Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz -PhysicalCores: 2 -ThreadsPerCore: 2 -LogicalCores: 4 -Family 6 Model: 42 -Features: CMOV,MMX,MMXEXT,SSE,SSE2,SSE3,SSSE3,SSE4.1,SSE4.2,AVX,AESNI,CLMUL -Cacheline bytes: 64 -We have Streaming SIMD Extensions -``` - -# private package - -In the "private" folder you can find an autogenerated version of the library you can include in your own packages. - -For this purpose all exports are removed, and functions and constants are lowercased. - -This is not a recommended way of using the library, but provided for convenience, if it is difficult for you to use external packages. - -# license - -This code is published under an MIT license. See LICENSE file for more information. diff --git a/vendor/github.com/klauspost/cpuid/cpuid.go b/vendor/github.com/klauspost/cpuid/cpuid.go deleted file mode 100644 index 60c681be..00000000 --- a/vendor/github.com/klauspost/cpuid/cpuid.go +++ /dev/null @@ -1,1040 +0,0 @@ -// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. - -// Package cpuid provides information about the CPU running the current program. -// -// CPU features are detected on startup, and kept for fast access through the life of the application. -// Currently x86 / x64 (AMD64) is supported. -// -// You can access the CPU information by accessing the shared CPU variable of the cpuid library. -// -// Package home: https://github.com/klauspost/cpuid -package cpuid - -import "strings" - -// Vendor is a representation of a CPU vendor. -type Vendor int - -const ( - Other Vendor = iota - Intel - AMD - VIA - Transmeta - NSC - KVM // Kernel-based Virtual Machine - MSVM // Microsoft Hyper-V or Windows Virtual PC - VMware - XenHVM -) - -const ( - CMOV = 1 << iota // i686 CMOV - NX // NX (No-Execute) bit - AMD3DNOW // AMD 3DNOW - AMD3DNOWEXT // AMD 3DNowExt - MMX // standard MMX - MMXEXT // SSE integer functions or AMD MMX ext - SSE // SSE functions - SSE2 // P4 SSE functions - SSE3 // Prescott SSE3 functions - SSSE3 // Conroe SSSE3 functions - SSE4 // Penryn SSE4.1 functions - SSE4A // AMD Barcelona microarchitecture SSE4a instructions - SSE42 // Nehalem SSE4.2 functions - AVX // AVX functions - AVX2 // AVX2 functions - FMA3 // Intel FMA 3 - FMA4 // Bulldozer FMA4 functions - XOP // Bulldozer XOP functions - F16C // Half-precision floating-point conversion - BMI1 // Bit Manipulation Instruction Set 1 - BMI2 // Bit Manipulation Instruction Set 2 - TBM // AMD Trailing Bit Manipulation - LZCNT // LZCNT instruction - POPCNT // POPCNT instruction - AESNI // Advanced Encryption Standard New Instructions - CLMUL // Carry-less Multiplication - HTT // Hyperthreading (enabled) - HLE // Hardware Lock Elision - RTM // Restricted Transactional Memory - RDRAND // RDRAND instruction is available - RDSEED // RDSEED instruction is available - ADX // Intel ADX (Multi-Precision Add-Carry Instruction Extensions) - SHA // Intel SHA Extensions - AVX512F // AVX-512 Foundation - AVX512DQ // AVX-512 Doubleword and Quadword Instructions - AVX512IFMA // AVX-512 Integer Fused Multiply-Add Instructions - AVX512PF // AVX-512 Prefetch Instructions - AVX512ER // AVX-512 Exponential and Reciprocal Instructions - AVX512CD // AVX-512 Conflict Detection Instructions - AVX512BW // AVX-512 Byte and Word Instructions - AVX512VL // AVX-512 Vector Length Extensions - AVX512VBMI // AVX-512 Vector Bit Manipulation Instructions - MPX // Intel MPX (Memory Protection Extensions) - ERMS // Enhanced REP MOVSB/STOSB - RDTSCP // RDTSCP Instruction - CX16 // CMPXCHG16B Instruction - SGX // Software Guard Extensions - IBPB // Indirect Branch Restricted Speculation (IBRS) and Indirect Branch Predictor Barrier (IBPB) - STIBP // Single Thread Indirect Branch Predictors - - // Performance indicators - SSE2SLOW // SSE2 is supported, but usually not faster - SSE3SLOW // SSE3 is supported, but usually not faster - ATOM // Atom processor, some SSSE3 instructions are slower -) - -var flagNames = map[Flags]string{ - CMOV: "CMOV", // i686 CMOV - NX: "NX", // NX (No-Execute) bit - AMD3DNOW: "AMD3DNOW", // AMD 3DNOW - AMD3DNOWEXT: "AMD3DNOWEXT", // AMD 3DNowExt - MMX: "MMX", // Standard MMX - MMXEXT: "MMXEXT", // SSE integer functions or AMD MMX ext - SSE: "SSE", // SSE functions - SSE2: "SSE2", // P4 SSE2 functions - SSE3: "SSE3", // Prescott SSE3 functions - SSSE3: "SSSE3", // Conroe SSSE3 functions - SSE4: "SSE4.1", // Penryn SSE4.1 functions - SSE4A: "SSE4A", // AMD Barcelona microarchitecture SSE4a instructions - SSE42: "SSE4.2", // Nehalem SSE4.2 functions - AVX: "AVX", // AVX functions - AVX2: "AVX2", // AVX functions - FMA3: "FMA3", // Intel FMA 3 - FMA4: "FMA4", // Bulldozer FMA4 functions - XOP: "XOP", // Bulldozer XOP functions - F16C: "F16C", // Half-precision floating-point conversion - BMI1: "BMI1", // Bit Manipulation Instruction Set 1 - BMI2: "BMI2", // Bit Manipulation Instruction Set 2 - TBM: "TBM", // AMD Trailing Bit Manipulation - LZCNT: "LZCNT", // LZCNT instruction - POPCNT: "POPCNT", // POPCNT instruction - AESNI: "AESNI", // Advanced Encryption Standard New Instructions - CLMUL: "CLMUL", // Carry-less Multiplication - HTT: "HTT", // Hyperthreading (enabled) - HLE: "HLE", // Hardware Lock Elision - RTM: "RTM", // Restricted Transactional Memory - RDRAND: "RDRAND", // RDRAND instruction is available - RDSEED: "RDSEED", // RDSEED instruction is available - ADX: "ADX", // Intel ADX (Multi-Precision Add-Carry Instruction Extensions) - SHA: "SHA", // Intel SHA Extensions - AVX512F: "AVX512F", // AVX-512 Foundation - AVX512DQ: "AVX512DQ", // AVX-512 Doubleword and Quadword Instructions - AVX512IFMA: "AVX512IFMA", // AVX-512 Integer Fused Multiply-Add Instructions - AVX512PF: "AVX512PF", // AVX-512 Prefetch Instructions - AVX512ER: "AVX512ER", // AVX-512 Exponential and Reciprocal Instructions - AVX512CD: "AVX512CD", // AVX-512 Conflict Detection Instructions - AVX512BW: "AVX512BW", // AVX-512 Byte and Word Instructions - AVX512VL: "AVX512VL", // AVX-512 Vector Length Extensions - AVX512VBMI: "AVX512VBMI", // AVX-512 Vector Bit Manipulation Instructions - MPX: "MPX", // Intel MPX (Memory Protection Extensions) - ERMS: "ERMS", // Enhanced REP MOVSB/STOSB - RDTSCP: "RDTSCP", // RDTSCP Instruction - CX16: "CX16", // CMPXCHG16B Instruction - SGX: "SGX", // Software Guard Extensions - IBPB: "IBPB", // Indirect Branch Restricted Speculation and Indirect Branch Predictor Barrier - STIBP: "STIBP", // Single Thread Indirect Branch Predictors - - // Performance indicators - SSE2SLOW: "SSE2SLOW", // SSE2 supported, but usually not faster - SSE3SLOW: "SSE3SLOW", // SSE3 supported, but usually not faster - ATOM: "ATOM", // Atom processor, some SSSE3 instructions are slower - -} - -// CPUInfo contains information about the detected system CPU. -type CPUInfo struct { - BrandName string // Brand name reported by the CPU - VendorID Vendor // Comparable CPU vendor ID - Features Flags // Features of the CPU - PhysicalCores int // Number of physical processor cores in your CPU. Will be 0 if undetectable. - ThreadsPerCore int // Number of threads per physical core. Will be 1 if undetectable. - LogicalCores int // Number of physical cores times threads that can run on each core through the use of hyperthreading. Will be 0 if undetectable. - Family int // CPU family number - Model int // CPU model number - CacheLine int // Cache line size in bytes. Will be 0 if undetectable. - Cache struct { - L1I int // L1 Instruction Cache (per core or shared). Will be -1 if undetected - L1D int // L1 Data Cache (per core or shared). Will be -1 if undetected - L2 int // L2 Cache (per core or shared). Will be -1 if undetected - L3 int // L3 Instruction Cache (per core or shared). Will be -1 if undetected - } - SGX SGXSupport - maxFunc uint32 - maxExFunc uint32 -} - -var cpuid func(op uint32) (eax, ebx, ecx, edx uint32) -var cpuidex func(op, op2 uint32) (eax, ebx, ecx, edx uint32) -var xgetbv func(index uint32) (eax, edx uint32) -var rdtscpAsm func() (eax, ebx, ecx, edx uint32) - -// CPU contains information about the CPU as detected on startup, -// or when Detect last was called. -// -// Use this as the primary entry point to you data, -// this way queries are -var CPU CPUInfo - -func init() { - initCPU() - Detect() -} - -// Detect will re-detect current CPU info. -// This will replace the content of the exported CPU variable. -// -// Unless you expect the CPU to change while you are running your program -// you should not need to call this function. -// If you call this, you must ensure that no other goroutine is accessing the -// exported CPU variable. -func Detect() { - CPU.maxFunc = maxFunctionID() - CPU.maxExFunc = maxExtendedFunction() - CPU.BrandName = brandName() - CPU.CacheLine = cacheLine() - CPU.Family, CPU.Model = familyModel() - CPU.Features = support() - CPU.SGX = hasSGX(CPU.Features&SGX != 0) - CPU.ThreadsPerCore = threadsPerCore() - CPU.LogicalCores = logicalCores() - CPU.PhysicalCores = physicalCores() - CPU.VendorID = vendorID() - CPU.cacheSize() -} - -// Generated here: http://play.golang.org/p/BxFH2Gdc0G - -// Cmov indicates support of CMOV instructions -func (c CPUInfo) Cmov() bool { - return c.Features&CMOV != 0 -} - -// Amd3dnow indicates support of AMD 3DNOW! instructions -func (c CPUInfo) Amd3dnow() bool { - return c.Features&AMD3DNOW != 0 -} - -// Amd3dnowExt indicates support of AMD 3DNOW! Extended instructions -func (c CPUInfo) Amd3dnowExt() bool { - return c.Features&AMD3DNOWEXT != 0 -} - -// MMX indicates support of MMX instructions -func (c CPUInfo) MMX() bool { - return c.Features&MMX != 0 -} - -// MMXExt indicates support of MMXEXT instructions -// (SSE integer functions or AMD MMX ext) -func (c CPUInfo) MMXExt() bool { - return c.Features&MMXEXT != 0 -} - -// SSE indicates support of SSE instructions -func (c CPUInfo) SSE() bool { - return c.Features&SSE != 0 -} - -// SSE2 indicates support of SSE 2 instructions -func (c CPUInfo) SSE2() bool { - return c.Features&SSE2 != 0 -} - -// SSE3 indicates support of SSE 3 instructions -func (c CPUInfo) SSE3() bool { - return c.Features&SSE3 != 0 -} - -// SSSE3 indicates support of SSSE 3 instructions -func (c CPUInfo) SSSE3() bool { - return c.Features&SSSE3 != 0 -} - -// SSE4 indicates support of SSE 4 (also called SSE 4.1) instructions -func (c CPUInfo) SSE4() bool { - return c.Features&SSE4 != 0 -} - -// SSE42 indicates support of SSE4.2 instructions -func (c CPUInfo) SSE42() bool { - return c.Features&SSE42 != 0 -} - -// AVX indicates support of AVX instructions -// and operating system support of AVX instructions -func (c CPUInfo) AVX() bool { - return c.Features&AVX != 0 -} - -// AVX2 indicates support of AVX2 instructions -func (c CPUInfo) AVX2() bool { - return c.Features&AVX2 != 0 -} - -// FMA3 indicates support of FMA3 instructions -func (c CPUInfo) FMA3() bool { - return c.Features&FMA3 != 0 -} - -// FMA4 indicates support of FMA4 instructions -func (c CPUInfo) FMA4() bool { - return c.Features&FMA4 != 0 -} - -// XOP indicates support of XOP instructions -func (c CPUInfo) XOP() bool { - return c.Features&XOP != 0 -} - -// F16C indicates support of F16C instructions -func (c CPUInfo) F16C() bool { - return c.Features&F16C != 0 -} - -// BMI1 indicates support of BMI1 instructions -func (c CPUInfo) BMI1() bool { - return c.Features&BMI1 != 0 -} - -// BMI2 indicates support of BMI2 instructions -func (c CPUInfo) BMI2() bool { - return c.Features&BMI2 != 0 -} - -// TBM indicates support of TBM instructions -// (AMD Trailing Bit Manipulation) -func (c CPUInfo) TBM() bool { - return c.Features&TBM != 0 -} - -// Lzcnt indicates support of LZCNT instruction -func (c CPUInfo) Lzcnt() bool { - return c.Features&LZCNT != 0 -} - -// Popcnt indicates support of POPCNT instruction -func (c CPUInfo) Popcnt() bool { - return c.Features&POPCNT != 0 -} - -// HTT indicates the processor has Hyperthreading enabled -func (c CPUInfo) HTT() bool { - return c.Features&HTT != 0 -} - -// SSE2Slow indicates that SSE2 may be slow on this processor -func (c CPUInfo) SSE2Slow() bool { - return c.Features&SSE2SLOW != 0 -} - -// SSE3Slow indicates that SSE3 may be slow on this processor -func (c CPUInfo) SSE3Slow() bool { - return c.Features&SSE3SLOW != 0 -} - -// AesNi indicates support of AES-NI instructions -// (Advanced Encryption Standard New Instructions) -func (c CPUInfo) AesNi() bool { - return c.Features&AESNI != 0 -} - -// Clmul indicates support of CLMUL instructions -// (Carry-less Multiplication) -func (c CPUInfo) Clmul() bool { - return c.Features&CLMUL != 0 -} - -// NX indicates support of NX (No-Execute) bit -func (c CPUInfo) NX() bool { - return c.Features&NX != 0 -} - -// SSE4A indicates support of AMD Barcelona microarchitecture SSE4a instructions -func (c CPUInfo) SSE4A() bool { - return c.Features&SSE4A != 0 -} - -// HLE indicates support of Hardware Lock Elision -func (c CPUInfo) HLE() bool { - return c.Features&HLE != 0 -} - -// RTM indicates support of Restricted Transactional Memory -func (c CPUInfo) RTM() bool { - return c.Features&RTM != 0 -} - -// Rdrand indicates support of RDRAND instruction is available -func (c CPUInfo) Rdrand() bool { - return c.Features&RDRAND != 0 -} - -// Rdseed indicates support of RDSEED instruction is available -func (c CPUInfo) Rdseed() bool { - return c.Features&RDSEED != 0 -} - -// ADX indicates support of Intel ADX (Multi-Precision Add-Carry Instruction Extensions) -func (c CPUInfo) ADX() bool { - return c.Features&ADX != 0 -} - -// SHA indicates support of Intel SHA Extensions -func (c CPUInfo) SHA() bool { - return c.Features&SHA != 0 -} - -// AVX512F indicates support of AVX-512 Foundation -func (c CPUInfo) AVX512F() bool { - return c.Features&AVX512F != 0 -} - -// AVX512DQ indicates support of AVX-512 Doubleword and Quadword Instructions -func (c CPUInfo) AVX512DQ() bool { - return c.Features&AVX512DQ != 0 -} - -// AVX512IFMA indicates support of AVX-512 Integer Fused Multiply-Add Instructions -func (c CPUInfo) AVX512IFMA() bool { - return c.Features&AVX512IFMA != 0 -} - -// AVX512PF indicates support of AVX-512 Prefetch Instructions -func (c CPUInfo) AVX512PF() bool { - return c.Features&AVX512PF != 0 -} - -// AVX512ER indicates support of AVX-512 Exponential and Reciprocal Instructions -func (c CPUInfo) AVX512ER() bool { - return c.Features&AVX512ER != 0 -} - -// AVX512CD indicates support of AVX-512 Conflict Detection Instructions -func (c CPUInfo) AVX512CD() bool { - return c.Features&AVX512CD != 0 -} - -// AVX512BW indicates support of AVX-512 Byte and Word Instructions -func (c CPUInfo) AVX512BW() bool { - return c.Features&AVX512BW != 0 -} - -// AVX512VL indicates support of AVX-512 Vector Length Extensions -func (c CPUInfo) AVX512VL() bool { - return c.Features&AVX512VL != 0 -} - -// AVX512VBMI indicates support of AVX-512 Vector Bit Manipulation Instructions -func (c CPUInfo) AVX512VBMI() bool { - return c.Features&AVX512VBMI != 0 -} - -// MPX indicates support of Intel MPX (Memory Protection Extensions) -func (c CPUInfo) MPX() bool { - return c.Features&MPX != 0 -} - -// ERMS indicates support of Enhanced REP MOVSB/STOSB -func (c CPUInfo) ERMS() bool { - return c.Features&ERMS != 0 -} - -// RDTSCP Instruction is available. -func (c CPUInfo) RDTSCP() bool { - return c.Features&RDTSCP != 0 -} - -// CX16 indicates if CMPXCHG16B instruction is available. -func (c CPUInfo) CX16() bool { - return c.Features&CX16 != 0 -} - -// TSX is split into HLE (Hardware Lock Elision) and RTM (Restricted Transactional Memory) detection. -// So TSX simply checks that. -func (c CPUInfo) TSX() bool { - return c.Features&(HLE|RTM) == HLE|RTM -} - -// Atom indicates an Atom processor -func (c CPUInfo) Atom() bool { - return c.Features&ATOM != 0 -} - -// Intel returns true if vendor is recognized as Intel -func (c CPUInfo) Intel() bool { - return c.VendorID == Intel -} - -// AMD returns true if vendor is recognized as AMD -func (c CPUInfo) AMD() bool { - return c.VendorID == AMD -} - -// Transmeta returns true if vendor is recognized as Transmeta -func (c CPUInfo) Transmeta() bool { - return c.VendorID == Transmeta -} - -// NSC returns true if vendor is recognized as National Semiconductor -func (c CPUInfo) NSC() bool { - return c.VendorID == NSC -} - -// VIA returns true if vendor is recognized as VIA -func (c CPUInfo) VIA() bool { - return c.VendorID == VIA -} - -// RTCounter returns the 64-bit time-stamp counter -// Uses the RDTSCP instruction. The value 0 is returned -// if the CPU does not support the instruction. -func (c CPUInfo) RTCounter() uint64 { - if !c.RDTSCP() { - return 0 - } - a, _, _, d := rdtscpAsm() - return uint64(a) | (uint64(d) << 32) -} - -// Ia32TscAux returns the IA32_TSC_AUX part of the RDTSCP. -// This variable is OS dependent, but on Linux contains information -// about the current cpu/core the code is running on. -// If the RDTSCP instruction isn't supported on the CPU, the value 0 is returned. -func (c CPUInfo) Ia32TscAux() uint32 { - if !c.RDTSCP() { - return 0 - } - _, _, ecx, _ := rdtscpAsm() - return ecx -} - -// LogicalCPU will return the Logical CPU the code is currently executing on. -// This is likely to change when the OS re-schedules the running thread -// to another CPU. -// If the current core cannot be detected, -1 will be returned. -func (c CPUInfo) LogicalCPU() int { - if c.maxFunc < 1 { - return -1 - } - _, ebx, _, _ := cpuid(1) - return int(ebx >> 24) -} - -// VM Will return true if the cpu id indicates we are in -// a virtual machine. This is only a hint, and will very likely -// have many false negatives. -func (c CPUInfo) VM() bool { - switch c.VendorID { - case MSVM, KVM, VMware, XenHVM: - return true - } - return false -} - -// Flags contains detected cpu features and caracteristics -type Flags uint64 - -// String returns a string representation of the detected -// CPU features. -func (f Flags) String() string { - return strings.Join(f.Strings(), ",") -} - -// Strings returns and array of the detected features. -func (f Flags) Strings() []string { - s := support() - r := make([]string, 0, 20) - for i := uint(0); i < 64; i++ { - key := Flags(1 << i) - val := flagNames[key] - if s&key != 0 { - r = append(r, val) - } - } - return r -} - -func maxExtendedFunction() uint32 { - eax, _, _, _ := cpuid(0x80000000) - return eax -} - -func maxFunctionID() uint32 { - a, _, _, _ := cpuid(0) - return a -} - -func brandName() string { - if maxExtendedFunction() >= 0x80000004 { - v := make([]uint32, 0, 48) - for i := uint32(0); i < 3; i++ { - a, b, c, d := cpuid(0x80000002 + i) - v = append(v, a, b, c, d) - } - return strings.Trim(string(valAsString(v...)), " ") - } - return "unknown" -} - -func threadsPerCore() int { - mfi := maxFunctionID() - if mfi < 0x4 || vendorID() != Intel { - return 1 - } - - if mfi < 0xb { - _, b, _, d := cpuid(1) - if (d & (1 << 28)) != 0 { - // v will contain logical core count - v := (b >> 16) & 255 - if v > 1 { - a4, _, _, _ := cpuid(4) - // physical cores - v2 := (a4 >> 26) + 1 - if v2 > 0 { - return int(v) / int(v2) - } - } - } - return 1 - } - _, b, _, _ := cpuidex(0xb, 0) - if b&0xffff == 0 { - return 1 - } - return int(b & 0xffff) -} - -func logicalCores() int { - mfi := maxFunctionID() - switch vendorID() { - case Intel: - // Use this on old Intel processors - if mfi < 0xb { - if mfi < 1 { - return 0 - } - // CPUID.1:EBX[23:16] represents the maximum number of addressable IDs (initial APIC ID) - // that can be assigned to logical processors in a physical package. - // The value may not be the same as the number of logical processors that are present in the hardware of a physical package. - _, ebx, _, _ := cpuid(1) - logical := (ebx >> 16) & 0xff - return int(logical) - } - _, b, _, _ := cpuidex(0xb, 1) - return int(b & 0xffff) - case AMD: - _, b, _, _ := cpuid(1) - return int((b >> 16) & 0xff) - default: - return 0 - } -} - -func familyModel() (int, int) { - if maxFunctionID() < 0x1 { - return 0, 0 - } - eax, _, _, _ := cpuid(1) - family := ((eax >> 8) & 0xf) + ((eax >> 20) & 0xff) - model := ((eax >> 4) & 0xf) + ((eax >> 12) & 0xf0) - return int(family), int(model) -} - -func physicalCores() int { - switch vendorID() { - case Intel: - return logicalCores() / threadsPerCore() - case AMD: - if maxExtendedFunction() >= 0x80000008 { - _, _, c, _ := cpuid(0x80000008) - return int(c&0xff) + 1 - } - } - return 0 -} - -// Except from http://en.wikipedia.org/wiki/CPUID#EAX.3D0:_Get_vendor_ID -var vendorMapping = map[string]Vendor{ - "AMDisbetter!": AMD, - "AuthenticAMD": AMD, - "CentaurHauls": VIA, - "GenuineIntel": Intel, - "TransmetaCPU": Transmeta, - "GenuineTMx86": Transmeta, - "Geode by NSC": NSC, - "VIA VIA VIA ": VIA, - "KVMKVMKVMKVM": KVM, - "Microsoft Hv": MSVM, - "VMwareVMware": VMware, - "XenVMMXenVMM": XenHVM, -} - -func vendorID() Vendor { - _, b, c, d := cpuid(0) - v := valAsString(b, d, c) - vend, ok := vendorMapping[string(v)] - if !ok { - return Other - } - return vend -} - -func cacheLine() int { - if maxFunctionID() < 0x1 { - return 0 - } - - _, ebx, _, _ := cpuid(1) - cache := (ebx & 0xff00) >> 5 // cflush size - if cache == 0 && maxExtendedFunction() >= 0x80000006 { - _, _, ecx, _ := cpuid(0x80000006) - cache = ecx & 0xff // cacheline size - } - // TODO: Read from Cache and TLB Information - return int(cache) -} - -func (c *CPUInfo) cacheSize() { - c.Cache.L1D = -1 - c.Cache.L1I = -1 - c.Cache.L2 = -1 - c.Cache.L3 = -1 - vendor := vendorID() - switch vendor { - case Intel: - if maxFunctionID() < 4 { - return - } - for i := uint32(0); ; i++ { - eax, ebx, ecx, _ := cpuidex(4, i) - cacheType := eax & 15 - if cacheType == 0 { - break - } - cacheLevel := (eax >> 5) & 7 - coherency := int(ebx&0xfff) + 1 - partitions := int((ebx>>12)&0x3ff) + 1 - associativity := int((ebx>>22)&0x3ff) + 1 - sets := int(ecx) + 1 - size := associativity * partitions * coherency * sets - switch cacheLevel { - case 1: - if cacheType == 1 { - // 1 = Data Cache - c.Cache.L1D = size - } else if cacheType == 2 { - // 2 = Instruction Cache - c.Cache.L1I = size - } else { - if c.Cache.L1D < 0 { - c.Cache.L1I = size - } - if c.Cache.L1I < 0 { - c.Cache.L1I = size - } - } - case 2: - c.Cache.L2 = size - case 3: - c.Cache.L3 = size - } - } - case AMD: - // Untested. - if maxExtendedFunction() < 0x80000005 { - return - } - _, _, ecx, edx := cpuid(0x80000005) - c.Cache.L1D = int(((ecx >> 24) & 0xFF) * 1024) - c.Cache.L1I = int(((edx >> 24) & 0xFF) * 1024) - - if maxExtendedFunction() < 0x80000006 { - return - } - _, _, ecx, _ = cpuid(0x80000006) - c.Cache.L2 = int(((ecx >> 16) & 0xFFFF) * 1024) - } - - return -} - -type SGXSupport struct { - Available bool - SGX1Supported bool - SGX2Supported bool - MaxEnclaveSizeNot64 int64 - MaxEnclaveSize64 int64 -} - -func hasSGX(available bool) (rval SGXSupport) { - rval.Available = available - - if !available { - return - } - - a, _, _, d := cpuidex(0x12, 0) - rval.SGX1Supported = a&0x01 != 0 - rval.SGX2Supported = a&0x02 != 0 - rval.MaxEnclaveSizeNot64 = 1 << (d & 0xFF) // pow 2 - rval.MaxEnclaveSize64 = 1 << ((d >> 8) & 0xFF) // pow 2 - - return -} - -func support() Flags { - mfi := maxFunctionID() - vend := vendorID() - if mfi < 0x1 { - return 0 - } - rval := uint64(0) - _, _, c, d := cpuid(1) - if (d & (1 << 15)) != 0 { - rval |= CMOV - } - if (d & (1 << 23)) != 0 { - rval |= MMX - } - if (d & (1 << 25)) != 0 { - rval |= MMXEXT - } - if (d & (1 << 25)) != 0 { - rval |= SSE - } - if (d & (1 << 26)) != 0 { - rval |= SSE2 - } - if (c & 1) != 0 { - rval |= SSE3 - } - if (c & 0x00000200) != 0 { - rval |= SSSE3 - } - if (c & 0x00080000) != 0 { - rval |= SSE4 - } - if (c & 0x00100000) != 0 { - rval |= SSE42 - } - if (c & (1 << 25)) != 0 { - rval |= AESNI - } - if (c & (1 << 1)) != 0 { - rval |= CLMUL - } - if c&(1<<23) != 0 { - rval |= POPCNT - } - if c&(1<<30) != 0 { - rval |= RDRAND - } - if c&(1<<29) != 0 { - rval |= F16C - } - if c&(1<<13) != 0 { - rval |= CX16 - } - if vend == Intel && (d&(1<<28)) != 0 && mfi >= 4 { - if threadsPerCore() > 1 { - rval |= HTT - } - } - - // Check XGETBV, OXSAVE and AVX bits - if c&(1<<26) != 0 && c&(1<<27) != 0 && c&(1<<28) != 0 { - // Check for OS support - eax, _ := xgetbv(0) - if (eax & 0x6) == 0x6 { - rval |= AVX - if (c & 0x00001000) != 0 { - rval |= FMA3 - } - } - } - - // Check AVX2, AVX2 requires OS support, but BMI1/2 don't. - if mfi >= 7 { - _, ebx, ecx, edx := cpuidex(7, 0) - if (rval&AVX) != 0 && (ebx&0x00000020) != 0 { - rval |= AVX2 - } - if (ebx & 0x00000008) != 0 { - rval |= BMI1 - if (ebx & 0x00000100) != 0 { - rval |= BMI2 - } - } - if ebx&(1<<2) != 0 { - rval |= SGX - } - if ebx&(1<<4) != 0 { - rval |= HLE - } - if ebx&(1<<9) != 0 { - rval |= ERMS - } - if ebx&(1<<11) != 0 { - rval |= RTM - } - if ebx&(1<<14) != 0 { - rval |= MPX - } - if ebx&(1<<18) != 0 { - rval |= RDSEED - } - if ebx&(1<<19) != 0 { - rval |= ADX - } - if ebx&(1<<29) != 0 { - rval |= SHA - } - if edx&(1<<26) != 0 { - rval |= IBPB - } - if edx&(1<<27) != 0 { - rval |= STIBP - } - - // Only detect AVX-512 features if XGETBV is supported - if c&((1<<26)|(1<<27)) == (1<<26)|(1<<27) { - // Check for OS support - eax, _ := xgetbv(0) - - // Verify that XCR0[7:5] = ‘111b’ (OPMASK state, upper 256-bit of ZMM0-ZMM15 and - // ZMM16-ZMM31 state are enabled by OS) - /// and that XCR0[2:1] = ‘11b’ (XMM state and YMM state are enabled by OS). - if (eax>>5)&7 == 7 && (eax>>1)&3 == 3 { - if ebx&(1<<16) != 0 { - rval |= AVX512F - } - if ebx&(1<<17) != 0 { - rval |= AVX512DQ - } - if ebx&(1<<21) != 0 { - rval |= AVX512IFMA - } - if ebx&(1<<26) != 0 { - rval |= AVX512PF - } - if ebx&(1<<27) != 0 { - rval |= AVX512ER - } - if ebx&(1<<28) != 0 { - rval |= AVX512CD - } - if ebx&(1<<30) != 0 { - rval |= AVX512BW - } - if ebx&(1<<31) != 0 { - rval |= AVX512VL - } - // ecx - if ecx&(1<<1) != 0 { - rval |= AVX512VBMI - } - } - } - } - - if maxExtendedFunction() >= 0x80000001 { - _, _, c, d := cpuid(0x80000001) - if (c & (1 << 5)) != 0 { - rval |= LZCNT - rval |= POPCNT - } - if (d & (1 << 31)) != 0 { - rval |= AMD3DNOW - } - if (d & (1 << 30)) != 0 { - rval |= AMD3DNOWEXT - } - if (d & (1 << 23)) != 0 { - rval |= MMX - } - if (d & (1 << 22)) != 0 { - rval |= MMXEXT - } - if (c & (1 << 6)) != 0 { - rval |= SSE4A - } - if d&(1<<20) != 0 { - rval |= NX - } - if d&(1<<27) != 0 { - rval |= RDTSCP - } - - /* Allow for selectively disabling SSE2 functions on AMD processors - with SSE2 support but not SSE4a. This includes Athlon64, some - Opteron, and some Sempron processors. MMX, SSE, or 3DNow! are faster - than SSE2 often enough to utilize this special-case flag. - AV_CPU_FLAG_SSE2 and AV_CPU_FLAG_SSE2SLOW are both set in this case - so that SSE2 is used unless explicitly disabled by checking - AV_CPU_FLAG_SSE2SLOW. */ - if vendorID() != Intel && - rval&SSE2 != 0 && (c&0x00000040) == 0 { - rval |= SSE2SLOW - } - - /* XOP and FMA4 use the AVX instruction coding scheme, so they can't be - * used unless the OS has AVX support. */ - if (rval & AVX) != 0 { - if (c & 0x00000800) != 0 { - rval |= XOP - } - if (c & 0x00010000) != 0 { - rval |= FMA4 - } - } - - if vendorID() == Intel { - family, model := familyModel() - if family == 6 && (model == 9 || model == 13 || model == 14) { - /* 6/9 (pentium-m "banias"), 6/13 (pentium-m "dothan"), and - * 6/14 (core1 "yonah") theoretically support sse2, but it's - * usually slower than mmx. */ - if (rval & SSE2) != 0 { - rval |= SSE2SLOW - } - if (rval & SSE3) != 0 { - rval |= SSE3SLOW - } - } - /* The Atom processor has SSSE3 support, which is useful in many cases, - * but sometimes the SSSE3 version is slower than the SSE2 equivalent - * on the Atom, but is generally faster on other processors supporting - * SSSE3. This flag allows for selectively disabling certain SSSE3 - * functions on the Atom. */ - if family == 6 && model == 28 { - rval |= ATOM - } - } - } - return Flags(rval) -} - -func valAsString(values ...uint32) []byte { - r := make([]byte, 4*len(values)) - for i, v := range values { - dst := r[i*4:] - dst[0] = byte(v & 0xff) - dst[1] = byte((v >> 8) & 0xff) - dst[2] = byte((v >> 16) & 0xff) - dst[3] = byte((v >> 24) & 0xff) - switch { - case dst[0] == 0: - return r[:i*4] - case dst[1] == 0: - return r[:i*4+1] - case dst[2] == 0: - return r[:i*4+2] - case dst[3] == 0: - return r[:i*4+3] - } - } - return r -} diff --git a/vendor/github.com/klauspost/cpuid/cpuid_386.s b/vendor/github.com/klauspost/cpuid/cpuid_386.s deleted file mode 100644 index 4d731711..00000000 --- a/vendor/github.com/klauspost/cpuid/cpuid_386.s +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. - -// +build 386,!gccgo - -// func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) -TEXT ·asmCpuid(SB), 7, $0 - XORL CX, CX - MOVL op+0(FP), AX - CPUID - MOVL AX, eax+4(FP) - MOVL BX, ebx+8(FP) - MOVL CX, ecx+12(FP) - MOVL DX, edx+16(FP) - RET - -// func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) -TEXT ·asmCpuidex(SB), 7, $0 - MOVL op+0(FP), AX - MOVL op2+4(FP), CX - CPUID - MOVL AX, eax+8(FP) - MOVL BX, ebx+12(FP) - MOVL CX, ecx+16(FP) - MOVL DX, edx+20(FP) - RET - -// func xgetbv(index uint32) (eax, edx uint32) -TEXT ·asmXgetbv(SB), 7, $0 - MOVL index+0(FP), CX - BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV - MOVL AX, eax+4(FP) - MOVL DX, edx+8(FP) - RET - -// func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) -TEXT ·asmRdtscpAsm(SB), 7, $0 - BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP - MOVL AX, eax+0(FP) - MOVL BX, ebx+4(FP) - MOVL CX, ecx+8(FP) - MOVL DX, edx+12(FP) - RET diff --git a/vendor/github.com/klauspost/cpuid/cpuid_amd64.s b/vendor/github.com/klauspost/cpuid/cpuid_amd64.s deleted file mode 100644 index 3c1d60e4..00000000 --- a/vendor/github.com/klauspost/cpuid/cpuid_amd64.s +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. - -//+build amd64,!gccgo - -// func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) -TEXT ·asmCpuid(SB), 7, $0 - XORQ CX, CX - MOVL op+0(FP), AX - CPUID - MOVL AX, eax+8(FP) - MOVL BX, ebx+12(FP) - MOVL CX, ecx+16(FP) - MOVL DX, edx+20(FP) - RET - -// func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) -TEXT ·asmCpuidex(SB), 7, $0 - MOVL op+0(FP), AX - MOVL op2+4(FP), CX - CPUID - MOVL AX, eax+8(FP) - MOVL BX, ebx+12(FP) - MOVL CX, ecx+16(FP) - MOVL DX, edx+20(FP) - RET - -// func asmXgetbv(index uint32) (eax, edx uint32) -TEXT ·asmXgetbv(SB), 7, $0 - MOVL index+0(FP), CX - BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV - MOVL AX, eax+8(FP) - MOVL DX, edx+12(FP) - RET - -// func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) -TEXT ·asmRdtscpAsm(SB), 7, $0 - BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP - MOVL AX, eax+0(FP) - MOVL BX, ebx+4(FP) - MOVL CX, ecx+8(FP) - MOVL DX, edx+12(FP) - RET diff --git a/vendor/github.com/klauspost/cpuid/detect_intel.go b/vendor/github.com/klauspost/cpuid/detect_intel.go deleted file mode 100644 index a5f04dd6..00000000 --- a/vendor/github.com/klauspost/cpuid/detect_intel.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. - -// +build 386,!gccgo amd64,!gccgo - -package cpuid - -func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) -func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) -func asmXgetbv(index uint32) (eax, edx uint32) -func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) - -func initCPU() { - cpuid = asmCpuid - cpuidex = asmCpuidex - xgetbv = asmXgetbv - rdtscpAsm = asmRdtscpAsm -} diff --git a/vendor/github.com/klauspost/cpuid/detect_ref.go b/vendor/github.com/klauspost/cpuid/detect_ref.go deleted file mode 100644 index 909c5d9a..00000000 --- a/vendor/github.com/klauspost/cpuid/detect_ref.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. - -// +build !amd64,!386 gccgo - -package cpuid - -func initCPU() { - cpuid = func(op uint32) (eax, ebx, ecx, edx uint32) { - return 0, 0, 0, 0 - } - - cpuidex = func(op, op2 uint32) (eax, ebx, ecx, edx uint32) { - return 0, 0, 0, 0 - } - - xgetbv = func(index uint32) (eax, edx uint32) { - return 0, 0 - } - - rdtscpAsm = func() (eax, ebx, ecx, edx uint32) { - return 0, 0, 0, 0 - } -} diff --git a/vendor/github.com/klauspost/cpuid/generate.go b/vendor/github.com/klauspost/cpuid/generate.go deleted file mode 100644 index 90e7a98d..00000000 --- a/vendor/github.com/klauspost/cpuid/generate.go +++ /dev/null @@ -1,4 +0,0 @@ -package cpuid - -//go:generate go run private-gen.go -//go:generate gofmt -w ./private diff --git a/vendor/github.com/klauspost/cpuid/private-gen.go b/vendor/github.com/klauspost/cpuid/private-gen.go deleted file mode 100644 index 437333d2..00000000 --- a/vendor/github.com/klauspost/cpuid/private-gen.go +++ /dev/null @@ -1,476 +0,0 @@ -// +build ignore - -package main - -import ( - "bytes" - "fmt" - "go/ast" - "go/parser" - "go/printer" - "go/token" - "io" - "io/ioutil" - "log" - "os" - "reflect" - "strings" - "unicode" - "unicode/utf8" -) - -var inFiles = []string{"cpuid.go", "cpuid_test.go"} -var copyFiles = []string{"cpuid_amd64.s", "cpuid_386.s", "detect_ref.go", "detect_intel.go"} -var fileSet = token.NewFileSet() -var reWrites = []rewrite{ - initRewrite("CPUInfo -> cpuInfo"), - initRewrite("Vendor -> vendor"), - initRewrite("Flags -> flags"), - initRewrite("Detect -> detect"), - initRewrite("CPU -> cpu"), -} -var excludeNames = map[string]bool{"string": true, "join": true, "trim": true, - // cpuid_test.go - "t": true, "println": true, "logf": true, "log": true, "fatalf": true, "fatal": true, -} - -var excludePrefixes = []string{"test", "benchmark"} - -func main() { - Package := "private" - parserMode := parser.ParseComments - exported := make(map[string]rewrite) - for _, file := range inFiles { - in, err := os.Open(file) - if err != nil { - log.Fatalf("opening input", err) - } - - src, err := ioutil.ReadAll(in) - if err != nil { - log.Fatalf("reading input", err) - } - - astfile, err := parser.ParseFile(fileSet, file, src, parserMode) - if err != nil { - log.Fatalf("parsing input", err) - } - - for _, rw := range reWrites { - astfile = rw(astfile) - } - - // Inspect the AST and print all identifiers and literals. - var startDecl token.Pos - var endDecl token.Pos - ast.Inspect(astfile, func(n ast.Node) bool { - var s string - switch x := n.(type) { - case *ast.Ident: - if x.IsExported() { - t := strings.ToLower(x.Name) - for _, pre := range excludePrefixes { - if strings.HasPrefix(t, pre) { - return true - } - } - if excludeNames[t] != true { - //if x.Pos() > startDecl && x.Pos() < endDecl { - exported[x.Name] = initRewrite(x.Name + " -> " + t) - } - } - - case *ast.GenDecl: - if x.Tok == token.CONST && x.Lparen > 0 { - startDecl = x.Lparen - endDecl = x.Rparen - // fmt.Printf("Decl:%s -> %s\n", fileSet.Position(startDecl), fileSet.Position(endDecl)) - } - } - if s != "" { - fmt.Printf("%s:\t%s\n", fileSet.Position(n.Pos()), s) - } - return true - }) - - for _, rw := range exported { - astfile = rw(astfile) - } - - var buf bytes.Buffer - - printer.Fprint(&buf, fileSet, astfile) - - // Remove package documentation and insert information - s := buf.String() - ind := strings.Index(buf.String(), "\npackage cpuid") - s = s[ind:] - s = "// Generated, DO NOT EDIT,\n" + - "// but copy it to your own project and rename the package.\n" + - "// See more at http://github.com/klauspost/cpuid\n" + - s - - outputName := Package + string(os.PathSeparator) + file - - err = ioutil.WriteFile(outputName, []byte(s), 0644) - if err != nil { - log.Fatalf("writing output: %s", err) - } - log.Println("Generated", outputName) - } - - for _, file := range copyFiles { - dst := "" - if strings.HasPrefix(file, "cpuid") { - dst = Package + string(os.PathSeparator) + file - } else { - dst = Package + string(os.PathSeparator) + "cpuid_" + file - } - err := copyFile(file, dst) - if err != nil { - log.Fatalf("copying file: %s", err) - } - log.Println("Copied", dst) - } -} - -// CopyFile copies a file from src to dst. If src and dst files exist, and are -// the same, then return success. Copy the file contents from src to dst. -func copyFile(src, dst string) (err error) { - sfi, err := os.Stat(src) - if err != nil { - return - } - if !sfi.Mode().IsRegular() { - // cannot copy non-regular files (e.g., directories, - // symlinks, devices, etc.) - return fmt.Errorf("CopyFile: non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String()) - } - dfi, err := os.Stat(dst) - if err != nil { - if !os.IsNotExist(err) { - return - } - } else { - if !(dfi.Mode().IsRegular()) { - return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String()) - } - if os.SameFile(sfi, dfi) { - return - } - } - err = copyFileContents(src, dst) - return -} - -// copyFileContents copies the contents of the file named src to the file named -// by dst. The file will be created if it does not already exist. If the -// destination file exists, all it's contents will be replaced by the contents -// of the source file. -func copyFileContents(src, dst string) (err error) { - in, err := os.Open(src) - if err != nil { - return - } - defer in.Close() - out, err := os.Create(dst) - if err != nil { - return - } - defer func() { - cerr := out.Close() - if err == nil { - err = cerr - } - }() - if _, err = io.Copy(out, in); err != nil { - return - } - err = out.Sync() - return -} - -type rewrite func(*ast.File) *ast.File - -// Mostly copied from gofmt -func initRewrite(rewriteRule string) rewrite { - f := strings.Split(rewriteRule, "->") - if len(f) != 2 { - fmt.Fprintf(os.Stderr, "rewrite rule must be of the form 'pattern -> replacement'\n") - os.Exit(2) - } - pattern := parseExpr(f[0], "pattern") - replace := parseExpr(f[1], "replacement") - return func(p *ast.File) *ast.File { return rewriteFile(pattern, replace, p) } -} - -// parseExpr parses s as an expression. -// It might make sense to expand this to allow statement patterns, -// but there are problems with preserving formatting and also -// with what a wildcard for a statement looks like. -func parseExpr(s, what string) ast.Expr { - x, err := parser.ParseExpr(s) - if err != nil { - fmt.Fprintf(os.Stderr, "parsing %s %s at %s\n", what, s, err) - os.Exit(2) - } - return x -} - -// Keep this function for debugging. -/* -func dump(msg string, val reflect.Value) { - fmt.Printf("%s:\n", msg) - ast.Print(fileSet, val.Interface()) - fmt.Println() -} -*/ - -// rewriteFile applies the rewrite rule 'pattern -> replace' to an entire file. -func rewriteFile(pattern, replace ast.Expr, p *ast.File) *ast.File { - cmap := ast.NewCommentMap(fileSet, p, p.Comments) - m := make(map[string]reflect.Value) - pat := reflect.ValueOf(pattern) - repl := reflect.ValueOf(replace) - - var rewriteVal func(val reflect.Value) reflect.Value - rewriteVal = func(val reflect.Value) reflect.Value { - // don't bother if val is invalid to start with - if !val.IsValid() { - return reflect.Value{} - } - for k := range m { - delete(m, k) - } - val = apply(rewriteVal, val) - if match(m, pat, val) { - val = subst(m, repl, reflect.ValueOf(val.Interface().(ast.Node).Pos())) - } - return val - } - - r := apply(rewriteVal, reflect.ValueOf(p)).Interface().(*ast.File) - r.Comments = cmap.Filter(r).Comments() // recreate comments list - return r -} - -// set is a wrapper for x.Set(y); it protects the caller from panics if x cannot be changed to y. -func set(x, y reflect.Value) { - // don't bother if x cannot be set or y is invalid - if !x.CanSet() || !y.IsValid() { - return - } - defer func() { - if x := recover(); x != nil { - if s, ok := x.(string); ok && - (strings.Contains(s, "type mismatch") || strings.Contains(s, "not assignable")) { - // x cannot be set to y - ignore this rewrite - return - } - panic(x) - } - }() - x.Set(y) -} - -// Values/types for special cases. -var ( - objectPtrNil = reflect.ValueOf((*ast.Object)(nil)) - scopePtrNil = reflect.ValueOf((*ast.Scope)(nil)) - - identType = reflect.TypeOf((*ast.Ident)(nil)) - objectPtrType = reflect.TypeOf((*ast.Object)(nil)) - positionType = reflect.TypeOf(token.NoPos) - callExprType = reflect.TypeOf((*ast.CallExpr)(nil)) - scopePtrType = reflect.TypeOf((*ast.Scope)(nil)) -) - -// apply replaces each AST field x in val with f(x), returning val. -// To avoid extra conversions, f operates on the reflect.Value form. -func apply(f func(reflect.Value) reflect.Value, val reflect.Value) reflect.Value { - if !val.IsValid() { - return reflect.Value{} - } - - // *ast.Objects introduce cycles and are likely incorrect after - // rewrite; don't follow them but replace with nil instead - if val.Type() == objectPtrType { - return objectPtrNil - } - - // similarly for scopes: they are likely incorrect after a rewrite; - // replace them with nil - if val.Type() == scopePtrType { - return scopePtrNil - } - - switch v := reflect.Indirect(val); v.Kind() { - case reflect.Slice: - for i := 0; i < v.Len(); i++ { - e := v.Index(i) - set(e, f(e)) - } - case reflect.Struct: - for i := 0; i < v.NumField(); i++ { - e := v.Field(i) - set(e, f(e)) - } - case reflect.Interface: - e := v.Elem() - set(v, f(e)) - } - return val -} - -func isWildcard(s string) bool { - rune, size := utf8.DecodeRuneInString(s) - return size == len(s) && unicode.IsLower(rune) -} - -// match returns true if pattern matches val, -// recording wildcard submatches in m. -// If m == nil, match checks whether pattern == val. -func match(m map[string]reflect.Value, pattern, val reflect.Value) bool { - // Wildcard matches any expression. If it appears multiple - // times in the pattern, it must match the same expression - // each time. - if m != nil && pattern.IsValid() && pattern.Type() == identType { - name := pattern.Interface().(*ast.Ident).Name - if isWildcard(name) && val.IsValid() { - // wildcards only match valid (non-nil) expressions. - if _, ok := val.Interface().(ast.Expr); ok && !val.IsNil() { - if old, ok := m[name]; ok { - return match(nil, old, val) - } - m[name] = val - return true - } - } - } - - // Otherwise, pattern and val must match recursively. - if !pattern.IsValid() || !val.IsValid() { - return !pattern.IsValid() && !val.IsValid() - } - if pattern.Type() != val.Type() { - return false - } - - // Special cases. - switch pattern.Type() { - case identType: - // For identifiers, only the names need to match - // (and none of the other *ast.Object information). - // This is a common case, handle it all here instead - // of recursing down any further via reflection. - p := pattern.Interface().(*ast.Ident) - v := val.Interface().(*ast.Ident) - return p == nil && v == nil || p != nil && v != nil && p.Name == v.Name - case objectPtrType, positionType: - // object pointers and token positions always match - return true - case callExprType: - // For calls, the Ellipsis fields (token.Position) must - // match since that is how f(x) and f(x...) are different. - // Check them here but fall through for the remaining fields. - p := pattern.Interface().(*ast.CallExpr) - v := val.Interface().(*ast.CallExpr) - if p.Ellipsis.IsValid() != v.Ellipsis.IsValid() { - return false - } - } - - p := reflect.Indirect(pattern) - v := reflect.Indirect(val) - if !p.IsValid() || !v.IsValid() { - return !p.IsValid() && !v.IsValid() - } - - switch p.Kind() { - case reflect.Slice: - if p.Len() != v.Len() { - return false - } - for i := 0; i < p.Len(); i++ { - if !match(m, p.Index(i), v.Index(i)) { - return false - } - } - return true - - case reflect.Struct: - for i := 0; i < p.NumField(); i++ { - if !match(m, p.Field(i), v.Field(i)) { - return false - } - } - return true - - case reflect.Interface: - return match(m, p.Elem(), v.Elem()) - } - - // Handle token integers, etc. - return p.Interface() == v.Interface() -} - -// subst returns a copy of pattern with values from m substituted in place -// of wildcards and pos used as the position of tokens from the pattern. -// if m == nil, subst returns a copy of pattern and doesn't change the line -// number information. -func subst(m map[string]reflect.Value, pattern reflect.Value, pos reflect.Value) reflect.Value { - if !pattern.IsValid() { - return reflect.Value{} - } - - // Wildcard gets replaced with map value. - if m != nil && pattern.Type() == identType { - name := pattern.Interface().(*ast.Ident).Name - if isWildcard(name) { - if old, ok := m[name]; ok { - return subst(nil, old, reflect.Value{}) - } - } - } - - if pos.IsValid() && pattern.Type() == positionType { - // use new position only if old position was valid in the first place - if old := pattern.Interface().(token.Pos); !old.IsValid() { - return pattern - } - return pos - } - - // Otherwise copy. - switch p := pattern; p.Kind() { - case reflect.Slice: - v := reflect.MakeSlice(p.Type(), p.Len(), p.Len()) - for i := 0; i < p.Len(); i++ { - v.Index(i).Set(subst(m, p.Index(i), pos)) - } - return v - - case reflect.Struct: - v := reflect.New(p.Type()).Elem() - for i := 0; i < p.NumField(); i++ { - v.Field(i).Set(subst(m, p.Field(i), pos)) - } - return v - - case reflect.Ptr: - v := reflect.New(p.Type()).Elem() - if elem := p.Elem(); elem.IsValid() { - v.Set(subst(m, elem, pos).Addr()) - } - return v - - case reflect.Interface: - v := reflect.New(p.Type()).Elem() - if elem := p.Elem(); elem.IsValid() { - v.Set(subst(m, elem, pos)) - } - return v - } - - return pattern -} diff --git a/vendor/github.com/valyala/bytebufferpool/.travis.yml b/vendor/github.com/valyala/bytebufferpool/.travis.yml deleted file mode 100644 index 6a6ec2eb..00000000 --- a/vendor/github.com/valyala/bytebufferpool/.travis.yml +++ /dev/null @@ -1,15 +0,0 @@ -language: go - -go: - - 1.6 - -script: - # build test for supported platforms - - GOOS=linux go build - - GOOS=darwin go build - - GOOS=freebsd go build - - GOOS=windows go build - - GOARCH=386 go build - - # run tests on a standard platform - - go test -v ./... diff --git a/vendor/github.com/valyala/bytebufferpool/LICENSE b/vendor/github.com/valyala/bytebufferpool/LICENSE deleted file mode 100644 index f7c935c2..00000000 --- a/vendor/github.com/valyala/bytebufferpool/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2016 Aliaksandr Valialkin, VertaMedia - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/vendor/github.com/valyala/bytebufferpool/README.md b/vendor/github.com/valyala/bytebufferpool/README.md deleted file mode 100644 index 061357e8..00000000 --- a/vendor/github.com/valyala/bytebufferpool/README.md +++ /dev/null @@ -1,21 +0,0 @@ -[![Build Status](https://travis-ci.org/valyala/bytebufferpool.svg)](https://travis-ci.org/valyala/bytebufferpool) -[![GoDoc](https://godoc.org/github.com/valyala/bytebufferpool?status.svg)](http://godoc.org/github.com/valyala/bytebufferpool) -[![Go Report](http://goreportcard.com/badge/valyala/bytebufferpool)](http://goreportcard.com/report/valyala/bytebufferpool) - -# bytebufferpool - -An implementation of a pool of byte buffers with anti-memory-waste protection. - -The pool may waste limited amount of memory due to fragmentation. -This amount equals to the maximum total size of the byte buffers -in concurrent use. - -# Benchmark results -Currently bytebufferpool is fastest and most effective buffer pool written in Go. - -You can find results [here](https://omgnull.github.io/go-benchmark/buffer/). - -# bytebufferpool users - -* [fasthttp](https://github.com/valyala/fasthttp) -* [quicktemplate](https://github.com/valyala/quicktemplate) diff --git a/vendor/github.com/valyala/bytebufferpool/bytebuffer.go b/vendor/github.com/valyala/bytebufferpool/bytebuffer.go deleted file mode 100644 index 07a055a2..00000000 --- a/vendor/github.com/valyala/bytebufferpool/bytebuffer.go +++ /dev/null @@ -1,111 +0,0 @@ -package bytebufferpool - -import "io" - -// ByteBuffer provides byte buffer, which can be used for minimizing -// memory allocations. -// -// ByteBuffer may be used with functions appending data to the given []byte -// slice. See example code for details. -// -// Use Get for obtaining an empty byte buffer. -type ByteBuffer struct { - - // B is a byte buffer to use in append-like workloads. - // See example code for details. - B []byte -} - -// Len returns the size of the byte buffer. -func (b *ByteBuffer) Len() int { - return len(b.B) -} - -// ReadFrom implements io.ReaderFrom. -// -// The function appends all the data read from r to b. -func (b *ByteBuffer) ReadFrom(r io.Reader) (int64, error) { - p := b.B - nStart := int64(len(p)) - nMax := int64(cap(p)) - n := nStart - if nMax == 0 { - nMax = 64 - p = make([]byte, nMax) - } else { - p = p[:nMax] - } - for { - if n == nMax { - nMax *= 2 - bNew := make([]byte, nMax) - copy(bNew, p) - p = bNew - } - nn, err := r.Read(p[n:]) - n += int64(nn) - if err != nil { - b.B = p[:n] - n -= nStart - if err == io.EOF { - return n, nil - } - return n, err - } - } -} - -// WriteTo implements io.WriterTo. -func (b *ByteBuffer) WriteTo(w io.Writer) (int64, error) { - n, err := w.Write(b.B) - return int64(n), err -} - -// Bytes returns b.B, i.e. all the bytes accumulated in the buffer. -// -// The purpose of this function is bytes.Buffer compatibility. -func (b *ByteBuffer) Bytes() []byte { - return b.B -} - -// Write implements io.Writer - it appends p to ByteBuffer.B -func (b *ByteBuffer) Write(p []byte) (int, error) { - b.B = append(b.B, p...) - return len(p), nil -} - -// WriteByte appends the byte c to the buffer. -// -// The purpose of this function is bytes.Buffer compatibility. -// -// The function always returns nil. -func (b *ByteBuffer) WriteByte(c byte) error { - b.B = append(b.B, c) - return nil -} - -// WriteString appends s to ByteBuffer.B. -func (b *ByteBuffer) WriteString(s string) (int, error) { - b.B = append(b.B, s...) - return len(s), nil -} - -// Set sets ByteBuffer.B to p. -func (b *ByteBuffer) Set(p []byte) { - b.B = append(b.B[:0], p...) -} - -// SetString sets ByteBuffer.B to s. -func (b *ByteBuffer) SetString(s string) { - b.B = append(b.B[:0], s...) -} - -// String returns string representation of ByteBuffer.B. -func (b *ByteBuffer) String() string { - return string(b.B) -} - -// Reset makes ByteBuffer.B empty. -func (b *ByteBuffer) Reset() { - b.B = b.B[:0] -} diff --git a/vendor/github.com/valyala/bytebufferpool/doc.go b/vendor/github.com/valyala/bytebufferpool/doc.go deleted file mode 100644 index e511b7c5..00000000 --- a/vendor/github.com/valyala/bytebufferpool/doc.go +++ /dev/null @@ -1,7 +0,0 @@ -// Package bytebufferpool implements a pool of byte buffers -// with anti-fragmentation protection. -// -// The pool may waste limited amount of memory due to fragmentation. -// This amount equals to the maximum total size of the byte buffers -// in concurrent use. -package bytebufferpool diff --git a/vendor/github.com/valyala/bytebufferpool/pool.go b/vendor/github.com/valyala/bytebufferpool/pool.go deleted file mode 100644 index 8bb4134d..00000000 --- a/vendor/github.com/valyala/bytebufferpool/pool.go +++ /dev/null @@ -1,151 +0,0 @@ -package bytebufferpool - -import ( - "sort" - "sync" - "sync/atomic" -) - -const ( - minBitSize = 6 // 2**6=64 is a CPU cache line size - steps = 20 - - minSize = 1 << minBitSize - maxSize = 1 << (minBitSize + steps - 1) - - calibrateCallsThreshold = 42000 - maxPercentile = 0.95 -) - -// Pool represents byte buffer pool. -// -// Distinct pools may be used for distinct types of byte buffers. -// Properly determined byte buffer types with their own pools may help reducing -// memory waste. -type Pool struct { - calls [steps]uint64 - calibrating uint64 - - defaultSize uint64 - maxSize uint64 - - pool sync.Pool -} - -var defaultPool Pool - -// Get returns an empty byte buffer from the pool. -// -// Got byte buffer may be returned to the pool via Put call. -// This reduces the number of memory allocations required for byte buffer -// management. -func Get() *ByteBuffer { return defaultPool.Get() } - -// Get returns new byte buffer with zero length. -// -// The byte buffer may be returned to the pool via Put after the use -// in order to minimize GC overhead. -func (p *Pool) Get() *ByteBuffer { - v := p.pool.Get() - if v != nil { - return v.(*ByteBuffer) - } - return &ByteBuffer{ - B: make([]byte, 0, atomic.LoadUint64(&p.defaultSize)), - } -} - -// Put returns byte buffer to the pool. -// -// ByteBuffer.B mustn't be touched after returning it to the pool. -// Otherwise data races will occur. -func Put(b *ByteBuffer) { defaultPool.Put(b) } - -// Put releases byte buffer obtained via Get to the pool. -// -// The buffer mustn't be accessed after returning to the pool. -func (p *Pool) Put(b *ByteBuffer) { - idx := index(len(b.B)) - - if atomic.AddUint64(&p.calls[idx], 1) > calibrateCallsThreshold { - p.calibrate() - } - - maxSize := int(atomic.LoadUint64(&p.maxSize)) - if maxSize == 0 || cap(b.B) <= maxSize { - b.Reset() - p.pool.Put(b) - } -} - -func (p *Pool) calibrate() { - if !atomic.CompareAndSwapUint64(&p.calibrating, 0, 1) { - return - } - - a := make(callSizes, 0, steps) - var callsSum uint64 - for i := uint64(0); i < steps; i++ { - calls := atomic.SwapUint64(&p.calls[i], 0) - callsSum += calls - a = append(a, callSize{ - calls: calls, - size: minSize << i, - }) - } - sort.Sort(a) - - defaultSize := a[0].size - maxSize := defaultSize - - maxSum := uint64(float64(callsSum) * maxPercentile) - callsSum = 0 - for i := 0; i < steps; i++ { - if callsSum > maxSum { - break - } - callsSum += a[i].calls - size := a[i].size - if size > maxSize { - maxSize = size - } - } - - atomic.StoreUint64(&p.defaultSize, defaultSize) - atomic.StoreUint64(&p.maxSize, maxSize) - - atomic.StoreUint64(&p.calibrating, 0) -} - -type callSize struct { - calls uint64 - size uint64 -} - -type callSizes []callSize - -func (ci callSizes) Len() int { - return len(ci) -} - -func (ci callSizes) Less(i, j int) bool { - return ci[i].calls > ci[j].calls -} - -func (ci callSizes) Swap(i, j int) { - ci[i], ci[j] = ci[j], ci[i] -} - -func index(n int) int { - n-- - n >>= minBitSize - idx := 0 - for n > 0 { - n >>= 1 - idx++ - } - if idx >= steps { - idx = steps - 1 - } - return idx -} diff --git a/vendor/github.com/valyala/fasthttp/.gitignore b/vendor/github.com/valyala/fasthttp/.gitignore deleted file mode 100644 index 7b58ce45..00000000 --- a/vendor/github.com/valyala/fasthttp/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -tags -*.pprof -*.fasthttp.gz diff --git a/vendor/github.com/valyala/fasthttp/.travis.yml b/vendor/github.com/valyala/fasthttp/.travis.yml deleted file mode 100644 index 104ead04..00000000 --- a/vendor/github.com/valyala/fasthttp/.travis.yml +++ /dev/null @@ -1,36 +0,0 @@ -language: go - -go: - - tip - - 1.11.x - - 1.10.x - - 1.9.x - -os: - - linux - - osx - -matrix: - allow_failures: - - tip - fast_finish: true - -before_install: - - go get -t -v ./... - # - go get -v golang.org/x/tools/cmd/goimports - -script: - # TODO(@kirilldanshin) - # - test -z "$(goimports -d $(find . -type f -name '*.go' -not -path "./vendor/*"))" - # build test for supported platforms - - GOOS=linux go build - - GOOS=darwin go build - - GOOS=freebsd go build - - GOOS=windows go build - - GOARCH=386 go build - - # run tests on a standard platform - - go test -v ./... - - # run tests with the race detector as well - - go test -race -v ./... diff --git a/vendor/github.com/valyala/fasthttp/LICENSE b/vendor/github.com/valyala/fasthttp/LICENSE deleted file mode 100644 index b098914a..00000000 --- a/vendor/github.com/valyala/fasthttp/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015-present Aliaksandr Valialkin, VertaMedia -Copyright (c) 2018-present Kirill Danshin -Copyright (c) 2018-present Erik Dubbelboer -Copyright (c) 2018-present FastHTTP Authors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/vendor/github.com/valyala/fasthttp/README.md b/vendor/github.com/valyala/fasthttp/README.md deleted file mode 100644 index 5fcb6398..00000000 --- a/vendor/github.com/valyala/fasthttp/README.md +++ /dev/null @@ -1,585 +0,0 @@ -[![Build Status](https://travis-ci.org/valyala/fasthttp.svg)](https://travis-ci.org/valyala/fasthttp) -[![GoDoc](https://godoc.org/github.com/valyala/fasthttp?status.svg)](http://godoc.org/github.com/valyala/fasthttp) -[![Go Report](https://goreportcard.com/badge/github.com/valyala/fasthttp)](https://goreportcard.com/report/github.com/valyala/fasthttp) - -# fasthttp -Fast HTTP implementation for Go. - -Currently fasthttp is successfully used by [VertaMedia](https://vertamedia.com/) -in a production serving up to 200K rps from more than 1.5M concurrent keep-alive -connections per physical server. - -[TechEmpower Benchmark round 12 results](https://www.techempower.com/benchmarks/#section=data-r12&hw=peak&test=plaintext) - -[Server Benchmarks](#http-server-performance-comparison-with-nethttp) - -[Client Benchmarks](#http-client-comparison-with-nethttp) - -[Install](#install) - -[Documentation](https://godoc.org/github.com/valyala/fasthttp) - -[Examples from docs](https://godoc.org/github.com/valyala/fasthttp#pkg-examples) - -[Code examples](examples) - -[Awesome fasthttp tools](https://github.com/fasthttp) - -[Switching from net/http to fasthttp](#switching-from-nethttp-to-fasthttp) - -[Fasthttp best practices](#fasthttp-best-practices) - -[Tricks with byte buffers](#tricks-with-byte-buffers) - -[Related projects](#related-projects) - -[FAQ](#faq) - -# HTTP server performance comparison with [net/http](https://golang.org/pkg/net/http/) - -In short, fasthttp server is up to 10 times faster than net/http. -Below are benchmark results. - -*GOMAXPROCS=1* - -net/http server: -``` -$ GOMAXPROCS=1 go test -bench=NetHTTPServerGet -benchmem -benchtime=10s -BenchmarkNetHTTPServerGet1ReqPerConn 1000000 12052 ns/op 2297 B/op 29 allocs/op -BenchmarkNetHTTPServerGet2ReqPerConn 1000000 12278 ns/op 2327 B/op 24 allocs/op -BenchmarkNetHTTPServerGet10ReqPerConn 2000000 8903 ns/op 2112 B/op 19 allocs/op -BenchmarkNetHTTPServerGet10KReqPerConn 2000000 8451 ns/op 2058 B/op 18 allocs/op -BenchmarkNetHTTPServerGet1ReqPerConn10KClients 500000 26733 ns/op 3229 B/op 29 allocs/op -BenchmarkNetHTTPServerGet2ReqPerConn10KClients 1000000 23351 ns/op 3211 B/op 24 allocs/op -BenchmarkNetHTTPServerGet10ReqPerConn10KClients 1000000 13390 ns/op 2483 B/op 19 allocs/op -BenchmarkNetHTTPServerGet100ReqPerConn10KClients 1000000 13484 ns/op 2171 B/op 18 allocs/op -``` - -fasthttp server: -``` -$ GOMAXPROCS=1 go test -bench=kServerGet -benchmem -benchtime=10s -BenchmarkServerGet1ReqPerConn 10000000 1559 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet2ReqPerConn 10000000 1248 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet10ReqPerConn 20000000 797 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet10KReqPerConn 20000000 716 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet1ReqPerConn10KClients 10000000 1974 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet2ReqPerConn10KClients 10000000 1352 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet10ReqPerConn10KClients 20000000 789 ns/op 2 B/op 0 allocs/op -BenchmarkServerGet100ReqPerConn10KClients 20000000 604 ns/op 0 B/op 0 allocs/op -``` - -*GOMAXPROCS=4* - -net/http server: -``` -$ GOMAXPROCS=4 go test -bench=NetHTTPServerGet -benchmem -benchtime=10s -BenchmarkNetHTTPServerGet1ReqPerConn-4 3000000 4529 ns/op 2389 B/op 29 allocs/op -BenchmarkNetHTTPServerGet2ReqPerConn-4 5000000 3896 ns/op 2418 B/op 24 allocs/op -BenchmarkNetHTTPServerGet10ReqPerConn-4 5000000 3145 ns/op 2160 B/op 19 allocs/op -BenchmarkNetHTTPServerGet10KReqPerConn-4 5000000 3054 ns/op 2065 B/op 18 allocs/op -BenchmarkNetHTTPServerGet1ReqPerConn10KClients-4 1000000 10321 ns/op 3710 B/op 30 allocs/op -BenchmarkNetHTTPServerGet2ReqPerConn10KClients-4 2000000 7556 ns/op 3296 B/op 24 allocs/op -BenchmarkNetHTTPServerGet10ReqPerConn10KClients-4 5000000 3905 ns/op 2349 B/op 19 allocs/op -BenchmarkNetHTTPServerGet100ReqPerConn10KClients-4 5000000 3435 ns/op 2130 B/op 18 allocs/op -``` - -fasthttp server: -``` -$ GOMAXPROCS=4 go test -bench=kServerGet -benchmem -benchtime=10s -BenchmarkServerGet1ReqPerConn-4 10000000 1141 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet2ReqPerConn-4 20000000 707 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet10ReqPerConn-4 30000000 341 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet10KReqPerConn-4 50000000 310 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet1ReqPerConn10KClients-4 10000000 1119 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet2ReqPerConn10KClients-4 20000000 644 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet10ReqPerConn10KClients-4 30000000 346 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet100ReqPerConn10KClients-4 50000000 282 ns/op 0 B/op 0 allocs/op -``` - -# HTTP client comparison with net/http - -In short, fasthttp client is up to 10 times faster than net/http. -Below are benchmark results. - -*GOMAXPROCS=1* - -net/http client: -``` -$ GOMAXPROCS=1 go test -bench='HTTPClient(Do|GetEndToEnd)' -benchmem -benchtime=10s -BenchmarkNetHTTPClientDoFastServer 1000000 12567 ns/op 2616 B/op 35 allocs/op -BenchmarkNetHTTPClientGetEndToEnd1TCP 200000 67030 ns/op 5028 B/op 56 allocs/op -BenchmarkNetHTTPClientGetEndToEnd10TCP 300000 51098 ns/op 5031 B/op 56 allocs/op -BenchmarkNetHTTPClientGetEndToEnd100TCP 300000 45096 ns/op 5026 B/op 55 allocs/op -BenchmarkNetHTTPClientGetEndToEnd1Inmemory 500000 24779 ns/op 5035 B/op 57 allocs/op -BenchmarkNetHTTPClientGetEndToEnd10Inmemory 1000000 26425 ns/op 5035 B/op 57 allocs/op -BenchmarkNetHTTPClientGetEndToEnd100Inmemory 500000 28515 ns/op 5045 B/op 57 allocs/op -BenchmarkNetHTTPClientGetEndToEnd1000Inmemory 500000 39511 ns/op 5096 B/op 56 allocs/op -``` - -fasthttp client: -``` -$ GOMAXPROCS=1 go test -bench='kClient(Do|GetEndToEnd)' -benchmem -benchtime=10s -BenchmarkClientDoFastServer 20000000 865 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd1TCP 1000000 18711 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd10TCP 1000000 14664 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd100TCP 1000000 14043 ns/op 1 B/op 0 allocs/op -BenchmarkClientGetEndToEnd1Inmemory 5000000 3965 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd10Inmemory 3000000 4060 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd100Inmemory 5000000 3396 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd1000Inmemory 5000000 3306 ns/op 2 B/op 0 allocs/op -``` - -*GOMAXPROCS=4* - -net/http client: -``` -$ GOMAXPROCS=4 go test -bench='HTTPClient(Do|GetEndToEnd)' -benchmem -benchtime=10s -BenchmarkNetHTTPClientDoFastServer-4 2000000 8774 ns/op 2619 B/op 35 allocs/op -BenchmarkNetHTTPClientGetEndToEnd1TCP-4 500000 22951 ns/op 5047 B/op 56 allocs/op -BenchmarkNetHTTPClientGetEndToEnd10TCP-4 1000000 19182 ns/op 5037 B/op 55 allocs/op -BenchmarkNetHTTPClientGetEndToEnd100TCP-4 1000000 16535 ns/op 5031 B/op 55 allocs/op -BenchmarkNetHTTPClientGetEndToEnd1Inmemory-4 1000000 14495 ns/op 5038 B/op 56 allocs/op -BenchmarkNetHTTPClientGetEndToEnd10Inmemory-4 1000000 10237 ns/op 5034 B/op 56 allocs/op -BenchmarkNetHTTPClientGetEndToEnd100Inmemory-4 1000000 10125 ns/op 5045 B/op 56 allocs/op -BenchmarkNetHTTPClientGetEndToEnd1000Inmemory-4 1000000 11132 ns/op 5136 B/op 56 allocs/op -``` - -fasthttp client: -``` -$ GOMAXPROCS=4 go test -bench='kClient(Do|GetEndToEnd)' -benchmem -benchtime=10s -BenchmarkClientDoFastServer-4 50000000 397 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd1TCP-4 2000000 7388 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd10TCP-4 2000000 6689 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd100TCP-4 3000000 4927 ns/op 1 B/op 0 allocs/op -BenchmarkClientGetEndToEnd1Inmemory-4 10000000 1604 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd10Inmemory-4 10000000 1458 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd100Inmemory-4 10000000 1329 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd1000Inmemory-4 10000000 1316 ns/op 5 B/op 0 allocs/op -``` - - -# Install - -``` -go get -u github.com/valyala/fasthttp -``` - - -# Switching from net/http to fasthttp - -Unfortunately, fasthttp doesn't provide API identical to net/http. -See the [FAQ](#faq) for details. -There is [net/http -> fasthttp handler converter](https://godoc.org/github.com/valyala/fasthttp/fasthttpadaptor), -but it is better to write fasthttp request handlers by hand in order to use -all of the fasthttp advantages (especially high performance :) ). - -Important points: - -* Fasthttp works with [RequestHandler functions](https://godoc.org/github.com/valyala/fasthttp#RequestHandler) -instead of objects implementing [Handler interface](https://golang.org/pkg/net/http/#Handler). -Fortunately, it is easy to pass bound struct methods to fasthttp: - - ```go - type MyHandler struct { - foobar string - } - - // request handler in net/http style, i.e. method bound to MyHandler struct. - func (h *MyHandler) HandleFastHTTP(ctx *fasthttp.RequestCtx) { - // notice that we may access MyHandler properties here - see h.foobar. - fmt.Fprintf(ctx, "Hello, world! Requested path is %q. Foobar is %q", - ctx.Path(), h.foobar) - } - - // request handler in fasthttp style, i.e. just plain function. - func fastHTTPHandler(ctx *fasthttp.RequestCtx) { - fmt.Fprintf(ctx, "Hi there! RequestURI is %q", ctx.RequestURI()) - } - - // pass bound struct method to fasthttp - myHandler := &MyHandler{ - foobar: "foobar", - } - fasthttp.ListenAndServe(":8080", myHandler.HandleFastHTTP) - - // pass plain function to fasthttp - fasthttp.ListenAndServe(":8081", fastHTTPHandler) - ``` - -* The [RequestHandler](https://godoc.org/github.com/valyala/fasthttp#RequestHandler) -accepts only one argument - [RequestCtx](https://godoc.org/github.com/valyala/fasthttp#RequestCtx). -It contains all the functionality required for http request processing -and response writing. Below is an example of a simple request handler conversion -from net/http to fasthttp. - - ```go - // net/http request handler - requestHandler := func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/foo": - fooHandler(w, r) - case "/bar": - barHandler(w, r) - default: - http.Error(w, "Unsupported path", http.StatusNotFound) - } - } - ``` - - ```go - // the corresponding fasthttp request handler - requestHandler := func(ctx *fasthttp.RequestCtx) { - switch string(ctx.Path()) { - case "/foo": - fooHandler(ctx) - case "/bar": - barHandler(ctx) - default: - ctx.Error("Unsupported path", fasthttp.StatusNotFound) - } - } - ``` - -* Fasthttp allows setting response headers and writing response body -in an arbitrary order. There is no 'headers first, then body' restriction -like in net/http. The following code is valid for fasthttp: - - ```go - requestHandler := func(ctx *fasthttp.RequestCtx) { - // set some headers and status code first - ctx.SetContentType("foo/bar") - ctx.SetStatusCode(fasthttp.StatusOK) - - // then write the first part of body - fmt.Fprintf(ctx, "this is the first part of body\n") - - // then set more headers - ctx.Response.Header.Set("Foo-Bar", "baz") - - // then write more body - fmt.Fprintf(ctx, "this is the second part of body\n") - - // then override already written body - ctx.SetBody([]byte("this is completely new body contents")) - - // then update status code - ctx.SetStatusCode(fasthttp.StatusNotFound) - - // basically, anything may be updated many times before - // returning from RequestHandler. - // - // Unlike net/http fasthttp doesn't put response to the wire until - // returning from RequestHandler. - } - ``` - -* Fasthttp doesn't provide [ServeMux](https://golang.org/pkg/net/http/#ServeMux), -but there are more powerful third-party routers and web frameworks -with fasthttp support: - - * [fasthttp-routing](https://github.com/qiangxue/fasthttp-routing) - * [fasthttprouter](https://github.com/buaazp/fasthttprouter) - * [lu](https://github.com/vincentLiuxiang/lu) - * [atreugo](https://github.com/savsgio/atreugo) - - Net/http code with simple ServeMux is trivially converted to fasthttp code: - - ```go - // net/http code - - m := &http.ServeMux{} - m.HandleFunc("/foo", fooHandlerFunc) - m.HandleFunc("/bar", barHandlerFunc) - m.Handle("/baz", bazHandler) - - http.ListenAndServe(":80", m) - ``` - - ```go - // the corresponding fasthttp code - m := func(ctx *fasthttp.RequestCtx) { - switch string(ctx.Path()) { - case "/foo": - fooHandlerFunc(ctx) - case "/bar": - barHandlerFunc(ctx) - case "/baz": - bazHandler.HandlerFunc(ctx) - default: - ctx.Error("not found", fasthttp.StatusNotFound) - } - } - - fasthttp.ListenAndServe(":80", m) - ``` - -* net/http -> fasthttp conversion table: - - * All the pseudocode below assumes w, r and ctx have these types: - ```go - var ( - w http.ResponseWriter - r *http.Request - ctx *fasthttp.RequestCtx - ) - ``` - * r.Body -> [ctx.PostBody()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.PostBody) - * r.URL.Path -> [ctx.Path()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Path) - * r.URL -> [ctx.URI()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.URI) - * r.Method -> [ctx.Method()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Method) - * r.Header -> [ctx.Request.Header](https://godoc.org/github.com/valyala/fasthttp#RequestHeader) - * r.Header.Get() -> [ctx.Request.Header.Peek()](https://godoc.org/github.com/valyala/fasthttp#RequestHeader.Peek) - * r.Host -> [ctx.Host()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Host) - * r.Form -> [ctx.QueryArgs()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.QueryArgs) + - [ctx.PostArgs()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.PostArgs) - * r.PostForm -> [ctx.PostArgs()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.PostArgs) - * r.FormValue() -> [ctx.FormValue()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.FormValue) - * r.FormFile() -> [ctx.FormFile()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.FormFile) - * r.MultipartForm -> [ctx.MultipartForm()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.MultipartForm) - * r.RemoteAddr -> [ctx.RemoteAddr()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.RemoteAddr) - * r.RequestURI -> [ctx.RequestURI()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.RequestURI) - * r.TLS -> [ctx.IsTLS()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.IsTLS) - * r.Cookie() -> [ctx.Request.Header.Cookie()](https://godoc.org/github.com/valyala/fasthttp#RequestHeader.Cookie) - * r.Referer() -> [ctx.Referer()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Referer) - * r.UserAgent() -> [ctx.UserAgent()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.UserAgent) - * w.Header() -> [ctx.Response.Header](https://godoc.org/github.com/valyala/fasthttp#ResponseHeader) - * w.Header().Set() -> [ctx.Response.Header.Set()](https://godoc.org/github.com/valyala/fasthttp#ResponseHeader.Set) - * w.Header().Set("Content-Type") -> [ctx.SetContentType()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.SetContentType) - * w.Header().Set("Set-Cookie") -> [ctx.Response.Header.SetCookie()](https://godoc.org/github.com/valyala/fasthttp#ResponseHeader.SetCookie) - * w.Write() -> [ctx.Write()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Write), - [ctx.SetBody()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.SetBody), - [ctx.SetBodyStream()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.SetBodyStream), - [ctx.SetBodyStreamWriter()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.SetBodyStreamWriter) - * w.WriteHeader() -> [ctx.SetStatusCode()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.SetStatusCode) - * w.(http.Hijacker).Hijack() -> [ctx.Hijack()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Hijack) - * http.Error() -> [ctx.Error()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Error) - * http.FileServer() -> [fasthttp.FSHandler()](https://godoc.org/github.com/valyala/fasthttp#FSHandler), - [fasthttp.FS](https://godoc.org/github.com/valyala/fasthttp#FS) - * http.ServeFile() -> [fasthttp.ServeFile()](https://godoc.org/github.com/valyala/fasthttp#ServeFile) - * http.Redirect() -> [ctx.Redirect()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Redirect) - * http.NotFound() -> [ctx.NotFound()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.NotFound) - * http.StripPrefix() -> [fasthttp.PathRewriteFunc](https://godoc.org/github.com/valyala/fasthttp#PathRewriteFunc) - -* *VERY IMPORTANT!* Fasthttp disallows holding references -to [RequestCtx](https://godoc.org/github.com/valyala/fasthttp#RequestCtx) or to its' -members after returning from [RequestHandler](https://godoc.org/github.com/valyala/fasthttp#RequestHandler). -Otherwise [data races](http://blog.golang.org/race-detector) are inevitable. -Carefully inspect all the net/http request handlers converted to fasthttp whether -they retain references to RequestCtx or to its' members after returning. -RequestCtx provides the following _band aids_ for this case: - - * Wrap RequestHandler into [TimeoutHandler](https://godoc.org/github.com/valyala/fasthttp#TimeoutHandler). - * Call [TimeoutError](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.TimeoutError) - before returning from RequestHandler if there are references to RequestCtx or to its' members. - See [the example](https://godoc.org/github.com/valyala/fasthttp#example-RequestCtx-TimeoutError) - for more details. - -Use this brilliant tool - [race detector](http://blog.golang.org/race-detector) - -for detecting and eliminating data races in your program. If you detected -data race related to fasthttp in your program, then there is high probability -you forgot calling [TimeoutError](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.TimeoutError) -before returning from [RequestHandler](https://godoc.org/github.com/valyala/fasthttp#RequestHandler). - -* Blind switching from net/http to fasthttp won't give you performance boost. -While fasthttp is optimized for speed, its' performance may be easily saturated -by slow [RequestHandler](https://godoc.org/github.com/valyala/fasthttp#RequestHandler). -So [profile](http://blog.golang.org/profiling-go-programs) and optimize your -code after switching to fasthttp. For instance, use [quicktemplate](https://github.com/valyala/quicktemplate) -instead of [html/template](https://golang.org/pkg/html/template/). - -* See also [fasthttputil](https://godoc.org/github.com/valyala/fasthttp/fasthttputil), -[fasthttpadaptor](https://godoc.org/github.com/valyala/fasthttp/fasthttpadaptor) and -[expvarhandler](https://godoc.org/github.com/valyala/fasthttp/expvarhandler). - - -# Performance optimization tips for multi-core systems - -* Use [reuseport](https://godoc.org/github.com/valyala/fasthttp/reuseport) listener. -* Run a separate server instance per CPU core with GOMAXPROCS=1. -* Pin each server instance to a separate CPU core using [taskset](http://linux.die.net/man/1/taskset). -* Ensure the interrupts of multiqueue network card are evenly distributed between CPU cores. - See [this article](https://blog.cloudflare.com/how-to-achieve-low-latency/) for details. -* Use Go 1.6 as it provides some considerable performance improvements. - - -# Fasthttp best practices - -* Do not allocate objects and `[]byte` buffers - just reuse them as much - as possible. Fasthttp API design encourages this. -* [sync.Pool](https://golang.org/pkg/sync/#Pool) is your best friend. -* [Profile your program](http://blog.golang.org/profiling-go-programs) - in production. - `go tool pprof --alloc_objects your-program mem.pprof` usually gives better - insights for optimization opportunities than `go tool pprof your-program cpu.pprof`. -* Write [tests and benchmarks](https://golang.org/pkg/testing/) for hot paths. -* Avoid conversion between `[]byte` and `string`, since this may result in memory - allocation+copy. Fasthttp API provides functions for both `[]byte` and `string` - - use these functions instead of converting manually between `[]byte` and `string`. - There are some exceptions - see [this wiki page](https://github.com/golang/go/wiki/CompilerOptimizations#string-and-byte) - for more details. -* Verify your tests and production code under - [race detector](https://golang.org/doc/articles/race_detector.html) on a regular basis. -* Prefer [quicktemplate](https://github.com/valyala/quicktemplate) instead of - [html/template](https://golang.org/pkg/html/template/) in your webserver. - - -# Tricks with `[]byte` buffers - -The following tricks are used by fasthttp. Use them in your code too. - -* Standard Go functions accept nil buffers -```go -var ( - // both buffers are uninitialized - dst []byte - src []byte -) -dst = append(dst, src...) // is legal if dst is nil and/or src is nil -copy(dst, src) // is legal if dst is nil and/or src is nil -(string(src) == "") // is true if src is nil -(len(src) == 0) // is true if src is nil -src = src[:0] // works like a charm with nil src - -// this for loop doesn't panic if src is nil -for i, ch := range src { - doSomething(i, ch) -} -``` - -So throw away nil checks for `[]byte` buffers from you code. For example, -```go -srcLen := 0 -if src != nil { - srcLen = len(src) -} -``` - -becomes - -```go -srcLen := len(src) -``` - -* String may be appended to `[]byte` buffer with `append` -```go -dst = append(dst, "foobar"...) -``` - -* `[]byte` buffer may be extended to its' capacity. -```go -buf := make([]byte, 100) -a := buf[:10] // len(a) == 10, cap(a) == 100. -b := a[:100] // is valid, since cap(a) == 100. -``` - -* All fasthttp functions accept nil `[]byte` buffer -```go -statusCode, body, err := fasthttp.Get(nil, "http://google.com/") -uintBuf := fasthttp.AppendUint(nil, 1234) -``` - -# Related projects - - * [fasthttp](https://github.com/fasthttp) - various useful - helpers for projects based on fasthttp. - * [fasthttp-routing](https://github.com/qiangxue/fasthttp-routing) - fast and - powerful routing package for fasthttp servers. - * [fasthttprouter](https://github.com/buaazp/fasthttprouter) - a high - performance fasthttp request router that scales well. - * [gramework](https://github.com/gramework/gramework) - a web framework made by one of fasthttp maintainers - * [lu](https://github.com/vincentLiuxiang/lu) - a high performance - go middleware web framework which is based on fasthttp. - * [websocket](https://github.com/fasthttp/websocket) - Gorilla-based - websocket implementation for fasthttp. - * [fasthttpsession](https://github.com/phachon/fasthttpsession) - a fast and powerful session package for fasthttp servers. - * [atreugo](https://github.com/savsgio/atreugo) - Micro-framework to make simple the use of routing and middlewares. - * [kratgo](https://github.com/savsgio/kratgo) - Simple, lightweight and ultra-fast HTTP Cache to speed up your websites. - - -# FAQ - -* *Why creating yet another http package instead of optimizing net/http?* - - Because net/http API limits many optimization opportunities. - For example: - * net/http Request object lifetime isn't limited by request handler execution - time. So the server must create a new request object per each request instead - of reusing existing objects like fasthttp does. - * net/http headers are stored in a `map[string][]string`. So the server - must parse all the headers, convert them from `[]byte` to `string` and put - them into the map before calling user-provided request handler. - This all requires unnecessary memory allocations avoided by fasthttp. - * net/http client API requires creating a new response object per each request. - -* *Why fasthttp API is incompatible with net/http?* - - Because net/http API limits many optimization opportunities. See the answer - above for more details. Also certain net/http API parts are suboptimal - for use: - * Compare [net/http connection hijacking](https://golang.org/pkg/net/http/#Hijacker) - to [fasthttp connection hijacking](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Hijack). - * Compare [net/http Request.Body reading](https://golang.org/pkg/net/http/#Request) - to [fasthttp request body reading](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.PostBody). - -* *Why fasthttp doesn't support HTTP/2.0 and WebSockets?* - - [HTTP/2.0 support](https://github.com/fasthttp/http2) is in progress. [WebSockets](https://github.com/fasthttp/websockets) has been done already. - Third parties also may use [RequestCtx.Hijack](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Hijack) - for implementing these goodies. - -* *Are there known net/http advantages comparing to fasthttp?* - - Yes: - * net/http supports [HTTP/2.0 starting from go1.6](https://http2.golang.org/). - * net/http API is stable, while fasthttp API constantly evolves. - * net/http handles more HTTP corner cases. - * net/http should contain less bugs, since it is used and tested by much - wider audience. - * net/http works on Go older than 1.5. - -* *Why fasthttp API prefers returning `[]byte` instead of `string`?* - - Because `[]byte` to `string` conversion isn't free - it requires memory - allocation and copy. Feel free wrapping returned `[]byte` result into - `string()` if you prefer working with strings instead of byte slices. - But be aware that this has non-zero overhead. - -* *Which GO versions are supported by fasthttp?* - - Go1.5+. Older versions won't be supported, since their standard package - [miss useful functions](https://github.com/valyala/fasthttp/issues/5). - - **NOTE**: Go 1.9.7 is the oldest tested version. We recommend you to update as soon as you can. As of 1.11.3 we will drop 1.9.x support. - -* *Please provide real benchmark data and server information* - - See [this issue](https://github.com/valyala/fasthttp/issues/4). - -* *Are there plans to add request routing to fasthttp?* - - There are no plans to add request routing into fasthttp. - Use third-party routers and web frameworks with fasthttp support: - - * [fasthttp-routing](https://github.com/qiangxue/fasthttp-routing) - * [fasthttprouter](https://github.com/buaazp/fasthttprouter) - * [gramework](https://github.com/gramework/gramework) - * [lu](https://github.com/vincentLiuxiang/lu) - * [atreugo](https://github.com/savsgio/atreugo) - - See also [this issue](https://github.com/valyala/fasthttp/issues/9) for more info. - -* *I detected data race in fasthttp!* - - Cool! [File a bug](https://github.com/valyala/fasthttp/issues/new). But before - doing this check the following in your code: - - * Make sure there are no references to [RequestCtx](https://godoc.org/github.com/valyala/fasthttp#RequestCtx) - or to its' members after returning from [RequestHandler](https://godoc.org/github.com/valyala/fasthttp#RequestHandler). - * Make sure you call [TimeoutError](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.TimeoutError) - before returning from [RequestHandler](https://godoc.org/github.com/valyala/fasthttp#RequestHandler) - if there are references to [RequestCtx](https://godoc.org/github.com/valyala/fasthttp#RequestCtx) - or to its' members, which may be accessed by other goroutines. - -* *I didn't find an answer for my question here* - - Try exploring [these questions](https://github.com/valyala/fasthttp/issues?q=label%3Aquestion). diff --git a/vendor/github.com/valyala/fasthttp/TODO b/vendor/github.com/valyala/fasthttp/TODO deleted file mode 100644 index ce7505f1..00000000 --- a/vendor/github.com/valyala/fasthttp/TODO +++ /dev/null @@ -1,4 +0,0 @@ -- SessionClient with referer and cookies support. -- ProxyHandler similar to FSHandler. -- WebSockets. See https://tools.ietf.org/html/rfc6455 . -- HTTP/2.0. See https://tools.ietf.org/html/rfc7540 . diff --git a/vendor/github.com/valyala/fasthttp/args.go b/vendor/github.com/valyala/fasthttp/args.go deleted file mode 100644 index e5865cd2..00000000 --- a/vendor/github.com/valyala/fasthttp/args.go +++ /dev/null @@ -1,588 +0,0 @@ -package fasthttp - -import ( - "bytes" - "errors" - "io" - "sort" - "sync" - - "github.com/valyala/bytebufferpool" -) - -const ( - argsNoValue = true - argsHasValue = false -) - -// AcquireArgs returns an empty Args object from the pool. -// -// The returned Args may be returned to the pool with ReleaseArgs -// when no longer needed. This allows reducing GC load. -func AcquireArgs() *Args { - return argsPool.Get().(*Args) -} - -// ReleaseArgs returns the object acquired via AcquireArgs to the pool. -// -// Do not access the released Args object, otherwise data races may occur. -func ReleaseArgs(a *Args) { - a.Reset() - argsPool.Put(a) -} - -var argsPool = &sync.Pool{ - New: func() interface{} { - return &Args{} - }, -} - -// Args represents query arguments. -// -// It is forbidden copying Args instances. Create new instances instead -// and use CopyTo(). -// -// Args instance MUST NOT be used from concurrently running goroutines. -type Args struct { - noCopy noCopy - - args []argsKV - buf []byte -} - -type argsKV struct { - key []byte - value []byte - noValue bool -} - -// Reset clears query args. -func (a *Args) Reset() { - a.args = a.args[:0] -} - -// CopyTo copies all args to dst. -func (a *Args) CopyTo(dst *Args) { - dst.Reset() - dst.args = copyArgs(dst.args, a.args) -} - -// VisitAll calls f for each existing arg. -// -// f must not retain references to key and value after returning. -// Make key and/or value copies if you need storing them after returning. -func (a *Args) VisitAll(f func(key, value []byte)) { - visitArgs(a.args, f) -} - -// Len returns the number of query args. -func (a *Args) Len() int { - return len(a.args) -} - -// Parse parses the given string containing query args. -func (a *Args) Parse(s string) { - a.buf = append(a.buf[:0], s...) - a.ParseBytes(a.buf) -} - -// ParseBytes parses the given b containing query args. -func (a *Args) ParseBytes(b []byte) { - a.Reset() - - var s argsScanner - s.b = b - - var kv *argsKV - a.args, kv = allocArg(a.args) - for s.next(kv) { - if len(kv.key) > 0 || len(kv.value) > 0 { - a.args, kv = allocArg(a.args) - } - } - a.args = releaseArg(a.args) -} - -// String returns string representation of query args. -func (a *Args) String() string { - return string(a.QueryString()) -} - -// QueryString returns query string for the args. -// -// The returned value is valid until the next call to Args methods. -func (a *Args) QueryString() []byte { - a.buf = a.AppendBytes(a.buf[:0]) - return a.buf -} - -// Sort sorts Args by key and then value using 'f' as comparison function. -// -// For example args.Sort(bytes.Compare) -func (a *Args) Sort(f func(x, y []byte) int) { - sort.SliceStable(a.args, func(i, j int) bool { - n := f(a.args[i].key, a.args[j].key) - if n == 0 { - return f(a.args[i].value, a.args[j].value) == -1 - } - return n == -1 - }) -} - -// AppendBytes appends query string to dst and returns the extended dst. -func (a *Args) AppendBytes(dst []byte) []byte { - for i, n := 0, len(a.args); i < n; i++ { - kv := &a.args[i] - dst = AppendQuotedArg(dst, kv.key) - if !kv.noValue { - dst = append(dst, '=') - if len(kv.value) > 0 { - dst = AppendQuotedArg(dst, kv.value) - } - } - if i+1 < n { - dst = append(dst, '&') - } - } - return dst -} - -// WriteTo writes query string to w. -// -// WriteTo implements io.WriterTo interface. -func (a *Args) WriteTo(w io.Writer) (int64, error) { - n, err := w.Write(a.QueryString()) - return int64(n), err -} - -// Del deletes argument with the given key from query args. -func (a *Args) Del(key string) { - a.args = delAllArgs(a.args, key) -} - -// DelBytes deletes argument with the given key from query args. -func (a *Args) DelBytes(key []byte) { - a.args = delAllArgs(a.args, b2s(key)) -} - -// Add adds 'key=value' argument. -// -// Multiple values for the same key may be added. -func (a *Args) Add(key, value string) { - a.args = appendArg(a.args, key, value, argsHasValue) -} - -// AddBytesK adds 'key=value' argument. -// -// Multiple values for the same key may be added. -func (a *Args) AddBytesK(key []byte, value string) { - a.args = appendArg(a.args, b2s(key), value, argsHasValue) -} - -// AddBytesV adds 'key=value' argument. -// -// Multiple values for the same key may be added. -func (a *Args) AddBytesV(key string, value []byte) { - a.args = appendArg(a.args, key, b2s(value), argsHasValue) -} - -// AddBytesKV adds 'key=value' argument. -// -// Multiple values for the same key may be added. -func (a *Args) AddBytesKV(key, value []byte) { - a.args = appendArg(a.args, b2s(key), b2s(value), argsHasValue) -} - -// AddNoValue adds only 'key' as argument without the '='. -// -// Multiple values for the same key may be added. -func (a *Args) AddNoValue(key string) { - a.args = appendArg(a.args, key, "", argsNoValue) -} - -// AddBytesKNoValue adds only 'key' as argument without the '='. -// -// Multiple values for the same key may be added. -func (a *Args) AddBytesKNoValue(key []byte) { - a.args = appendArg(a.args, b2s(key), "", argsNoValue) -} - -// Set sets 'key=value' argument. -func (a *Args) Set(key, value string) { - a.args = setArg(a.args, key, value, argsHasValue) -} - -// SetBytesK sets 'key=value' argument. -func (a *Args) SetBytesK(key []byte, value string) { - a.args = setArg(a.args, b2s(key), value, argsHasValue) -} - -// SetBytesV sets 'key=value' argument. -func (a *Args) SetBytesV(key string, value []byte) { - a.args = setArg(a.args, key, b2s(value), argsHasValue) -} - -// SetBytesKV sets 'key=value' argument. -func (a *Args) SetBytesKV(key, value []byte) { - a.args = setArgBytes(a.args, key, value, argsHasValue) -} - -// SetNoValue sets only 'key' as argument without the '='. -// -// Only key in argumemt, like key1&key2 -func (a *Args) SetNoValue(key string) { - a.args = setArg(a.args, key, "", argsNoValue) -} - -// SetBytesKNoValue sets 'key' argument. -func (a *Args) SetBytesKNoValue(key []byte) { - a.args = setArg(a.args, b2s(key), "", argsNoValue) -} - -// Peek returns query arg value for the given key. -// -// Returned value is valid until the next Args call. -func (a *Args) Peek(key string) []byte { - return peekArgStr(a.args, key) -} - -// PeekBytes returns query arg value for the given key. -// -// Returned value is valid until the next Args call. -func (a *Args) PeekBytes(key []byte) []byte { - return peekArgBytes(a.args, key) -} - -// PeekMulti returns all the arg values for the given key. -func (a *Args) PeekMulti(key string) [][]byte { - var values [][]byte - a.VisitAll(func(k, v []byte) { - if string(k) == key { - values = append(values, v) - } - }) - return values -} - -// PeekMultiBytes returns all the arg values for the given key. -func (a *Args) PeekMultiBytes(key []byte) [][]byte { - return a.PeekMulti(b2s(key)) -} - -// Has returns true if the given key exists in Args. -func (a *Args) Has(key string) bool { - return hasArg(a.args, key) -} - -// HasBytes returns true if the given key exists in Args. -func (a *Args) HasBytes(key []byte) bool { - return hasArg(a.args, b2s(key)) -} - -// ErrNoArgValue is returned when Args value with the given key is missing. -var ErrNoArgValue = errors.New("no Args value for the given key") - -// GetUint returns uint value for the given key. -func (a *Args) GetUint(key string) (int, error) { - value := a.Peek(key) - if len(value) == 0 { - return -1, ErrNoArgValue - } - return ParseUint(value) -} - -// SetUint sets uint value for the given key. -func (a *Args) SetUint(key string, value int) { - bb := bytebufferpool.Get() - bb.B = AppendUint(bb.B[:0], value) - a.SetBytesV(key, bb.B) - bytebufferpool.Put(bb) -} - -// SetUintBytes sets uint value for the given key. -func (a *Args) SetUintBytes(key []byte, value int) { - a.SetUint(b2s(key), value) -} - -// GetUintOrZero returns uint value for the given key. -// -// Zero (0) is returned on error. -func (a *Args) GetUintOrZero(key string) int { - n, err := a.GetUint(key) - if err != nil { - n = 0 - } - return n -} - -// GetUfloat returns ufloat value for the given key. -func (a *Args) GetUfloat(key string) (float64, error) { - value := a.Peek(key) - if len(value) == 0 { - return -1, ErrNoArgValue - } - return ParseUfloat(value) -} - -// GetUfloatOrZero returns ufloat value for the given key. -// -// Zero (0) is returned on error. -func (a *Args) GetUfloatOrZero(key string) float64 { - f, err := a.GetUfloat(key) - if err != nil { - f = 0 - } - return f -} - -// GetBool returns boolean value for the given key. -// -// true is returned for "1", "t", "T", "true", "TRUE", "True", "y", "yes", "Y", "YES", "Yes", -// otherwise false is returned. -func (a *Args) GetBool(key string) bool { - switch b2s(a.Peek(key)) { - // Support the same true cases as strconv.ParseBool - // See: https://github.com/golang/go/blob/4e1b11e2c9bdb0ddea1141eed487be1a626ff5be/src/strconv/atob.go#L12 - // and Y and Yes versions. - case "1", "t", "T", "true", "TRUE", "True", "y", "yes", "Y", "YES", "Yes": - return true - default: - return false - } -} - -func visitArgs(args []argsKV, f func(k, v []byte)) { - for i, n := 0, len(args); i < n; i++ { - kv := &args[i] - f(kv.key, kv.value) - } -} - -func copyArgs(dst, src []argsKV) []argsKV { - if cap(dst) < len(src) { - tmp := make([]argsKV, len(src)) - copy(tmp, dst) - dst = tmp - } - n := len(src) - dst = dst[:n] - for i := 0; i < n; i++ { - dstKV := &dst[i] - srcKV := &src[i] - dstKV.key = append(dstKV.key[:0], srcKV.key...) - if srcKV.noValue { - dstKV.value = dstKV.value[:0] - } else { - dstKV.value = append(dstKV.value[:0], srcKV.value...) - } - dstKV.noValue = srcKV.noValue - } - return dst -} - -func delAllArgsBytes(args []argsKV, key []byte) []argsKV { - return delAllArgs(args, b2s(key)) -} - -func delAllArgs(args []argsKV, key string) []argsKV { - for i, n := 0, len(args); i < n; i++ { - kv := &args[i] - if key == string(kv.key) { - tmp := *kv - copy(args[i:], args[i+1:]) - n-- - args[n] = tmp - args = args[:n] - } - } - return args -} - -func setArgBytes(h []argsKV, key, value []byte, noValue bool) []argsKV { - return setArg(h, b2s(key), b2s(value), noValue) -} - -func setArg(h []argsKV, key, value string, noValue bool) []argsKV { - n := len(h) - for i := 0; i < n; i++ { - kv := &h[i] - if key == string(kv.key) { - if noValue { - kv.value = kv.value[:0] - } else { - kv.value = append(kv.value[:0], value...) - } - kv.noValue = noValue - return h - } - } - return appendArg(h, key, value, noValue) -} - -func appendArgBytes(h []argsKV, key, value []byte, noValue bool) []argsKV { - return appendArg(h, b2s(key), b2s(value), noValue) -} - -func appendArg(args []argsKV, key, value string, noValue bool) []argsKV { - var kv *argsKV - args, kv = allocArg(args) - kv.key = append(kv.key[:0], key...) - if noValue { - kv.value = kv.value[:0] - } else { - kv.value = append(kv.value[:0], value...) - } - kv.noValue = noValue - return args -} - -func allocArg(h []argsKV) ([]argsKV, *argsKV) { - n := len(h) - if cap(h) > n { - h = h[:n+1] - } else { - h = append(h, argsKV{}) - } - return h, &h[n] -} - -func releaseArg(h []argsKV) []argsKV { - return h[:len(h)-1] -} - -func hasArg(h []argsKV, key string) bool { - for i, n := 0, len(h); i < n; i++ { - kv := &h[i] - if key == string(kv.key) { - return true - } - } - return false -} - -func peekArgBytes(h []argsKV, k []byte) []byte { - for i, n := 0, len(h); i < n; i++ { - kv := &h[i] - if bytes.Equal(kv.key, k) { - return kv.value - } - } - return nil -} - -func peekArgStr(h []argsKV, k string) []byte { - for i, n := 0, len(h); i < n; i++ { - kv := &h[i] - if string(kv.key) == k { - return kv.value - } - } - return nil -} - -type argsScanner struct { - b []byte -} - -func (s *argsScanner) next(kv *argsKV) bool { - if len(s.b) == 0 { - return false - } - kv.noValue = argsHasValue - - isKey := true - k := 0 - for i, c := range s.b { - switch c { - case '=': - if isKey { - isKey = false - kv.key = decodeArgAppend(kv.key[:0], s.b[:i]) - k = i + 1 - } - case '&': - if isKey { - kv.key = decodeArgAppend(kv.key[:0], s.b[:i]) - kv.value = kv.value[:0] - kv.noValue = argsNoValue - } else { - kv.value = decodeArgAppend(kv.value[:0], s.b[k:i]) - } - s.b = s.b[i+1:] - return true - } - } - - if isKey { - kv.key = decodeArgAppend(kv.key[:0], s.b) - kv.value = kv.value[:0] - kv.noValue = argsNoValue - } else { - kv.value = decodeArgAppend(kv.value[:0], s.b[k:]) - } - s.b = s.b[len(s.b):] - return true -} - -func decodeArgAppend(dst, src []byte) []byte { - if bytes.IndexByte(src, '%') < 0 && bytes.IndexByte(src, '+') < 0 { - // fast path: src doesn't contain encoded chars - return append(dst, src...) - } - - // slow path - for i := 0; i < len(src); i++ { - c := src[i] - if c == '%' { - if i+2 >= len(src) { - return append(dst, src[i:]...) - } - x2 := hex2intTable[src[i+2]] - x1 := hex2intTable[src[i+1]] - if x1 == 16 || x2 == 16 { - dst = append(dst, '%') - } else { - dst = append(dst, x1<<4|x2) - i += 2 - } - } else if c == '+' { - dst = append(dst, ' ') - } else { - dst = append(dst, c) - } - } - return dst -} - -// decodeArgAppendNoPlus is almost identical to decodeArgAppend, but it doesn't -// substitute '+' with ' '. -// -// The function is copy-pasted from decodeArgAppend due to the performance -// reasons only. -func decodeArgAppendNoPlus(dst, src []byte) []byte { - if bytes.IndexByte(src, '%') < 0 { - // fast path: src doesn't contain encoded chars - return append(dst, src...) - } - - // slow path - for i := 0; i < len(src); i++ { - c := src[i] - if c == '%' { - if i+2 >= len(src) { - return append(dst, src[i:]...) - } - x2 := hex2intTable[src[i+2]] - x1 := hex2intTable[src[i+1]] - if x1 == 16 || x2 == 16 { - dst = append(dst, '%') - } else { - dst = append(dst, x1<<4|x2) - i += 2 - } - } else { - dst = append(dst, c) - } - } - return dst -} diff --git a/vendor/github.com/valyala/fasthttp/bytesconv.go b/vendor/github.com/valyala/fasthttp/bytesconv.go deleted file mode 100644 index 8c0e1545..00000000 --- a/vendor/github.com/valyala/fasthttp/bytesconv.go +++ /dev/null @@ -1,437 +0,0 @@ -package fasthttp - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - "math" - "net" - "reflect" - "strings" - "sync" - "time" - "unsafe" -) - -// AppendHTMLEscape appends html-escaped s to dst and returns the extended dst. -func AppendHTMLEscape(dst []byte, s string) []byte { - if strings.IndexByte(s, '<') < 0 && - strings.IndexByte(s, '>') < 0 && - strings.IndexByte(s, '"') < 0 && - strings.IndexByte(s, '\'') < 0 { - - // fast path - nothing to escape - return append(dst, s...) - } - - // slow path - var prev int - var sub string - for i, n := 0, len(s); i < n; i++ { - sub = "" - switch s[i] { - case '<': - sub = "<" - case '>': - sub = ">" - case '"': - sub = """ - case '\'': - sub = "'" - } - if len(sub) > 0 { - dst = append(dst, s[prev:i]...) - dst = append(dst, sub...) - prev = i + 1 - } - } - return append(dst, s[prev:]...) -} - -// AppendHTMLEscapeBytes appends html-escaped s to dst and returns -// the extended dst. -func AppendHTMLEscapeBytes(dst, s []byte) []byte { - return AppendHTMLEscape(dst, b2s(s)) -} - -// AppendIPv4 appends string representation of the given ip v4 to dst -// and returns the extended dst. -func AppendIPv4(dst []byte, ip net.IP) []byte { - ip = ip.To4() - if ip == nil { - return append(dst, "non-v4 ip passed to AppendIPv4"...) - } - - dst = AppendUint(dst, int(ip[0])) - for i := 1; i < 4; i++ { - dst = append(dst, '.') - dst = AppendUint(dst, int(ip[i])) - } - return dst -} - -var errEmptyIPStr = errors.New("empty ip address string") - -// ParseIPv4 parses ip address from ipStr into dst and returns the extended dst. -func ParseIPv4(dst net.IP, ipStr []byte) (net.IP, error) { - if len(ipStr) == 0 { - return dst, errEmptyIPStr - } - if len(dst) < net.IPv4len { - dst = make([]byte, net.IPv4len) - } - copy(dst, net.IPv4zero) - dst = dst.To4() - if dst == nil { - panic("BUG: dst must not be nil") - } - - b := ipStr - for i := 0; i < 3; i++ { - n := bytes.IndexByte(b, '.') - if n < 0 { - return dst, fmt.Errorf("cannot find dot in ipStr %q", ipStr) - } - v, err := ParseUint(b[:n]) - if err != nil { - return dst, fmt.Errorf("cannot parse ipStr %q: %s", ipStr, err) - } - if v > 255 { - return dst, fmt.Errorf("cannot parse ipStr %q: ip part cannot exceed 255: parsed %d", ipStr, v) - } - dst[i] = byte(v) - b = b[n+1:] - } - v, err := ParseUint(b) - if err != nil { - return dst, fmt.Errorf("cannot parse ipStr %q: %s", ipStr, err) - } - if v > 255 { - return dst, fmt.Errorf("cannot parse ipStr %q: ip part cannot exceed 255: parsed %d", ipStr, v) - } - dst[3] = byte(v) - - return dst, nil -} - -// AppendHTTPDate appends HTTP-compliant (RFC1123) representation of date -// to dst and returns the extended dst. -func AppendHTTPDate(dst []byte, date time.Time) []byte { - dst = date.In(time.UTC).AppendFormat(dst, time.RFC1123) - copy(dst[len(dst)-3:], strGMT) - return dst -} - -// ParseHTTPDate parses HTTP-compliant (RFC1123) date. -func ParseHTTPDate(date []byte) (time.Time, error) { - return time.Parse(time.RFC1123, b2s(date)) -} - -// AppendUint appends n to dst and returns the extended dst. -func AppendUint(dst []byte, n int) []byte { - if n < 0 { - panic("BUG: int must be positive") - } - - var b [20]byte - buf := b[:] - i := len(buf) - var q int - for n >= 10 { - i-- - q = n / 10 - buf[i] = '0' + byte(n-q*10) - n = q - } - i-- - buf[i] = '0' + byte(n) - - dst = append(dst, buf[i:]...) - return dst -} - -// ParseUint parses uint from buf. -func ParseUint(buf []byte) (int, error) { - v, n, err := parseUintBuf(buf) - if n != len(buf) { - return -1, errUnexpectedTrailingChar - } - return v, err -} - -var ( - errEmptyInt = errors.New("empty integer") - errUnexpectedFirstChar = errors.New("unexpected first char found. Expecting 0-9") - errUnexpectedTrailingChar = errors.New("unexpected trailing char found. Expecting 0-9") - errTooLongInt = errors.New("too long int") -) - -func parseUintBuf(b []byte) (int, int, error) { - n := len(b) - if n == 0 { - return -1, 0, errEmptyInt - } - v := 0 - for i := 0; i < n; i++ { - c := b[i] - k := c - '0' - if k > 9 { - if i == 0 { - return -1, i, errUnexpectedFirstChar - } - return v, i, nil - } - // Test for overflow. - if v*10 < v { - return -1, i, errTooLongInt - } - v = 10*v + int(k) - } - return v, n, nil -} - -var ( - errEmptyFloat = errors.New("empty float number") - errDuplicateFloatPoint = errors.New("duplicate point found in float number") - errUnexpectedFloatEnd = errors.New("unexpected end of float number") - errInvalidFloatExponent = errors.New("invalid float number exponent") - errUnexpectedFloatChar = errors.New("unexpected char found in float number") -) - -// ParseUfloat parses unsigned float from buf. -func ParseUfloat(buf []byte) (float64, error) { - if len(buf) == 0 { - return -1, errEmptyFloat - } - b := buf - var v uint64 - var offset = 1.0 - var pointFound bool - for i, c := range b { - if c < '0' || c > '9' { - if c == '.' { - if pointFound { - return -1, errDuplicateFloatPoint - } - pointFound = true - continue - } - if c == 'e' || c == 'E' { - if i+1 >= len(b) { - return -1, errUnexpectedFloatEnd - } - b = b[i+1:] - minus := -1 - switch b[0] { - case '+': - b = b[1:] - minus = 1 - case '-': - b = b[1:] - default: - minus = 1 - } - vv, err := ParseUint(b) - if err != nil { - return -1, errInvalidFloatExponent - } - return float64(v) * offset * math.Pow10(minus*int(vv)), nil - } - return -1, errUnexpectedFloatChar - } - v = 10*v + uint64(c-'0') - if pointFound { - offset /= 10 - } - } - return float64(v) * offset, nil -} - -var ( - errEmptyHexNum = errors.New("empty hex number") - errTooLargeHexNum = errors.New("too large hex number") -) - -func readHexInt(r *bufio.Reader) (int, error) { - n := 0 - i := 0 - var k int - for { - c, err := r.ReadByte() - if err != nil { - if err == io.EOF && i > 0 { - return n, nil - } - return -1, err - } - k = int(hex2intTable[c]) - if k == 16 { - if i == 0 { - return -1, errEmptyHexNum - } - r.UnreadByte() - return n, nil - } - if i >= maxHexIntChars { - return -1, errTooLargeHexNum - } - n = (n << 4) | k - i++ - } -} - -var hexIntBufPool sync.Pool - -func writeHexInt(w *bufio.Writer, n int) error { - if n < 0 { - panic("BUG: int must be positive") - } - - v := hexIntBufPool.Get() - if v == nil { - v = make([]byte, maxHexIntChars+1) - } - buf := v.([]byte) - i := len(buf) - 1 - for { - buf[i] = int2hexbyte(n & 0xf) - n >>= 4 - if n == 0 { - break - } - i-- - } - _, err := w.Write(buf[i:]) - hexIntBufPool.Put(v) - return err -} - -func int2hexbyte(n int) byte { - if n < 10 { - return '0' + byte(n) - } - return 'a' + byte(n) - 10 -} - -func hexCharUpper(c byte) byte { - if c < 10 { - return '0' + c - } - return c - 10 + 'A' -} - -var hex2intTable = func() []byte { - b := make([]byte, 256) - for i := 0; i < 256; i++ { - c := byte(16) - if i >= '0' && i <= '9' { - c = byte(i) - '0' - } else if i >= 'a' && i <= 'f' { - c = byte(i) - 'a' + 10 - } else if i >= 'A' && i <= 'F' { - c = byte(i) - 'A' + 10 - } - b[i] = c - } - return b -}() - -const toLower = 'a' - 'A' - -var toLowerTable = func() [256]byte { - var a [256]byte - for i := 0; i < 256; i++ { - c := byte(i) - if c >= 'A' && c <= 'Z' { - c += toLower - } - a[i] = c - } - return a -}() - -var toUpperTable = func() [256]byte { - var a [256]byte - for i := 0; i < 256; i++ { - c := byte(i) - if c >= 'a' && c <= 'z' { - c -= toLower - } - a[i] = c - } - return a -}() - -func lowercaseBytes(b []byte) { - for i := 0; i < len(b); i++ { - p := &b[i] - *p = toLowerTable[*p] - } -} - -// b2s converts byte slice to a string without memory allocation. -// See https://groups.google.com/forum/#!msg/Golang-Nuts/ENgbUzYvCuU/90yGx7GUAgAJ . -// -// Note it may break if string and/or slice header will change -// in the future go versions. -func b2s(b []byte) string { - return *(*string)(unsafe.Pointer(&b)) -} - -// s2b converts string to a byte slice without memory allocation. -// -// Note it may break if string and/or slice header will change -// in the future go versions. -func s2b(s string) []byte { - sh := (*reflect.StringHeader)(unsafe.Pointer(&s)) - bh := reflect.SliceHeader{ - Data: sh.Data, - Len: sh.Len, - Cap: sh.Len, - } - return *(*[]byte)(unsafe.Pointer(&bh)) -} - -// AppendUnquotedArg appends url-decoded src to dst and returns appended dst. -// -// dst may point to src. In this case src will be overwritten. -func AppendUnquotedArg(dst, src []byte) []byte { - return decodeArgAppend(dst, src) -} - -// AppendQuotedArg appends url-encoded src to dst and returns appended dst. -func AppendQuotedArg(dst, src []byte) []byte { - for _, c := range src { - // See http://www.w3.org/TR/html5/forms.html#form-submission-algorithm - if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || - c == '*' || c == '-' || c == '.' || c == '_' { - dst = append(dst, c) - } else { - dst = append(dst, '%', hexCharUpper(c>>4), hexCharUpper(c&15)) - } - } - return dst -} - -func appendQuotedPath(dst, src []byte) []byte { - for _, c := range src { - // From the spec: http://tools.ietf.org/html/rfc3986#section-3.3 - // an path can contain zero or more of pchar that is defined as follows: - // pchar = unreserved / pct-encoded / sub-delims / ":" / "@" - // pct-encoded = "%" HEXDIG HEXDIG - // unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" - // sub-delims = "!" / "$" / "&" / "'" / "(" / ")" - // / "*" / "+" / "," / ";" / "=" - if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || - c == '-' || c == '.' || c == '_' || c == '~' || c == '!' || c == '$' || - c == '&' || c == '\'' || c == '(' || c == ')' || c == '*' || c == '+' || - c == ',' || c == ';' || c == '=' || c == ':' || c == '@' || c == '/' { - dst = append(dst, c) - } else { - dst = append(dst, '%', hexCharUpper(c>>4), hexCharUpper(c&15)) - } - } - return dst -} diff --git a/vendor/github.com/valyala/fasthttp/bytesconv_32.go b/vendor/github.com/valyala/fasthttp/bytesconv_32.go deleted file mode 100644 index 7fd6f5f1..00000000 --- a/vendor/github.com/valyala/fasthttp/bytesconv_32.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build !amd64,!arm64,!ppc64 - -package fasthttp - -const ( - maxHexIntChars = 7 -) diff --git a/vendor/github.com/valyala/fasthttp/bytesconv_64.go b/vendor/github.com/valyala/fasthttp/bytesconv_64.go deleted file mode 100644 index edf7309c..00000000 --- a/vendor/github.com/valyala/fasthttp/bytesconv_64.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build amd64 arm64 ppc64 - -package fasthttp - -const ( - maxHexIntChars = 15 -) diff --git a/vendor/github.com/valyala/fasthttp/client.go b/vendor/github.com/valyala/fasthttp/client.go deleted file mode 100644 index 89e98082..00000000 --- a/vendor/github.com/valyala/fasthttp/client.go +++ /dev/null @@ -1,2257 +0,0 @@ -package fasthttp - -import ( - "bufio" - "bytes" - "crypto/tls" - "errors" - "fmt" - "io" - "net" - "strings" - "sync" - "sync/atomic" - "time" -) - -// Do performs the given http request and fills the given http response. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// Client determines the server to be requested in the following order: -// -// - from RequestURI if it contains full url with scheme and host; -// - from Host header otherwise. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// ErrNoFreeConns is returned if all DefaultMaxConnsPerHost connections -// to the requested host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func Do(req *Request, resp *Response) error { - return defaultClient.Do(req, resp) -} - -// DoTimeout performs the given request and waits for response during -// the given timeout duration. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// Client determines the server to be requested in the following order: -// -// - from RequestURI if it contains full url with scheme and host; -// - from Host header otherwise. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// ErrTimeout is returned if the response wasn't returned during -// the given timeout. -// -// ErrNoFreeConns is returned if all DefaultMaxConnsPerHost connections -// to the requested host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -// -// Warning: DoTimeout does not terminate the request itself. The request will -// continue in the background and the response will be discarded. -// If requests take too long and the connection pool gets filled up please -// try using a Client and setting a ReadTimeout. -func DoTimeout(req *Request, resp *Response, timeout time.Duration) error { - return defaultClient.DoTimeout(req, resp, timeout) -} - -// DoDeadline performs the given request and waits for response until -// the given deadline. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// Client determines the server to be requested in the following order: -// -// - from RequestURI if it contains full url with scheme and host; -// - from Host header otherwise. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// ErrTimeout is returned if the response wasn't returned until -// the given deadline. -// -// ErrNoFreeConns is returned if all DefaultMaxConnsPerHost connections -// to the requested host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func DoDeadline(req *Request, resp *Response, deadline time.Time) error { - return defaultClient.DoDeadline(req, resp, deadline) -} - -// Get returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -func Get(dst []byte, url string) (statusCode int, body []byte, err error) { - return defaultClient.Get(dst, url) -} - -// GetTimeout returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// ErrTimeout error is returned if url contents couldn't be fetched -// during the given timeout. -func GetTimeout(dst []byte, url string, timeout time.Duration) (statusCode int, body []byte, err error) { - return defaultClient.GetTimeout(dst, url, timeout) -} - -// GetDeadline returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// ErrTimeout error is returned if url contents couldn't be fetched -// until the given deadline. -func GetDeadline(dst []byte, url string, deadline time.Time) (statusCode int, body []byte, err error) { - return defaultClient.GetDeadline(dst, url, deadline) -} - -// Post sends POST request to the given url with the given POST arguments. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// Empty POST body is sent if postArgs is nil. -func Post(dst []byte, url string, postArgs *Args) (statusCode int, body []byte, err error) { - return defaultClient.Post(dst, url, postArgs) -} - -var defaultClient Client - -// Client implements http client. -// -// Copying Client by value is prohibited. Create new instance instead. -// -// It is safe calling Client methods from concurrently running goroutines. -type Client struct { - noCopy noCopy - - // Client name. Used in User-Agent request header. - // - // Default client name is used if not set. - Name string - - // NoDefaultUserAgentHeader when set to true, causes the default - // User-Agent header to be excluded from the Request. - NoDefaultUserAgentHeader bool - - // Callback for establishing new connections to hosts. - // - // Default Dial is used if not set. - Dial DialFunc - - // Attempt to connect to both ipv4 and ipv6 addresses if set to true. - // - // This option is used only if default TCP dialer is used, - // i.e. if Dial is blank. - // - // By default client connects only to ipv4 addresses, - // since unfortunately ipv6 remains broken in many networks worldwide :) - DialDualStack bool - - // TLS config for https connections. - // - // Default TLS config is used if not set. - TLSConfig *tls.Config - - // Maximum number of connections per each host which may be established. - // - // DefaultMaxConnsPerHost is used if not set. - MaxConnsPerHost int - - // Idle keep-alive connections are closed after this duration. - // - // By default idle connections are closed - // after DefaultMaxIdleConnDuration. - MaxIdleConnDuration time.Duration - - // Maximum number of attempts for idempotent calls - // - // DefaultMaxIdemponentCallAttempts is used if not set. - MaxIdemponentCallAttempts int - - // Per-connection buffer size for responses' reading. - // This also limits the maximum header size. - // - // Default buffer size is used if 0. - ReadBufferSize int - - // Per-connection buffer size for requests' writing. - // - // Default buffer size is used if 0. - WriteBufferSize int - - // Maximum duration for full response reading (including body). - // - // By default response read timeout is unlimited. - ReadTimeout time.Duration - - // Maximum duration for full request writing (including body). - // - // By default request write timeout is unlimited. - WriteTimeout time.Duration - - // Maximum response body size. - // - // The client returns ErrBodyTooLarge if this limit is greater than 0 - // and response body is greater than the limit. - // - // By default response body size is unlimited. - MaxResponseBodySize int - - // Header names are passed as-is without normalization - // if this option is set. - // - // Disabled header names' normalization may be useful only for proxying - // responses to other clients expecting case-sensitive - // header names. See https://github.com/valyala/fasthttp/issues/57 - // for details. - // - // By default request and response header names are normalized, i.e. - // The first letter and the first letters following dashes - // are uppercased, while all the other letters are lowercased. - // Examples: - // - // * HOST -> Host - // * content-type -> Content-Type - // * cONTENT-lenGTH -> Content-Length - DisableHeaderNamesNormalizing bool - - mLock sync.Mutex - m map[string]*HostClient - ms map[string]*HostClient -} - -// Get returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -func (c *Client) Get(dst []byte, url string) (statusCode int, body []byte, err error) { - return clientGetURL(dst, url, c) -} - -// GetTimeout returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// ErrTimeout error is returned if url contents couldn't be fetched -// during the given timeout. -func (c *Client) GetTimeout(dst []byte, url string, timeout time.Duration) (statusCode int, body []byte, err error) { - return clientGetURLTimeout(dst, url, timeout, c) -} - -// GetDeadline returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// ErrTimeout error is returned if url contents couldn't be fetched -// until the given deadline. -func (c *Client) GetDeadline(dst []byte, url string, deadline time.Time) (statusCode int, body []byte, err error) { - return clientGetURLDeadline(dst, url, deadline, c) -} - -// Post sends POST request to the given url with the given POST arguments. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// Empty POST body is sent if postArgs is nil. -func (c *Client) Post(dst []byte, url string, postArgs *Args) (statusCode int, body []byte, err error) { - return clientPostURL(dst, url, postArgs, c) -} - -// DoTimeout performs the given request and waits for response during -// the given timeout duration. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// Client determines the server to be requested in the following order: -// -// - from RequestURI if it contains full url with scheme and host; -// - from Host header otherwise. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// ErrTimeout is returned if the response wasn't returned during -// the given timeout. -// -// ErrNoFreeConns is returned if all Client.MaxConnsPerHost connections -// to the requested host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -// -// Warning: DoTimeout does not terminate the request itself. The request will -// continue in the background and the response will be discarded. -// If requests take too long and the connection pool gets filled up please -// try setting a ReadTimeout. -func (c *Client) DoTimeout(req *Request, resp *Response, timeout time.Duration) error { - return clientDoTimeout(req, resp, timeout, c) -} - -// DoDeadline performs the given request and waits for response until -// the given deadline. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// Client determines the server to be requested in the following order: -// -// - from RequestURI if it contains full url with scheme and host; -// - from Host header otherwise. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// ErrTimeout is returned if the response wasn't returned until -// the given deadline. -// -// ErrNoFreeConns is returned if all Client.MaxConnsPerHost connections -// to the requested host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func (c *Client) DoDeadline(req *Request, resp *Response, deadline time.Time) error { - return clientDoDeadline(req, resp, deadline, c) -} - -// Do performs the given http request and fills the given http response. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// Client determines the server to be requested in the following order: -// -// - from RequestURI if it contains full url with scheme and host; -// - from Host header otherwise. -// -// Response is ignored if resp is nil. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// ErrNoFreeConns is returned if all Client.MaxConnsPerHost connections -// to the requested host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func (c *Client) Do(req *Request, resp *Response) error { - uri := req.URI() - host := uri.Host() - - isTLS := false - scheme := uri.Scheme() - if bytes.Equal(scheme, strHTTPS) { - isTLS = true - } else if !bytes.Equal(scheme, strHTTP) { - return fmt.Errorf("unsupported protocol %q. http and https are supported", scheme) - } - - startCleaner := false - - c.mLock.Lock() - m := c.m - if isTLS { - m = c.ms - } - if m == nil { - m = make(map[string]*HostClient) - if isTLS { - c.ms = m - } else { - c.m = m - } - } - hc := m[string(host)] - if hc == nil { - hc = &HostClient{ - Addr: addMissingPort(string(host), isTLS), - Name: c.Name, - NoDefaultUserAgentHeader: c.NoDefaultUserAgentHeader, - Dial: c.Dial, - DialDualStack: c.DialDualStack, - IsTLS: isTLS, - TLSConfig: c.TLSConfig, - MaxConns: c.MaxConnsPerHost, - MaxIdleConnDuration: c.MaxIdleConnDuration, - MaxIdemponentCallAttempts: c.MaxIdemponentCallAttempts, - ReadBufferSize: c.ReadBufferSize, - WriteBufferSize: c.WriteBufferSize, - ReadTimeout: c.ReadTimeout, - WriteTimeout: c.WriteTimeout, - MaxResponseBodySize: c.MaxResponseBodySize, - DisableHeaderNamesNormalizing: c.DisableHeaderNamesNormalizing, - } - m[string(host)] = hc - if len(m) == 1 { - startCleaner = true - } - } - c.mLock.Unlock() - - if startCleaner { - go c.mCleaner(m) - } - - return hc.Do(req, resp) -} - -func (c *Client) mCleaner(m map[string]*HostClient) { - mustStop := false - - for { - c.mLock.Lock() - for k, v := range m { - v.connsLock.Lock() - shouldRemove := v.connsCount == 0 - v.connsLock.Unlock() - - if shouldRemove { - delete(m, k) - } - } - if len(m) == 0 { - mustStop = true - } - c.mLock.Unlock() - - if mustStop { - break - } - time.Sleep(10 * time.Second) - } -} - -// DefaultMaxConnsPerHost is the maximum number of concurrent connections -// http client may establish per host by default (i.e. if -// Client.MaxConnsPerHost isn't set). -const DefaultMaxConnsPerHost = 512 - -// DefaultMaxIdleConnDuration is the default duration before idle keep-alive -// connection is closed. -const DefaultMaxIdleConnDuration = 10 * time.Second - -// DefaultMaxIdemponentCallAttempts is the default idempotent calls attempts count. -const DefaultMaxIdemponentCallAttempts = 5 - -// DialFunc must establish connection to addr. -// -// There is no need in establishing TLS (SSL) connection for https. -// The client automatically converts connection to TLS -// if HostClient.IsTLS is set. -// -// TCP address passed to DialFunc always contains host and port. -// Example TCP addr values: -// -// - foobar.com:80 -// - foobar.com:443 -// - foobar.com:8080 -type DialFunc func(addr string) (net.Conn, error) - -// HostClient balances http requests among hosts listed in Addr. -// -// HostClient may be used for balancing load among multiple upstream hosts. -// While multiple addresses passed to HostClient.Addr may be used for balancing -// load among them, it would be better using LBClient instead, since HostClient -// may unevenly balance load among upstream hosts. -// -// It is forbidden copying HostClient instances. Create new instances instead. -// -// It is safe calling HostClient methods from concurrently running goroutines. -type HostClient struct { - noCopy noCopy - - // Comma-separated list of upstream HTTP server host addresses, - // which are passed to Dial in a round-robin manner. - // - // Each address may contain port if default dialer is used. - // For example, - // - // - foobar.com:80 - // - foobar.com:443 - // - foobar.com:8080 - Addr string - - // Client name. Used in User-Agent request header. - Name string - - // NoDefaultUserAgentHeader when set to true, causes the default - // User-Agent header to be excluded from the Request. - NoDefaultUserAgentHeader bool - - // Callback for establishing new connection to the host. - // - // Default Dial is used if not set. - Dial DialFunc - - // Attempt to connect to both ipv4 and ipv6 host addresses - // if set to true. - // - // This option is used only if default TCP dialer is used, - // i.e. if Dial is blank. - // - // By default client connects only to ipv4 addresses, - // since unfortunately ipv6 remains broken in many networks worldwide :) - DialDualStack bool - - // Whether to use TLS (aka SSL or HTTPS) for host connections. - IsTLS bool - - // Optional TLS config. - TLSConfig *tls.Config - - // Maximum number of connections which may be established to all hosts - // listed in Addr. - // - // You can change this value while the HostClient is being used - // using HostClient.SetMaxConns(value) - // - // DefaultMaxConnsPerHost is used if not set. - MaxConns int - - // Keep-alive connections are closed after this duration. - // - // By default connection duration is unlimited. - MaxConnDuration time.Duration - - // Idle keep-alive connections are closed after this duration. - // - // By default idle connections are closed - // after DefaultMaxIdleConnDuration. - MaxIdleConnDuration time.Duration - - // Maximum number of attempts for idempotent calls - // - // DefaultMaxIdemponentCallAttempts is used if not set. - MaxIdemponentCallAttempts int - - // Per-connection buffer size for responses' reading. - // This also limits the maximum header size. - // - // Default buffer size is used if 0. - ReadBufferSize int - - // Per-connection buffer size for requests' writing. - // - // Default buffer size is used if 0. - WriteBufferSize int - - // Maximum duration for full response reading (including body). - // - // By default response read timeout is unlimited. - ReadTimeout time.Duration - - // Maximum duration for full request writing (including body). - // - // By default request write timeout is unlimited. - WriteTimeout time.Duration - - // Maximum response body size. - // - // The client returns ErrBodyTooLarge if this limit is greater than 0 - // and response body is greater than the limit. - // - // By default response body size is unlimited. - MaxResponseBodySize int - - // Header names are passed as-is without normalization - // if this option is set. - // - // Disabled header names' normalization may be useful only for proxying - // responses to other clients expecting case-sensitive - // header names. See https://github.com/valyala/fasthttp/issues/57 - // for details. - // - // By default request and response header names are normalized, i.e. - // The first letter and the first letters following dashes - // are uppercased, while all the other letters are lowercased. - // Examples: - // - // * HOST -> Host - // * content-type -> Content-Type - // * cONTENT-lenGTH -> Content-Length - DisableHeaderNamesNormalizing bool - - clientName atomic.Value - lastUseTime uint32 - - connsLock sync.Mutex - connsCount int - conns []*clientConn - - addrsLock sync.Mutex - addrs []string - addrIdx uint32 - - tlsConfigMap map[string]*tls.Config - tlsConfigMapLock sync.Mutex - - readerPool sync.Pool - writerPool sync.Pool - - pendingRequests int32 - - connsCleanerRun bool -} - -type clientConn struct { - c net.Conn - - createdTime time.Time - lastUseTime time.Time -} - -var startTimeUnix = time.Now().Unix() - -// LastUseTime returns time the client was last used -func (c *HostClient) LastUseTime() time.Time { - n := atomic.LoadUint32(&c.lastUseTime) - return time.Unix(startTimeUnix+int64(n), 0) -} - -// Get returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -func (c *HostClient) Get(dst []byte, url string) (statusCode int, body []byte, err error) { - return clientGetURL(dst, url, c) -} - -// GetTimeout returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// ErrTimeout error is returned if url contents couldn't be fetched -// during the given timeout. -func (c *HostClient) GetTimeout(dst []byte, url string, timeout time.Duration) (statusCode int, body []byte, err error) { - return clientGetURLTimeout(dst, url, timeout, c) -} - -// GetDeadline returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// ErrTimeout error is returned if url contents couldn't be fetched -// until the given deadline. -func (c *HostClient) GetDeadline(dst []byte, url string, deadline time.Time) (statusCode int, body []byte, err error) { - return clientGetURLDeadline(dst, url, deadline, c) -} - -// Post sends POST request to the given url with the given POST arguments. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// Empty POST body is sent if postArgs is nil. -func (c *HostClient) Post(dst []byte, url string, postArgs *Args) (statusCode int, body []byte, err error) { - return clientPostURL(dst, url, postArgs, c) -} - -type clientDoer interface { - Do(req *Request, resp *Response) error -} - -func clientGetURL(dst []byte, url string, c clientDoer) (statusCode int, body []byte, err error) { - req := AcquireRequest() - - statusCode, body, err = doRequestFollowRedirects(req, dst, url, c) - - ReleaseRequest(req) - return statusCode, body, err -} - -func clientGetURLTimeout(dst []byte, url string, timeout time.Duration, c clientDoer) (statusCode int, body []byte, err error) { - deadline := time.Now().Add(timeout) - return clientGetURLDeadline(dst, url, deadline, c) -} - -type clientURLResponse struct { - statusCode int - body []byte - err error -} - -func clientGetURLDeadline(dst []byte, url string, deadline time.Time, c clientDoer) (statusCode int, body []byte, err error) { - timeout := -time.Since(deadline) - if timeout <= 0 { - return 0, dst, ErrTimeout - } - - var ch chan clientURLResponse - chv := clientURLResponseChPool.Get() - if chv == nil { - chv = make(chan clientURLResponse, 1) - } - ch = chv.(chan clientURLResponse) - - req := AcquireRequest() - - // Note that the request continues execution on ErrTimeout until - // client-specific ReadTimeout exceeds. This helps limiting load - // on slow hosts by MaxConns* concurrent requests. - // - // Without this 'hack' the load on slow host could exceed MaxConns* - // concurrent requests, since timed out requests on client side - // usually continue execution on the host. - go func() { - statusCodeCopy, bodyCopy, errCopy := doRequestFollowRedirects(req, dst, url, c) - ch <- clientURLResponse{ - statusCode: statusCodeCopy, - body: bodyCopy, - err: errCopy, - } - }() - - tc := AcquireTimer(timeout) - select { - case resp := <-ch: - ReleaseRequest(req) - clientURLResponseChPool.Put(chv) - statusCode = resp.statusCode - body = resp.body - err = resp.err - case <-tc.C: - body = dst - err = ErrTimeout - } - ReleaseTimer(tc) - - return statusCode, body, err -} - -var clientURLResponseChPool sync.Pool - -func clientPostURL(dst []byte, url string, postArgs *Args, c clientDoer) (statusCode int, body []byte, err error) { - req := AcquireRequest() - req.Header.SetMethodBytes(strPost) - req.Header.SetContentTypeBytes(strPostArgsContentType) - if postArgs != nil { - postArgs.WriteTo(req.BodyWriter()) - } - - statusCode, body, err = doRequestFollowRedirects(req, dst, url, c) - - ReleaseRequest(req) - return statusCode, body, err -} - -var ( - errMissingLocation = errors.New("missing Location header for http redirect") - errTooManyRedirects = errors.New("too many redirects detected when doing the request") -) - -const maxRedirectsCount = 16 - -func doRequestFollowRedirects(req *Request, dst []byte, url string, c clientDoer) (statusCode int, body []byte, err error) { - resp := AcquireResponse() - bodyBuf := resp.bodyBuffer() - resp.keepBodyBuffer = true - oldBody := bodyBuf.B - bodyBuf.B = dst - scheme := req.uri.Scheme() - req.schemaUpdate = false - - redirectsCount := 0 - for { - // In case redirect to different scheme - if redirectsCount > 0 && !bytes.Equal(scheme, req.uri.Scheme()) { - if strings.HasPrefix(url, string(strHTTPS)) { - req.isTLS = true - req.uri.SetSchemeBytes(strHTTPS) - } else { - req.isTLS = false - req.uri.SetSchemeBytes(strHTTP) - } - scheme = req.uri.Scheme() - req.schemaUpdate = true - } - - req.parsedURI = false - req.Header.host = req.Header.host[:0] - req.SetRequestURI(url) - - if err = c.Do(req, resp); err != nil { - break - } - statusCode = resp.Header.StatusCode() - if statusCode != StatusMovedPermanently && - statusCode != StatusFound && - statusCode != StatusSeeOther && - statusCode != StatusTemporaryRedirect && - statusCode != StatusPermanentRedirect { - break - } - - redirectsCount++ - if redirectsCount > maxRedirectsCount { - err = errTooManyRedirects - break - } - location := resp.Header.peek(strLocation) - if len(location) == 0 { - err = errMissingLocation - break - } - url = getRedirectURL(url, location) - } - - body = bodyBuf.B - bodyBuf.B = oldBody - resp.keepBodyBuffer = false - ReleaseResponse(resp) - - return statusCode, body, err -} - -func getRedirectURL(baseURL string, location []byte) string { - u := AcquireURI() - u.Update(baseURL) - u.UpdateBytes(location) - redirectURL := u.String() - ReleaseURI(u) - return redirectURL -} - -var ( - requestPool sync.Pool - responsePool sync.Pool -) - -// AcquireRequest returns an empty Request instance from request pool. -// -// The returned Request instance may be passed to ReleaseRequest when it is -// no longer needed. This allows Request recycling, reduces GC pressure -// and usually improves performance. -func AcquireRequest() *Request { - v := requestPool.Get() - if v == nil { - return &Request{} - } - return v.(*Request) -} - -// ReleaseRequest returns req acquired via AcquireRequest to request pool. -// -// It is forbidden accessing req and/or its' members after returning -// it to request pool. -func ReleaseRequest(req *Request) { - req.Reset() - requestPool.Put(req) -} - -// AcquireResponse returns an empty Response instance from response pool. -// -// The returned Response instance may be passed to ReleaseResponse when it is -// no longer needed. This allows Response recycling, reduces GC pressure -// and usually improves performance. -func AcquireResponse() *Response { - v := responsePool.Get() - if v == nil { - return &Response{} - } - return v.(*Response) -} - -// ReleaseResponse return resp acquired via AcquireResponse to response pool. -// -// It is forbidden accessing resp and/or its' members after returning -// it to response pool. -func ReleaseResponse(resp *Response) { - resp.Reset() - responsePool.Put(resp) -} - -// DoTimeout performs the given request and waits for response during -// the given timeout duration. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// ErrTimeout is returned if the response wasn't returned during -// the given timeout. -// -// ErrNoFreeConns is returned if all HostClient.MaxConns connections -// to the host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -// -// Warning: DoTimeout does not terminate the request itself. The request will -// continue in the background and the response will be discarded. -// If requests take too long and the connection pool gets filled up please -// try setting a ReadTimeout. -func (c *HostClient) DoTimeout(req *Request, resp *Response, timeout time.Duration) error { - return clientDoTimeout(req, resp, timeout, c) -} - -// DoDeadline performs the given request and waits for response until -// the given deadline. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// ErrTimeout is returned if the response wasn't returned until -// the given deadline. -// -// ErrNoFreeConns is returned if all HostClient.MaxConns connections -// to the host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func (c *HostClient) DoDeadline(req *Request, resp *Response, deadline time.Time) error { - return clientDoDeadline(req, resp, deadline, c) -} - -func clientDoTimeout(req *Request, resp *Response, timeout time.Duration, c clientDoer) error { - deadline := time.Now().Add(timeout) - return clientDoDeadline(req, resp, deadline, c) -} - -func clientDoDeadline(req *Request, resp *Response, deadline time.Time, c clientDoer) error { - timeout := -time.Since(deadline) - if timeout <= 0 { - return ErrTimeout - } - - var ch chan error - chv := errorChPool.Get() - if chv == nil { - chv = make(chan error, 1) - } - ch = chv.(chan error) - - // Make req and resp copies, since on timeout they no longer - // may be accessed. - reqCopy := AcquireRequest() - req.copyToSkipBody(reqCopy) - swapRequestBody(req, reqCopy) - respCopy := AcquireResponse() - // Not calling resp.copyToSkipBody(respCopy) here to avoid - // unexpected messing with headers - respCopy.SkipBody = resp.SkipBody - - // Note that the request continues execution on ErrTimeout until - // client-specific ReadTimeout exceeds. This helps limiting load - // on slow hosts by MaxConns* concurrent requests. - // - // Without this 'hack' the load on slow host could exceed MaxConns* - // concurrent requests, since timed out requests on client side - // usually continue execution on the host. - - var cleanup int32 - go func() { - errDo := c.Do(reqCopy, respCopy) - if atomic.LoadInt32(&cleanup) == 1 { - ReleaseResponse(respCopy) - ReleaseRequest(reqCopy) - errorChPool.Put(chv) - } else { - ch <- errDo - } - }() - - tc := AcquireTimer(timeout) - var err error - select { - case err = <-ch: - if resp != nil { - respCopy.copyToSkipBody(resp) - swapResponseBody(resp, respCopy) - } - swapRequestBody(reqCopy, req) - ReleaseResponse(respCopy) - ReleaseRequest(reqCopy) - errorChPool.Put(chv) - case <-tc.C: - atomic.StoreInt32(&cleanup, 1) - err = ErrTimeout - } - ReleaseTimer(tc) - - return err -} - -var errorChPool sync.Pool - -// Do performs the given http request and sets the corresponding response. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// ErrNoFreeConns is returned if all HostClient.MaxConns connections -// to the host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func (c *HostClient) Do(req *Request, resp *Response) error { - var err error - var retry bool - maxAttempts := c.MaxIdemponentCallAttempts - if maxAttempts <= 0 { - maxAttempts = DefaultMaxIdemponentCallAttempts - } - attempts := 0 - - atomic.AddInt32(&c.pendingRequests, 1) - for { - retry, err = c.do(req, resp) - if err == nil || !retry { - break - } - - if !isIdempotent(req) { - // Retry non-idempotent requests if the server closes - // the connection before sending the response. - // - // This case is possible if the server closes the idle - // keep-alive connection on timeout. - // - // Apache and nginx usually do this. - if err != io.EOF { - break - } - } - attempts++ - if attempts >= maxAttempts { - break - } - } - atomic.AddInt32(&c.pendingRequests, -1) - - if err == io.EOF { - err = ErrConnectionClosed - } - return err -} - -// PendingRequests returns the current number of requests the client -// is executing. -// -// This function may be used for balancing load among multiple HostClient -// instances. -func (c *HostClient) PendingRequests() int { - return int(atomic.LoadInt32(&c.pendingRequests)) -} - -func isIdempotent(req *Request) bool { - return req.Header.IsGet() || req.Header.IsHead() || req.Header.IsPut() -} - -func (c *HostClient) do(req *Request, resp *Response) (bool, error) { - nilResp := false - if resp == nil { - nilResp = true - resp = AcquireResponse() - } - - ok, err := c.doNonNilReqResp(req, resp) - - if nilResp { - ReleaseResponse(resp) - } - - return ok, err -} - -func (c *HostClient) doNonNilReqResp(req *Request, resp *Response) (bool, error) { - if req == nil { - panic("BUG: req cannot be nil") - } - if resp == nil { - panic("BUG: resp cannot be nil") - } - - atomic.StoreUint32(&c.lastUseTime, uint32(time.Now().Unix()-startTimeUnix)) - - // Free up resources occupied by response before sending the request, - // so the GC may reclaim these resources (e.g. response body). - resp.Reset() - - // If we detected a redirect to another schema - if req.schemaUpdate { - c.IsTLS = bytes.Equal(req.URI().Scheme(), strHTTPS) - c.Addr = addMissingPort(string(req.Host()), c.IsTLS) - c.addrIdx = 0 - c.addrs = nil - req.schemaUpdate = false - req.SetConnectionClose() - } - - cc, err := c.acquireConn() - if err != nil { - return false, err - } - conn := cc.c - - resp.parseNetConn(conn) - - if c.WriteTimeout > 0 { - // Set Deadline every time, since golang has fixed the performance issue - // See https://github.com/golang/go/issues/15133#issuecomment-271571395 for details - currentTime := time.Now() - if err = conn.SetWriteDeadline(currentTime.Add(c.WriteTimeout)); err != nil { - c.closeConn(cc) - return true, err - } - } - - resetConnection := false - if c.MaxConnDuration > 0 && time.Since(cc.createdTime) > c.MaxConnDuration && !req.ConnectionClose() { - req.SetConnectionClose() - resetConnection = true - } - - userAgentOld := req.Header.UserAgent() - if len(userAgentOld) == 0 { - req.Header.userAgent = c.getClientName() - } - bw := c.acquireWriter(conn) - err = req.Write(bw) - - if resetConnection { - req.Header.ResetConnectionClose() - } - - if err == nil { - err = bw.Flush() - } - if err != nil { - c.releaseWriter(bw) - c.closeConn(cc) - return true, err - } - c.releaseWriter(bw) - - if c.ReadTimeout > 0 { - // Set Deadline every time, since golang has fixed the performance issue - // See https://github.com/golang/go/issues/15133#issuecomment-271571395 for details - currentTime := time.Now() - if err = conn.SetReadDeadline(currentTime.Add(c.ReadTimeout)); err != nil { - c.closeConn(cc) - return true, err - } - } - - if !req.Header.IsGet() && req.Header.IsHead() { - resp.SkipBody = true - } - if c.DisableHeaderNamesNormalizing { - resp.Header.DisableNormalizing() - } - - br := c.acquireReader(conn) - if err = resp.ReadLimitBody(br, c.MaxResponseBodySize); err != nil { - c.releaseReader(br) - c.closeConn(cc) - // Don't retry in case of ErrBodyTooLarge since we will just get the same again. - retry := err != ErrBodyTooLarge - return retry, err - } - c.releaseReader(br) - - if resetConnection || req.ConnectionClose() || resp.ConnectionClose() { - c.closeConn(cc) - } else { - c.releaseConn(cc) - } - - return false, err -} - -var ( - // ErrNoFreeConns is returned when no free connections available - // to the given host. - // - // Increase the allowed number of connections per host if you - // see this error. - ErrNoFreeConns = errors.New("no free connections available to host") - - // ErrTimeout is returned from timed out calls. - ErrTimeout = errors.New("timeout") - - // ErrConnectionClosed may be returned from client methods if the server - // closes connection before returning the first response byte. - // - // If you see this error, then either fix the server by returning - // 'Connection: close' response header before closing the connection - // or add 'Connection: close' request header before sending requests - // to broken server. - ErrConnectionClosed = errors.New("the server closed connection before returning the first response byte. " + - "Make sure the server returns 'Connection: close' response header before closing the connection") -) - -func (c *HostClient) SetMaxConns(newMaxConns int) { - c.connsLock.Lock() - c.MaxConns = newMaxConns - c.connsLock.Unlock() -} - -func (c *HostClient) acquireConn() (*clientConn, error) { - var cc *clientConn - createConn := false - startCleaner := false - - var n int - c.connsLock.Lock() - n = len(c.conns) - if n == 0 { - maxConns := c.MaxConns - if maxConns <= 0 { - maxConns = DefaultMaxConnsPerHost - } - if c.connsCount < maxConns { - c.connsCount++ - createConn = true - if !c.connsCleanerRun { - startCleaner = true - c.connsCleanerRun = true - } - } - } else { - n-- - cc = c.conns[n] - c.conns[n] = nil - c.conns = c.conns[:n] - } - c.connsLock.Unlock() - - if cc != nil { - return cc, nil - } - if !createConn { - return nil, ErrNoFreeConns - } - - if startCleaner { - go c.connsCleaner() - } - - conn, err := c.dialHostHard() - if err != nil { - c.decConnsCount() - return nil, err - } - cc = acquireClientConn(conn) - - return cc, nil -} - -func (c *HostClient) connsCleaner() { - var ( - scratch []*clientConn - maxIdleConnDuration = c.MaxIdleConnDuration - ) - if maxIdleConnDuration <= 0 { - maxIdleConnDuration = DefaultMaxIdleConnDuration - } - for { - currentTime := time.Now() - - // Determine idle connections to be closed. - c.connsLock.Lock() - conns := c.conns - n := len(conns) - i := 0 - for i < n && currentTime.Sub(conns[i].lastUseTime) > maxIdleConnDuration { - i++ - } - sleepFor := maxIdleConnDuration - if i < n { - // + 1 so we actually sleep past the expiration time and not up to it. - // Otherwise the > check above would still fail. - sleepFor = maxIdleConnDuration - currentTime.Sub(conns[i].lastUseTime) + 1 - } - scratch = append(scratch[:0], conns[:i]...) - if i > 0 { - m := copy(conns, conns[i:]) - for i = m; i < n; i++ { - conns[i] = nil - } - c.conns = conns[:m] - } - c.connsLock.Unlock() - - // Close idle connections. - for i, cc := range scratch { - c.closeConn(cc) - scratch[i] = nil - } - - // Determine whether to stop the connsCleaner. - c.connsLock.Lock() - mustStop := c.connsCount == 0 - if mustStop { - c.connsCleanerRun = false - } - c.connsLock.Unlock() - if mustStop { - break - } - - time.Sleep(sleepFor) - } -} - -func (c *HostClient) closeConn(cc *clientConn) { - c.decConnsCount() - cc.c.Close() - releaseClientConn(cc) -} - -func (c *HostClient) decConnsCount() { - c.connsLock.Lock() - c.connsCount-- - c.connsLock.Unlock() -} - -func acquireClientConn(conn net.Conn) *clientConn { - v := clientConnPool.Get() - if v == nil { - v = &clientConn{} - } - cc := v.(*clientConn) - cc.c = conn - cc.createdTime = time.Now() - return cc -} - -func releaseClientConn(cc *clientConn) { - // Reset all fields. - *cc = clientConn{} - clientConnPool.Put(cc) -} - -var clientConnPool sync.Pool - -func (c *HostClient) releaseConn(cc *clientConn) { - cc.lastUseTime = time.Now() - c.connsLock.Lock() - c.conns = append(c.conns, cc) - c.connsLock.Unlock() -} - -func (c *HostClient) acquireWriter(conn net.Conn) *bufio.Writer { - v := c.writerPool.Get() - if v == nil { - n := c.WriteBufferSize - if n <= 0 { - n = defaultWriteBufferSize - } - return bufio.NewWriterSize(conn, n) - } - bw := v.(*bufio.Writer) - bw.Reset(conn) - return bw -} - -func (c *HostClient) releaseWriter(bw *bufio.Writer) { - c.writerPool.Put(bw) -} - -func (c *HostClient) acquireReader(conn net.Conn) *bufio.Reader { - v := c.readerPool.Get() - if v == nil { - n := c.ReadBufferSize - if n <= 0 { - n = defaultReadBufferSize - } - return bufio.NewReaderSize(conn, n) - } - br := v.(*bufio.Reader) - br.Reset(conn) - return br -} - -func (c *HostClient) releaseReader(br *bufio.Reader) { - c.readerPool.Put(br) -} - -func newClientTLSConfig(c *tls.Config, addr string) *tls.Config { - if c == nil { - c = &tls.Config{} - } else { - // TODO: substitute this with c.Clone() after go1.8 becomes mainstream :) - c = &tls.Config{ - Rand: c.Rand, - Time: c.Time, - Certificates: c.Certificates, - NameToCertificate: c.NameToCertificate, - GetCertificate: c.GetCertificate, - RootCAs: c.RootCAs, - NextProtos: c.NextProtos, - ServerName: c.ServerName, - - // Do not copy ClientAuth, since it is server-related stuff - // Do not copy ClientCAs, since it is server-related stuff - - InsecureSkipVerify: c.InsecureSkipVerify, - CipherSuites: c.CipherSuites, - - // Do not copy PreferServerCipherSuites - this is server stuff - - SessionTicketsDisabled: c.SessionTicketsDisabled, - - // Do not copy SessionTicketKey - this is server stuff - - ClientSessionCache: c.ClientSessionCache, - MinVersion: c.MinVersion, - MaxVersion: c.MaxVersion, - CurvePreferences: c.CurvePreferences, - } - } - - if c.ClientSessionCache == nil { - c.ClientSessionCache = tls.NewLRUClientSessionCache(0) - } - - if len(c.ServerName) == 0 { - serverName := tlsServerName(addr) - if serverName == "*" { - c.InsecureSkipVerify = true - } else { - c.ServerName = serverName - } - } - return c -} - -func tlsServerName(addr string) string { - if !strings.Contains(addr, ":") { - return addr - } - host, _, err := net.SplitHostPort(addr) - if err != nil { - return "*" - } - return host -} - -func (c *HostClient) nextAddr() string { - c.addrsLock.Lock() - if c.addrs == nil { - c.addrs = strings.Split(c.Addr, ",") - } - addr := c.addrs[0] - if len(c.addrs) > 1 { - addr = c.addrs[c.addrIdx%uint32(len(c.addrs))] - c.addrIdx++ - } - c.addrsLock.Unlock() - return addr -} - -func (c *HostClient) dialHostHard() (conn net.Conn, err error) { - // attempt to dial all the available hosts before giving up. - - c.addrsLock.Lock() - n := len(c.addrs) - c.addrsLock.Unlock() - - if n == 0 { - // It looks like c.addrs isn't initialized yet. - n = 1 - } - - timeout := c.ReadTimeout + c.WriteTimeout - if timeout <= 0 { - timeout = DefaultDialTimeout - } - deadline := time.Now().Add(timeout) - for n > 0 { - addr := c.nextAddr() - tlsConfig := c.cachedTLSConfig(addr) - conn, err = dialAddr(addr, c.Dial, c.DialDualStack, c.IsTLS, tlsConfig) - if err == nil { - return conn, nil - } - if time.Since(deadline) >= 0 { - break - } - n-- - } - return nil, err -} - -func (c *HostClient) cachedTLSConfig(addr string) *tls.Config { - if !c.IsTLS { - return nil - } - - c.tlsConfigMapLock.Lock() - if c.tlsConfigMap == nil { - c.tlsConfigMap = make(map[string]*tls.Config) - } - cfg := c.tlsConfigMap[addr] - if cfg == nil { - cfg = newClientTLSConfig(c.TLSConfig, addr) - c.tlsConfigMap[addr] = cfg - } - c.tlsConfigMapLock.Unlock() - - return cfg -} - -func dialAddr(addr string, dial DialFunc, dialDualStack, isTLS bool, tlsConfig *tls.Config) (net.Conn, error) { - if dial == nil { - if dialDualStack { - dial = DialDualStack - } else { - dial = Dial - } - addr = addMissingPort(addr, isTLS) - } - conn, err := dial(addr) - if err != nil { - return nil, err - } - if conn == nil { - panic("BUG: DialFunc returned (nil, nil)") - } - if isTLS { - conn = tls.Client(conn, tlsConfig) - } - return conn, nil -} - -func (c *HostClient) getClientName() []byte { - v := c.clientName.Load() - var clientName []byte - if v == nil { - clientName = []byte(c.Name) - if len(clientName) == 0 && !c.NoDefaultUserAgentHeader { - clientName = defaultUserAgent - } - c.clientName.Store(clientName) - } else { - clientName = v.([]byte) - } - return clientName -} - -func addMissingPort(addr string, isTLS bool) string { - n := strings.Index(addr, ":") - if n >= 0 { - return addr - } - port := 80 - if isTLS { - port = 443 - } - return fmt.Sprintf("%s:%d", addr, port) -} - -// PipelineClient pipelines requests over a limited set of concurrent -// connections to the given Addr. -// -// This client may be used in highly loaded HTTP-based RPC systems for reducing -// context switches and network level overhead. -// See https://en.wikipedia.org/wiki/HTTP_pipelining for details. -// -// It is forbidden copying PipelineClient instances. Create new instances -// instead. -// -// It is safe calling PipelineClient methods from concurrently running -// goroutines. -type PipelineClient struct { - noCopy noCopy - - // Address of the host to connect to. - Addr string - - // The maximum number of concurrent connections to the Addr. - // - // A single connection is used by default. - MaxConns int - - // The maximum number of pending pipelined requests over - // a single connection to Addr. - // - // DefaultMaxPendingRequests is used by default. - MaxPendingRequests int - - // The maximum delay before sending pipelined requests as a batch - // to the server. - // - // By default requests are sent immediately to the server. - MaxBatchDelay time.Duration - - // Callback for connection establishing to the host. - // - // Default Dial is used if not set. - Dial DialFunc - - // Attempt to connect to both ipv4 and ipv6 host addresses - // if set to true. - // - // This option is used only if default TCP dialer is used, - // i.e. if Dial is blank. - // - // By default client connects only to ipv4 addresses, - // since unfortunately ipv6 remains broken in many networks worldwide :) - DialDualStack bool - - // Whether to use TLS (aka SSL or HTTPS) for host connections. - IsTLS bool - - // Optional TLS config. - TLSConfig *tls.Config - - // Idle connection to the host is closed after this duration. - // - // By default idle connection is closed after - // DefaultMaxIdleConnDuration. - MaxIdleConnDuration time.Duration - - // Buffer size for responses' reading. - // This also limits the maximum header size. - // - // Default buffer size is used if 0. - ReadBufferSize int - - // Buffer size for requests' writing. - // - // Default buffer size is used if 0. - WriteBufferSize int - - // Maximum duration for full response reading (including body). - // - // By default response read timeout is unlimited. - ReadTimeout time.Duration - - // Maximum duration for full request writing (including body). - // - // By default request write timeout is unlimited. - WriteTimeout time.Duration - - // Logger for logging client errors. - // - // By default standard logger from log package is used. - Logger Logger - - connClients []*pipelineConnClient - connClientsLock sync.Mutex -} - -type pipelineConnClient struct { - noCopy noCopy - - Addr string - MaxPendingRequests int - MaxBatchDelay time.Duration - Dial DialFunc - DialDualStack bool - IsTLS bool - TLSConfig *tls.Config - MaxIdleConnDuration time.Duration - ReadBufferSize int - WriteBufferSize int - ReadTimeout time.Duration - WriteTimeout time.Duration - Logger Logger - - workPool sync.Pool - - chLock sync.Mutex - chW chan *pipelineWork - chR chan *pipelineWork - - tlsConfigLock sync.Mutex - tlsConfig *tls.Config -} - -type pipelineWork struct { - reqCopy Request - respCopy Response - req *Request - resp *Response - t *time.Timer - deadline time.Time - err error - done chan struct{} -} - -// DoTimeout performs the given request and waits for response during -// the given timeout duration. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// The function doesn't follow redirects. -// -// Response is ignored if resp is nil. -// -// ErrTimeout is returned if the response wasn't returned during -// the given timeout. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -// -// Warning: DoTimeout does not terminate the request itself. The request will -// continue in the background and the response will be discarded. -// If requests take too long and the connection pool gets filled up please -// try setting a ReadTimeout. -func (c *PipelineClient) DoTimeout(req *Request, resp *Response, timeout time.Duration) error { - return c.DoDeadline(req, resp, time.Now().Add(timeout)) -} - -// DoDeadline performs the given request and waits for response until -// the given deadline. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// The function doesn't follow redirects. -// -// Response is ignored if resp is nil. -// -// ErrTimeout is returned if the response wasn't returned until -// the given deadline. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func (c *PipelineClient) DoDeadline(req *Request, resp *Response, deadline time.Time) error { - return c.getConnClient().DoDeadline(req, resp, deadline) -} - -func (c *pipelineConnClient) DoDeadline(req *Request, resp *Response, deadline time.Time) error { - c.init() - - timeout := -time.Since(deadline) - if timeout < 0 { - return ErrTimeout - } - - w := acquirePipelineWork(&c.workPool, timeout) - w.req = &w.reqCopy - w.resp = &w.respCopy - - // Make a copy of the request in order to avoid data races on timeouts - req.copyToSkipBody(&w.reqCopy) - swapRequestBody(req, &w.reqCopy) - - // Put the request to outgoing queue - select { - case c.chW <- w: - // Fast path: len(c.ch) < cap(c.ch) - default: - // Slow path - select { - case c.chW <- w: - case <-w.t.C: - releasePipelineWork(&c.workPool, w) - return ErrTimeout - } - } - - // Wait for the response - var err error - select { - case <-w.done: - if resp != nil { - w.respCopy.copyToSkipBody(resp) - swapResponseBody(resp, &w.respCopy) - } - err = w.err - releasePipelineWork(&c.workPool, w) - case <-w.t.C: - err = ErrTimeout - } - - return err -} - -// Do performs the given http request and sets the corresponding response. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func (c *PipelineClient) Do(req *Request, resp *Response) error { - return c.getConnClient().Do(req, resp) -} - -func (c *pipelineConnClient) Do(req *Request, resp *Response) error { - c.init() - - w := acquirePipelineWork(&c.workPool, 0) - w.req = req - if resp != nil { - w.resp = resp - } else { - w.resp = &w.respCopy - } - - // Put the request to outgoing queue - select { - case c.chW <- w: - default: - // Try substituting the oldest w with the current one. - select { - case wOld := <-c.chW: - wOld.err = ErrPipelineOverflow - wOld.done <- struct{}{} - default: - } - select { - case c.chW <- w: - default: - releasePipelineWork(&c.workPool, w) - return ErrPipelineOverflow - } - } - - // Wait for the response - <-w.done - err := w.err - - releasePipelineWork(&c.workPool, w) - - return err -} - -func (c *PipelineClient) getConnClient() *pipelineConnClient { - c.connClientsLock.Lock() - cc := c.getConnClientUnlocked() - c.connClientsLock.Unlock() - return cc -} - -func (c *PipelineClient) getConnClientUnlocked() *pipelineConnClient { - if len(c.connClients) == 0 { - return c.newConnClient() - } - - // Return the client with the minimum number of pending requests. - minCC := c.connClients[0] - minReqs := minCC.PendingRequests() - if minReqs == 0 { - return minCC - } - for i := 1; i < len(c.connClients); i++ { - cc := c.connClients[i] - reqs := cc.PendingRequests() - if reqs == 0 { - return cc - } - if reqs < minReqs { - minCC = cc - minReqs = reqs - } - } - - maxConns := c.MaxConns - if maxConns <= 0 { - maxConns = 1 - } - if len(c.connClients) < maxConns { - return c.newConnClient() - } - return minCC -} - -func (c *PipelineClient) newConnClient() *pipelineConnClient { - cc := &pipelineConnClient{ - Addr: c.Addr, - MaxPendingRequests: c.MaxPendingRequests, - MaxBatchDelay: c.MaxBatchDelay, - Dial: c.Dial, - DialDualStack: c.DialDualStack, - IsTLS: c.IsTLS, - TLSConfig: c.TLSConfig, - MaxIdleConnDuration: c.MaxIdleConnDuration, - ReadBufferSize: c.ReadBufferSize, - WriteBufferSize: c.WriteBufferSize, - ReadTimeout: c.ReadTimeout, - WriteTimeout: c.WriteTimeout, - Logger: c.Logger, - } - c.connClients = append(c.connClients, cc) - return cc -} - -// ErrPipelineOverflow may be returned from PipelineClient.Do* -// if the requests' queue is overflown. -var ErrPipelineOverflow = errors.New("pipelined requests' queue has been overflown. Increase MaxConns and/or MaxPendingRequests") - -// DefaultMaxPendingRequests is the default value -// for PipelineClient.MaxPendingRequests. -const DefaultMaxPendingRequests = 1024 - -func (c *pipelineConnClient) init() { - c.chLock.Lock() - if c.chR == nil { - maxPendingRequests := c.MaxPendingRequests - if maxPendingRequests <= 0 { - maxPendingRequests = DefaultMaxPendingRequests - } - c.chR = make(chan *pipelineWork, maxPendingRequests) - if c.chW == nil { - c.chW = make(chan *pipelineWork, maxPendingRequests) - } - go func() { - if err := c.worker(); err != nil { - c.logger().Printf("error in PipelineClient(%q): %s", c.Addr, err) - if netErr, ok := err.(net.Error); ok && netErr.Temporary() { - // Throttle client reconnections on temporary errors - time.Sleep(time.Second) - } - } - - c.chLock.Lock() - // Do not reset c.chW to nil, since it may contain - // pending requests, which could be served on the next - // connection to the host. - c.chR = nil - c.chLock.Unlock() - }() - } - c.chLock.Unlock() -} - -func (c *pipelineConnClient) worker() error { - tlsConfig := c.cachedTLSConfig() - conn, err := dialAddr(c.Addr, c.Dial, c.DialDualStack, c.IsTLS, tlsConfig) - if err != nil { - return err - } - - // Start reader and writer - stopW := make(chan struct{}) - doneW := make(chan error) - go func() { - doneW <- c.writer(conn, stopW) - }() - stopR := make(chan struct{}) - doneR := make(chan error) - go func() { - doneR <- c.reader(conn, stopR) - }() - - // Wait until reader and writer are stopped - select { - case err = <-doneW: - conn.Close() - close(stopR) - <-doneR - case err = <-doneR: - conn.Close() - close(stopW) - <-doneW - } - - // Notify pending readers - for len(c.chR) > 0 { - w := <-c.chR - w.err = errPipelineConnStopped - w.done <- struct{}{} - } - - return err -} - -func (c *pipelineConnClient) cachedTLSConfig() *tls.Config { - if !c.IsTLS { - return nil - } - - c.tlsConfigLock.Lock() - cfg := c.tlsConfig - if cfg == nil { - cfg = newClientTLSConfig(c.TLSConfig, c.Addr) - c.tlsConfig = cfg - } - c.tlsConfigLock.Unlock() - - return cfg -} - -func (c *pipelineConnClient) writer(conn net.Conn, stopCh <-chan struct{}) error { - writeBufferSize := c.WriteBufferSize - if writeBufferSize <= 0 { - writeBufferSize = defaultWriteBufferSize - } - bw := bufio.NewWriterSize(conn, writeBufferSize) - defer bw.Flush() - chR := c.chR - chW := c.chW - writeTimeout := c.WriteTimeout - - maxIdleConnDuration := c.MaxIdleConnDuration - if maxIdleConnDuration <= 0 { - maxIdleConnDuration = DefaultMaxIdleConnDuration - } - maxBatchDelay := c.MaxBatchDelay - - var ( - stopTimer = time.NewTimer(time.Hour) - flushTimer = time.NewTimer(time.Hour) - flushTimerCh <-chan time.Time - instantTimerCh = make(chan time.Time) - - w *pipelineWork - err error - ) - close(instantTimerCh) - for { - againChW: - select { - case w = <-chW: - // Fast path: len(chW) > 0 - default: - // Slow path - stopTimer.Reset(maxIdleConnDuration) - select { - case w = <-chW: - case <-stopTimer.C: - return nil - case <-stopCh: - return nil - case <-flushTimerCh: - if err = bw.Flush(); err != nil { - return err - } - flushTimerCh = nil - goto againChW - } - } - - if !w.deadline.IsZero() && time.Since(w.deadline) >= 0 { - w.err = ErrTimeout - w.done <- struct{}{} - continue - } - - w.resp.parseNetConn(conn) - - if writeTimeout > 0 { - // Set Deadline every time, since golang has fixed the performance issue - // See https://github.com/golang/go/issues/15133#issuecomment-271571395 for details - currentTime := time.Now() - if err = conn.SetWriteDeadline(currentTime.Add(writeTimeout)); err != nil { - w.err = err - w.done <- struct{}{} - return err - } - } - if err = w.req.Write(bw); err != nil { - w.err = err - w.done <- struct{}{} - return err - } - if flushTimerCh == nil && (len(chW) == 0 || len(chR) == cap(chR)) { - if maxBatchDelay > 0 { - flushTimer.Reset(maxBatchDelay) - flushTimerCh = flushTimer.C - } else { - flushTimerCh = instantTimerCh - } - } - - againChR: - select { - case chR <- w: - // Fast path: len(chR) < cap(chR) - default: - // Slow path - select { - case chR <- w: - case <-stopCh: - w.err = errPipelineConnStopped - w.done <- struct{}{} - return nil - case <-flushTimerCh: - if err = bw.Flush(); err != nil { - w.err = err - w.done <- struct{}{} - return err - } - flushTimerCh = nil - goto againChR - } - } - } -} - -func (c *pipelineConnClient) reader(conn net.Conn, stopCh <-chan struct{}) error { - readBufferSize := c.ReadBufferSize - if readBufferSize <= 0 { - readBufferSize = defaultReadBufferSize - } - br := bufio.NewReaderSize(conn, readBufferSize) - chR := c.chR - readTimeout := c.ReadTimeout - - var ( - w *pipelineWork - err error - ) - for { - select { - case w = <-chR: - // Fast path: len(chR) > 0 - default: - // Slow path - select { - case w = <-chR: - case <-stopCh: - return nil - } - } - - if readTimeout > 0 { - // Set Deadline every time, since golang has fixed the performance issue - // See https://github.com/golang/go/issues/15133#issuecomment-271571395 for details - currentTime := time.Now() - if err = conn.SetReadDeadline(currentTime.Add(readTimeout)); err != nil { - w.err = err - w.done <- struct{}{} - return err - } - } - if err = w.resp.Read(br); err != nil { - w.err = err - w.done <- struct{}{} - return err - } - - w.done <- struct{}{} - } -} - -func (c *pipelineConnClient) logger() Logger { - if c.Logger != nil { - return c.Logger - } - return defaultLogger -} - -// PendingRequests returns the current number of pending requests pipelined -// to the server. -// -// This number may exceed MaxPendingRequests*MaxConns by up to two times, since -// each connection to the server may keep up to MaxPendingRequests requests -// in the queue before sending them to the server. -// -// This function may be used for balancing load among multiple PipelineClient -// instances. -func (c *PipelineClient) PendingRequests() int { - c.connClientsLock.Lock() - n := 0 - for _, cc := range c.connClients { - n += cc.PendingRequests() - } - c.connClientsLock.Unlock() - return n -} - -func (c *pipelineConnClient) PendingRequests() int { - c.init() - - c.chLock.Lock() - n := len(c.chR) + len(c.chW) - c.chLock.Unlock() - return n -} - -var errPipelineConnStopped = errors.New("pipeline connection has been stopped") - -func acquirePipelineWork(pool *sync.Pool, timeout time.Duration) *pipelineWork { - v := pool.Get() - if v == nil { - v = &pipelineWork{ - done: make(chan struct{}, 1), - } - } - w := v.(*pipelineWork) - if timeout > 0 { - if w.t == nil { - w.t = time.NewTimer(timeout) - } else { - w.t.Reset(timeout) - } - w.deadline = time.Now().Add(timeout) - } else { - w.deadline = zeroTime - } - return w -} - -func releasePipelineWork(pool *sync.Pool, w *pipelineWork) { - if w.t != nil { - w.t.Stop() - } - w.reqCopy.Reset() - w.respCopy.Reset() - w.req = nil - w.resp = nil - w.err = nil - pool.Put(w) -} diff --git a/vendor/github.com/valyala/fasthttp/coarseTime.go b/vendor/github.com/valyala/fasthttp/coarseTime.go deleted file mode 100644 index 4679df68..00000000 --- a/vendor/github.com/valyala/fasthttp/coarseTime.go +++ /dev/null @@ -1,13 +0,0 @@ -package fasthttp - -import ( - "time" -) - -// CoarseTimeNow returns the current time truncated to the nearest second. -// -// Deprecated: This is slower than calling time.Now() directly. -// This is now time.Now().Truncate(time.Second) shortcut. -func CoarseTimeNow() time.Time { - return time.Now().Truncate(time.Second) -} diff --git a/vendor/github.com/valyala/fasthttp/compress.go b/vendor/github.com/valyala/fasthttp/compress.go deleted file mode 100644 index 73a40d3b..00000000 --- a/vendor/github.com/valyala/fasthttp/compress.go +++ /dev/null @@ -1,438 +0,0 @@ -package fasthttp - -import ( - "bytes" - "fmt" - "io" - "os" - "sync" - - "github.com/klauspost/compress/flate" - "github.com/klauspost/compress/gzip" - "github.com/klauspost/compress/zlib" - "github.com/valyala/bytebufferpool" - "github.com/valyala/fasthttp/stackless" -) - -// Supported compression levels. -const ( - CompressNoCompression = flate.NoCompression - CompressBestSpeed = flate.BestSpeed - CompressBestCompression = flate.BestCompression - CompressDefaultCompression = 6 // flate.DefaultCompression - CompressHuffmanOnly = -2 // flate.HuffmanOnly -) - -func acquireGzipReader(r io.Reader) (*gzip.Reader, error) { - v := gzipReaderPool.Get() - if v == nil { - return gzip.NewReader(r) - } - zr := v.(*gzip.Reader) - if err := zr.Reset(r); err != nil { - return nil, err - } - return zr, nil -} - -func releaseGzipReader(zr *gzip.Reader) { - zr.Close() - gzipReaderPool.Put(zr) -} - -var gzipReaderPool sync.Pool - -func acquireFlateReader(r io.Reader) (io.ReadCloser, error) { - v := flateReaderPool.Get() - if v == nil { - zr, err := zlib.NewReader(r) - if err != nil { - return nil, err - } - return zr, nil - } - zr := v.(io.ReadCloser) - if err := resetFlateReader(zr, r); err != nil { - return nil, err - } - return zr, nil -} - -func releaseFlateReader(zr io.ReadCloser) { - zr.Close() - flateReaderPool.Put(zr) -} - -func resetFlateReader(zr io.ReadCloser, r io.Reader) error { - zrr, ok := zr.(zlib.Resetter) - if !ok { - panic("BUG: zlib.Reader doesn't implement zlib.Resetter???") - } - return zrr.Reset(r, nil) -} - -var flateReaderPool sync.Pool - -func acquireStacklessGzipWriter(w io.Writer, level int) stackless.Writer { - nLevel := normalizeCompressLevel(level) - p := stacklessGzipWriterPoolMap[nLevel] - v := p.Get() - if v == nil { - return stackless.NewWriter(w, func(w io.Writer) stackless.Writer { - return acquireRealGzipWriter(w, level) - }) - } - sw := v.(stackless.Writer) - sw.Reset(w) - return sw -} - -func releaseStacklessGzipWriter(sw stackless.Writer, level int) { - sw.Close() - nLevel := normalizeCompressLevel(level) - p := stacklessGzipWriterPoolMap[nLevel] - p.Put(sw) -} - -func acquireRealGzipWriter(w io.Writer, level int) *gzip.Writer { - nLevel := normalizeCompressLevel(level) - p := realGzipWriterPoolMap[nLevel] - v := p.Get() - if v == nil { - zw, err := gzip.NewWriterLevel(w, level) - if err != nil { - panic(fmt.Sprintf("BUG: unexpected error from gzip.NewWriterLevel(%d): %s", level, err)) - } - return zw - } - zw := v.(*gzip.Writer) - zw.Reset(w) - return zw -} - -func releaseRealGzipWriter(zw *gzip.Writer, level int) { - zw.Close() - nLevel := normalizeCompressLevel(level) - p := realGzipWriterPoolMap[nLevel] - p.Put(zw) -} - -var ( - stacklessGzipWriterPoolMap = newCompressWriterPoolMap() - realGzipWriterPoolMap = newCompressWriterPoolMap() -) - -// AppendGzipBytesLevel appends gzipped src to dst using the given -// compression level and returns the resulting dst. -// -// Supported compression levels are: -// -// * CompressNoCompression -// * CompressBestSpeed -// * CompressBestCompression -// * CompressDefaultCompression -// * CompressHuffmanOnly -func AppendGzipBytesLevel(dst, src []byte, level int) []byte { - w := &byteSliceWriter{dst} - WriteGzipLevel(w, src, level) - return w.b -} - -// WriteGzipLevel writes gzipped p to w using the given compression level -// and returns the number of compressed bytes written to w. -// -// Supported compression levels are: -// -// * CompressNoCompression -// * CompressBestSpeed -// * CompressBestCompression -// * CompressDefaultCompression -// * CompressHuffmanOnly -func WriteGzipLevel(w io.Writer, p []byte, level int) (int, error) { - switch w.(type) { - case *byteSliceWriter, - *bytes.Buffer, - *bytebufferpool.ByteBuffer: - // These writers don't block, so we can just use stacklessWriteGzip - ctx := &compressCtx{ - w: w, - p: p, - level: level, - } - stacklessWriteGzip(ctx) - return len(p), nil - default: - zw := acquireStacklessGzipWriter(w, level) - n, err := zw.Write(p) - releaseStacklessGzipWriter(zw, level) - return n, err - } -} - -var stacklessWriteGzip = stackless.NewFunc(nonblockingWriteGzip) - -func nonblockingWriteGzip(ctxv interface{}) { - ctx := ctxv.(*compressCtx) - zw := acquireRealGzipWriter(ctx.w, ctx.level) - - _, err := zw.Write(ctx.p) - if err != nil { - panic(fmt.Sprintf("BUG: gzip.Writer.Write for len(p)=%d returned unexpected error: %s", len(ctx.p), err)) - } - - releaseRealGzipWriter(zw, ctx.level) -} - -// WriteGzip writes gzipped p to w and returns the number of compressed -// bytes written to w. -func WriteGzip(w io.Writer, p []byte) (int, error) { - return WriteGzipLevel(w, p, CompressDefaultCompression) -} - -// AppendGzipBytes appends gzipped src to dst and returns the resulting dst. -func AppendGzipBytes(dst, src []byte) []byte { - return AppendGzipBytesLevel(dst, src, CompressDefaultCompression) -} - -// WriteGunzip writes ungzipped p to w and returns the number of uncompressed -// bytes written to w. -func WriteGunzip(w io.Writer, p []byte) (int, error) { - r := &byteSliceReader{p} - zr, err := acquireGzipReader(r) - if err != nil { - return 0, err - } - n, err := copyZeroAlloc(w, zr) - releaseGzipReader(zr) - nn := int(n) - if int64(nn) != n { - return 0, fmt.Errorf("too much data gunzipped: %d", n) - } - return nn, err -} - -// AppendGunzipBytes appends gunzipped src to dst and returns the resulting dst. -func AppendGunzipBytes(dst, src []byte) ([]byte, error) { - w := &byteSliceWriter{dst} - _, err := WriteGunzip(w, src) - return w.b, err -} - -// AppendDeflateBytesLevel appends deflated src to dst using the given -// compression level and returns the resulting dst. -// -// Supported compression levels are: -// -// * CompressNoCompression -// * CompressBestSpeed -// * CompressBestCompression -// * CompressDefaultCompression -// * CompressHuffmanOnly -func AppendDeflateBytesLevel(dst, src []byte, level int) []byte { - w := &byteSliceWriter{dst} - WriteDeflateLevel(w, src, level) - return w.b -} - -// WriteDeflateLevel writes deflated p to w using the given compression level -// and returns the number of compressed bytes written to w. -// -// Supported compression levels are: -// -// * CompressNoCompression -// * CompressBestSpeed -// * CompressBestCompression -// * CompressDefaultCompression -// * CompressHuffmanOnly -func WriteDeflateLevel(w io.Writer, p []byte, level int) (int, error) { - switch w.(type) { - case *byteSliceWriter, - *bytes.Buffer, - *bytebufferpool.ByteBuffer: - // These writers don't block, so we can just use stacklessWriteDeflate - ctx := &compressCtx{ - w: w, - p: p, - level: level, - } - stacklessWriteDeflate(ctx) - return len(p), nil - default: - zw := acquireStacklessDeflateWriter(w, level) - n, err := zw.Write(p) - releaseStacklessDeflateWriter(zw, level) - return n, err - } -} - -var stacklessWriteDeflate = stackless.NewFunc(nonblockingWriteDeflate) - -func nonblockingWriteDeflate(ctxv interface{}) { - ctx := ctxv.(*compressCtx) - zw := acquireRealDeflateWriter(ctx.w, ctx.level) - - _, err := zw.Write(ctx.p) - if err != nil { - panic(fmt.Sprintf("BUG: zlib.Writer.Write for len(p)=%d returned unexpected error: %s", len(ctx.p), err)) - } - - releaseRealDeflateWriter(zw, ctx.level) -} - -type compressCtx struct { - w io.Writer - p []byte - level int -} - -// WriteDeflate writes deflated p to w and returns the number of compressed -// bytes written to w. -func WriteDeflate(w io.Writer, p []byte) (int, error) { - return WriteDeflateLevel(w, p, CompressDefaultCompression) -} - -// AppendDeflateBytes appends deflated src to dst and returns the resulting dst. -func AppendDeflateBytes(dst, src []byte) []byte { - return AppendDeflateBytesLevel(dst, src, CompressDefaultCompression) -} - -// WriteInflate writes inflated p to w and returns the number of uncompressed -// bytes written to w. -func WriteInflate(w io.Writer, p []byte) (int, error) { - r := &byteSliceReader{p} - zr, err := acquireFlateReader(r) - if err != nil { - return 0, err - } - n, err := copyZeroAlloc(w, zr) - releaseFlateReader(zr) - nn := int(n) - if int64(nn) != n { - return 0, fmt.Errorf("too much data inflated: %d", n) - } - return nn, err -} - -// AppendInflateBytes appends inflated src to dst and returns the resulting dst. -func AppendInflateBytes(dst, src []byte) ([]byte, error) { - w := &byteSliceWriter{dst} - _, err := WriteInflate(w, src) - return w.b, err -} - -type byteSliceWriter struct { - b []byte -} - -func (w *byteSliceWriter) Write(p []byte) (int, error) { - w.b = append(w.b, p...) - return len(p), nil -} - -type byteSliceReader struct { - b []byte -} - -func (r *byteSliceReader) Read(p []byte) (int, error) { - if len(r.b) == 0 { - return 0, io.EOF - } - n := copy(p, r.b) - r.b = r.b[n:] - return n, nil -} - -func acquireStacklessDeflateWriter(w io.Writer, level int) stackless.Writer { - nLevel := normalizeCompressLevel(level) - p := stacklessDeflateWriterPoolMap[nLevel] - v := p.Get() - if v == nil { - return stackless.NewWriter(w, func(w io.Writer) stackless.Writer { - return acquireRealDeflateWriter(w, level) - }) - } - sw := v.(stackless.Writer) - sw.Reset(w) - return sw -} - -func releaseStacklessDeflateWriter(sw stackless.Writer, level int) { - sw.Close() - nLevel := normalizeCompressLevel(level) - p := stacklessDeflateWriterPoolMap[nLevel] - p.Put(sw) -} - -func acquireRealDeflateWriter(w io.Writer, level int) *zlib.Writer { - nLevel := normalizeCompressLevel(level) - p := realDeflateWriterPoolMap[nLevel] - v := p.Get() - if v == nil { - zw, err := zlib.NewWriterLevel(w, level) - if err != nil { - panic(fmt.Sprintf("BUG: unexpected error from zlib.NewWriterLevel(%d): %s", level, err)) - } - return zw - } - zw := v.(*zlib.Writer) - zw.Reset(w) - return zw -} - -func releaseRealDeflateWriter(zw *zlib.Writer, level int) { - zw.Close() - nLevel := normalizeCompressLevel(level) - p := realDeflateWriterPoolMap[nLevel] - p.Put(zw) -} - -var ( - stacklessDeflateWriterPoolMap = newCompressWriterPoolMap() - realDeflateWriterPoolMap = newCompressWriterPoolMap() -) - -func newCompressWriterPoolMap() []*sync.Pool { - // Initialize pools for all the compression levels defined - // in https://golang.org/pkg/compress/flate/#pkg-constants . - // Compression levels are normalized with normalizeCompressLevel, - // so the fit [0..11]. - var m []*sync.Pool - for i := 0; i < 12; i++ { - m = append(m, &sync.Pool{}) - } - return m -} - -func isFileCompressible(f *os.File, minCompressRatio float64) bool { - // Try compressing the first 4kb of of the file - // and see if it can be compressed by more than - // the given minCompressRatio. - b := bytebufferpool.Get() - zw := acquireStacklessGzipWriter(b, CompressDefaultCompression) - lr := &io.LimitedReader{ - R: f, - N: 4096, - } - _, err := copyZeroAlloc(zw, lr) - releaseStacklessGzipWriter(zw, CompressDefaultCompression) - f.Seek(0, 0) - if err != nil { - return false - } - - n := 4096 - lr.N - zn := len(b.B) - bytebufferpool.Put(b) - return float64(zn) < float64(n)*minCompressRatio -} - -// normalizes compression level into [0..11], so it could be used as an index -// in *PoolMap. -func normalizeCompressLevel(level int) int { - // -2 is the lowest compression level - CompressHuffmanOnly - // 9 is the highest compression level - CompressBestCompression - if level < -2 || level > 9 { - level = CompressDefaultCompression - } - return level + 2 -} diff --git a/vendor/github.com/valyala/fasthttp/cookie.go b/vendor/github.com/valyala/fasthttp/cookie.go deleted file mode 100644 index 8137643c..00000000 --- a/vendor/github.com/valyala/fasthttp/cookie.go +++ /dev/null @@ -1,534 +0,0 @@ -package fasthttp - -import ( - "bytes" - "errors" - "io" - "sync" - "time" -) - -var zeroTime time.Time - -var ( - // CookieExpireDelete may be set on Cookie.Expire for expiring the given cookie. - CookieExpireDelete = time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) - - // CookieExpireUnlimited indicates that the cookie doesn't expire. - CookieExpireUnlimited = zeroTime -) - -// CookieSameSite is an enum for the mode in which the SameSite flag should be set for the given cookie. -// See https://tools.ietf.org/html/draft-ietf-httpbis-cookie-same-site-00 for details. -type CookieSameSite int - -const ( - // CookieSameSiteDisabled removes the SameSite flag - CookieSameSiteDisabled CookieSameSite = iota - // CookieSameSiteDefaultMode sets the SameSite flag - CookieSameSiteDefaultMode - // CookieSameSiteLaxMode sets the SameSite flag with the "Lax" parameter - CookieSameSiteLaxMode - // CookieSameSiteStrictMode sets the SameSite flag with the "Strict" parameter - CookieSameSiteStrictMode -) - -// AcquireCookie returns an empty Cookie object from the pool. -// -// The returned object may be returned back to the pool with ReleaseCookie. -// This allows reducing GC load. -func AcquireCookie() *Cookie { - return cookiePool.Get().(*Cookie) -} - -// ReleaseCookie returns the Cookie object acquired with AcquireCookie back -// to the pool. -// -// Do not access released Cookie object, otherwise data races may occur. -func ReleaseCookie(c *Cookie) { - c.Reset() - cookiePool.Put(c) -} - -var cookiePool = &sync.Pool{ - New: func() interface{} { - return &Cookie{} - }, -} - -// Cookie represents HTTP response cookie. -// -// Do not copy Cookie objects. Create new object and use CopyTo instead. -// -// Cookie instance MUST NOT be used from concurrently running goroutines. -type Cookie struct { - noCopy noCopy - - key []byte - value []byte - expire time.Time - maxAge int - domain []byte - path []byte - - httpOnly bool - secure bool - sameSite CookieSameSite - - bufKV argsKV - buf []byte -} - -// CopyTo copies src cookie to c. -func (c *Cookie) CopyTo(src *Cookie) { - c.Reset() - c.key = append(c.key[:0], src.key...) - c.value = append(c.value[:0], src.value...) - c.expire = src.expire - c.maxAge = src.maxAge - c.domain = append(c.domain[:0], src.domain...) - c.path = append(c.path[:0], src.path...) - c.httpOnly = src.httpOnly - c.secure = src.secure - c.sameSite = src.sameSite -} - -// HTTPOnly returns true if the cookie is http only. -func (c *Cookie) HTTPOnly() bool { - return c.httpOnly -} - -// SetHTTPOnly sets cookie's httpOnly flag to the given value. -func (c *Cookie) SetHTTPOnly(httpOnly bool) { - c.httpOnly = httpOnly -} - -// Secure returns true if the cookie is secure. -func (c *Cookie) Secure() bool { - return c.secure -} - -// SetSecure sets cookie's secure flag to the given value. -func (c *Cookie) SetSecure(secure bool) { - c.secure = secure -} - -// SameSite returns the SameSite mode. -func (c *Cookie) SameSite() CookieSameSite { - return c.sameSite -} - -// SetSameSite sets the cookie's SameSite flag to the given value. -func (c *Cookie) SetSameSite(mode CookieSameSite) { - c.sameSite = mode -} - -// Path returns cookie path. -func (c *Cookie) Path() []byte { - return c.path -} - -// SetPath sets cookie path. -func (c *Cookie) SetPath(path string) { - c.buf = append(c.buf[:0], path...) - c.path = normalizePath(c.path, c.buf) -} - -// SetPathBytes sets cookie path. -func (c *Cookie) SetPathBytes(path []byte) { - c.buf = append(c.buf[:0], path...) - c.path = normalizePath(c.path, c.buf) -} - -// Domain returns cookie domain. -// -// The returned domain is valid until the next Cookie modification method call. -func (c *Cookie) Domain() []byte { - return c.domain -} - -// SetDomain sets cookie domain. -func (c *Cookie) SetDomain(domain string) { - c.domain = append(c.domain[:0], domain...) -} - -// SetDomainBytes sets cookie domain. -func (c *Cookie) SetDomainBytes(domain []byte) { - c.domain = append(c.domain[:0], domain...) -} - -// MaxAge returns the seconds until the cookie is meant to expire or 0 -// if no max age. -func (c *Cookie) MaxAge() int { - return c.maxAge -} - -// SetMaxAge sets cookie expiration time based on seconds. This takes precedence -// over any absolute expiry set on the cookie -// -// Set max age to 0 to unset -func (c *Cookie) SetMaxAge(seconds int) { - c.maxAge = seconds -} - -// Expire returns cookie expiration time. -// -// CookieExpireUnlimited is returned if cookie doesn't expire -func (c *Cookie) Expire() time.Time { - expire := c.expire - if expire.IsZero() { - expire = CookieExpireUnlimited - } - return expire -} - -// SetExpire sets cookie expiration time. -// -// Set expiration time to CookieExpireDelete for expiring (deleting) -// the cookie on the client. -// -// By default cookie lifetime is limited by browser session. -func (c *Cookie) SetExpire(expire time.Time) { - c.expire = expire -} - -// Value returns cookie value. -// -// The returned value is valid until the next Cookie modification method call. -func (c *Cookie) Value() []byte { - return c.value -} - -// SetValue sets cookie value. -func (c *Cookie) SetValue(value string) { - c.value = append(c.value[:0], value...) -} - -// SetValueBytes sets cookie value. -func (c *Cookie) SetValueBytes(value []byte) { - c.value = append(c.value[:0], value...) -} - -// Key returns cookie name. -// -// The returned value is valid until the next Cookie modification method call. -func (c *Cookie) Key() []byte { - return c.key -} - -// SetKey sets cookie name. -func (c *Cookie) SetKey(key string) { - c.key = append(c.key[:0], key...) -} - -// SetKeyBytes sets cookie name. -func (c *Cookie) SetKeyBytes(key []byte) { - c.key = append(c.key[:0], key...) -} - -// Reset clears the cookie. -func (c *Cookie) Reset() { - c.key = c.key[:0] - c.value = c.value[:0] - c.expire = zeroTime - c.maxAge = 0 - c.domain = c.domain[:0] - c.path = c.path[:0] - c.httpOnly = false - c.secure = false - c.sameSite = CookieSameSiteDisabled -} - -// AppendBytes appends cookie representation to dst and returns -// the extended dst. -func (c *Cookie) AppendBytes(dst []byte) []byte { - if len(c.key) > 0 { - dst = append(dst, c.key...) - dst = append(dst, '=') - } - dst = append(dst, c.value...) - - if c.maxAge > 0 { - dst = append(dst, ';', ' ') - dst = append(dst, strCookieMaxAge...) - dst = append(dst, '=') - dst = AppendUint(dst, c.maxAge) - } else if !c.expire.IsZero() { - c.bufKV.value = AppendHTTPDate(c.bufKV.value[:0], c.expire) - dst = append(dst, ';', ' ') - dst = append(dst, strCookieExpires...) - dst = append(dst, '=') - dst = append(dst, c.bufKV.value...) - } - if len(c.domain) > 0 { - dst = appendCookiePart(dst, strCookieDomain, c.domain) - } - if len(c.path) > 0 { - dst = appendCookiePart(dst, strCookiePath, c.path) - } - if c.httpOnly { - dst = append(dst, ';', ' ') - dst = append(dst, strCookieHTTPOnly...) - } - if c.secure { - dst = append(dst, ';', ' ') - dst = append(dst, strCookieSecure...) - } - switch c.sameSite { - case CookieSameSiteDefaultMode: - dst = append(dst, ';', ' ') - dst = append(dst, strCookieSameSite...) - case CookieSameSiteLaxMode: - dst = append(dst, ';', ' ') - dst = append(dst, strCookieSameSite...) - dst = append(dst, '=') - dst = append(dst, strCookieSameSiteLax...) - case CookieSameSiteStrictMode: - dst = append(dst, ';', ' ') - dst = append(dst, strCookieSameSite...) - dst = append(dst, '=') - dst = append(dst, strCookieSameSiteStrict...) - } - return dst -} - -// Cookie returns cookie representation. -// -// The returned value is valid until the next call to Cookie methods. -func (c *Cookie) Cookie() []byte { - c.buf = c.AppendBytes(c.buf[:0]) - return c.buf -} - -// String returns cookie representation. -func (c *Cookie) String() string { - return string(c.Cookie()) -} - -// WriteTo writes cookie representation to w. -// -// WriteTo implements io.WriterTo interface. -func (c *Cookie) WriteTo(w io.Writer) (int64, error) { - n, err := w.Write(c.Cookie()) - return int64(n), err -} - -var errNoCookies = errors.New("no cookies found") - -// Parse parses Set-Cookie header. -func (c *Cookie) Parse(src string) error { - c.buf = append(c.buf[:0], src...) - return c.ParseBytes(c.buf) -} - -// ParseBytes parses Set-Cookie header. -func (c *Cookie) ParseBytes(src []byte) error { - c.Reset() - - var s cookieScanner - s.b = src - - kv := &c.bufKV - if !s.next(kv) { - return errNoCookies - } - - c.key = append(c.key[:0], kv.key...) - c.value = append(c.value[:0], kv.value...) - - for s.next(kv) { - if len(kv.key) != 0 { - // Case insensitive switch on first char - switch kv.key[0] | 0x20 { - case 'm': - if caseInsensitiveCompare(strCookieMaxAge, kv.key) { - maxAge, err := ParseUint(kv.value) - if err != nil { - return err - } - c.maxAge = maxAge - } - - case 'e': // "expires" - if caseInsensitiveCompare(strCookieExpires, kv.key) { - v := b2s(kv.value) - // Try the same two formats as net/http - // See: https://github.com/golang/go/blob/00379be17e63a5b75b3237819392d2dc3b313a27/src/net/http/cookie.go#L133-L135 - exptime, err := time.ParseInLocation(time.RFC1123, v, time.UTC) - if err != nil { - exptime, err = time.Parse("Mon, 02-Jan-2006 15:04:05 MST", v) - if err != nil { - return err - } - } - c.expire = exptime - } - - case 'd': // "domain" - if caseInsensitiveCompare(strCookieDomain, kv.key) { - c.domain = append(c.domain[:0], kv.value...) - } - - case 'p': // "path" - if caseInsensitiveCompare(strCookiePath, kv.key) { - c.path = append(c.path[:0], kv.value...) - } - - case 's': // "samesite" - if caseInsensitiveCompare(strCookieSameSite, kv.key) { - // Case insensitive switch on first char - switch kv.value[0] | 0x20 { - case 'l': // "lax" - if caseInsensitiveCompare(strCookieSameSiteLax, kv.value) { - c.sameSite = CookieSameSiteLaxMode - } - case 's': // "strict" - if caseInsensitiveCompare(strCookieSameSiteStrict, kv.value) { - c.sameSite = CookieSameSiteStrictMode - } - } - } - } - - } else if len(kv.value) != 0 { - // Case insensitive switch on first char - switch kv.value[0] | 0x20 { - case 'h': // "httponly" - if caseInsensitiveCompare(strCookieHTTPOnly, kv.value) { - c.httpOnly = true - } - - case 's': // "secure" - if caseInsensitiveCompare(strCookieSecure, kv.value) { - c.secure = true - } else if caseInsensitiveCompare(strCookieSameSite, kv.value) { - c.sameSite = CookieSameSiteDefaultMode - } - } - } // else empty or no match - } - return nil -} - -func appendCookiePart(dst, key, value []byte) []byte { - dst = append(dst, ';', ' ') - dst = append(dst, key...) - dst = append(dst, '=') - return append(dst, value...) -} - -func getCookieKey(dst, src []byte) []byte { - n := bytes.IndexByte(src, '=') - if n >= 0 { - src = src[:n] - } - return decodeCookieArg(dst, src, false) -} - -func appendRequestCookieBytes(dst []byte, cookies []argsKV) []byte { - for i, n := 0, len(cookies); i < n; i++ { - kv := &cookies[i] - if len(kv.key) > 0 { - dst = append(dst, kv.key...) - dst = append(dst, '=') - } - dst = append(dst, kv.value...) - if i+1 < n { - dst = append(dst, ';', ' ') - } - } - return dst -} - -// For Response we can not use the above function as response cookies -// already contain the key= in the value. -func appendResponseCookieBytes(dst []byte, cookies []argsKV) []byte { - for i, n := 0, len(cookies); i < n; i++ { - kv := &cookies[i] - dst = append(dst, kv.value...) - if i+1 < n { - dst = append(dst, ';', ' ') - } - } - return dst -} - -func parseRequestCookies(cookies []argsKV, src []byte) []argsKV { - var s cookieScanner - s.b = src - var kv *argsKV - cookies, kv = allocArg(cookies) - for s.next(kv) { - if len(kv.key) > 0 || len(kv.value) > 0 { - cookies, kv = allocArg(cookies) - } - } - return releaseArg(cookies) -} - -type cookieScanner struct { - b []byte -} - -func (s *cookieScanner) next(kv *argsKV) bool { - b := s.b - if len(b) == 0 { - return false - } - - isKey := true - k := 0 - for i, c := range b { - switch c { - case '=': - if isKey { - isKey = false - kv.key = decodeCookieArg(kv.key, b[:i], false) - k = i + 1 - } - case ';': - if isKey { - kv.key = kv.key[:0] - } - kv.value = decodeCookieArg(kv.value, b[k:i], true) - s.b = b[i+1:] - return true - } - } - - if isKey { - kv.key = kv.key[:0] - } - kv.value = decodeCookieArg(kv.value, b[k:], true) - s.b = b[len(b):] - return true -} - -func decodeCookieArg(dst, src []byte, skipQuotes bool) []byte { - for len(src) > 0 && src[0] == ' ' { - src = src[1:] - } - for len(src) > 0 && src[len(src)-1] == ' ' { - src = src[:len(src)-1] - } - if skipQuotes { - if len(src) > 1 && src[0] == '"' && src[len(src)-1] == '"' { - src = src[1 : len(src)-1] - } - } - return append(dst[:0], src...) -} - -// caseInsensitiveCompare does a case insensitive equality comparison of -// two []byte. Assumes only letters need to be matched. -func caseInsensitiveCompare(a, b []byte) bool { - if len(a) != len(b) { - return false - } - for i := 0; i < len(a); i++ { - if a[i]|0x20 != b[i]|0x20 { - return false - } - } - return true -} diff --git a/vendor/github.com/valyala/fasthttp/doc.go b/vendor/github.com/valyala/fasthttp/doc.go deleted file mode 100644 index efcd4a03..00000000 --- a/vendor/github.com/valyala/fasthttp/doc.go +++ /dev/null @@ -1,37 +0,0 @@ -/* -Package fasthttp provides fast HTTP server and client API. - -Fasthttp provides the following features: - - * Optimized for speed. Easily handles more than 100K qps and more than 1M - concurrent keep-alive connections on modern hardware. - * Optimized for low memory usage. - * Easy 'Connection: Upgrade' support via RequestCtx.Hijack. - * Server provides the following anti-DoS limits: - - * The number of concurrent connections. - * The number of concurrent connections per client IP. - * The number of requests per connection. - * Request read timeout. - * Response write timeout. - * Maximum request header size. - * Maximum request body size. - * Maximum request execution time. - * Maximum keep-alive connection lifetime. - * Early filtering out non-GET requests. - - * A lot of additional useful info is exposed to request handler: - - * Server and client address. - * Per-request logger. - * Unique request id. - * Request start time. - * Connection start time. - * Request sequence number for the current connection. - - * Client supports automatic retry on idempotent requests' failure. - * Fasthttp API is designed with the ability to extend existing client - and server implementations or to write custom client and server - implementations from scratch. -*/ -package fasthttp diff --git a/vendor/github.com/valyala/fasthttp/fasthttputil/doc.go b/vendor/github.com/valyala/fasthttp/fasthttputil/doc.go deleted file mode 100644 index 9cf69e71..00000000 --- a/vendor/github.com/valyala/fasthttp/fasthttputil/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package fasthttputil provides utility functions for fasthttp. -package fasthttputil diff --git a/vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.key b/vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.key deleted file mode 100644 index 7e201fc4..00000000 --- a/vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.key +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIBpQbZ6a5jL1Yh4wdP6yZk4MKjYWArD/QOLENFw8vbELoAoGCCqGSM49 -AwEHoUQDQgAEKQCZWgE2IBhb47ot8MIs1D4KSisHYlZ41IWyeutpjb0fjwwIhimh -pl1Qld1/d2j3Z3vVyfa5yD+ncV7qCFZuSg== ------END EC PRIVATE KEY----- diff --git a/vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.pem b/vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.pem deleted file mode 100644 index ca1a7f2e..00000000 --- a/vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.pem +++ /dev/null @@ -1,10 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBbTCCAROgAwIBAgIQPo718S+K+G7hc1SgTEU4QDAKBggqhkjOPQQDAjASMRAw -DgYDVQQKEwdBY21lIENvMB4XDTE3MDQyMDIxMDExNFoXDTE4MDQyMDIxMDExNFow -EjEQMA4GA1UEChMHQWNtZSBDbzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCkA -mVoBNiAYW+O6LfDCLNQ+CkorB2JWeNSFsnrraY29H48MCIYpoaZdUJXdf3do92d7 -1cn2ucg/p3Fe6ghWbkqjSzBJMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggr -BgEFBQcDATAMBgNVHRMBAf8EAjAAMBQGA1UdEQQNMAuCCWxvY2FsaG9zdDAKBggq -hkjOPQQDAgNIADBFAiEAoLAIQkvSuIcHUqyWroA6yWYw2fznlRH/uO9/hMCxUCEC -IClRYb/5O9eD/Eq/ozPnwNpsQHOeYefEhadJ/P82y0lG ------END CERTIFICATE----- diff --git a/vendor/github.com/valyala/fasthttp/fasthttputil/inmemory_listener.go b/vendor/github.com/valyala/fasthttp/fasthttputil/inmemory_listener.go deleted file mode 100644 index 1b1a5f36..00000000 --- a/vendor/github.com/valyala/fasthttp/fasthttputil/inmemory_listener.go +++ /dev/null @@ -1,94 +0,0 @@ -package fasthttputil - -import ( - "fmt" - "net" - "sync" -) - -// InmemoryListener provides in-memory dialer<->net.Listener implementation. -// -// It may be used either for fast in-process client<->server communications -// without network stack overhead or for client<->server tests. -type InmemoryListener struct { - lock sync.Mutex - closed bool - conns chan acceptConn -} - -type acceptConn struct { - conn net.Conn - accepted chan struct{} -} - -// NewInmemoryListener returns new in-memory dialer<->net.Listener. -func NewInmemoryListener() *InmemoryListener { - return &InmemoryListener{ - conns: make(chan acceptConn, 1024), - } -} - -// Accept implements net.Listener's Accept. -// -// It is safe calling Accept from concurrently running goroutines. -// -// Accept returns new connection per each Dial call. -func (ln *InmemoryListener) Accept() (net.Conn, error) { - c, ok := <-ln.conns - if !ok { - return nil, fmt.Errorf("InmemoryListener is already closed: use of closed network connection") - } - close(c.accepted) - return c.conn, nil -} - -// Close implements net.Listener's Close. -func (ln *InmemoryListener) Close() error { - var err error - - ln.lock.Lock() - if !ln.closed { - close(ln.conns) - ln.closed = true - } else { - err = fmt.Errorf("InmemoryListener is already closed") - } - ln.lock.Unlock() - return err -} - -// Addr implements net.Listener's Addr. -func (ln *InmemoryListener) Addr() net.Addr { - return &net.UnixAddr{ - Name: "InmemoryListener", - Net: "memory", - } -} - -// Dial creates new client<->server connection. -// Just like a real Dial it only returns once the server -// has accepted the connection. -// -// It is safe calling Dial from concurrently running goroutines. -func (ln *InmemoryListener) Dial() (net.Conn, error) { - pc := NewPipeConns() - cConn := pc.Conn1() - sConn := pc.Conn2() - ln.lock.Lock() - accepted := make(chan struct{}) - if !ln.closed { - ln.conns <- acceptConn{sConn, accepted} - // Wait until the connection has been accepted. - <-accepted - } else { - sConn.Close() - cConn.Close() - cConn = nil - } - ln.lock.Unlock() - - if cConn == nil { - return nil, fmt.Errorf("InmemoryListener is already closed") - } - return cConn, nil -} diff --git a/vendor/github.com/valyala/fasthttp/fasthttputil/pipeconns.go b/vendor/github.com/valyala/fasthttp/fasthttputil/pipeconns.go deleted file mode 100644 index aa92b6ff..00000000 --- a/vendor/github.com/valyala/fasthttp/fasthttputil/pipeconns.go +++ /dev/null @@ -1,283 +0,0 @@ -package fasthttputil - -import ( - "errors" - "io" - "net" - "sync" - "time" -) - -// NewPipeConns returns new bi-directional connection pipe. -func NewPipeConns() *PipeConns { - ch1 := make(chan *byteBuffer, 4) - ch2 := make(chan *byteBuffer, 4) - - pc := &PipeConns{ - stopCh: make(chan struct{}), - } - pc.c1.rCh = ch1 - pc.c1.wCh = ch2 - pc.c2.rCh = ch2 - pc.c2.wCh = ch1 - pc.c1.pc = pc - pc.c2.pc = pc - return pc -} - -// PipeConns provides bi-directional connection pipe, -// which use in-process memory as a transport. -// -// PipeConns must be created by calling NewPipeConns. -// -// PipeConns has the following additional features comparing to connections -// returned from net.Pipe(): -// -// * It is faster. -// * It buffers Write calls, so there is no need to have concurrent goroutine -// calling Read in order to unblock each Write call. -// * It supports read and write deadlines. -// -type PipeConns struct { - c1 pipeConn - c2 pipeConn - stopCh chan struct{} - stopChLock sync.Mutex -} - -// Conn1 returns the first end of bi-directional pipe. -// -// Data written to Conn1 may be read from Conn2. -// Data written to Conn2 may be read from Conn1. -func (pc *PipeConns) Conn1() net.Conn { - return &pc.c1 -} - -// Conn2 returns the second end of bi-directional pipe. -// -// Data written to Conn2 may be read from Conn1. -// Data written to Conn1 may be read from Conn2. -func (pc *PipeConns) Conn2() net.Conn { - return &pc.c2 -} - -// Close closes pipe connections. -func (pc *PipeConns) Close() error { - pc.stopChLock.Lock() - select { - case <-pc.stopCh: - default: - close(pc.stopCh) - } - pc.stopChLock.Unlock() - - return nil -} - -type pipeConn struct { - b *byteBuffer - bb []byte - - rCh chan *byteBuffer - wCh chan *byteBuffer - pc *PipeConns - - readDeadlineTimer *time.Timer - writeDeadlineTimer *time.Timer - - readDeadlineCh <-chan time.Time - writeDeadlineCh <-chan time.Time -} - -func (c *pipeConn) Write(p []byte) (int, error) { - b := acquireByteBuffer() - b.b = append(b.b[:0], p...) - - select { - case <-c.pc.stopCh: - releaseByteBuffer(b) - return 0, errConnectionClosed - default: - } - - select { - case c.wCh <- b: - default: - select { - case c.wCh <- b: - case <-c.writeDeadlineCh: - c.writeDeadlineCh = closedDeadlineCh - return 0, ErrTimeout - case <-c.pc.stopCh: - releaseByteBuffer(b) - return 0, errConnectionClosed - } - } - - return len(p), nil -} - -func (c *pipeConn) Read(p []byte) (int, error) { - mayBlock := true - nn := 0 - for len(p) > 0 { - n, err := c.read(p, mayBlock) - nn += n - if err != nil { - if !mayBlock && err == errWouldBlock { - err = nil - } - return nn, err - } - p = p[n:] - mayBlock = false - } - - return nn, nil -} - -func (c *pipeConn) read(p []byte, mayBlock bool) (int, error) { - if len(c.bb) == 0 { - if err := c.readNextByteBuffer(mayBlock); err != nil { - return 0, err - } - } - n := copy(p, c.bb) - c.bb = c.bb[n:] - - return n, nil -} - -func (c *pipeConn) readNextByteBuffer(mayBlock bool) error { - releaseByteBuffer(c.b) - c.b = nil - - select { - case c.b = <-c.rCh: - default: - if !mayBlock { - return errWouldBlock - } - select { - case c.b = <-c.rCh: - case <-c.readDeadlineCh: - c.readDeadlineCh = closedDeadlineCh - // rCh may contain data when deadline is reached. - // Read the data before returning ErrTimeout. - select { - case c.b = <-c.rCh: - default: - return ErrTimeout - } - case <-c.pc.stopCh: - // rCh may contain data when stopCh is closed. - // Read the data before returning EOF. - select { - case c.b = <-c.rCh: - default: - return io.EOF - } - } - } - - c.bb = c.b.b - return nil -} - -var ( - errWouldBlock = errors.New("would block") - errConnectionClosed = errors.New("connection closed") - - // ErrTimeout is returned from Read() or Write() on timeout. - ErrTimeout = errors.New("timeout") -) - -func (c *pipeConn) Close() error { - return c.pc.Close() -} - -func (c *pipeConn) LocalAddr() net.Addr { - return pipeAddr(0) -} - -func (c *pipeConn) RemoteAddr() net.Addr { - return pipeAddr(0) -} - -func (c *pipeConn) SetDeadline(deadline time.Time) error { - c.SetReadDeadline(deadline) - c.SetWriteDeadline(deadline) - return nil -} - -func (c *pipeConn) SetReadDeadline(deadline time.Time) error { - if c.readDeadlineTimer == nil { - c.readDeadlineTimer = time.NewTimer(time.Hour) - } - c.readDeadlineCh = updateTimer(c.readDeadlineTimer, deadline) - return nil -} - -func (c *pipeConn) SetWriteDeadline(deadline time.Time) error { - if c.writeDeadlineTimer == nil { - c.writeDeadlineTimer = time.NewTimer(time.Hour) - } - c.writeDeadlineCh = updateTimer(c.writeDeadlineTimer, deadline) - return nil -} - -func updateTimer(t *time.Timer, deadline time.Time) <-chan time.Time { - if !t.Stop() { - select { - case <-t.C: - default: - } - } - if deadline.IsZero() { - return nil - } - d := -time.Since(deadline) - if d <= 0 { - return closedDeadlineCh - } - t.Reset(d) - return t.C -} - -var closedDeadlineCh = func() <-chan time.Time { - ch := make(chan time.Time) - close(ch) - return ch -}() - -type pipeAddr int - -func (pipeAddr) Network() string { - return "pipe" -} - -func (pipeAddr) String() string { - return "pipe" -} - -type byteBuffer struct { - b []byte -} - -func acquireByteBuffer() *byteBuffer { - return byteBufferPool.Get().(*byteBuffer) -} - -func releaseByteBuffer(b *byteBuffer) { - if b != nil { - byteBufferPool.Put(b) - } -} - -var byteBufferPool = &sync.Pool{ - New: func() interface{} { - return &byteBuffer{ - b: make([]byte, 1024), - } - }, -} diff --git a/vendor/github.com/valyala/fasthttp/fasthttputil/rsa.key b/vendor/github.com/valyala/fasthttp/fasthttputil/rsa.key deleted file mode 100644 index 00a79a3b..00000000 --- a/vendor/github.com/valyala/fasthttp/fasthttputil/rsa.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQD4IQusAs8PJdnG -3mURt/AXtgC+ceqLOatJ49JJE1VPTkMAy+oE1f1XvkMrYsHqmDf6GWVzgVXryL4U -wq2/nJSm56ddhN55nI8oSN3dtywUB8/ShelEN73nlN77PeD9tl6NksPwWaKrqxq0 -FlabRPZSQCfmgZbhDV8Sa8mfCkFU0G0lit6kLGceCKMvmW+9Bz7ebsYmVdmVMxmf -IJStFD44lWFTdUc65WISKEdW2ELcUefb0zOLw+0PCbXFGJH5x5ktksW8+BBk2Hkg -GeQRL/qPCccthbScO0VgNj3zJ3ZZL0ObSDAbvNDG85joeNjDNq5DT/BAZ0bOSbEF -sh+f9BAzAgMBAAECggEBAJWv2cq7Jw6MVwSRxYca38xuD6TUNBopgBvjREixURW2 -sNUaLuMb9Omp7fuOaE2N5rcJ+xnjPGIxh/oeN5MQctz9gwn3zf6vY+15h97pUb4D -uGvYPRDaT8YVGS+X9NMZ4ZCmqW2lpWzKnCFoGHcy8yZLbcaxBsRdvKzwOYGoPiFb -K2QuhXZ/1UPmqK9i2DFKtj40X6vBszTNboFxOVpXrPu0FJwLVSDf2hSZ4fMM0DH3 -YqwKcYf5te+hxGKgrqRA3tn0NCWii0in6QIwXMC+kMw1ebg/tZKqyDLMNptAK8J+ -DVw9m5X1seUHS5ehU/g2jrQrtK5WYn7MrFK4lBzlRwECgYEA/d1TeANYECDWRRDk -B0aaRZs87Rwl/J9PsvbsKvtU/bX+OfSOUjOa9iQBqn0LmU8GqusEET/QVUfocVwV -Bggf/5qDLxz100Rj0ags/yE/kNr0Bb31kkkKHFMnCT06YasR7qKllwrAlPJvQv9x -IzBKq+T/Dx08Wep9bCRSFhzRCnsCgYEA+jdeZXTDr/Vz+D2B3nAw1frqYFfGnEVY -wqmoK3VXMDkGuxsloO2rN+SyiUo3JNiQNPDub/t7175GH5pmKtZOlftePANsUjBj -wZ1D0rI5Bxu/71ibIUYIRVmXsTEQkh/ozoh3jXCZ9+bLgYiYx7789IUZZSokFQ3D -FICUT9KJ36kCgYAGoq9Y1rWJjmIrYfqj2guUQC+CfxbbGIrrwZqAsRsSmpwvhZ3m -tiSZxG0quKQB+NfSxdvQW5ulbwC7Xc3K35F+i9pb8+TVBdeaFkw+yu6vaZmxQLrX -fQM/pEjD7A7HmMIaO7QaU5SfEAsqdCTP56Y8AftMuNXn/8IRfo2KuGwaWwKBgFpU -ILzJoVdlad9E/Rw7LjYhZfkv1uBVXIyxyKcfrkEXZSmozDXDdxsvcZCEfVHM6Ipk -K/+7LuMcqp4AFEAEq8wTOdq6daFaHLkpt/FZK6M4TlruhtpFOPkoNc3e45eM83OT -6mziKINJC1CQ6m65sQHpBtjxlKMRG8rL/D6wx9s5AoGBAMRlqNPMwglT3hvDmsAt -9Lf9pdmhERUlHhD8bj8mDaBj2Aqv7f6VRJaYZqP403pKKQexuqcn80mtjkSAPFkN -Cj7BVt/RXm5uoxDTnfi26RF9F6yNDEJ7UU9+peBr99aazF/fTgW/1GcMkQnum8uV -c257YgaWmjK9uB0Y2r2VxS0G ------END PRIVATE KEY----- diff --git a/vendor/github.com/valyala/fasthttp/fasthttputil/rsa.pem b/vendor/github.com/valyala/fasthttp/fasthttputil/rsa.pem deleted file mode 100644 index 93e77cd9..00000000 --- a/vendor/github.com/valyala/fasthttp/fasthttputil/rsa.pem +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICujCCAaKgAwIBAgIJAMbXnKZ/cikUMA0GCSqGSIb3DQEBCwUAMBUxEzARBgNV -BAMTCnVidW50dS5uYW4wHhcNMTUwMjA0MDgwMTM5WhcNMjUwMjAxMDgwMTM5WjAV -MRMwEQYDVQQDEwp1YnVudHUubmFuMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEA+CELrALPDyXZxt5lEbfwF7YAvnHqizmrSePSSRNVT05DAMvqBNX9V75D -K2LB6pg3+hllc4FV68i+FMKtv5yUpuenXYTeeZyPKEjd3bcsFAfP0oXpRDe955Te -+z3g/bZejZLD8Fmiq6satBZWm0T2UkAn5oGW4Q1fEmvJnwpBVNBtJYrepCxnHgij -L5lvvQc+3m7GJlXZlTMZnyCUrRQ+OJVhU3VHOuViEihHVthC3FHn29Mzi8PtDwm1 -xRiR+ceZLZLFvPgQZNh5IBnkES/6jwnHLYW0nDtFYDY98yd2WS9Dm0gwG7zQxvOY -6HjYwzauQ0/wQGdGzkmxBbIfn/QQMwIDAQABow0wCzAJBgNVHRMEAjAAMA0GCSqG -SIb3DQEBCwUAA4IBAQBQjKm/4KN/iTgXbLTL3i7zaxYXFLXsnT1tF+ay4VA8aj98 -L3JwRTciZ3A5iy/W4VSCt3eASwOaPWHKqDBB5RTtL73LoAqsWmO3APOGQAbixcQ2 -45GXi05OKeyiYRi1Nvq7Unv9jUkRDHUYVPZVSAjCpsXzPhFkmZoTRxmx5l0ZF7Li -K91lI5h+eFq0dwZwrmlPambyh1vQUi70VHv8DNToVU29kel7YLbxGbuqETfhrcy6 -X+Mha6RYITkAn5FqsZcKMsc9eYGEF4l3XV+oS7q6xfTxktYJMFTI18J0lQ2Lv/CI -whdMnYGntDQBE/iFCrJEGNsKGc38796GBOb5j+zd ------END CERTIFICATE----- diff --git a/vendor/github.com/valyala/fasthttp/fs.go b/vendor/github.com/valyala/fasthttp/fs.go deleted file mode 100644 index 1e9b4ab1..00000000 --- a/vendor/github.com/valyala/fasthttp/fs.go +++ /dev/null @@ -1,1271 +0,0 @@ -package fasthttp - -import ( - "bytes" - "errors" - "fmt" - "html" - "io" - "io/ioutil" - "mime" - "net/http" - "os" - "path/filepath" - "sort" - "strings" - "sync" - "time" - - "github.com/klauspost/compress/gzip" - "github.com/valyala/bytebufferpool" -) - -// ServeFileBytesUncompressed returns HTTP response containing file contents -// from the given path. -// -// Directory contents is returned if path points to directory. -// -// ServeFileBytes may be used for saving network traffic when serving files -// with good compression ratio. -// -// See also RequestCtx.SendFileBytes. -func ServeFileBytesUncompressed(ctx *RequestCtx, path []byte) { - ServeFileUncompressed(ctx, b2s(path)) -} - -// ServeFileUncompressed returns HTTP response containing file contents -// from the given path. -// -// Directory contents is returned if path points to directory. -// -// ServeFile may be used for saving network traffic when serving files -// with good compression ratio. -// -// See also RequestCtx.SendFile. -func ServeFileUncompressed(ctx *RequestCtx, path string) { - ctx.Request.Header.DelBytes(strAcceptEncoding) - ServeFile(ctx, path) -} - -// ServeFileBytes returns HTTP response containing compressed file contents -// from the given path. -// -// HTTP response may contain uncompressed file contents in the following cases: -// -// * Missing 'Accept-Encoding: gzip' request header. -// * No write access to directory containing the file. -// -// Directory contents is returned if path points to directory. -// -// Use ServeFileBytesUncompressed is you don't need serving compressed -// file contents. -// -// See also RequestCtx.SendFileBytes. -func ServeFileBytes(ctx *RequestCtx, path []byte) { - ServeFile(ctx, b2s(path)) -} - -// ServeFile returns HTTP response containing compressed file contents -// from the given path. -// -// HTTP response may contain uncompressed file contents in the following cases: -// -// * Missing 'Accept-Encoding: gzip' request header. -// * No write access to directory containing the file. -// -// Directory contents is returned if path points to directory. -// -// Use ServeFileUncompressed is you don't need serving compressed file contents. -// -// See also RequestCtx.SendFile. -func ServeFile(ctx *RequestCtx, path string) { - rootFSOnce.Do(func() { - rootFSHandler = rootFS.NewRequestHandler() - }) - if len(path) == 0 || path[0] != '/' { - // extend relative path to absolute path - var err error - if path, err = filepath.Abs(path); err != nil { - ctx.Logger().Printf("cannot resolve path %q to absolute file path: %s", path, err) - ctx.Error("Internal Server Error", StatusInternalServerError) - return - } - } - ctx.Request.SetRequestURI(path) - rootFSHandler(ctx) -} - -var ( - rootFSOnce sync.Once - rootFS = &FS{ - Root: "/", - GenerateIndexPages: true, - Compress: true, - AcceptByteRange: true, - } - rootFSHandler RequestHandler -) - -// PathRewriteFunc must return new request path based on arbitrary ctx -// info such as ctx.Path(). -// -// Path rewriter is used in FS for translating the current request -// to the local filesystem path relative to FS.Root. -// -// The returned path must not contain '/../' substrings due to security reasons, -// since such paths may refer files outside FS.Root. -// -// The returned path may refer to ctx members. For example, ctx.Path(). -type PathRewriteFunc func(ctx *RequestCtx) []byte - -// NewVHostPathRewriter returns path rewriter, which strips slashesCount -// leading slashes from the path and prepends the path with request's host, -// thus simplifying virtual hosting for static files. -// -// Examples: -// -// * host=foobar.com, slashesCount=0, original path="/foo/bar". -// Resulting path: "/foobar.com/foo/bar" -// -// * host=img.aaa.com, slashesCount=1, original path="/images/123/456.jpg" -// Resulting path: "/img.aaa.com/123/456.jpg" -// -func NewVHostPathRewriter(slashesCount int) PathRewriteFunc { - return func(ctx *RequestCtx) []byte { - path := stripLeadingSlashes(ctx.Path(), slashesCount) - host := ctx.Host() - if n := bytes.IndexByte(host, '/'); n >= 0 { - host = nil - } - if len(host) == 0 { - host = strInvalidHost - } - b := bytebufferpool.Get() - b.B = append(b.B, '/') - b.B = append(b.B, host...) - b.B = append(b.B, path...) - ctx.URI().SetPathBytes(b.B) - bytebufferpool.Put(b) - - return ctx.Path() - } -} - -var strInvalidHost = []byte("invalid-host") - -// NewPathSlashesStripper returns path rewriter, which strips slashesCount -// leading slashes from the path. -// -// Examples: -// -// * slashesCount = 0, original path: "/foo/bar", result: "/foo/bar" -// * slashesCount = 1, original path: "/foo/bar", result: "/bar" -// * slashesCount = 2, original path: "/foo/bar", result: "" -// -// The returned path rewriter may be used as FS.PathRewrite . -func NewPathSlashesStripper(slashesCount int) PathRewriteFunc { - return func(ctx *RequestCtx) []byte { - return stripLeadingSlashes(ctx.Path(), slashesCount) - } -} - -// NewPathPrefixStripper returns path rewriter, which removes prefixSize bytes -// from the path prefix. -// -// Examples: -// -// * prefixSize = 0, original path: "/foo/bar", result: "/foo/bar" -// * prefixSize = 3, original path: "/foo/bar", result: "o/bar" -// * prefixSize = 7, original path: "/foo/bar", result: "r" -// -// The returned path rewriter may be used as FS.PathRewrite . -func NewPathPrefixStripper(prefixSize int) PathRewriteFunc { - return func(ctx *RequestCtx) []byte { - path := ctx.Path() - if len(path) >= prefixSize { - path = path[prefixSize:] - } - return path - } -} - -// FS represents settings for request handler serving static files -// from the local filesystem. -// -// It is prohibited copying FS values. Create new values instead. -type FS struct { - noCopy noCopy - - // Path to the root directory to serve files from. - Root string - - // List of index file names to try opening during directory access. - // - // For example: - // - // * index.html - // * index.htm - // * my-super-index.xml - // - // By default the list is empty. - IndexNames []string - - // Index pages for directories without files matching IndexNames - // are automatically generated if set. - // - // Directory index generation may be quite slow for directories - // with many files (more than 1K), so it is discouraged enabling - // index pages' generation for such directories. - // - // By default index pages aren't generated. - GenerateIndexPages bool - - // Transparently compresses responses if set to true. - // - // The server tries minimizing CPU usage by caching compressed files. - // It adds CompressedFileSuffix suffix to the original file name and - // tries saving the resulting compressed file under the new file name. - // So it is advisable to give the server write access to Root - // and to all inner folders in order to minimize CPU usage when serving - // compressed responses. - // - // Transparent compression is disabled by default. - Compress bool - - // Enables byte range requests if set to true. - // - // Byte range requests are disabled by default. - AcceptByteRange bool - - // Path rewriting function. - // - // By default request path is not modified. - PathRewrite PathRewriteFunc - - // PathNotFound fires when file is not found in filesystem - // this functions tries to replace "Cannot open requested path" - // server response giving to the programmer the control of server flow. - // - // By default PathNotFound returns - // "Cannot open requested path" - PathNotFound RequestHandler - - // Expiration duration for inactive file handlers. - // - // FSHandlerCacheDuration is used by default. - CacheDuration time.Duration - - // Suffix to add to the name of cached compressed file. - // - // This value has sense only if Compress is set. - // - // FSCompressedFileSuffix is used by default. - CompressedFileSuffix string - - once sync.Once - h RequestHandler -} - -// FSCompressedFileSuffix is the suffix FS adds to the original file names -// when trying to store compressed file under the new file name. -// See FS.Compress for details. -const FSCompressedFileSuffix = ".fasthttp.gz" - -// FSHandlerCacheDuration is the default expiration duration for inactive -// file handlers opened by FS. -const FSHandlerCacheDuration = 10 * time.Second - -// FSHandler returns request handler serving static files from -// the given root folder. -// -// stripSlashes indicates how many leading slashes must be stripped -// from requested path before searching requested file in the root folder. -// Examples: -// -// * stripSlashes = 0, original path: "/foo/bar", result: "/foo/bar" -// * stripSlashes = 1, original path: "/foo/bar", result: "/bar" -// * stripSlashes = 2, original path: "/foo/bar", result: "" -// -// The returned request handler automatically generates index pages -// for directories without index.html. -// -// The returned handler caches requested file handles -// for FSHandlerCacheDuration. -// Make sure your program has enough 'max open files' limit aka -// 'ulimit -n' if root folder contains many files. -// -// Do not create multiple request handler instances for the same -// (root, stripSlashes) arguments - just reuse a single instance. -// Otherwise goroutine leak will occur. -func FSHandler(root string, stripSlashes int) RequestHandler { - fs := &FS{ - Root: root, - IndexNames: []string{"index.html"}, - GenerateIndexPages: true, - AcceptByteRange: true, - } - if stripSlashes > 0 { - fs.PathRewrite = NewPathSlashesStripper(stripSlashes) - } - return fs.NewRequestHandler() -} - -// NewRequestHandler returns new request handler with the given FS settings. -// -// The returned handler caches requested file handles -// for FS.CacheDuration. -// Make sure your program has enough 'max open files' limit aka -// 'ulimit -n' if FS.Root folder contains many files. -// -// Do not create multiple request handlers from a single FS instance - -// just reuse a single request handler. -func (fs *FS) NewRequestHandler() RequestHandler { - fs.once.Do(fs.initRequestHandler) - return fs.h -} - -func (fs *FS) initRequestHandler() { - root := fs.Root - - // serve files from the current working directory if root is empty - if len(root) == 0 { - root = "." - } - - // strip trailing slashes from the root path - for len(root) > 0 && root[len(root)-1] == '/' { - root = root[:len(root)-1] - } - - cacheDuration := fs.CacheDuration - if cacheDuration <= 0 { - cacheDuration = FSHandlerCacheDuration - } - compressedFileSuffix := fs.CompressedFileSuffix - if len(compressedFileSuffix) == 0 { - compressedFileSuffix = FSCompressedFileSuffix - } - - h := &fsHandler{ - root: root, - indexNames: fs.IndexNames, - pathRewrite: fs.PathRewrite, - generateIndexPages: fs.GenerateIndexPages, - compress: fs.Compress, - pathNotFound: fs.PathNotFound, - acceptByteRange: fs.AcceptByteRange, - cacheDuration: cacheDuration, - compressedFileSuffix: compressedFileSuffix, - cache: make(map[string]*fsFile), - compressedCache: make(map[string]*fsFile), - } - - go func() { - var pendingFiles []*fsFile - for { - time.Sleep(cacheDuration / 2) - pendingFiles = h.cleanCache(pendingFiles) - } - }() - - fs.h = h.handleRequest -} - -type fsHandler struct { - root string - indexNames []string - pathRewrite PathRewriteFunc - pathNotFound RequestHandler - generateIndexPages bool - compress bool - acceptByteRange bool - cacheDuration time.Duration - compressedFileSuffix string - - cache map[string]*fsFile - compressedCache map[string]*fsFile - cacheLock sync.Mutex - - smallFileReaderPool sync.Pool -} - -type fsFile struct { - h *fsHandler - f *os.File - dirIndex []byte - contentType string - contentLength int - compressed bool - - lastModified time.Time - lastModifiedStr []byte - - t time.Time - readersCount int - - bigFiles []*bigFileReader - bigFilesLock sync.Mutex -} - -func (ff *fsFile) NewReader() (io.Reader, error) { - if ff.isBig() { - r, err := ff.bigFileReader() - if err != nil { - ff.decReadersCount() - } - return r, err - } - return ff.smallFileReader(), nil -} - -func (ff *fsFile) smallFileReader() io.Reader { - v := ff.h.smallFileReaderPool.Get() - if v == nil { - v = &fsSmallFileReader{} - } - r := v.(*fsSmallFileReader) - r.ff = ff - r.endPos = ff.contentLength - if r.startPos > 0 { - panic("BUG: fsSmallFileReader with non-nil startPos found in the pool") - } - return r -} - -// files bigger than this size are sent with sendfile -const maxSmallFileSize = 2 * 4096 - -func (ff *fsFile) isBig() bool { - return ff.contentLength > maxSmallFileSize && len(ff.dirIndex) == 0 -} - -func (ff *fsFile) bigFileReader() (io.Reader, error) { - if ff.f == nil { - panic("BUG: ff.f must be non-nil in bigFileReader") - } - - var r io.Reader - - ff.bigFilesLock.Lock() - n := len(ff.bigFiles) - if n > 0 { - r = ff.bigFiles[n-1] - ff.bigFiles = ff.bigFiles[:n-1] - } - ff.bigFilesLock.Unlock() - - if r != nil { - return r, nil - } - - f, err := os.Open(ff.f.Name()) - if err != nil { - return nil, fmt.Errorf("cannot open already opened file: %s", err) - } - return &bigFileReader{ - f: f, - ff: ff, - r: f, - }, nil -} - -func (ff *fsFile) Release() { - if ff.f != nil { - ff.f.Close() - - if ff.isBig() { - ff.bigFilesLock.Lock() - for _, r := range ff.bigFiles { - r.f.Close() - } - ff.bigFilesLock.Unlock() - } - } -} - -func (ff *fsFile) decReadersCount() { - ff.h.cacheLock.Lock() - ff.readersCount-- - if ff.readersCount < 0 { - panic("BUG: negative fsFile.readersCount!") - } - ff.h.cacheLock.Unlock() -} - -// bigFileReader attempts to trigger sendfile -// for sending big files over the wire. -type bigFileReader struct { - f *os.File - ff *fsFile - r io.Reader - lr io.LimitedReader -} - -func (r *bigFileReader) UpdateByteRange(startPos, endPos int) error { - if _, err := r.f.Seek(int64(startPos), 0); err != nil { - return err - } - r.r = &r.lr - r.lr.R = r.f - r.lr.N = int64(endPos - startPos + 1) - return nil -} - -func (r *bigFileReader) Read(p []byte) (int, error) { - return r.r.Read(p) -} - -func (r *bigFileReader) WriteTo(w io.Writer) (int64, error) { - if rf, ok := w.(io.ReaderFrom); ok { - // fast path. Senfile must be triggered - return rf.ReadFrom(r.r) - } - - // slow path - return copyZeroAlloc(w, r.r) -} - -func (r *bigFileReader) Close() error { - r.r = r.f - n, err := r.f.Seek(0, 0) - if err == nil { - if n != 0 { - panic("BUG: File.Seek(0,0) returned (non-zero, nil)") - } - - ff := r.ff - ff.bigFilesLock.Lock() - ff.bigFiles = append(ff.bigFiles, r) - ff.bigFilesLock.Unlock() - } else { - r.f.Close() - } - r.ff.decReadersCount() - return err -} - -type fsSmallFileReader struct { - ff *fsFile - startPos int - endPos int -} - -func (r *fsSmallFileReader) Close() error { - ff := r.ff - ff.decReadersCount() - r.ff = nil - r.startPos = 0 - r.endPos = 0 - ff.h.smallFileReaderPool.Put(r) - return nil -} - -func (r *fsSmallFileReader) UpdateByteRange(startPos, endPos int) error { - r.startPos = startPos - r.endPos = endPos + 1 - return nil -} - -func (r *fsSmallFileReader) Read(p []byte) (int, error) { - tailLen := r.endPos - r.startPos - if tailLen <= 0 { - return 0, io.EOF - } - if len(p) > tailLen { - p = p[:tailLen] - } - - ff := r.ff - if ff.f != nil { - n, err := ff.f.ReadAt(p, int64(r.startPos)) - r.startPos += n - return n, err - } - - n := copy(p, ff.dirIndex[r.startPos:]) - r.startPos += n - return n, nil -} - -func (r *fsSmallFileReader) WriteTo(w io.Writer) (int64, error) { - ff := r.ff - - var n int - var err error - if ff.f == nil { - n, err = w.Write(ff.dirIndex[r.startPos:r.endPos]) - return int64(n), err - } - - if rf, ok := w.(io.ReaderFrom); ok { - return rf.ReadFrom(r) - } - - curPos := r.startPos - bufv := copyBufPool.Get() - buf := bufv.([]byte) - for err == nil { - tailLen := r.endPos - curPos - if tailLen <= 0 { - break - } - if len(buf) > tailLen { - buf = buf[:tailLen] - } - n, err = ff.f.ReadAt(buf, int64(curPos)) - nw, errw := w.Write(buf[:n]) - curPos += nw - if errw == nil && nw != n { - panic("BUG: Write(p) returned (n, nil), where n != len(p)") - } - if err == nil { - err = errw - } - } - copyBufPool.Put(bufv) - - if err == io.EOF { - err = nil - } - return int64(curPos - r.startPos), err -} - -func (h *fsHandler) cleanCache(pendingFiles []*fsFile) []*fsFile { - var filesToRelease []*fsFile - - h.cacheLock.Lock() - - // Close files which couldn't be closed before due to non-zero - // readers count on the previous run. - var remainingFiles []*fsFile - for _, ff := range pendingFiles { - if ff.readersCount > 0 { - remainingFiles = append(remainingFiles, ff) - } else { - filesToRelease = append(filesToRelease, ff) - } - } - pendingFiles = remainingFiles - - pendingFiles, filesToRelease = cleanCacheNolock(h.cache, pendingFiles, filesToRelease, h.cacheDuration) - pendingFiles, filesToRelease = cleanCacheNolock(h.compressedCache, pendingFiles, filesToRelease, h.cacheDuration) - - h.cacheLock.Unlock() - - for _, ff := range filesToRelease { - ff.Release() - } - - return pendingFiles -} - -func cleanCacheNolock(cache map[string]*fsFile, pendingFiles, filesToRelease []*fsFile, cacheDuration time.Duration) ([]*fsFile, []*fsFile) { - t := time.Now() - for k, ff := range cache { - if t.Sub(ff.t) > cacheDuration { - if ff.readersCount > 0 { - // There are pending readers on stale file handle, - // so we cannot close it. Put it into pendingFiles - // so it will be closed later. - pendingFiles = append(pendingFiles, ff) - } else { - filesToRelease = append(filesToRelease, ff) - } - delete(cache, k) - } - } - return pendingFiles, filesToRelease -} - -func (h *fsHandler) handleRequest(ctx *RequestCtx) { - var path []byte - if h.pathRewrite != nil { - path = h.pathRewrite(ctx) - } else { - path = ctx.Path() - } - path = stripTrailingSlashes(path) - - if n := bytes.IndexByte(path, 0); n >= 0 { - ctx.Logger().Printf("cannot serve path with nil byte at position %d: %q", n, path) - ctx.Error("Are you a hacker?", StatusBadRequest) - return - } - if h.pathRewrite != nil { - // There is no need to check for '/../' if path = ctx.Path(), - // since ctx.Path must normalize and sanitize the path. - - if n := bytes.Index(path, strSlashDotDotSlash); n >= 0 { - ctx.Logger().Printf("cannot serve path with '/../' at position %d due to security reasons: %q", n, path) - ctx.Error("Internal Server Error", StatusInternalServerError) - return - } - } - - mustCompress := false - fileCache := h.cache - byteRange := ctx.Request.Header.peek(strRange) - if len(byteRange) == 0 && h.compress && ctx.Request.Header.HasAcceptEncodingBytes(strGzip) { - mustCompress = true - fileCache = h.compressedCache - } - - h.cacheLock.Lock() - ff, ok := fileCache[string(path)] - if ok { - ff.readersCount++ - } - h.cacheLock.Unlock() - - if !ok { - pathStr := string(path) - filePath := h.root + pathStr - var err error - ff, err = h.openFSFile(filePath, mustCompress) - if mustCompress && err == errNoCreatePermission { - ctx.Logger().Printf("insufficient permissions for saving compressed file for %q. Serving uncompressed file. "+ - "Allow write access to the directory with this file in order to improve fasthttp performance", filePath) - mustCompress = false - ff, err = h.openFSFile(filePath, mustCompress) - } - if err == errDirIndexRequired { - ff, err = h.openIndexFile(ctx, filePath, mustCompress) - if err != nil { - ctx.Logger().Printf("cannot open dir index %q: %s", filePath, err) - ctx.Error("Directory index is forbidden", StatusForbidden) - return - } - } else if err != nil { - ctx.Logger().Printf("cannot open file %q: %s", filePath, err) - if h.pathNotFound == nil { - ctx.Error("Cannot open requested path", StatusNotFound) - } else { - ctx.SetStatusCode(StatusNotFound) - h.pathNotFound(ctx) - } - return - } - - h.cacheLock.Lock() - ff1, ok := fileCache[pathStr] - if !ok { - fileCache[pathStr] = ff - ff.readersCount++ - } else { - ff1.readersCount++ - } - h.cacheLock.Unlock() - - if ok { - // The file has been already opened by another - // goroutine, so close the current file and use - // the file opened by another goroutine instead. - ff.Release() - ff = ff1 - } - } - - if !ctx.IfModifiedSince(ff.lastModified) { - ff.decReadersCount() - ctx.NotModified() - return - } - - r, err := ff.NewReader() - if err != nil { - ctx.Logger().Printf("cannot obtain file reader for path=%q: %s", path, err) - ctx.Error("Internal Server Error", StatusInternalServerError) - return - } - - hdr := &ctx.Response.Header - if ff.compressed { - hdr.SetCanonical(strContentEncoding, strGzip) - } - - statusCode := StatusOK - contentLength := ff.contentLength - if h.acceptByteRange { - hdr.SetCanonical(strAcceptRanges, strBytes) - if len(byteRange) > 0 { - startPos, endPos, err := ParseByteRange(byteRange, contentLength) - if err != nil { - r.(io.Closer).Close() - ctx.Logger().Printf("cannot parse byte range %q for path=%q: %s", byteRange, path, err) - ctx.Error("Range Not Satisfiable", StatusRequestedRangeNotSatisfiable) - return - } - - if err = r.(byteRangeUpdater).UpdateByteRange(startPos, endPos); err != nil { - r.(io.Closer).Close() - ctx.Logger().Printf("cannot seek byte range %q for path=%q: %s", byteRange, path, err) - ctx.Error("Internal Server Error", StatusInternalServerError) - return - } - - hdr.SetContentRange(startPos, endPos, contentLength) - contentLength = endPos - startPos + 1 - statusCode = StatusPartialContent - } - } - - hdr.SetCanonical(strLastModified, ff.lastModifiedStr) - if !ctx.IsHead() { - ctx.SetBodyStream(r, contentLength) - } else { - ctx.Response.ResetBody() - ctx.Response.SkipBody = true - ctx.Response.Header.SetContentLength(contentLength) - if rc, ok := r.(io.Closer); ok { - if err := rc.Close(); err != nil { - ctx.Logger().Printf("cannot close file reader: %s", err) - ctx.Error("Internal Server Error", StatusInternalServerError) - return - } - } - } - hdr.noDefaultContentType = true - if len(hdr.ContentType()) == 0 { - ctx.SetContentType(ff.contentType) - } - ctx.SetStatusCode(statusCode) -} - -type byteRangeUpdater interface { - UpdateByteRange(startPos, endPos int) error -} - -// ParseByteRange parses 'Range: bytes=...' header value. -// -// It follows https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35 . -func ParseByteRange(byteRange []byte, contentLength int) (startPos, endPos int, err error) { - b := byteRange - if !bytes.HasPrefix(b, strBytes) { - return 0, 0, fmt.Errorf("unsupported range units: %q. Expecting %q", byteRange, strBytes) - } - - b = b[len(strBytes):] - if len(b) == 0 || b[0] != '=' { - return 0, 0, fmt.Errorf("missing byte range in %q", byteRange) - } - b = b[1:] - - n := bytes.IndexByte(b, '-') - if n < 0 { - return 0, 0, fmt.Errorf("missing the end position of byte range in %q", byteRange) - } - - if n == 0 { - v, err := ParseUint(b[n+1:]) - if err != nil { - return 0, 0, err - } - startPos := contentLength - v - if startPos < 0 { - startPos = 0 - } - return startPos, contentLength - 1, nil - } - - if startPos, err = ParseUint(b[:n]); err != nil { - return 0, 0, err - } - if startPos >= contentLength { - return 0, 0, fmt.Errorf("the start position of byte range cannot exceed %d. byte range %q", contentLength-1, byteRange) - } - - b = b[n+1:] - if len(b) == 0 { - return startPos, contentLength - 1, nil - } - - if endPos, err = ParseUint(b); err != nil { - return 0, 0, err - } - if endPos >= contentLength { - endPos = contentLength - 1 - } - if endPos < startPos { - return 0, 0, fmt.Errorf("the start position of byte range cannot exceed the end position. byte range %q", byteRange) - } - return startPos, endPos, nil -} - -func (h *fsHandler) openIndexFile(ctx *RequestCtx, dirPath string, mustCompress bool) (*fsFile, error) { - for _, indexName := range h.indexNames { - indexFilePath := dirPath + "/" + indexName - ff, err := h.openFSFile(indexFilePath, mustCompress) - if err == nil { - return ff, nil - } - if !os.IsNotExist(err) { - return nil, fmt.Errorf("cannot open file %q: %s", indexFilePath, err) - } - } - - if !h.generateIndexPages { - return nil, fmt.Errorf("cannot access directory without index page. Directory %q", dirPath) - } - - return h.createDirIndex(ctx.URI(), dirPath, mustCompress) -} - -var ( - errDirIndexRequired = errors.New("directory index required") - errNoCreatePermission = errors.New("no 'create file' permissions") -) - -func (h *fsHandler) createDirIndex(base *URI, dirPath string, mustCompress bool) (*fsFile, error) { - w := &bytebufferpool.ByteBuffer{} - - basePathEscaped := html.EscapeString(string(base.Path())) - fmt.Fprintf(w, "%s", basePathEscaped) - fmt.Fprintf(w, "

%s

", basePathEscaped) - fmt.Fprintf(w, "
    ") - - if len(basePathEscaped) > 1 { - var parentURI URI - base.CopyTo(&parentURI) - parentURI.Update(string(base.Path()) + "/..") - parentPathEscaped := html.EscapeString(string(parentURI.Path())) - fmt.Fprintf(w, `
  • ..
    • `, parentPathEscaped) - } - - f, err := os.Open(dirPath) - if err != nil { - return nil, err - } - - fileinfos, err := f.Readdir(0) - f.Close() - if err != nil { - return nil, err - } - - fm := make(map[string]os.FileInfo, len(fileinfos)) - filenames := make([]string, 0, len(fileinfos)) - for _, fi := range fileinfos { - name := fi.Name() - if strings.HasSuffix(name, h.compressedFileSuffix) { - // Do not show compressed files on index page. - continue - } - fm[name] = fi - filenames = append(filenames, name) - } - - var u URI - base.CopyTo(&u) - u.Update(string(u.Path()) + "/") - - sort.Strings(filenames) - for _, name := range filenames { - u.Update(name) - pathEscaped := html.EscapeString(string(u.Path())) - fi := fm[name] - auxStr := "dir" - className := "dir" - if !fi.IsDir() { - auxStr = fmt.Sprintf("file, %d bytes", fi.Size()) - className = "file" - } - fmt.Fprintf(w, `
  • %s, %s, last modified %s
    • `, - pathEscaped, className, html.EscapeString(name), auxStr, fsModTime(fi.ModTime())) - } - - fmt.Fprintf(w, "
") - - if mustCompress { - var zbuf bytebufferpool.ByteBuffer - zbuf.B = AppendGzipBytesLevel(zbuf.B, w.B, CompressDefaultCompression) - w = &zbuf - } - - dirIndex := w.B - lastModified := time.Now() - ff := &fsFile{ - h: h, - dirIndex: dirIndex, - contentType: "text/html; charset=utf-8", - contentLength: len(dirIndex), - compressed: mustCompress, - lastModified: lastModified, - lastModifiedStr: AppendHTTPDate(nil, lastModified), - - t: lastModified, - } - return ff, nil -} - -const ( - fsMinCompressRatio = 0.8 - fsMaxCompressibleFileSize = 8 * 1024 * 1024 -) - -func (h *fsHandler) compressAndOpenFSFile(filePath string) (*fsFile, error) { - f, err := os.Open(filePath) - if err != nil { - return nil, err - } - - fileInfo, err := f.Stat() - if err != nil { - f.Close() - return nil, fmt.Errorf("cannot obtain info for file %q: %s", filePath, err) - } - - if fileInfo.IsDir() { - f.Close() - return nil, errDirIndexRequired - } - - if strings.HasSuffix(filePath, h.compressedFileSuffix) || - fileInfo.Size() > fsMaxCompressibleFileSize || - !isFileCompressible(f, fsMinCompressRatio) { - return h.newFSFile(f, fileInfo, false) - } - - compressedFilePath := filePath + h.compressedFileSuffix - absPath, err := filepath.Abs(compressedFilePath) - if err != nil { - f.Close() - return nil, fmt.Errorf("cannot determine absolute path for %q: %s", compressedFilePath, err) - } - - flock := getFileLock(absPath) - flock.Lock() - ff, err := h.compressFileNolock(f, fileInfo, filePath, compressedFilePath) - flock.Unlock() - - return ff, err -} - -func (h *fsHandler) compressFileNolock(f *os.File, fileInfo os.FileInfo, filePath, compressedFilePath string) (*fsFile, error) { - // Attempt to open compressed file created by another concurrent - // goroutine. - // It is safe opening such a file, since the file creation - // is guarded by file mutex - see getFileLock call. - if _, err := os.Stat(compressedFilePath); err == nil { - f.Close() - return h.newCompressedFSFile(compressedFilePath) - } - - // Create temporary file, so concurrent goroutines don't use - // it until it is created. - tmpFilePath := compressedFilePath + ".tmp" - zf, err := os.Create(tmpFilePath) - if err != nil { - f.Close() - if !os.IsPermission(err) { - return nil, fmt.Errorf("cannot create temporary file %q: %s", tmpFilePath, err) - } - return nil, errNoCreatePermission - } - - zw := acquireStacklessGzipWriter(zf, CompressDefaultCompression) - _, err = copyZeroAlloc(zw, f) - if err1 := zw.Flush(); err == nil { - err = err1 - } - releaseStacklessGzipWriter(zw, CompressDefaultCompression) - zf.Close() - f.Close() - if err != nil { - return nil, fmt.Errorf("error when compressing file %q to %q: %s", filePath, tmpFilePath, err) - } - if err = os.Chtimes(tmpFilePath, time.Now(), fileInfo.ModTime()); err != nil { - return nil, fmt.Errorf("cannot change modification time to %s for tmp file %q: %s", - fileInfo.ModTime(), tmpFilePath, err) - } - if err = os.Rename(tmpFilePath, compressedFilePath); err != nil { - return nil, fmt.Errorf("cannot move compressed file from %q to %q: %s", tmpFilePath, compressedFilePath, err) - } - return h.newCompressedFSFile(compressedFilePath) -} - -func (h *fsHandler) newCompressedFSFile(filePath string) (*fsFile, error) { - f, err := os.Open(filePath) - if err != nil { - return nil, fmt.Errorf("cannot open compressed file %q: %s", filePath, err) - } - fileInfo, err := f.Stat() - if err != nil { - f.Close() - return nil, fmt.Errorf("cannot obtain info for compressed file %q: %s", filePath, err) - } - return h.newFSFile(f, fileInfo, true) -} - -func (h *fsHandler) openFSFile(filePath string, mustCompress bool) (*fsFile, error) { - filePathOriginal := filePath - if mustCompress { - filePath += h.compressedFileSuffix - } - - f, err := os.Open(filePath) - if err != nil { - if mustCompress && os.IsNotExist(err) { - return h.compressAndOpenFSFile(filePathOriginal) - } - return nil, err - } - - fileInfo, err := f.Stat() - if err != nil { - f.Close() - return nil, fmt.Errorf("cannot obtain info for file %q: %s", filePath, err) - } - - if fileInfo.IsDir() { - f.Close() - if mustCompress { - return nil, fmt.Errorf("directory with unexpected suffix found: %q. Suffix: %q", - filePath, h.compressedFileSuffix) - } - return nil, errDirIndexRequired - } - - if mustCompress { - fileInfoOriginal, err := os.Stat(filePathOriginal) - if err != nil { - f.Close() - return nil, fmt.Errorf("cannot obtain info for original file %q: %s", filePathOriginal, err) - } - - if fileInfoOriginal.ModTime() != fileInfo.ModTime() { - // The compressed file became stale. Re-create it. - f.Close() - os.Remove(filePath) - return h.compressAndOpenFSFile(filePathOriginal) - } - } - - return h.newFSFile(f, fileInfo, mustCompress) -} - -func (h *fsHandler) newFSFile(f *os.File, fileInfo os.FileInfo, compressed bool) (*fsFile, error) { - n := fileInfo.Size() - contentLength := int(n) - if n != int64(contentLength) { - f.Close() - return nil, fmt.Errorf("too big file: %d bytes", n) - } - - // detect content-type - ext := fileExtension(fileInfo.Name(), compressed, h.compressedFileSuffix) - contentType := mime.TypeByExtension(ext) - if len(contentType) == 0 { - data, err := readFileHeader(f, compressed) - if err != nil { - return nil, fmt.Errorf("cannot read header of the file %q: %s", f.Name(), err) - } - contentType = http.DetectContentType(data) - } - - lastModified := fileInfo.ModTime() - ff := &fsFile{ - h: h, - f: f, - contentType: contentType, - contentLength: contentLength, - compressed: compressed, - lastModified: lastModified, - lastModifiedStr: AppendHTTPDate(nil, lastModified), - - t: time.Now(), - } - return ff, nil -} - -func readFileHeader(f *os.File, compressed bool) ([]byte, error) { - r := io.Reader(f) - var zr *gzip.Reader - if compressed { - var err error - if zr, err = acquireGzipReader(f); err != nil { - return nil, err - } - r = zr - } - - lr := &io.LimitedReader{ - R: r, - N: 512, - } - data, err := ioutil.ReadAll(lr) - f.Seek(0, 0) - - if zr != nil { - releaseGzipReader(zr) - } - - return data, err -} - -func stripLeadingSlashes(path []byte, stripSlashes int) []byte { - for stripSlashes > 0 && len(path) > 0 { - if path[0] != '/' { - panic("BUG: path must start with slash") - } - n := bytes.IndexByte(path[1:], '/') - if n < 0 { - path = path[:0] - break - } - path = path[n+1:] - stripSlashes-- - } - return path -} - -func stripTrailingSlashes(path []byte) []byte { - for len(path) > 0 && path[len(path)-1] == '/' { - path = path[:len(path)-1] - } - return path -} - -func fileExtension(path string, compressed bool, compressedFileSuffix string) string { - if compressed && strings.HasSuffix(path, compressedFileSuffix) { - path = path[:len(path)-len(compressedFileSuffix)] - } - n := strings.LastIndexByte(path, '.') - if n < 0 { - return "" - } - return path[n:] -} - -// FileLastModified returns last modified time for the file. -func FileLastModified(path string) (time.Time, error) { - f, err := os.Open(path) - if err != nil { - return zeroTime, err - } - fileInfo, err := f.Stat() - f.Close() - if err != nil { - return zeroTime, err - } - return fsModTime(fileInfo.ModTime()), nil -} - -func fsModTime(t time.Time) time.Time { - return t.In(time.UTC).Truncate(time.Second) -} - -var ( - filesLockMap = make(map[string]*sync.Mutex) - filesLockMapLock sync.Mutex -) - -func getFileLock(absPath string) *sync.Mutex { - filesLockMapLock.Lock() - flock := filesLockMap[absPath] - if flock == nil { - flock = &sync.Mutex{} - filesLockMap[absPath] = flock - } - filesLockMapLock.Unlock() - return flock -} diff --git a/vendor/github.com/valyala/fasthttp/go.mod b/vendor/github.com/valyala/fasthttp/go.mod deleted file mode 100644 index 8434ca1e..00000000 --- a/vendor/github.com/valyala/fasthttp/go.mod +++ /dev/null @@ -1,9 +0,0 @@ -module github.com/valyala/fasthttp - -require ( - github.com/klauspost/compress v1.4.0 - github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e // indirect - github.com/valyala/bytebufferpool v1.0.0 - github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a - golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 -) diff --git a/vendor/github.com/valyala/fasthttp/go.sum b/vendor/github.com/valyala/fasthttp/go.sum deleted file mode 100644 index 93f38fcf..00000000 --- a/vendor/github.com/valyala/fasthttp/go.sum +++ /dev/null @@ -1,10 +0,0 @@ -github.com/klauspost/compress v1.4.0 h1:8nsMz3tWa9SWWPL60G1V6CUsf4lLjWLTNEtibhe8gh8= -github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e h1:+lIPJOWl+jSiJOc70QXJ07+2eg2Jy2EC7Mi11BWujeM= -github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 h1:czFLhve3vsQetD6JOJ8NZZvGQIXlnN3/yXxbT6/awxI= -golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/vendor/github.com/valyala/fasthttp/header.go b/vendor/github.com/valyala/fasthttp/header.go deleted file mode 100644 index 190ac323..00000000 --- a/vendor/github.com/valyala/fasthttp/header.go +++ /dev/null @@ -1,2200 +0,0 @@ -package fasthttp - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - "sync/atomic" - "time" -) - -// ResponseHeader represents HTTP response header. -// -// It is forbidden copying ResponseHeader instances. -// Create new instances instead and use CopyTo. -// -// ResponseHeader instance MUST NOT be used from concurrently running -// goroutines. -type ResponseHeader struct { - noCopy noCopy - - disableNormalizing bool - noHTTP11 bool - connectionClose bool - noDefaultContentType bool - - statusCode int - contentLength int - contentLengthBytes []byte - - contentType []byte - server []byte - - h []argsKV - bufKV argsKV - - cookies []argsKV -} - -// RequestHeader represents HTTP request header. -// -// It is forbidden copying RequestHeader instances. -// Create new instances instead and use CopyTo. -// -// RequestHeader instance MUST NOT be used from concurrently running -// goroutines. -type RequestHeader struct { - noCopy noCopy - - disableNormalizing bool - noHTTP11 bool - connectionClose bool - - // These two fields have been moved close to other bool fields - // for reducing RequestHeader object size. - cookiesCollected bool - rawHeadersParsed bool - - contentLength int - contentLengthBytes []byte - - method []byte - requestURI []byte - host []byte - contentType []byte - userAgent []byte - - h []argsKV - bufKV argsKV - - cookies []argsKV - - rawHeaders []byte - - // stores an immutable copy of headers as they were received from the - // wire. - rawHeadersCopy []byte -} - -// SetContentRange sets 'Content-Range: bytes startPos-endPos/contentLength' -// header. -func (h *ResponseHeader) SetContentRange(startPos, endPos, contentLength int) { - b := h.bufKV.value[:0] - b = append(b, strBytes...) - b = append(b, ' ') - b = AppendUint(b, startPos) - b = append(b, '-') - b = AppendUint(b, endPos) - b = append(b, '/') - b = AppendUint(b, contentLength) - h.bufKV.value = b - - h.SetCanonical(strContentRange, h.bufKV.value) -} - -// SetByteRange sets 'Range: bytes=startPos-endPos' header. -// -// * If startPos is negative, then 'bytes=-startPos' value is set. -// * If endPos is negative, then 'bytes=startPos-' value is set. -func (h *RequestHeader) SetByteRange(startPos, endPos int) { - h.parseRawHeaders() - - b := h.bufKV.value[:0] - b = append(b, strBytes...) - b = append(b, '=') - if startPos >= 0 { - b = AppendUint(b, startPos) - } else { - endPos = -startPos - } - b = append(b, '-') - if endPos >= 0 { - b = AppendUint(b, endPos) - } - h.bufKV.value = b - - h.SetCanonical(strRange, h.bufKV.value) -} - -// StatusCode returns response status code. -func (h *ResponseHeader) StatusCode() int { - if h.statusCode == 0 { - return StatusOK - } - return h.statusCode -} - -// SetStatusCode sets response status code. -func (h *ResponseHeader) SetStatusCode(statusCode int) { - h.statusCode = statusCode -} - -// SetLastModified sets 'Last-Modified' header to the given value. -func (h *ResponseHeader) SetLastModified(t time.Time) { - h.bufKV.value = AppendHTTPDate(h.bufKV.value[:0], t) - h.SetCanonical(strLastModified, h.bufKV.value) -} - -// ConnectionClose returns true if 'Connection: close' header is set. -func (h *ResponseHeader) ConnectionClose() bool { - return h.connectionClose -} - -// SetConnectionClose sets 'Connection: close' header. -func (h *ResponseHeader) SetConnectionClose() { - h.connectionClose = true -} - -// ResetConnectionClose clears 'Connection: close' header if it exists. -func (h *ResponseHeader) ResetConnectionClose() { - if h.connectionClose { - h.connectionClose = false - h.h = delAllArgsBytes(h.h, strConnection) - } -} - -// ConnectionClose returns true if 'Connection: close' header is set. -func (h *RequestHeader) ConnectionClose() bool { - h.parseRawHeaders() - return h.connectionClose -} - -// SetConnectionClose sets 'Connection: close' header. -func (h *RequestHeader) SetConnectionClose() { - // h.parseRawHeaders() isn't called for performance reasons. - h.connectionClose = true -} - -// ResetConnectionClose clears 'Connection: close' header if it exists. -func (h *RequestHeader) ResetConnectionClose() { - h.parseRawHeaders() - if h.connectionClose { - h.connectionClose = false - h.h = delAllArgsBytes(h.h, strConnection) - } -} - -// ConnectionUpgrade returns true if 'Connection: Upgrade' header is set. -func (h *ResponseHeader) ConnectionUpgrade() bool { - return hasHeaderValue(h.Peek("Connection"), strUpgrade) -} - -// ConnectionUpgrade returns true if 'Connection: Upgrade' header is set. -func (h *RequestHeader) ConnectionUpgrade() bool { - h.parseRawHeaders() - return hasHeaderValue(h.Peek("Connection"), strUpgrade) -} - -// PeekCookie is able to returns cookie by a given key from response. -func (h *ResponseHeader) PeekCookie(key string) []byte { - return peekArgStr(h.cookies, key) -} - -// ContentLength returns Content-Length header value. -// -// It may be negative: -// -1 means Transfer-Encoding: chunked. -// -2 means Transfer-Encoding: identity. -func (h *ResponseHeader) ContentLength() int { - return h.contentLength -} - -// SetContentLength sets Content-Length header value. -// -// Content-Length may be negative: -// -1 means Transfer-Encoding: chunked. -// -2 means Transfer-Encoding: identity. -func (h *ResponseHeader) SetContentLength(contentLength int) { - if h.mustSkipContentLength() { - return - } - h.contentLength = contentLength - if contentLength >= 0 { - h.contentLengthBytes = AppendUint(h.contentLengthBytes[:0], contentLength) - h.h = delAllArgsBytes(h.h, strTransferEncoding) - } else { - h.contentLengthBytes = h.contentLengthBytes[:0] - value := strChunked - if contentLength == -2 { - h.SetConnectionClose() - value = strIdentity - } - h.h = setArgBytes(h.h, strTransferEncoding, value, argsHasValue) - } -} - -func (h *ResponseHeader) mustSkipContentLength() bool { - // From http/1.1 specs: - // All 1xx (informational), 204 (no content), and 304 (not modified) responses MUST NOT include a message-body - statusCode := h.StatusCode() - - // Fast path. - if statusCode < 100 || statusCode == StatusOK { - return false - } - - // Slow path. - return statusCode == StatusNotModified || statusCode == StatusNoContent || statusCode < 200 -} - -// ContentLength returns Content-Length header value. -// -// It may be negative: -// -1 means Transfer-Encoding: chunked. -func (h *RequestHeader) ContentLength() int { - if h.ignoreBody() { - return 0 - } - return h.realContentLength() -} - -// realContentLength returns the actual Content-Length set in the request, -// including positive lengths for GET/HEAD requests. -func (h *RequestHeader) realContentLength() int { - h.parseRawHeaders() - return h.contentLength -} - -// SetContentLength sets Content-Length header value. -// -// Negative content-length sets 'Transfer-Encoding: chunked' header. -func (h *RequestHeader) SetContentLength(contentLength int) { - h.parseRawHeaders() - h.contentLength = contentLength - if contentLength >= 0 { - h.contentLengthBytes = AppendUint(h.contentLengthBytes[:0], contentLength) - h.h = delAllArgsBytes(h.h, strTransferEncoding) - } else { - h.contentLengthBytes = h.contentLengthBytes[:0] - h.h = setArgBytes(h.h, strTransferEncoding, strChunked, argsHasValue) - } -} - -func (h *ResponseHeader) isCompressibleContentType() bool { - contentType := h.ContentType() - return bytes.HasPrefix(contentType, strTextSlash) || - bytes.HasPrefix(contentType, strApplicationSlash) -} - -// ContentType returns Content-Type header value. -func (h *ResponseHeader) ContentType() []byte { - contentType := h.contentType - if !h.noDefaultContentType && len(h.contentType) == 0 { - contentType = defaultContentType - } - return contentType -} - -// SetContentType sets Content-Type header value. -func (h *ResponseHeader) SetContentType(contentType string) { - h.contentType = append(h.contentType[:0], contentType...) -} - -// SetContentTypeBytes sets Content-Type header value. -func (h *ResponseHeader) SetContentTypeBytes(contentType []byte) { - h.contentType = append(h.contentType[:0], contentType...) -} - -// Server returns Server header value. -func (h *ResponseHeader) Server() []byte { - return h.server -} - -// SetServer sets Server header value. -func (h *ResponseHeader) SetServer(server string) { - h.server = append(h.server[:0], server...) -} - -// SetServerBytes sets Server header value. -func (h *ResponseHeader) SetServerBytes(server []byte) { - h.server = append(h.server[:0], server...) -} - -// ContentType returns Content-Type header value. -func (h *RequestHeader) ContentType() []byte { - h.parseRawHeaders() - return h.contentType -} - -// SetContentType sets Content-Type header value. -func (h *RequestHeader) SetContentType(contentType string) { - h.parseRawHeaders() - h.contentType = append(h.contentType[:0], contentType...) -} - -// SetContentTypeBytes sets Content-Type header value. -func (h *RequestHeader) SetContentTypeBytes(contentType []byte) { - h.parseRawHeaders() - h.contentType = append(h.contentType[:0], contentType...) -} - -// SetMultipartFormBoundary sets the following Content-Type: -// 'multipart/form-data; boundary=...' -// where ... is substituted by the given boundary. -func (h *RequestHeader) SetMultipartFormBoundary(boundary string) { - h.parseRawHeaders() - - b := h.bufKV.value[:0] - b = append(b, strMultipartFormData...) - b = append(b, ';', ' ') - b = append(b, strBoundary...) - b = append(b, '=') - b = append(b, boundary...) - h.bufKV.value = b - - h.SetContentTypeBytes(h.bufKV.value) -} - -// SetMultipartFormBoundaryBytes sets the following Content-Type: -// 'multipart/form-data; boundary=...' -// where ... is substituted by the given boundary. -func (h *RequestHeader) SetMultipartFormBoundaryBytes(boundary []byte) { - h.parseRawHeaders() - - b := h.bufKV.value[:0] - b = append(b, strMultipartFormData...) - b = append(b, ';', ' ') - b = append(b, strBoundary...) - b = append(b, '=') - b = append(b, boundary...) - h.bufKV.value = b - - h.SetContentTypeBytes(h.bufKV.value) -} - -// MultipartFormBoundary returns boundary part -// from 'multipart/form-data; boundary=...' Content-Type. -func (h *RequestHeader) MultipartFormBoundary() []byte { - b := h.ContentType() - if !bytes.HasPrefix(b, strMultipartFormData) { - return nil - } - b = b[len(strMultipartFormData):] - if len(b) == 0 || b[0] != ';' { - return nil - } - - var n int - for len(b) > 0 { - n++ - for len(b) > n && b[n] == ' ' { - n++ - } - b = b[n:] - if !bytes.HasPrefix(b, strBoundary) { - if n = bytes.IndexByte(b, ';'); n < 0 { - return nil - } - continue - } - - b = b[len(strBoundary):] - if len(b) == 0 || b[0] != '=' { - return nil - } - b = b[1:] - if n = bytes.IndexByte(b, ';'); n >= 0 { - b = b[:n] - } - if len(b) > 1 && b[0] == '"' && b[len(b)-1] == '"' { - b = b[1 : len(b)-1] - } - return b - } - return nil -} - -// Host returns Host header value. -func (h *RequestHeader) Host() []byte { - if len(h.host) > 0 { - return h.host - } - if !h.rawHeadersParsed { - // fast path without employing full headers parsing. - host := peekRawHeader(h.rawHeaders, strHost) - if len(host) > 0 { - h.host = append(h.host[:0], host...) - return h.host - } - } - - // slow path. - h.parseRawHeaders() - return h.host -} - -// SetHost sets Host header value. -func (h *RequestHeader) SetHost(host string) { - h.parseRawHeaders() - h.host = append(h.host[:0], host...) -} - -// SetHostBytes sets Host header value. -func (h *RequestHeader) SetHostBytes(host []byte) { - h.parseRawHeaders() - h.host = append(h.host[:0], host...) -} - -// UserAgent returns User-Agent header value. -func (h *RequestHeader) UserAgent() []byte { - h.parseRawHeaders() - return h.userAgent -} - -// SetUserAgent sets User-Agent header value. -func (h *RequestHeader) SetUserAgent(userAgent string) { - h.parseRawHeaders() - h.userAgent = append(h.userAgent[:0], userAgent...) -} - -// SetUserAgentBytes sets User-Agent header value. -func (h *RequestHeader) SetUserAgentBytes(userAgent []byte) { - h.parseRawHeaders() - h.userAgent = append(h.userAgent[:0], userAgent...) -} - -// Referer returns Referer header value. -func (h *RequestHeader) Referer() []byte { - return h.PeekBytes(strReferer) -} - -// SetReferer sets Referer header value. -func (h *RequestHeader) SetReferer(referer string) { - h.SetBytesK(strReferer, referer) -} - -// SetRefererBytes sets Referer header value. -func (h *RequestHeader) SetRefererBytes(referer []byte) { - h.SetCanonical(strReferer, referer) -} - -// Method returns HTTP request method. -func (h *RequestHeader) Method() []byte { - if len(h.method) == 0 { - return strGet - } - return h.method -} - -// SetMethod sets HTTP request method. -func (h *RequestHeader) SetMethod(method string) { - h.method = append(h.method[:0], method...) -} - -// SetMethodBytes sets HTTP request method. -func (h *RequestHeader) SetMethodBytes(method []byte) { - h.method = append(h.method[:0], method...) -} - -// RequestURI returns RequestURI from the first HTTP request line. -func (h *RequestHeader) RequestURI() []byte { - requestURI := h.requestURI - if len(requestURI) == 0 { - requestURI = strSlash - } - return requestURI -} - -// SetRequestURI sets RequestURI for the first HTTP request line. -// RequestURI must be properly encoded. -// Use URI.RequestURI for constructing proper RequestURI if unsure. -func (h *RequestHeader) SetRequestURI(requestURI string) { - h.requestURI = append(h.requestURI[:0], requestURI...) -} - -// SetRequestURIBytes sets RequestURI for the first HTTP request line. -// RequestURI must be properly encoded. -// Use URI.RequestURI for constructing proper RequestURI if unsure. -func (h *RequestHeader) SetRequestURIBytes(requestURI []byte) { - h.requestURI = append(h.requestURI[:0], requestURI...) -} - -// IsGet returns true if request method is GET. -func (h *RequestHeader) IsGet() bool { - return bytes.Equal(h.Method(), strGet) -} - -// IsPost returns true if request method is POST. -func (h *RequestHeader) IsPost() bool { - return bytes.Equal(h.Method(), strPost) -} - -// IsPut returns true if request method is PUT. -func (h *RequestHeader) IsPut() bool { - return bytes.Equal(h.Method(), strPut) -} - -// IsHead returns true if request method is HEAD. -func (h *RequestHeader) IsHead() bool { - return bytes.Equal(h.Method(), strHead) -} - -// IsDelete returns true if request method is DELETE. -func (h *RequestHeader) IsDelete() bool { - return bytes.Equal(h.Method(), strDelete) -} - -// IsConnect returns true if request method is CONNECT. -func (h *RequestHeader) IsConnect() bool { - return bytes.Equal(h.Method(), strConnect) -} - -// IsOptions returns true if request method is OPTIONS. -func (h *RequestHeader) IsOptions() bool { - return bytes.Equal(h.Method(), strOptions) -} - -// IsTrace returns true if request method is TRACE. -func (h *RequestHeader) IsTrace() bool { - return bytes.Equal(h.Method(), strTrace) -} - -// IsPatch returns true if request method is PATCH. -func (h *RequestHeader) IsPatch() bool { - return bytes.Equal(h.Method(), strPatch) -} - -// IsHTTP11 returns true if the request is HTTP/1.1. -func (h *RequestHeader) IsHTTP11() bool { - return !h.noHTTP11 -} - -// IsHTTP11 returns true if the response is HTTP/1.1. -func (h *ResponseHeader) IsHTTP11() bool { - return !h.noHTTP11 -} - -// HasAcceptEncoding returns true if the header contains -// the given Accept-Encoding value. -func (h *RequestHeader) HasAcceptEncoding(acceptEncoding string) bool { - h.bufKV.value = append(h.bufKV.value[:0], acceptEncoding...) - return h.HasAcceptEncodingBytes(h.bufKV.value) -} - -// HasAcceptEncodingBytes returns true if the header contains -// the given Accept-Encoding value. -func (h *RequestHeader) HasAcceptEncodingBytes(acceptEncoding []byte) bool { - ae := h.peek(strAcceptEncoding) - n := bytes.Index(ae, acceptEncoding) - if n < 0 { - return false - } - b := ae[n+len(acceptEncoding):] - if len(b) > 0 && b[0] != ',' { - return false - } - if n == 0 { - return true - } - return ae[n-1] == ' ' -} - -// Len returns the number of headers set, -// i.e. the number of times f is called in VisitAll. -func (h *ResponseHeader) Len() int { - n := 0 - h.VisitAll(func(k, v []byte) { n++ }) - return n -} - -// Len returns the number of headers set, -// i.e. the number of times f is called in VisitAll. -func (h *RequestHeader) Len() int { - n := 0 - h.VisitAll(func(k, v []byte) { n++ }) - return n -} - -// DisableNormalizing disables header names' normalization. -// -// By default all the header names are normalized by uppercasing -// the first letter and all the first letters following dashes, -// while lowercasing all the other letters. -// Examples: -// -// * CONNECTION -> Connection -// * conteNT-tYPE -> Content-Type -// * foo-bar-baz -> Foo-Bar-Baz -// -// Disable header names' normalization only if know what are you doing. -func (h *RequestHeader) DisableNormalizing() { - h.disableNormalizing = true -} - -// DisableNormalizing disables header names' normalization. -// -// By default all the header names are normalized by uppercasing -// the first letter and all the first letters following dashes, -// while lowercasing all the other letters. -// Examples: -// -// * CONNECTION -> Connection -// * conteNT-tYPE -> Content-Type -// * foo-bar-baz -> Foo-Bar-Baz -// -// Disable header names' normalization only if know what are you doing. -func (h *ResponseHeader) DisableNormalizing() { - h.disableNormalizing = true -} - -// Reset clears response header. -func (h *ResponseHeader) Reset() { - h.disableNormalizing = false - h.noDefaultContentType = false - h.resetSkipNormalize() -} - -func (h *ResponseHeader) resetSkipNormalize() { - h.noHTTP11 = false - h.connectionClose = false - - h.statusCode = 0 - h.contentLength = 0 - h.contentLengthBytes = h.contentLengthBytes[:0] - - h.contentType = h.contentType[:0] - h.server = h.server[:0] - - h.h = h.h[:0] - h.cookies = h.cookies[:0] -} - -// Reset clears request header. -func (h *RequestHeader) Reset() { - h.disableNormalizing = false - h.resetSkipNormalize() -} - -func (h *RequestHeader) resetSkipNormalize() { - h.noHTTP11 = false - h.connectionClose = false - - h.contentLength = 0 - h.contentLengthBytes = h.contentLengthBytes[:0] - - h.method = h.method[:0] - h.requestURI = h.requestURI[:0] - h.host = h.host[:0] - h.contentType = h.contentType[:0] - h.userAgent = h.userAgent[:0] - - h.h = h.h[:0] - h.cookies = h.cookies[:0] - h.cookiesCollected = false - - h.rawHeaders = h.rawHeaders[:0] - h.rawHeadersParsed = false -} - -// CopyTo copies all the headers to dst. -func (h *ResponseHeader) CopyTo(dst *ResponseHeader) { - dst.Reset() - - dst.disableNormalizing = h.disableNormalizing - dst.noHTTP11 = h.noHTTP11 - dst.connectionClose = h.connectionClose - dst.noDefaultContentType = h.noDefaultContentType - - dst.statusCode = h.statusCode - dst.contentLength = h.contentLength - dst.contentLengthBytes = append(dst.contentLengthBytes[:0], h.contentLengthBytes...) - dst.contentType = append(dst.contentType[:0], h.contentType...) - dst.server = append(dst.server[:0], h.server...) - dst.h = copyArgs(dst.h, h.h) - dst.cookies = copyArgs(dst.cookies, h.cookies) -} - -// CopyTo copies all the headers to dst. -func (h *RequestHeader) CopyTo(dst *RequestHeader) { - dst.Reset() - - dst.disableNormalizing = h.disableNormalizing - dst.noHTTP11 = h.noHTTP11 - dst.connectionClose = h.connectionClose - - dst.contentLength = h.contentLength - dst.contentLengthBytes = append(dst.contentLengthBytes[:0], h.contentLengthBytes...) - dst.method = append(dst.method[:0], h.method...) - dst.requestURI = append(dst.requestURI[:0], h.requestURI...) - dst.host = append(dst.host[:0], h.host...) - dst.contentType = append(dst.contentType[:0], h.contentType...) - dst.userAgent = append(dst.userAgent[:0], h.userAgent...) - dst.h = copyArgs(dst.h, h.h) - dst.cookies = copyArgs(dst.cookies, h.cookies) - dst.cookiesCollected = h.cookiesCollected - dst.rawHeaders = append(dst.rawHeaders[:0], h.rawHeaders...) - dst.rawHeadersParsed = h.rawHeadersParsed - dst.rawHeadersCopy = append(dst.rawHeadersCopy[:0], h.rawHeadersCopy...) -} - -// VisitAll calls f for each header. -// -// f must not retain references to key and/or value after returning. -// Copy key and/or value contents before returning if you need retaining them. -func (h *ResponseHeader) VisitAll(f func(key, value []byte)) { - if len(h.contentLengthBytes) > 0 { - f(strContentLength, h.contentLengthBytes) - } - contentType := h.ContentType() - if len(contentType) > 0 { - f(strContentType, contentType) - } - server := h.Server() - if len(server) > 0 { - f(strServer, server) - } - if len(h.cookies) > 0 { - visitArgs(h.cookies, func(k, v []byte) { - f(strSetCookie, v) - }) - } - visitArgs(h.h, f) - if h.ConnectionClose() { - f(strConnection, strClose) - } -} - -// VisitAllCookie calls f for each response cookie. -// -// Cookie name is passed in key and the whole Set-Cookie header value -// is passed in value on each f invocation. Value may be parsed -// with Cookie.ParseBytes(). -// -// f must not retain references to key and/or value after returning. -func (h *ResponseHeader) VisitAllCookie(f func(key, value []byte)) { - visitArgs(h.cookies, f) -} - -// VisitAllCookie calls f for each request cookie. -// -// f must not retain references to key and/or value after returning. -func (h *RequestHeader) VisitAllCookie(f func(key, value []byte)) { - h.parseRawHeaders() - h.collectCookies() - visitArgs(h.cookies, f) -} - -// VisitAll calls f for each header. -// -// f must not retain references to key and/or value after returning. -// Copy key and/or value contents before returning if you need retaining them. -// -// To get the headers in order they were received use VisitAllInOrder. -func (h *RequestHeader) VisitAll(f func(key, value []byte)) { - h.parseRawHeaders() - host := h.Host() - if len(host) > 0 { - f(strHost, host) - } - if len(h.contentLengthBytes) > 0 { - f(strContentLength, h.contentLengthBytes) - } - contentType := h.ContentType() - if len(contentType) > 0 { - f(strContentType, contentType) - } - userAgent := h.UserAgent() - if len(userAgent) > 0 { - f(strUserAgent, userAgent) - } - - h.collectCookies() - if len(h.cookies) > 0 { - h.bufKV.value = appendRequestCookieBytes(h.bufKV.value[:0], h.cookies) - f(strCookie, h.bufKV.value) - } - visitArgs(h.h, f) - if h.ConnectionClose() { - f(strConnection, strClose) - } -} - -// VisitAllInOrder calls f for each header in the order they were received. -// -// f must not retain references to key and/or value after returning. -// Copy key and/or value contents before returning if you need retaining them. -// -// This function is slightly slower than VisitAll because it has to reparse the -// raw headers to get the order. -func (h *RequestHeader) VisitAllInOrder(f func(key, value []byte)) { - h.parseRawHeaders() - var s headerScanner - s.b = h.rawHeaders - s.disableNormalizing = h.disableNormalizing - for s.next() { - if len(s.key) > 0 { - f(s.key, s.value) - } - } -} - -// Del deletes header with the given key. -func (h *ResponseHeader) Del(key string) { - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - h.del(k) -} - -// DelBytes deletes header with the given key. -func (h *ResponseHeader) DelBytes(key []byte) { - h.bufKV.key = append(h.bufKV.key[:0], key...) - normalizeHeaderKey(h.bufKV.key, h.disableNormalizing) - h.del(h.bufKV.key) -} - -func (h *ResponseHeader) del(key []byte) { - switch string(key) { - case "Content-Type": - h.contentType = h.contentType[:0] - case "Server": - h.server = h.server[:0] - case "Set-Cookie": - h.cookies = h.cookies[:0] - case "Content-Length": - h.contentLength = 0 - h.contentLengthBytes = h.contentLengthBytes[:0] - case "Connection": - h.connectionClose = false - } - h.h = delAllArgsBytes(h.h, key) -} - -// Del deletes header with the given key. -func (h *RequestHeader) Del(key string) { - h.parseRawHeaders() - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - h.del(k) -} - -// DelBytes deletes header with the given key. -func (h *RequestHeader) DelBytes(key []byte) { - h.parseRawHeaders() - h.bufKV.key = append(h.bufKV.key[:0], key...) - normalizeHeaderKey(h.bufKV.key, h.disableNormalizing) - h.del(h.bufKV.key) -} - -func (h *RequestHeader) del(key []byte) { - switch string(key) { - case "Host": - h.host = h.host[:0] - case "Content-Type": - h.contentType = h.contentType[:0] - case "User-Agent": - h.userAgent = h.userAgent[:0] - case "Cookie": - h.cookies = h.cookies[:0] - case "Content-Length": - h.contentLength = 0 - h.contentLengthBytes = h.contentLengthBytes[:0] - case "Connection": - h.connectionClose = false - } - h.h = delAllArgsBytes(h.h, key) -} - -// Add adds the given 'key: value' header. -// -// Multiple headers with the same key may be added with this function. -// Use Set for setting a single header for the given key. -func (h *ResponseHeader) Add(key, value string) { - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - h.h = appendArg(h.h, b2s(k), value, argsHasValue) -} - -// AddBytesK adds the given 'key: value' header. -// -// Multiple headers with the same key may be added with this function. -// Use SetBytesK for setting a single header for the given key. -func (h *ResponseHeader) AddBytesK(key []byte, value string) { - h.Add(b2s(key), value) -} - -// AddBytesV adds the given 'key: value' header. -// -// Multiple headers with the same key may be added with this function. -// Use SetBytesV for setting a single header for the given key. -func (h *ResponseHeader) AddBytesV(key string, value []byte) { - h.Add(key, b2s(value)) -} - -// AddBytesKV adds the given 'key: value' header. -// -// Multiple headers with the same key may be added with this function. -// Use SetBytesKV for setting a single header for the given key. -func (h *ResponseHeader) AddBytesKV(key, value []byte) { - h.Add(b2s(key), b2s(value)) -} - -// Set sets the given 'key: value' header. -// -// Use Add for setting multiple header values under the same key. -func (h *ResponseHeader) Set(key, value string) { - initHeaderKV(&h.bufKV, key, value, h.disableNormalizing) - h.SetCanonical(h.bufKV.key, h.bufKV.value) -} - -// SetBytesK sets the given 'key: value' header. -// -// Use AddBytesK for setting multiple header values under the same key. -func (h *ResponseHeader) SetBytesK(key []byte, value string) { - h.bufKV.value = append(h.bufKV.value[:0], value...) - h.SetBytesKV(key, h.bufKV.value) -} - -// SetBytesV sets the given 'key: value' header. -// -// Use AddBytesV for setting multiple header values under the same key. -func (h *ResponseHeader) SetBytesV(key string, value []byte) { - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - h.SetCanonical(k, value) -} - -// SetBytesKV sets the given 'key: value' header. -// -// Use AddBytesKV for setting multiple header values under the same key. -func (h *ResponseHeader) SetBytesKV(key, value []byte) { - h.bufKV.key = append(h.bufKV.key[:0], key...) - normalizeHeaderKey(h.bufKV.key, h.disableNormalizing) - h.SetCanonical(h.bufKV.key, value) -} - -// SetCanonical sets the given 'key: value' header assuming that -// key is in canonical form. -func (h *ResponseHeader) SetCanonical(key, value []byte) { - switch string(key) { - case "Content-Type": - h.SetContentTypeBytes(value) - case "Server": - h.SetServerBytes(value) - case "Set-Cookie": - var kv *argsKV - h.cookies, kv = allocArg(h.cookies) - kv.key = getCookieKey(kv.key, value) - kv.value = append(kv.value[:0], value...) - case "Content-Length": - if contentLength, err := parseContentLength(value); err == nil { - h.contentLength = contentLength - h.contentLengthBytes = append(h.contentLengthBytes[:0], value...) - } - case "Connection": - if bytes.Equal(strClose, value) { - h.SetConnectionClose() - } else { - h.ResetConnectionClose() - h.h = setArgBytes(h.h, key, value, argsHasValue) - } - case "Transfer-Encoding": - // Transfer-Encoding is managed automatically. - case "Date": - // Date is managed automatically. - default: - h.h = setArgBytes(h.h, key, value, argsHasValue) - } -} - -// SetCookie sets the given response cookie. -// -// It is save re-using the cookie after the function returns. -func (h *ResponseHeader) SetCookie(cookie *Cookie) { - h.cookies = setArgBytes(h.cookies, cookie.Key(), cookie.Cookie(), argsHasValue) -} - -// SetCookie sets 'key: value' cookies. -func (h *RequestHeader) SetCookie(key, value string) { - h.parseRawHeaders() - h.collectCookies() - h.cookies = setArg(h.cookies, key, value, argsHasValue) -} - -// SetCookieBytesK sets 'key: value' cookies. -func (h *RequestHeader) SetCookieBytesK(key []byte, value string) { - h.SetCookie(b2s(key), value) -} - -// SetCookieBytesKV sets 'key: value' cookies. -func (h *RequestHeader) SetCookieBytesKV(key, value []byte) { - h.SetCookie(b2s(key), b2s(value)) -} - -// DelClientCookie instructs the client to remove the given cookie. -// -// Use DelCookie if you want just removing the cookie from response header. -func (h *ResponseHeader) DelClientCookie(key string) { - h.DelCookie(key) - - c := AcquireCookie() - c.SetKey(key) - c.SetExpire(CookieExpireDelete) - h.SetCookie(c) - ReleaseCookie(c) -} - -// DelClientCookieBytes instructs the client to remove the given cookie. -// -// Use DelCookieBytes if you want just removing the cookie from response header. -func (h *ResponseHeader) DelClientCookieBytes(key []byte) { - h.DelClientCookie(b2s(key)) -} - -// DelCookie removes cookie under the given key from response header. -// -// Note that DelCookie doesn't remove the cookie from the client. -// Use DelClientCookie instead. -func (h *ResponseHeader) DelCookie(key string) { - h.cookies = delAllArgs(h.cookies, key) -} - -// DelCookieBytes removes cookie under the given key from response header. -// -// Note that DelCookieBytes doesn't remove the cookie from the client. -// Use DelClientCookieBytes instead. -func (h *ResponseHeader) DelCookieBytes(key []byte) { - h.DelCookie(b2s(key)) -} - -// DelCookie removes cookie under the given key. -func (h *RequestHeader) DelCookie(key string) { - h.parseRawHeaders() - h.collectCookies() - h.cookies = delAllArgs(h.cookies, key) -} - -// DelCookieBytes removes cookie under the given key. -func (h *RequestHeader) DelCookieBytes(key []byte) { - h.DelCookie(b2s(key)) -} - -// DelAllCookies removes all the cookies from response headers. -func (h *ResponseHeader) DelAllCookies() { - h.cookies = h.cookies[:0] -} - -// DelAllCookies removes all the cookies from request headers. -func (h *RequestHeader) DelAllCookies() { - h.parseRawHeaders() - h.collectCookies() - h.cookies = h.cookies[:0] -} - -// Add adds the given 'key: value' header. -// -// Multiple headers with the same key may be added with this function. -// Use Set for setting a single header for the given key. -func (h *RequestHeader) Add(key, value string) { - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - h.h = appendArg(h.h, b2s(k), value, argsHasValue) -} - -// AddBytesK adds the given 'key: value' header. -// -// Multiple headers with the same key may be added with this function. -// Use SetBytesK for setting a single header for the given key. -func (h *RequestHeader) AddBytesK(key []byte, value string) { - h.Add(b2s(key), value) -} - -// AddBytesV adds the given 'key: value' header. -// -// Multiple headers with the same key may be added with this function. -// Use SetBytesV for setting a single header for the given key. -func (h *RequestHeader) AddBytesV(key string, value []byte) { - h.Add(key, b2s(value)) -} - -// AddBytesKV adds the given 'key: value' header. -// -// Multiple headers with the same key may be added with this function. -// Use SetBytesKV for setting a single header for the given key. -func (h *RequestHeader) AddBytesKV(key, value []byte) { - h.Add(b2s(key), b2s(value)) -} - -// Set sets the given 'key: value' header. -// -// Use Add for setting multiple header values under the same key. -func (h *RequestHeader) Set(key, value string) { - initHeaderKV(&h.bufKV, key, value, h.disableNormalizing) - h.SetCanonical(h.bufKV.key, h.bufKV.value) -} - -// SetBytesK sets the given 'key: value' header. -// -// Use AddBytesK for setting multiple header values under the same key. -func (h *RequestHeader) SetBytesK(key []byte, value string) { - h.bufKV.value = append(h.bufKV.value[:0], value...) - h.SetBytesKV(key, h.bufKV.value) -} - -// SetBytesV sets the given 'key: value' header. -// -// Use AddBytesV for setting multiple header values under the same key. -func (h *RequestHeader) SetBytesV(key string, value []byte) { - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - h.SetCanonical(k, value) -} - -// SetBytesKV sets the given 'key: value' header. -// -// Use AddBytesKV for setting multiple header values under the same key. -func (h *RequestHeader) SetBytesKV(key, value []byte) { - h.bufKV.key = append(h.bufKV.key[:0], key...) - normalizeHeaderKey(h.bufKV.key, h.disableNormalizing) - h.SetCanonical(h.bufKV.key, value) -} - -// SetCanonical sets the given 'key: value' header assuming that -// key is in canonical form. -func (h *RequestHeader) SetCanonical(key, value []byte) { - h.parseRawHeaders() - switch string(key) { - case "Host": - h.SetHostBytes(value) - case "Content-Type": - h.SetContentTypeBytes(value) - case "User-Agent": - h.SetUserAgentBytes(value) - case "Cookie": - h.collectCookies() - h.cookies = parseRequestCookies(h.cookies, value) - case "Content-Length": - if contentLength, err := parseContentLength(value); err == nil { - h.contentLength = contentLength - h.contentLengthBytes = append(h.contentLengthBytes[:0], value...) - } - case "Connection": - if bytes.Equal(strClose, value) { - h.SetConnectionClose() - } else { - h.ResetConnectionClose() - h.h = setArgBytes(h.h, key, value, argsHasValue) - } - case "Transfer-Encoding": - // Transfer-Encoding is managed automatically. - default: - h.h = setArgBytes(h.h, key, value, argsHasValue) - } -} - -// Peek returns header value for the given key. -// -// Returned value is valid until the next call to ResponseHeader. -// Do not store references to returned value. Make copies instead. -func (h *ResponseHeader) Peek(key string) []byte { - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - return h.peek(k) -} - -// PeekBytes returns header value for the given key. -// -// Returned value is valid until the next call to ResponseHeader. -// Do not store references to returned value. Make copies instead. -func (h *ResponseHeader) PeekBytes(key []byte) []byte { - h.bufKV.key = append(h.bufKV.key[:0], key...) - normalizeHeaderKey(h.bufKV.key, h.disableNormalizing) - return h.peek(h.bufKV.key) -} - -// Peek returns header value for the given key. -// -// Returned value is valid until the next call to RequestHeader. -// Do not store references to returned value. Make copies instead. -func (h *RequestHeader) Peek(key string) []byte { - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - return h.peek(k) -} - -// PeekBytes returns header value for the given key. -// -// Returned value is valid until the next call to RequestHeader. -// Do not store references to returned value. Make copies instead. -func (h *RequestHeader) PeekBytes(key []byte) []byte { - h.bufKV.key = append(h.bufKV.key[:0], key...) - normalizeHeaderKey(h.bufKV.key, h.disableNormalizing) - return h.peek(h.bufKV.key) -} - -func (h *ResponseHeader) peek(key []byte) []byte { - switch string(key) { - case "Content-Type": - return h.ContentType() - case "Server": - return h.Server() - case "Connection": - if h.ConnectionClose() { - return strClose - } - return peekArgBytes(h.h, key) - case "Content-Length": - return h.contentLengthBytes - case "Set-Cookie": - return appendResponseCookieBytes(nil, h.cookies) - default: - return peekArgBytes(h.h, key) - } -} - -func (h *RequestHeader) peek(key []byte) []byte { - h.parseRawHeaders() - switch string(key) { - case "Host": - return h.Host() - case "Content-Type": - return h.ContentType() - case "User-Agent": - return h.UserAgent() - case "Connection": - if h.ConnectionClose() { - return strClose - } - return peekArgBytes(h.h, key) - case "Content-Length": - return h.contentLengthBytes - case "Cookie": - if h.cookiesCollected { - return appendRequestCookieBytes(nil, h.cookies) - } else { - return peekArgBytes(h.h, key) - } - default: - return peekArgBytes(h.h, key) - } -} - -// Cookie returns cookie for the given key. -func (h *RequestHeader) Cookie(key string) []byte { - h.parseRawHeaders() - h.collectCookies() - return peekArgStr(h.cookies, key) -} - -// CookieBytes returns cookie for the given key. -func (h *RequestHeader) CookieBytes(key []byte) []byte { - h.parseRawHeaders() - h.collectCookies() - return peekArgBytes(h.cookies, key) -} - -// Cookie fills cookie for the given cookie.Key. -// -// Returns false if cookie with the given cookie.Key is missing. -func (h *ResponseHeader) Cookie(cookie *Cookie) bool { - v := peekArgBytes(h.cookies, cookie.Key()) - if v == nil { - return false - } - cookie.ParseBytes(v) - return true -} - -// Read reads response header from r. -// -// io.EOF is returned if r is closed before reading the first header byte. -func (h *ResponseHeader) Read(r *bufio.Reader) error { - n := 1 - for { - err := h.tryRead(r, n) - if err == nil { - return nil - } - if err != errNeedMore { - h.resetSkipNormalize() - return err - } - n = r.Buffered() + 1 - } -} - -func (h *ResponseHeader) tryRead(r *bufio.Reader, n int) error { - h.resetSkipNormalize() - b, err := r.Peek(n) - if len(b) == 0 { - // treat all errors on the first byte read as EOF - if n == 1 || err == io.EOF { - return io.EOF - } - - // This is for go 1.6 bug. See https://github.com/golang/go/issues/14121 . - if err == bufio.ErrBufferFull { - return &ErrSmallBuffer{ - error: fmt.Errorf("error when reading response headers: %s", errSmallBuffer), - } - } - - return fmt.Errorf("error when reading response headers: %s", err) - } - b = mustPeekBuffered(r) - headersLen, errParse := h.parse(b) - if errParse != nil { - return headerError("response", err, errParse, b) - } - mustDiscard(r, headersLen) - return nil -} - -func headerError(typ string, err, errParse error, b []byte) error { - if errParse != errNeedMore { - return headerErrorMsg(typ, errParse, b) - } - if err == nil { - return errNeedMore - } - - // Buggy servers may leave trailing CRLFs after http body. - // Treat this case as EOF. - if isOnlyCRLF(b) { - return io.EOF - } - - if err != bufio.ErrBufferFull { - return headerErrorMsg(typ, err, b) - } - return &ErrSmallBuffer{ - error: headerErrorMsg(typ, errSmallBuffer, b), - } -} - -func headerErrorMsg(typ string, err error, b []byte) error { - return fmt.Errorf("error when reading %s headers: %s. Buffer size=%d, contents: %s", typ, err, len(b), bufferSnippet(b)) -} - -// Read reads request header from r. -// -// io.EOF is returned if r is closed before reading the first header byte. -func (h *RequestHeader) Read(r *bufio.Reader) error { - n := 1 - for { - err := h.tryRead(r, n) - if err == nil { - return nil - } - if err != errNeedMore { - h.resetSkipNormalize() - return err - } - n = r.Buffered() + 1 - } -} - -func (h *RequestHeader) tryRead(r *bufio.Reader, n int) error { - h.resetSkipNormalize() - b, err := r.Peek(n) - if len(b) == 0 { - if err == io.EOF { - return err - } - - if err == nil { - panic("bufio.Reader.Peek() returned nil, nil") - } - - // This is for go 1.6 bug. See https://github.com/golang/go/issues/14121 . - if err == bufio.ErrBufferFull { - return &ErrSmallBuffer{ - error: fmt.Errorf("error when reading request headers: %s", errSmallBuffer), - } - } - - if n == 1 { - // We didn't read a single byte. - return errNothingRead - } - - return fmt.Errorf("error when reading request headers: %s", err) - } - b = mustPeekBuffered(r) - headersLen, errParse := h.parse(b) - if errParse != nil { - return headerError("request", err, errParse, b) - } - mustDiscard(r, headersLen) - return nil -} - -func bufferSnippet(b []byte) string { - n := len(b) - start := 200 - end := n - start - if start >= end { - start = n - end = n - } - bStart, bEnd := b[:start], b[end:] - if len(bEnd) == 0 { - return fmt.Sprintf("%q", b) - } - return fmt.Sprintf("%q...%q", bStart, bEnd) -} - -func isOnlyCRLF(b []byte) bool { - for _, ch := range b { - if ch != '\r' && ch != '\n' { - return false - } - } - return true -} - -func init() { - refreshServerDate() - go func() { - for { - time.Sleep(time.Second) - refreshServerDate() - } - }() -} - -var serverDate atomic.Value - -func refreshServerDate() { - b := AppendHTTPDate(nil, time.Now()) - serverDate.Store(b) -} - -// Write writes response header to w. -func (h *ResponseHeader) Write(w *bufio.Writer) error { - _, err := w.Write(h.Header()) - return err -} - -// WriteTo writes response header to w. -// -// WriteTo implements io.WriterTo interface. -func (h *ResponseHeader) WriteTo(w io.Writer) (int64, error) { - n, err := w.Write(h.Header()) - return int64(n), err -} - -// Header returns response header representation. -// -// The returned value is valid until the next call to ResponseHeader methods. -func (h *ResponseHeader) Header() []byte { - h.bufKV.value = h.AppendBytes(h.bufKV.value[:0]) - return h.bufKV.value -} - -// String returns response header representation. -func (h *ResponseHeader) String() string { - return string(h.Header()) -} - -// AppendBytes appends response header representation to dst and returns -// the extended dst. -func (h *ResponseHeader) AppendBytes(dst []byte) []byte { - statusCode := h.StatusCode() - if statusCode < 0 { - statusCode = StatusOK - } - dst = append(dst, statusLine(statusCode)...) - - server := h.Server() - if len(server) != 0 { - dst = appendHeaderLine(dst, strServer, server) - } - dst = appendHeaderLine(dst, strDate, serverDate.Load().([]byte)) - - // Append Content-Type only for non-zero responses - // or if it is explicitly set. - // See https://github.com/valyala/fasthttp/issues/28 . - if h.ContentLength() != 0 || len(h.contentType) > 0 { - dst = appendHeaderLine(dst, strContentType, h.ContentType()) - } - - if len(h.contentLengthBytes) > 0 { - dst = appendHeaderLine(dst, strContentLength, h.contentLengthBytes) - } - - for i, n := 0, len(h.h); i < n; i++ { - kv := &h.h[i] - if !bytes.Equal(kv.key, strDate) { - dst = appendHeaderLine(dst, kv.key, kv.value) - } - } - - n := len(h.cookies) - if n > 0 { - for i := 0; i < n; i++ { - kv := &h.cookies[i] - dst = appendHeaderLine(dst, strSetCookie, kv.value) - } - } - - if h.ConnectionClose() { - dst = appendHeaderLine(dst, strConnection, strClose) - } - - return append(dst, strCRLF...) -} - -// Write writes request header to w. -func (h *RequestHeader) Write(w *bufio.Writer) error { - _, err := w.Write(h.Header()) - return err -} - -// WriteTo writes request header to w. -// -// WriteTo implements io.WriterTo interface. -func (h *RequestHeader) WriteTo(w io.Writer) (int64, error) { - n, err := w.Write(h.Header()) - return int64(n), err -} - -// Header returns request header representation. -// -// The returned representation is valid until the next call to RequestHeader methods. -func (h *RequestHeader) Header() []byte { - h.bufKV.value = h.AppendBytes(h.bufKV.value[:0]) - return h.bufKV.value -} - -// RawHeaders returns raw header key/value bytes. -// -// Depending on server configuration, header keys may be normalized to -// capital-case in place. -// -// This copy is set aside during parsing, so empty slice is returned for all -// cases where parsing did not happen. Similarly, request line is not stored -// during parsing and can not be returned. -// -// The slice is not safe to use after the handler returns. -func (h *RequestHeader) RawHeaders() []byte { - return h.rawHeadersCopy -} - -// String returns request header representation. -func (h *RequestHeader) String() string { - return string(h.Header()) -} - -// AppendBytes appends request header representation to dst and returns -// the extended dst. -func (h *RequestHeader) AppendBytes(dst []byte) []byte { - // there is no need in h.parseRawHeaders() here - raw headers are specially handled below. - dst = append(dst, h.Method()...) - dst = append(dst, ' ') - dst = append(dst, h.RequestURI()...) - dst = append(dst, ' ') - dst = append(dst, strHTTP11...) - dst = append(dst, strCRLF...) - - if !h.rawHeadersParsed && len(h.rawHeaders) > 0 { - return append(dst, h.rawHeaders...) - } - - userAgent := h.UserAgent() - if len(userAgent) > 0 { - dst = appendHeaderLine(dst, strUserAgent, userAgent) - } - - host := h.Host() - if len(host) > 0 { - dst = appendHeaderLine(dst, strHost, host) - } - - contentType := h.ContentType() - if !h.ignoreBody() { - if len(contentType) == 0 { - contentType = strPostArgsContentType - } - dst = appendHeaderLine(dst, strContentType, contentType) - - if len(h.contentLengthBytes) > 0 { - dst = appendHeaderLine(dst, strContentLength, h.contentLengthBytes) - } - } else if len(contentType) > 0 { - dst = appendHeaderLine(dst, strContentType, contentType) - } - - for i, n := 0, len(h.h); i < n; i++ { - kv := &h.h[i] - dst = appendHeaderLine(dst, kv.key, kv.value) - } - - // there is no need in h.collectCookies() here, since if cookies aren't collected yet, - // they all are located in h.h. - n := len(h.cookies) - if n > 0 { - dst = append(dst, strCookie...) - dst = append(dst, strColonSpace...) - dst = appendRequestCookieBytes(dst, h.cookies) - dst = append(dst, strCRLF...) - } - - if h.ConnectionClose() { - dst = appendHeaderLine(dst, strConnection, strClose) - } - - return append(dst, strCRLF...) -} - -func appendHeaderLine(dst, key, value []byte) []byte { - dst = append(dst, key...) - dst = append(dst, strColonSpace...) - dst = append(dst, value...) - return append(dst, strCRLF...) -} - -func (h *ResponseHeader) parse(buf []byte) (int, error) { - m, err := h.parseFirstLine(buf) - if err != nil { - return 0, err - } - n, err := h.parseHeaders(buf[m:]) - if err != nil { - return 0, err - } - return m + n, nil -} - -func (h *RequestHeader) ignoreBody() bool { - return h.IsGet() || h.IsHead() -} - -func (h *RequestHeader) parse(buf []byte) (int, error) { - m, err := h.parseFirstLine(buf) - if err != nil { - return 0, err - } - - var n int - var rawHeaders []byte - rawHeaders, n, err = readRawHeaders(h.rawHeaders[:0], buf[m:]) - if err != nil { - return 0, err - } - h.rawHeadersCopy = append(h.rawHeadersCopy[:0], rawHeaders...) - if !h.ignoreBody() || h.noHTTP11 { - n, err = h.parseHeaders(buf[m:]) - if err != nil { - return 0, err - } - h.rawHeaders = append(h.rawHeaders[:0], buf[m:m+n]...) - h.rawHeadersParsed = true - } else { - h.rawHeaders = rawHeaders - } - return m + n, nil -} - -func (h *ResponseHeader) parseFirstLine(buf []byte) (int, error) { - bNext := buf - var b []byte - var err error - for len(b) == 0 { - if b, bNext, err = nextLine(bNext); err != nil { - return 0, err - } - } - - // parse protocol - n := bytes.IndexByte(b, ' ') - if n < 0 { - return 0, fmt.Errorf("cannot find whitespace in the first line of response %q", buf) - } - h.noHTTP11 = !bytes.Equal(b[:n], strHTTP11) - b = b[n+1:] - - // parse status code - h.statusCode, n, err = parseUintBuf(b) - if err != nil { - return 0, fmt.Errorf("cannot parse response status code: %s. Response %q", err, buf) - } - if len(b) > n && b[n] != ' ' { - return 0, fmt.Errorf("unexpected char at the end of status code. Response %q", buf) - } - - return len(buf) - len(bNext), nil -} - -func (h *RequestHeader) parseFirstLine(buf []byte) (int, error) { - bNext := buf - var b []byte - var err error - for len(b) == 0 { - if b, bNext, err = nextLine(bNext); err != nil { - return 0, err - } - } - - // parse method - n := bytes.IndexByte(b, ' ') - if n <= 0 { - return 0, fmt.Errorf("cannot find http request method in %q", buf) - } - h.method = append(h.method[:0], b[:n]...) - b = b[n+1:] - - // parse requestURI - n = bytes.LastIndexByte(b, ' ') - if n < 0 { - h.noHTTP11 = true - n = len(b) - } else if n == 0 { - return 0, fmt.Errorf("requestURI cannot be empty in %q", buf) - } else if !bytes.Equal(b[n+1:], strHTTP11) { - h.noHTTP11 = true - } - h.requestURI = append(h.requestURI[:0], b[:n]...) - - return len(buf) - len(bNext), nil -} - -func peekRawHeader(buf, key []byte) []byte { - n := bytes.Index(buf, key) - if n < 0 { - return nil - } - if n > 0 && buf[n-1] != '\n' { - return nil - } - n += len(key) - if n >= len(buf) { - return nil - } - if buf[n] != ':' { - return nil - } - n++ - if buf[n] != ' ' { - return nil - } - n++ - buf = buf[n:] - n = bytes.IndexByte(buf, '\n') - if n < 0 { - return nil - } - if n > 0 && buf[n-1] == '\r' { - n-- - } - return buf[:n] -} - -func readRawHeaders(dst, buf []byte) ([]byte, int, error) { - n := bytes.IndexByte(buf, '\n') - if n < 0 { - return nil, 0, errNeedMore - } - if (n == 1 && buf[0] == '\r') || n == 0 { - // empty headers - return dst, n + 1, nil - } - - n++ - b := buf - m := n - for { - b = b[m:] - m = bytes.IndexByte(b, '\n') - if m < 0 { - return nil, 0, errNeedMore - } - m++ - n += m - if (m == 2 && b[0] == '\r') || m == 1 { - dst = append(dst, buf[:n]...) - return dst, n, nil - } - } -} - -func (h *ResponseHeader) parseHeaders(buf []byte) (int, error) { - // 'identity' content-length by default - h.contentLength = -2 - - var s headerScanner - s.b = buf - s.disableNormalizing = h.disableNormalizing - var err error - var kv *argsKV - for s.next() { - if len(s.key) > 0 { - switch s.key[0] | 0x20 { - case 'c': - if caseInsensitiveCompare(s.key, strContentType) { - h.contentType = append(h.contentType[:0], s.value...) - continue - } - if caseInsensitiveCompare(s.key, strContentLength) { - if h.contentLength != -1 { - if h.contentLength, err = parseContentLength(s.value); err != nil { - h.contentLength = -2 - } else { - h.contentLengthBytes = append(h.contentLengthBytes[:0], s.value...) - } - } - continue - } - if caseInsensitiveCompare(s.key, strConnection) { - if bytes.Equal(s.value, strClose) { - h.connectionClose = true - } else { - h.connectionClose = false - h.h = appendArgBytes(h.h, s.key, s.value, argsHasValue) - } - continue - } - case 's': - if caseInsensitiveCompare(s.key, strServer) { - h.server = append(h.server[:0], s.value...) - continue - } - if caseInsensitiveCompare(s.key, strSetCookie) { - h.cookies, kv = allocArg(h.cookies) - kv.key = getCookieKey(kv.key, s.value) - kv.value = append(kv.value[:0], s.value...) - continue - } - case 't': - if caseInsensitiveCompare(s.key, strTransferEncoding) { - if !bytes.Equal(s.value, strIdentity) { - h.contentLength = -1 - h.h = setArgBytes(h.h, strTransferEncoding, strChunked, argsHasValue) - } - continue - } - } - h.h = appendArgBytes(h.h, s.key, s.value, argsHasValue) - } - } - if s.err != nil { - h.connectionClose = true - return 0, s.err - } - - if h.contentLength < 0 { - h.contentLengthBytes = h.contentLengthBytes[:0] - } - if h.contentLength == -2 && !h.ConnectionUpgrade() && !h.mustSkipContentLength() { - h.h = setArgBytes(h.h, strTransferEncoding, strIdentity, argsHasValue) - h.connectionClose = true - } - if h.noHTTP11 && !h.connectionClose { - // close connection for non-http/1.1 response unless 'Connection: keep-alive' is set. - v := peekArgBytes(h.h, strConnection) - h.connectionClose = !hasHeaderValue(v, strKeepAlive) - } - - return len(buf) - len(s.b), nil -} - -func (h *RequestHeader) parseHeaders(buf []byte) (int, error) { - h.contentLength = -2 - - var s headerScanner - s.b = buf - s.disableNormalizing = h.disableNormalizing - var err error - for s.next() { - if len(s.key) > 0 { - switch s.key[0] | 0x20 { - case 'h': - if caseInsensitiveCompare(s.key, strHost) { - h.host = append(h.host[:0], s.value...) - continue - } - case 'u': - if caseInsensitiveCompare(s.key, strUserAgent) { - h.userAgent = append(h.userAgent[:0], s.value...) - continue - } - case 'c': - if caseInsensitiveCompare(s.key, strContentType) { - h.contentType = append(h.contentType[:0], s.value...) - continue - } - if caseInsensitiveCompare(s.key, strContentLength) { - if h.contentLength != -1 { - if h.contentLength, err = parseContentLength(s.value); err != nil { - h.contentLength = -2 - } else { - h.contentLengthBytes = append(h.contentLengthBytes[:0], s.value...) - } - } - continue - } - if caseInsensitiveCompare(s.key, strConnection) { - if bytes.Equal(s.value, strClose) { - h.connectionClose = true - } else { - h.connectionClose = false - h.h = appendArgBytes(h.h, s.key, s.value, argsHasValue) - } - continue - } - case 't': - if caseInsensitiveCompare(s.key, strTransferEncoding) { - if !bytes.Equal(s.value, strIdentity) { - h.contentLength = -1 - h.h = setArgBytes(h.h, strTransferEncoding, strChunked, argsHasValue) - } - continue - } - } - } - h.h = appendArgBytes(h.h, s.key, s.value, argsHasValue) - } - if s.err != nil { - h.connectionClose = true - return 0, s.err - } - - if h.contentLength < 0 { - h.contentLengthBytes = h.contentLengthBytes[:0] - } - if h.noHTTP11 && !h.connectionClose { - // close connection for non-http/1.1 request unless 'Connection: keep-alive' is set. - v := peekArgBytes(h.h, strConnection) - h.connectionClose = !hasHeaderValue(v, strKeepAlive) - } - return s.hLen, nil -} - -func (h *RequestHeader) parseRawHeaders() { - if h.rawHeadersParsed { - return - } - h.rawHeadersParsed = true - if len(h.rawHeaders) == 0 { - return - } - h.parseHeaders(h.rawHeaders) -} - -func (h *RequestHeader) collectCookies() { - if h.cookiesCollected { - return - } - - for i, n := 0, len(h.h); i < n; i++ { - kv := &h.h[i] - if bytes.Equal(kv.key, strCookie) { - h.cookies = parseRequestCookies(h.cookies, kv.value) - tmp := *kv - copy(h.h[i:], h.h[i+1:]) - n-- - i-- - h.h[n] = tmp - h.h = h.h[:n] - } - } - h.cookiesCollected = true -} - -func parseContentLength(b []byte) (int, error) { - v, n, err := parseUintBuf(b) - if err != nil { - return -1, err - } - if n != len(b) { - return -1, fmt.Errorf("non-numeric chars at the end of Content-Length") - } - return v, nil -} - -type headerScanner struct { - b []byte - key []byte - value []byte - err error - - // hLen stores header subslice len - hLen int - - disableNormalizing bool -} - -func (s *headerScanner) next() bool { - bLen := len(s.b) - if bLen >= 2 && s.b[0] == '\r' && s.b[1] == '\n' { - s.b = s.b[2:] - s.hLen += 2 - return false - } - if bLen >= 1 && s.b[0] == '\n' { - s.b = s.b[1:] - s.hLen++ - return false - } - n := bytes.IndexByte(s.b, ':') - if n < 0 { - s.err = errNeedMore - return false - } - s.key = s.b[:n] - normalizeHeaderKey(s.key, s.disableNormalizing) - n++ - for len(s.b) > n && s.b[n] == ' ' { - n++ - } - s.hLen += n - s.b = s.b[n:] - n = bytes.IndexByte(s.b, '\n') - if n < 0 { - s.err = errNeedMore - return false - } - s.value = s.b[:n] - s.hLen += n + 1 - s.b = s.b[n+1:] - - if n > 0 && s.value[n-1] == '\r' { - n-- - } - for n > 0 && s.value[n-1] == ' ' { - n-- - } - s.value = s.value[:n] - return true -} - -type headerValueScanner struct { - b []byte - value []byte -} - -func (s *headerValueScanner) next() bool { - b := s.b - if len(b) == 0 { - return false - } - n := bytes.IndexByte(b, ',') - if n < 0 { - s.value = stripSpace(b) - s.b = b[len(b):] - return true - } - s.value = stripSpace(b[:n]) - s.b = b[n+1:] - return true -} - -func stripSpace(b []byte) []byte { - for len(b) > 0 && b[0] == ' ' { - b = b[1:] - } - for len(b) > 0 && b[len(b)-1] == ' ' { - b = b[:len(b)-1] - } - return b -} - -func hasHeaderValue(s, value []byte) bool { - var vs headerValueScanner - vs.b = s - for vs.next() { - if caseInsensitiveCompare(vs.value, value) { - return true - } - } - return false -} - -func nextLine(b []byte) ([]byte, []byte, error) { - nNext := bytes.IndexByte(b, '\n') - if nNext < 0 { - return nil, nil, errNeedMore - } - n := nNext - if n > 0 && b[n-1] == '\r' { - n-- - } - return b[:n], b[nNext+1:], nil -} - -func initHeaderKV(kv *argsKV, key, value string, disableNormalizing bool) { - kv.key = getHeaderKeyBytes(kv, key, disableNormalizing) - kv.value = append(kv.value[:0], value...) -} - -func getHeaderKeyBytes(kv *argsKV, key string, disableNormalizing bool) []byte { - kv.key = append(kv.key[:0], key...) - normalizeHeaderKey(kv.key, disableNormalizing) - return kv.key -} - -func normalizeHeaderKey(b []byte, disableNormalizing bool) { - if disableNormalizing { - return - } - - n := len(b) - if n == 0 { - return - } - - b[0] = toUpperTable[b[0]] - for i := 1; i < n; i++ { - p := &b[i] - if *p == '-' { - i++ - if i < n { - b[i] = toUpperTable[b[i]] - } - continue - } - *p = toLowerTable[*p] - } -} - -// AppendNormalizedHeaderKey appends normalized header key (name) to dst -// and returns the resulting dst. -// -// Normalized header key starts with uppercase letter. The first letters -// after dashes are also uppercased. All the other letters are lowercased. -// Examples: -// -// * coNTENT-TYPe -> Content-Type -// * HOST -> Host -// * foo-bar-baz -> Foo-Bar-Baz -func AppendNormalizedHeaderKey(dst []byte, key string) []byte { - dst = append(dst, key...) - normalizeHeaderKey(dst[len(dst)-len(key):], false) - return dst -} - -// AppendNormalizedHeaderKeyBytes appends normalized header key (name) to dst -// and returns the resulting dst. -// -// Normalized header key starts with uppercase letter. The first letters -// after dashes are also uppercased. All the other letters are lowercased. -// Examples: -// -// * coNTENT-TYPe -> Content-Type -// * HOST -> Host -// * foo-bar-baz -> Foo-Bar-Baz -func AppendNormalizedHeaderKeyBytes(dst, key []byte) []byte { - return AppendNormalizedHeaderKey(dst, b2s(key)) -} - -var ( - errNeedMore = errors.New("need more data: cannot find trailing lf") - errSmallBuffer = errors.New("small read buffer. Increase ReadBufferSize") - errNothingRead = errors.New("read timeout with nothing read") -) - -// ErrSmallBuffer is returned when the provided buffer size is too small -// for reading request and/or response headers. -// -// ReadBufferSize value from Server or clients should reduce the number -// of such errors. -type ErrSmallBuffer struct { - error -} - -func mustPeekBuffered(r *bufio.Reader) []byte { - buf, err := r.Peek(r.Buffered()) - if len(buf) == 0 || err != nil { - panic(fmt.Sprintf("bufio.Reader.Peek() returned unexpected data (%q, %v)", buf, err)) - } - return buf -} - -func mustDiscard(r *bufio.Reader, n int) { - if _, err := r.Discard(n); err != nil { - panic(fmt.Sprintf("bufio.Reader.Discard(%d) failed: %s", n, err)) - } -} diff --git a/vendor/github.com/valyala/fasthttp/http.go b/vendor/github.com/valyala/fasthttp/http.go deleted file mode 100644 index 10dc4654..00000000 --- a/vendor/github.com/valyala/fasthttp/http.go +++ /dev/null @@ -1,1766 +0,0 @@ -package fasthttp - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - "mime/multipart" - "net" - "os" - "sync" - - "github.com/valyala/bytebufferpool" -) - -// Request represents HTTP request. -// -// It is forbidden copying Request instances. Create new instances -// and use CopyTo instead. -// -// Request instance MUST NOT be used from concurrently running goroutines. -type Request struct { - noCopy noCopy - - // Request header - // - // Copying Header by value is forbidden. Use pointer to Header instead. - Header RequestHeader - - uri URI - postArgs Args - - bodyStream io.Reader - w requestBodyWriter - body *bytebufferpool.ByteBuffer - - multipartForm *multipart.Form - multipartFormBoundary string - - // Group bool members in order to reduce Request object size. - parsedURI bool - parsedPostArgs bool - - keepBodyBuffer bool - - isTLS bool - - // To detect scheme changes in redirects - schemaUpdate bool -} - -// Response represents HTTP response. -// -// It is forbidden copying Response instances. Create new instances -// and use CopyTo instead. -// -// Response instance MUST NOT be used from concurrently running goroutines. -type Response struct { - noCopy noCopy - - // Response header - // - // Copying Header by value is forbidden. Use pointer to Header instead. - Header ResponseHeader - - bodyStream io.Reader - w responseBodyWriter - body *bytebufferpool.ByteBuffer - - // Response.Read() skips reading body if set to true. - // Use it for reading HEAD responses. - // - // Response.Write() skips writing body if set to true. - // Use it for writing HEAD responses. - SkipBody bool - - keepBodyBuffer bool - - // Remote TCPAddr from concurrently net.Conn - raddr net.Addr - // Local TCPAddr from concurrently net.Conn - laddr net.Addr -} - -// SetHost sets host for the request. -func (req *Request) SetHost(host string) { - req.URI().SetHost(host) -} - -// SetHostBytes sets host for the request. -func (req *Request) SetHostBytes(host []byte) { - req.URI().SetHostBytes(host) -} - -// Host returns the host for the given request. -func (req *Request) Host() []byte { - return req.URI().Host() -} - -// SetRequestURI sets RequestURI. -func (req *Request) SetRequestURI(requestURI string) { - req.Header.SetRequestURI(requestURI) - req.parsedURI = false -} - -// SetRequestURIBytes sets RequestURI. -func (req *Request) SetRequestURIBytes(requestURI []byte) { - req.Header.SetRequestURIBytes(requestURI) - req.parsedURI = false -} - -// RequestURI returns request's URI. -func (req *Request) RequestURI() []byte { - if req.parsedURI { - requestURI := req.uri.RequestURI() - req.SetRequestURIBytes(requestURI) - } - return req.Header.RequestURI() -} - -// StatusCode returns response status code. -func (resp *Response) StatusCode() int { - return resp.Header.StatusCode() -} - -// SetStatusCode sets response status code. -func (resp *Response) SetStatusCode(statusCode int) { - resp.Header.SetStatusCode(statusCode) -} - -// ConnectionClose returns true if 'Connection: close' header is set. -func (resp *Response) ConnectionClose() bool { - return resp.Header.ConnectionClose() -} - -// SetConnectionClose sets 'Connection: close' header. -func (resp *Response) SetConnectionClose() { - resp.Header.SetConnectionClose() -} - -// ConnectionClose returns true if 'Connection: close' header is set. -func (req *Request) ConnectionClose() bool { - return req.Header.ConnectionClose() -} - -// SetConnectionClose sets 'Connection: close' header. -func (req *Request) SetConnectionClose() { - req.Header.SetConnectionClose() -} - -// SendFile registers file on the given path to be used as response body -// when Write is called. -// -// Note that SendFile doesn't set Content-Type, so set it yourself -// with Header.SetContentType. -func (resp *Response) SendFile(path string) error { - f, err := os.Open(path) - if err != nil { - return err - } - fileInfo, err := f.Stat() - if err != nil { - f.Close() - return err - } - size64 := fileInfo.Size() - size := int(size64) - if int64(size) != size64 { - size = -1 - } - - resp.Header.SetLastModified(fileInfo.ModTime()) - resp.SetBodyStream(f, size) - return nil -} - -// SetBodyStream sets request body stream and, optionally body size. -// -// If bodySize is >= 0, then the bodyStream must provide exactly bodySize bytes -// before returning io.EOF. -// -// If bodySize < 0, then bodyStream is read until io.EOF. -// -// bodyStream.Close() is called after finishing reading all body data -// if it implements io.Closer. -// -// Note that GET and HEAD requests cannot have body. -// -// See also SetBodyStreamWriter. -func (req *Request) SetBodyStream(bodyStream io.Reader, bodySize int) { - req.ResetBody() - req.bodyStream = bodyStream - req.Header.SetContentLength(bodySize) -} - -// SetBodyStream sets response body stream and, optionally body size. -// -// If bodySize is >= 0, then the bodyStream must provide exactly bodySize bytes -// before returning io.EOF. -// -// If bodySize < 0, then bodyStream is read until io.EOF. -// -// bodyStream.Close() is called after finishing reading all body data -// if it implements io.Closer. -// -// See also SetBodyStreamWriter. -func (resp *Response) SetBodyStream(bodyStream io.Reader, bodySize int) { - resp.ResetBody() - resp.bodyStream = bodyStream - resp.Header.SetContentLength(bodySize) -} - -// IsBodyStream returns true if body is set via SetBodyStream* -func (req *Request) IsBodyStream() bool { - return req.bodyStream != nil -} - -// IsBodyStream returns true if body is set via SetBodyStream* -func (resp *Response) IsBodyStream() bool { - return resp.bodyStream != nil -} - -// SetBodyStreamWriter registers the given sw for populating request body. -// -// This function may be used in the following cases: -// -// * if request body is too big (more than 10MB). -// * if request body is streamed from slow external sources. -// * if request body must be streamed to the server in chunks -// (aka `http client push` or `chunked transfer-encoding`). -// -// Note that GET and HEAD requests cannot have body. -// -/// See also SetBodyStream. -func (req *Request) SetBodyStreamWriter(sw StreamWriter) { - sr := NewStreamReader(sw) - req.SetBodyStream(sr, -1) -} - -// SetBodyStreamWriter registers the given sw for populating response body. -// -// This function may be used in the following cases: -// -// * if response body is too big (more than 10MB). -// * if response body is streamed from slow external sources. -// * if response body must be streamed to the client in chunks -// (aka `http server push` or `chunked transfer-encoding`). -// -// See also SetBodyStream. -func (resp *Response) SetBodyStreamWriter(sw StreamWriter) { - sr := NewStreamReader(sw) - resp.SetBodyStream(sr, -1) -} - -// BodyWriter returns writer for populating response body. -// -// If used inside RequestHandler, the returned writer must not be used -// after returning from RequestHandler. Use RequestCtx.Write -// or SetBodyStreamWriter in this case. -func (resp *Response) BodyWriter() io.Writer { - resp.w.r = resp - return &resp.w -} - -// BodyWriter returns writer for populating request body. -func (req *Request) BodyWriter() io.Writer { - req.w.r = req - return &req.w -} - -type responseBodyWriter struct { - r *Response -} - -func (w *responseBodyWriter) Write(p []byte) (int, error) { - w.r.AppendBody(p) - return len(p), nil -} - -type requestBodyWriter struct { - r *Request -} - -func (w *requestBodyWriter) Write(p []byte) (int, error) { - w.r.AppendBody(p) - return len(p), nil -} - -func (resp *Response) parseNetConn(conn net.Conn) { - resp.raddr = conn.RemoteAddr() - resp.laddr = conn.LocalAddr() -} - -// RemoteAddr returns the remote network address. The Addr returned is shared -// by all invocations of RemoteAddr, so do not modify it. -func (resp *Response) RemoteAddr() net.Addr { - return resp.raddr -} - -// LocalAddr returns the local network address. The Addr returned is shared -// by all invocations of LocalAddr, so do not modify it. -func (resp *Response) LocalAddr() net.Addr { - return resp.laddr -} - -// Body returns response body. -// -// The returned body is valid until the response modification. -func (resp *Response) Body() []byte { - if resp.bodyStream != nil { - bodyBuf := resp.bodyBuffer() - bodyBuf.Reset() - _, err := copyZeroAlloc(bodyBuf, resp.bodyStream) - resp.closeBodyStream() - if err != nil { - bodyBuf.SetString(err.Error()) - } - } - return resp.bodyBytes() -} - -func (resp *Response) bodyBytes() []byte { - if resp.body == nil { - return nil - } - return resp.body.B -} - -func (req *Request) bodyBytes() []byte { - if req.body == nil { - return nil - } - return req.body.B -} - -func (resp *Response) bodyBuffer() *bytebufferpool.ByteBuffer { - if resp.body == nil { - resp.body = responseBodyPool.Get() - } - return resp.body -} - -func (req *Request) bodyBuffer() *bytebufferpool.ByteBuffer { - if req.body == nil { - req.body = requestBodyPool.Get() - } - return req.body -} - -var ( - responseBodyPool bytebufferpool.Pool - requestBodyPool bytebufferpool.Pool -) - -// BodyGunzip returns un-gzipped body data. -// -// This method may be used if the request header contains -// 'Content-Encoding: gzip' for reading un-gzipped body. -// Use Body for reading gzipped request body. -func (req *Request) BodyGunzip() ([]byte, error) { - return gunzipData(req.Body()) -} - -// BodyGunzip returns un-gzipped body data. -// -// This method may be used if the response header contains -// 'Content-Encoding: gzip' for reading un-gzipped body. -// Use Body for reading gzipped response body. -func (resp *Response) BodyGunzip() ([]byte, error) { - return gunzipData(resp.Body()) -} - -func gunzipData(p []byte) ([]byte, error) { - var bb bytebufferpool.ByteBuffer - _, err := WriteGunzip(&bb, p) - if err != nil { - return nil, err - } - return bb.B, nil -} - -// BodyInflate returns inflated body data. -// -// This method may be used if the response header contains -// 'Content-Encoding: deflate' for reading inflated request body. -// Use Body for reading deflated request body. -func (req *Request) BodyInflate() ([]byte, error) { - return inflateData(req.Body()) -} - -// BodyInflate returns inflated body data. -// -// This method may be used if the response header contains -// 'Content-Encoding: deflate' for reading inflated response body. -// Use Body for reading deflated response body. -func (resp *Response) BodyInflate() ([]byte, error) { - return inflateData(resp.Body()) -} - -func inflateData(p []byte) ([]byte, error) { - var bb bytebufferpool.ByteBuffer - _, err := WriteInflate(&bb, p) - if err != nil { - return nil, err - } - return bb.B, nil -} - -// BodyWriteTo writes request body to w. -func (req *Request) BodyWriteTo(w io.Writer) error { - if req.bodyStream != nil { - _, err := copyZeroAlloc(w, req.bodyStream) - req.closeBodyStream() - return err - } - if req.onlyMultipartForm() { - return WriteMultipartForm(w, req.multipartForm, req.multipartFormBoundary) - } - _, err := w.Write(req.bodyBytes()) - return err -} - -// BodyWriteTo writes response body to w. -func (resp *Response) BodyWriteTo(w io.Writer) error { - if resp.bodyStream != nil { - _, err := copyZeroAlloc(w, resp.bodyStream) - resp.closeBodyStream() - return err - } - _, err := w.Write(resp.bodyBytes()) - return err -} - -// AppendBody appends p to response body. -// -// It is safe re-using p after the function returns. -func (resp *Response) AppendBody(p []byte) { - resp.AppendBodyString(b2s(p)) -} - -// AppendBodyString appends s to response body. -func (resp *Response) AppendBodyString(s string) { - resp.closeBodyStream() - resp.bodyBuffer().WriteString(s) -} - -// SetBody sets response body. -// -// It is safe re-using body argument after the function returns. -func (resp *Response) SetBody(body []byte) { - resp.SetBodyString(b2s(body)) -} - -// SetBodyString sets response body. -func (resp *Response) SetBodyString(body string) { - resp.closeBodyStream() - bodyBuf := resp.bodyBuffer() - bodyBuf.Reset() - bodyBuf.WriteString(body) -} - -// ResetBody resets response body. -func (resp *Response) ResetBody() { - resp.closeBodyStream() - if resp.body != nil { - if resp.keepBodyBuffer { - resp.body.Reset() - } else { - responseBodyPool.Put(resp.body) - resp.body = nil - } - } -} - -// ReleaseBody retires the response body if it is greater than "size" bytes. -// -// This permits GC to reclaim the large buffer. If used, must be before -// ReleaseResponse. -// -// Use this method only if you really understand how it works. -// The majority of workloads don't need this method. -func (resp *Response) ReleaseBody(size int) { - if cap(resp.body.B) > size { - resp.closeBodyStream() - resp.body = nil - } -} - -// ReleaseBody retires the request body if it is greater than "size" bytes. -// -// This permits GC to reclaim the large buffer. If used, must be before -// ReleaseRequest. -// -// Use this method only if you really understand how it works. -// The majority of workloads don't need this method. -func (req *Request) ReleaseBody(size int) { - if cap(req.body.B) > size { - req.closeBodyStream() - req.body = nil - } -} - -// SwapBody swaps response body with the given body and returns -// the previous response body. -// -// It is forbidden to use the body passed to SwapBody after -// the function returns. -func (resp *Response) SwapBody(body []byte) []byte { - bb := resp.bodyBuffer() - - if resp.bodyStream != nil { - bb.Reset() - _, err := copyZeroAlloc(bb, resp.bodyStream) - resp.closeBodyStream() - if err != nil { - bb.Reset() - bb.SetString(err.Error()) - } - } - - oldBody := bb.B - bb.B = body - return oldBody -} - -// SwapBody swaps request body with the given body and returns -// the previous request body. -// -// It is forbidden to use the body passed to SwapBody after -// the function returns. -func (req *Request) SwapBody(body []byte) []byte { - bb := req.bodyBuffer() - - if req.bodyStream != nil { - bb.Reset() - _, err := copyZeroAlloc(bb, req.bodyStream) - req.closeBodyStream() - if err != nil { - bb.Reset() - bb.SetString(err.Error()) - } - } - - oldBody := bb.B - bb.B = body - return oldBody -} - -// Body returns request body. -// -// The returned body is valid until the request modification. -func (req *Request) Body() []byte { - if req.bodyStream != nil { - bodyBuf := req.bodyBuffer() - bodyBuf.Reset() - _, err := copyZeroAlloc(bodyBuf, req.bodyStream) - req.closeBodyStream() - if err != nil { - bodyBuf.SetString(err.Error()) - } - } else if req.onlyMultipartForm() { - body, err := marshalMultipartForm(req.multipartForm, req.multipartFormBoundary) - if err != nil { - return []byte(err.Error()) - } - return body - } - return req.bodyBytes() -} - -// AppendBody appends p to request body. -// -// It is safe re-using p after the function returns. -func (req *Request) AppendBody(p []byte) { - req.AppendBodyString(b2s(p)) -} - -// AppendBodyString appends s to request body. -func (req *Request) AppendBodyString(s string) { - req.RemoveMultipartFormFiles() - req.closeBodyStream() - req.bodyBuffer().WriteString(s) -} - -// SetBody sets request body. -// -// It is safe re-using body argument after the function returns. -func (req *Request) SetBody(body []byte) { - req.SetBodyString(b2s(body)) -} - -// SetBodyString sets request body. -func (req *Request) SetBodyString(body string) { - req.RemoveMultipartFormFiles() - req.closeBodyStream() - req.bodyBuffer().SetString(body) -} - -// ResetBody resets request body. -func (req *Request) ResetBody() { - req.RemoveMultipartFormFiles() - req.closeBodyStream() - if req.body != nil { - if req.keepBodyBuffer { - req.body.Reset() - } else { - requestBodyPool.Put(req.body) - req.body = nil - } - } -} - -// CopyTo copies req contents to dst except of body stream. -func (req *Request) CopyTo(dst *Request) { - req.copyToSkipBody(dst) - if req.body != nil { - dst.bodyBuffer().Set(req.body.B) - } else if dst.body != nil { - dst.body.Reset() - } -} - -func (req *Request) copyToSkipBody(dst *Request) { - dst.Reset() - req.Header.CopyTo(&dst.Header) - - req.uri.CopyTo(&dst.uri) - dst.parsedURI = req.parsedURI - - req.postArgs.CopyTo(&dst.postArgs) - dst.parsedPostArgs = req.parsedPostArgs - dst.isTLS = req.isTLS - - // do not copy multipartForm - it will be automatically - // re-created on the first call to MultipartForm. -} - -// CopyTo copies resp contents to dst except of body stream. -func (resp *Response) CopyTo(dst *Response) { - resp.copyToSkipBody(dst) - if resp.body != nil { - dst.bodyBuffer().Set(resp.body.B) - } else if dst.body != nil { - dst.body.Reset() - } -} - -func (resp *Response) copyToSkipBody(dst *Response) { - dst.Reset() - resp.Header.CopyTo(&dst.Header) - dst.SkipBody = resp.SkipBody - dst.raddr = resp.raddr - dst.laddr = resp.laddr -} - -func swapRequestBody(a, b *Request) { - a.body, b.body = b.body, a.body - a.bodyStream, b.bodyStream = b.bodyStream, a.bodyStream -} - -func swapResponseBody(a, b *Response) { - a.body, b.body = b.body, a.body - a.bodyStream, b.bodyStream = b.bodyStream, a.bodyStream -} - -// URI returns request URI -func (req *Request) URI() *URI { - req.parseURI() - return &req.uri -} - -func (req *Request) parseURI() { - if req.parsedURI { - return - } - req.parsedURI = true - - req.uri.parseQuick(req.Header.RequestURI(), &req.Header, req.isTLS) -} - -// PostArgs returns POST arguments. -func (req *Request) PostArgs() *Args { - req.parsePostArgs() - return &req.postArgs -} - -func (req *Request) parsePostArgs() { - if req.parsedPostArgs { - return - } - req.parsedPostArgs = true - - if !bytes.HasPrefix(req.Header.ContentType(), strPostArgsContentType) { - return - } - req.postArgs.ParseBytes(req.bodyBytes()) -} - -// ErrNoMultipartForm means that the request's Content-Type -// isn't 'multipart/form-data'. -var ErrNoMultipartForm = errors.New("request has no multipart/form-data Content-Type") - -// MultipartForm returns requests's multipart form. -// -// Returns ErrNoMultipartForm if request's Content-Type -// isn't 'multipart/form-data'. -// -// RemoveMultipartFormFiles must be called after returned multipart form -// is processed. -func (req *Request) MultipartForm() (*multipart.Form, error) { - if req.multipartForm != nil { - return req.multipartForm, nil - } - - req.multipartFormBoundary = string(req.Header.MultipartFormBoundary()) - if len(req.multipartFormBoundary) == 0 { - return nil, ErrNoMultipartForm - } - - ce := req.Header.peek(strContentEncoding) - body := req.bodyBytes() - if bytes.Equal(ce, strGzip) { - // Do not care about memory usage here. - var err error - if body, err = AppendGunzipBytes(nil, body); err != nil { - return nil, fmt.Errorf("cannot gunzip request body: %s", err) - } - } else if len(ce) > 0 { - return nil, fmt.Errorf("unsupported Content-Encoding: %q", ce) - } - - f, err := readMultipartForm(bytes.NewReader(body), req.multipartFormBoundary, len(body), len(body)) - if err != nil { - return nil, err - } - req.multipartForm = f - return f, nil -} - -func marshalMultipartForm(f *multipart.Form, boundary string) ([]byte, error) { - var buf bytebufferpool.ByteBuffer - if err := WriteMultipartForm(&buf, f, boundary); err != nil { - return nil, err - } - return buf.B, nil -} - -// WriteMultipartForm writes the given multipart form f with the given -// boundary to w. -func WriteMultipartForm(w io.Writer, f *multipart.Form, boundary string) error { - // Do not care about memory allocations here, since multipart - // form processing is slow. - if len(boundary) == 0 { - panic("BUG: form boundary cannot be empty") - } - - mw := multipart.NewWriter(w) - if err := mw.SetBoundary(boundary); err != nil { - return fmt.Errorf("cannot use form boundary %q: %s", boundary, err) - } - - // marshal values - for k, vv := range f.Value { - for _, v := range vv { - if err := mw.WriteField(k, v); err != nil { - return fmt.Errorf("cannot write form field %q value %q: %s", k, v, err) - } - } - } - - // marshal files - for k, fvv := range f.File { - for _, fv := range fvv { - vw, err := mw.CreatePart(fv.Header) - if err != nil { - return fmt.Errorf("cannot create form file %q (%q): %s", k, fv.Filename, err) - } - fh, err := fv.Open() - if err != nil { - return fmt.Errorf("cannot open form file %q (%q): %s", k, fv.Filename, err) - } - if _, err = copyZeroAlloc(vw, fh); err != nil { - return fmt.Errorf("error when copying form file %q (%q): %s", k, fv.Filename, err) - } - if err = fh.Close(); err != nil { - return fmt.Errorf("cannot close form file %q (%q): %s", k, fv.Filename, err) - } - } - } - - if err := mw.Close(); err != nil { - return fmt.Errorf("error when closing multipart form writer: %s", err) - } - - return nil -} - -func readMultipartForm(r io.Reader, boundary string, size, maxInMemoryFileSize int) (*multipart.Form, error) { - // Do not care about memory allocations here, since they are tiny - // compared to multipart data (aka multi-MB files) usually sent - // in multipart/form-data requests. - - if size <= 0 { - panic(fmt.Sprintf("BUG: form size must be greater than 0. Given %d", size)) - } - lr := io.LimitReader(r, int64(size)) - mr := multipart.NewReader(lr, boundary) - f, err := mr.ReadForm(int64(maxInMemoryFileSize)) - if err != nil { - return nil, fmt.Errorf("cannot read multipart/form-data body: %s", err) - } - return f, nil -} - -// Reset clears request contents. -func (req *Request) Reset() { - req.Header.Reset() - req.resetSkipHeader() -} - -func (req *Request) resetSkipHeader() { - req.ResetBody() - req.uri.Reset() - req.parsedURI = false - req.postArgs.Reset() - req.parsedPostArgs = false - req.isTLS = false -} - -// RemoveMultipartFormFiles removes multipart/form-data temporary files -// associated with the request. -func (req *Request) RemoveMultipartFormFiles() { - if req.multipartForm != nil { - // Do not check for error, since these files may be deleted or moved - // to new places by user code. - req.multipartForm.RemoveAll() - req.multipartForm = nil - } - req.multipartFormBoundary = "" -} - -// Reset clears response contents. -func (resp *Response) Reset() { - resp.Header.Reset() - resp.resetSkipHeader() - resp.SkipBody = false - resp.raddr = nil - resp.laddr = nil -} - -func (resp *Response) resetSkipHeader() { - resp.ResetBody() -} - -// Read reads request (including body) from the given r. -// -// RemoveMultipartFormFiles or Reset must be called after -// reading multipart/form-data request in order to delete temporarily -// uploaded files. -// -// If MayContinue returns true, the caller must: -// -// - Either send StatusExpectationFailed response if request headers don't -// satisfy the caller. -// - Or send StatusContinue response before reading request body -// with ContinueReadBody. -// - Or close the connection. -// -// io.EOF is returned if r is closed before reading the first header byte. -func (req *Request) Read(r *bufio.Reader) error { - return req.ReadLimitBody(r, 0) -} - -const defaultMaxInMemoryFileSize = 16 * 1024 * 1024 - -// ErrGetOnly is returned when server expects only GET requests, -// but some other type of request came (Server.GetOnly option is true). -var ErrGetOnly = errors.New("non-GET request received") - -// ReadLimitBody reads request from the given r, limiting the body size. -// -// If maxBodySize > 0 and the body size exceeds maxBodySize, -// then ErrBodyTooLarge is returned. -// -// RemoveMultipartFormFiles or Reset must be called after -// reading multipart/form-data request in order to delete temporarily -// uploaded files. -// -// If MayContinue returns true, the caller must: -// -// - Either send StatusExpectationFailed response if request headers don't -// satisfy the caller. -// - Or send StatusContinue response before reading request body -// with ContinueReadBody. -// - Or close the connection. -// -// io.EOF is returned if r is closed before reading the first header byte. -func (req *Request) ReadLimitBody(r *bufio.Reader, maxBodySize int) error { - req.resetSkipHeader() - return req.readLimitBody(r, maxBodySize, false) -} - -func (req *Request) readLimitBody(r *bufio.Reader, maxBodySize int, getOnly bool) error { - // Do not reset the request here - the caller must reset it before - // calling this method. - - err := req.Header.Read(r) - if err != nil { - return err - } - if getOnly && !req.Header.IsGet() { - return ErrGetOnly - } - - if req.MayContinue() { - // 'Expect: 100-continue' header found. Let the caller deciding - // whether to read request body or - // to return StatusExpectationFailed. - return nil - } - - return req.ContinueReadBody(r, maxBodySize) -} - -// MayContinue returns true if the request contains -// 'Expect: 100-continue' header. -// -// The caller must do one of the following actions if MayContinue returns true: -// -// - Either send StatusExpectationFailed response if request headers don't -// satisfy the caller. -// - Or send StatusContinue response before reading request body -// with ContinueReadBody. -// - Or close the connection. -func (req *Request) MayContinue() bool { - return bytes.Equal(req.Header.peek(strExpect), str100Continue) -} - -// ContinueReadBody reads request body if request header contains -// 'Expect: 100-continue'. -// -// The caller must send StatusContinue response before calling this method. -// -// If maxBodySize > 0 and the body size exceeds maxBodySize, -// then ErrBodyTooLarge is returned. -func (req *Request) ContinueReadBody(r *bufio.Reader, maxBodySize int) error { - var err error - contentLength := req.Header.realContentLength() - if contentLength > 0 { - if maxBodySize > 0 && contentLength > maxBodySize { - return ErrBodyTooLarge - } - - // Pre-read multipart form data of known length. - // This way we limit memory usage for large file uploads, since their contents - // is streamed into temporary files if file size exceeds defaultMaxInMemoryFileSize. - req.multipartFormBoundary = string(req.Header.MultipartFormBoundary()) - if len(req.multipartFormBoundary) > 0 && len(req.Header.peek(strContentEncoding)) == 0 { - req.multipartForm, err = readMultipartForm(r, req.multipartFormBoundary, contentLength, defaultMaxInMemoryFileSize) - if err != nil { - req.Reset() - } - return err - } - } - - if contentLength == -2 { - // identity body has no sense for http requests, since - // the end of body is determined by connection close. - // So just ignore request body for requests without - // 'Content-Length' and 'Transfer-Encoding' headers. - req.Header.SetContentLength(0) - return nil - } - - bodyBuf := req.bodyBuffer() - bodyBuf.Reset() - bodyBuf.B, err = readBody(r, contentLength, maxBodySize, bodyBuf.B) - if err != nil { - req.Reset() - return err - } - req.Header.SetContentLength(len(bodyBuf.B)) - return nil -} - -// Read reads response (including body) from the given r. -// -// io.EOF is returned if r is closed before reading the first header byte. -func (resp *Response) Read(r *bufio.Reader) error { - return resp.ReadLimitBody(r, 0) -} - -// ReadLimitBody reads response from the given r, limiting the body size. -// -// If maxBodySize > 0 and the body size exceeds maxBodySize, -// then ErrBodyTooLarge is returned. -// -// io.EOF is returned if r is closed before reading the first header byte. -func (resp *Response) ReadLimitBody(r *bufio.Reader, maxBodySize int) error { - resp.resetSkipHeader() - err := resp.Header.Read(r) - if err != nil { - return err - } - if resp.Header.StatusCode() == StatusContinue { - // Read the next response according to http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html . - if err = resp.Header.Read(r); err != nil { - return err - } - } - - if !resp.mustSkipBody() { - bodyBuf := resp.bodyBuffer() - bodyBuf.Reset() - bodyBuf.B, err = readBody(r, resp.Header.ContentLength(), maxBodySize, bodyBuf.B) - if err != nil { - return err - } - resp.Header.SetContentLength(len(bodyBuf.B)) - } - return nil -} - -func (resp *Response) mustSkipBody() bool { - return resp.SkipBody || resp.Header.mustSkipContentLength() -} - -var errRequestHostRequired = errors.New("missing required Host header in request") - -// WriteTo writes request to w. It implements io.WriterTo. -func (req *Request) WriteTo(w io.Writer) (int64, error) { - return writeBufio(req, w) -} - -// WriteTo writes response to w. It implements io.WriterTo. -func (resp *Response) WriteTo(w io.Writer) (int64, error) { - return writeBufio(resp, w) -} - -func writeBufio(hw httpWriter, w io.Writer) (int64, error) { - sw := acquireStatsWriter(w) - bw := acquireBufioWriter(sw) - err1 := hw.Write(bw) - err2 := bw.Flush() - releaseBufioWriter(bw) - n := sw.bytesWritten - releaseStatsWriter(sw) - - err := err1 - if err == nil { - err = err2 - } - return n, err -} - -type statsWriter struct { - w io.Writer - bytesWritten int64 -} - -func (w *statsWriter) Write(p []byte) (int, error) { - n, err := w.w.Write(p) - w.bytesWritten += int64(n) - return n, err -} - -func acquireStatsWriter(w io.Writer) *statsWriter { - v := statsWriterPool.Get() - if v == nil { - return &statsWriter{ - w: w, - } - } - sw := v.(*statsWriter) - sw.w = w - return sw -} - -func releaseStatsWriter(sw *statsWriter) { - sw.w = nil - sw.bytesWritten = 0 - statsWriterPool.Put(sw) -} - -var statsWriterPool sync.Pool - -func acquireBufioWriter(w io.Writer) *bufio.Writer { - v := bufioWriterPool.Get() - if v == nil { - return bufio.NewWriter(w) - } - bw := v.(*bufio.Writer) - bw.Reset(w) - return bw -} - -func releaseBufioWriter(bw *bufio.Writer) { - bufioWriterPool.Put(bw) -} - -var bufioWriterPool sync.Pool - -func (req *Request) onlyMultipartForm() bool { - return req.multipartForm != nil && (req.body == nil || len(req.body.B) == 0) -} - -// Write writes request to w. -// -// Write doesn't flush request to w for performance reasons. -// -// See also WriteTo. -func (req *Request) Write(w *bufio.Writer) error { - if len(req.Header.Host()) == 0 || req.parsedURI { - uri := req.URI() - host := uri.Host() - if len(host) == 0 { - return errRequestHostRequired - } - req.Header.SetHostBytes(host) - req.Header.SetRequestURIBytes(uri.RequestURI()) - } - - if req.bodyStream != nil { - return req.writeBodyStream(w) - } - - body := req.bodyBytes() - var err error - if req.onlyMultipartForm() { - body, err = marshalMultipartForm(req.multipartForm, req.multipartFormBoundary) - if err != nil { - return fmt.Errorf("error when marshaling multipart form: %s", err) - } - req.Header.SetMultipartFormBoundary(req.multipartFormBoundary) - } - - hasBody := !req.Header.ignoreBody() - if hasBody { - if len(body) == 0 { - body = req.postArgs.QueryString() - } - req.Header.SetContentLength(len(body)) - } - if err = req.Header.Write(w); err != nil { - return err - } - if hasBody { - _, err = w.Write(body) - } else if len(body) > 0 { - return fmt.Errorf("non-zero body for non-POST request. body=%q", body) - } - return err -} - -// WriteGzip writes response with gzipped body to w. -// -// The method gzips response body and sets 'Content-Encoding: gzip' -// header before writing response to w. -// -// WriteGzip doesn't flush response to w for performance reasons. -func (resp *Response) WriteGzip(w *bufio.Writer) error { - return resp.WriteGzipLevel(w, CompressDefaultCompression) -} - -// WriteGzipLevel writes response with gzipped body to w. -// -// Level is the desired compression level: -// -// * CompressNoCompression -// * CompressBestSpeed -// * CompressBestCompression -// * CompressDefaultCompression -// * CompressHuffmanOnly -// -// The method gzips response body and sets 'Content-Encoding: gzip' -// header before writing response to w. -// -// WriteGzipLevel doesn't flush response to w for performance reasons. -func (resp *Response) WriteGzipLevel(w *bufio.Writer, level int) error { - if err := resp.gzipBody(level); err != nil { - return err - } - return resp.Write(w) -} - -// WriteDeflate writes response with deflated body to w. -// -// The method deflates response body and sets 'Content-Encoding: deflate' -// header before writing response to w. -// -// WriteDeflate doesn't flush response to w for performance reasons. -func (resp *Response) WriteDeflate(w *bufio.Writer) error { - return resp.WriteDeflateLevel(w, CompressDefaultCompression) -} - -// WriteDeflateLevel writes response with deflated body to w. -// -// Level is the desired compression level: -// -// * CompressNoCompression -// * CompressBestSpeed -// * CompressBestCompression -// * CompressDefaultCompression -// * CompressHuffmanOnly -// -// The method deflates response body and sets 'Content-Encoding: deflate' -// header before writing response to w. -// -// WriteDeflateLevel doesn't flush response to w for performance reasons. -func (resp *Response) WriteDeflateLevel(w *bufio.Writer, level int) error { - if err := resp.deflateBody(level); err != nil { - return err - } - return resp.Write(w) -} - -func (resp *Response) gzipBody(level int) error { - if len(resp.Header.peek(strContentEncoding)) > 0 { - // It looks like the body is already compressed. - // Do not compress it again. - return nil - } - - if !resp.Header.isCompressibleContentType() { - // The content-type cannot be compressed. - return nil - } - - if resp.bodyStream != nil { - // Reset Content-Length to -1, since it is impossible - // to determine body size beforehand of streamed compression. - // For https://github.com/valyala/fasthttp/issues/176 . - resp.Header.SetContentLength(-1) - - // Do not care about memory allocations here, since gzip is slow - // and allocates a lot of memory by itself. - bs := resp.bodyStream - resp.bodyStream = NewStreamReader(func(sw *bufio.Writer) { - zw := acquireStacklessGzipWriter(sw, level) - fw := &flushWriter{ - wf: zw, - bw: sw, - } - copyZeroAlloc(fw, bs) - releaseStacklessGzipWriter(zw, level) - if bsc, ok := bs.(io.Closer); ok { - bsc.Close() - } - }) - } else { - bodyBytes := resp.bodyBytes() - if len(bodyBytes) < minCompressLen { - // There is no sense in spending CPU time on small body compression, - // since there is a very high probability that the compressed - // body size will be bigger than the original body size. - return nil - } - w := responseBodyPool.Get() - w.B = AppendGzipBytesLevel(w.B, bodyBytes, level) - - // Hack: swap resp.body with w. - if resp.body != nil { - responseBodyPool.Put(resp.body) - } - resp.body = w - } - resp.Header.SetCanonical(strContentEncoding, strGzip) - return nil -} - -func (resp *Response) deflateBody(level int) error { - if len(resp.Header.peek(strContentEncoding)) > 0 { - // It looks like the body is already compressed. - // Do not compress it again. - return nil - } - - if !resp.Header.isCompressibleContentType() { - // The content-type cannot be compressed. - return nil - } - - if resp.bodyStream != nil { - // Reset Content-Length to -1, since it is impossible - // to determine body size beforehand of streamed compression. - // For https://github.com/valyala/fasthttp/issues/176 . - resp.Header.SetContentLength(-1) - - // Do not care about memory allocations here, since flate is slow - // and allocates a lot of memory by itself. - bs := resp.bodyStream - resp.bodyStream = NewStreamReader(func(sw *bufio.Writer) { - zw := acquireStacklessDeflateWriter(sw, level) - fw := &flushWriter{ - wf: zw, - bw: sw, - } - copyZeroAlloc(fw, bs) - releaseStacklessDeflateWriter(zw, level) - if bsc, ok := bs.(io.Closer); ok { - bsc.Close() - } - }) - } else { - bodyBytes := resp.bodyBytes() - if len(bodyBytes) < minCompressLen { - // There is no sense in spending CPU time on small body compression, - // since there is a very high probability that the compressed - // body size will be bigger than the original body size. - return nil - } - w := responseBodyPool.Get() - w.B = AppendDeflateBytesLevel(w.B, bodyBytes, level) - - // Hack: swap resp.body with w. - if resp.body != nil { - responseBodyPool.Put(resp.body) - } - resp.body = w - } - resp.Header.SetCanonical(strContentEncoding, strDeflate) - return nil -} - -// Bodies with sizes smaller than minCompressLen aren't compressed at all -const minCompressLen = 200 - -type writeFlusher interface { - io.Writer - Flush() error -} - -type flushWriter struct { - wf writeFlusher - bw *bufio.Writer -} - -func (w *flushWriter) Write(p []byte) (int, error) { - n, err := w.wf.Write(p) - if err != nil { - return 0, err - } - if err = w.wf.Flush(); err != nil { - return 0, err - } - if err = w.bw.Flush(); err != nil { - return 0, err - } - return n, nil -} - -// Write writes response to w. -// -// Write doesn't flush response to w for performance reasons. -// -// See also WriteTo. -func (resp *Response) Write(w *bufio.Writer) error { - sendBody := !resp.mustSkipBody() - - if resp.bodyStream != nil { - return resp.writeBodyStream(w, sendBody) - } - - body := resp.bodyBytes() - bodyLen := len(body) - if sendBody || bodyLen > 0 { - resp.Header.SetContentLength(bodyLen) - } - if err := resp.Header.Write(w); err != nil { - return err - } - if sendBody { - if _, err := w.Write(body); err != nil { - return err - } - } - return nil -} - -func (req *Request) writeBodyStream(w *bufio.Writer) error { - var err error - - contentLength := req.Header.ContentLength() - if contentLength < 0 { - lrSize := limitedReaderSize(req.bodyStream) - if lrSize >= 0 { - contentLength = int(lrSize) - if int64(contentLength) != lrSize { - contentLength = -1 - } - if contentLength >= 0 { - req.Header.SetContentLength(contentLength) - } - } - } - if contentLength >= 0 { - if err = req.Header.Write(w); err == nil { - err = writeBodyFixedSize(w, req.bodyStream, int64(contentLength)) - } - } else { - req.Header.SetContentLength(-1) - if err = req.Header.Write(w); err == nil { - err = writeBodyChunked(w, req.bodyStream) - } - } - err1 := req.closeBodyStream() - if err == nil { - err = err1 - } - return err -} - -func (resp *Response) writeBodyStream(w *bufio.Writer, sendBody bool) error { - var err error - - contentLength := resp.Header.ContentLength() - if contentLength < 0 { - lrSize := limitedReaderSize(resp.bodyStream) - if lrSize >= 0 { - contentLength = int(lrSize) - if int64(contentLength) != lrSize { - contentLength = -1 - } - if contentLength >= 0 { - resp.Header.SetContentLength(contentLength) - } - } - } - if contentLength >= 0 { - if err = resp.Header.Write(w); err == nil && sendBody { - err = writeBodyFixedSize(w, resp.bodyStream, int64(contentLength)) - } - } else { - resp.Header.SetContentLength(-1) - if err = resp.Header.Write(w); err == nil && sendBody { - err = writeBodyChunked(w, resp.bodyStream) - } - } - err1 := resp.closeBodyStream() - if err == nil { - err = err1 - } - return err -} - -func (req *Request) closeBodyStream() error { - if req.bodyStream == nil { - return nil - } - var err error - if bsc, ok := req.bodyStream.(io.Closer); ok { - err = bsc.Close() - } - req.bodyStream = nil - return err -} - -func (resp *Response) closeBodyStream() error { - if resp.bodyStream == nil { - return nil - } - var err error - if bsc, ok := resp.bodyStream.(io.Closer); ok { - err = bsc.Close() - } - resp.bodyStream = nil - return err -} - -// String returns request representation. -// -// Returns error message instead of request representation on error. -// -// Use Write instead of String for performance-critical code. -func (req *Request) String() string { - return getHTTPString(req) -} - -// String returns response representation. -// -// Returns error message instead of response representation on error. -// -// Use Write instead of String for performance-critical code. -func (resp *Response) String() string { - return getHTTPString(resp) -} - -func getHTTPString(hw httpWriter) string { - w := bytebufferpool.Get() - bw := bufio.NewWriter(w) - if err := hw.Write(bw); err != nil { - return err.Error() - } - if err := bw.Flush(); err != nil { - return err.Error() - } - s := string(w.B) - bytebufferpool.Put(w) - return s -} - -type httpWriter interface { - Write(w *bufio.Writer) error -} - -func writeBodyChunked(w *bufio.Writer, r io.Reader) error { - vbuf := copyBufPool.Get() - buf := vbuf.([]byte) - - var err error - var n int - for { - n, err = r.Read(buf) - if n == 0 { - if err == nil { - panic("BUG: io.Reader returned 0, nil") - } - if err == io.EOF { - if err = writeChunk(w, buf[:0]); err != nil { - break - } - err = nil - } - break - } - if err = writeChunk(w, buf[:n]); err != nil { - break - } - } - - copyBufPool.Put(vbuf) - return err -} - -func limitedReaderSize(r io.Reader) int64 { - lr, ok := r.(*io.LimitedReader) - if !ok { - return -1 - } - return lr.N -} - -func writeBodyFixedSize(w *bufio.Writer, r io.Reader, size int64) error { - if size > maxSmallFileSize { - // w buffer must be empty for triggering - // sendfile path in bufio.Writer.ReadFrom. - if err := w.Flush(); err != nil { - return err - } - } - - // Unwrap a single limited reader for triggering sendfile path - // in net.TCPConn.ReadFrom. - lr, ok := r.(*io.LimitedReader) - if ok { - r = lr.R - } - - n, err := copyZeroAlloc(w, r) - - if ok { - lr.N -= n - } - - if n != size && err == nil { - err = fmt.Errorf("copied %d bytes from body stream instead of %d bytes", n, size) - } - return err -} - -func copyZeroAlloc(w io.Writer, r io.Reader) (int64, error) { - vbuf := copyBufPool.Get() - buf := vbuf.([]byte) - n, err := io.CopyBuffer(w, r, buf) - copyBufPool.Put(vbuf) - return n, err -} - -var copyBufPool = sync.Pool{ - New: func() interface{} { - return make([]byte, 4096) - }, -} - -func writeChunk(w *bufio.Writer, b []byte) error { - n := len(b) - writeHexInt(w, n) - w.Write(strCRLF) - w.Write(b) - _, err := w.Write(strCRLF) - err1 := w.Flush() - if err == nil { - err = err1 - } - return err -} - -// ErrBodyTooLarge is returned if either request or response body exceeds -// the given limit. -var ErrBodyTooLarge = errors.New("body size exceeds the given limit") - -func readBody(r *bufio.Reader, contentLength int, maxBodySize int, dst []byte) ([]byte, error) { - dst = dst[:0] - if contentLength >= 0 { - if maxBodySize > 0 && contentLength > maxBodySize { - return dst, ErrBodyTooLarge - } - return appendBodyFixedSize(r, dst, contentLength) - } - if contentLength == -1 { - return readBodyChunked(r, maxBodySize, dst) - } - return readBodyIdentity(r, maxBodySize, dst) -} - -func readBodyIdentity(r *bufio.Reader, maxBodySize int, dst []byte) ([]byte, error) { - dst = dst[:cap(dst)] - if len(dst) == 0 { - dst = make([]byte, 1024) - } - offset := 0 - for { - nn, err := r.Read(dst[offset:]) - if nn <= 0 { - if err != nil { - if err == io.EOF { - return dst[:offset], nil - } - return dst[:offset], err - } - panic(fmt.Sprintf("BUG: bufio.Read() returned (%d, nil)", nn)) - } - offset += nn - if maxBodySize > 0 && offset > maxBodySize { - return dst[:offset], ErrBodyTooLarge - } - if len(dst) == offset { - n := round2(2 * offset) - if maxBodySize > 0 && n > maxBodySize { - n = maxBodySize + 1 - } - b := make([]byte, n) - copy(b, dst) - dst = b - } - } -} - -func appendBodyFixedSize(r *bufio.Reader, dst []byte, n int) ([]byte, error) { - if n == 0 { - return dst, nil - } - - offset := len(dst) - dstLen := offset + n - if cap(dst) < dstLen { - b := make([]byte, round2(dstLen)) - copy(b, dst) - dst = b - } - dst = dst[:dstLen] - - for { - nn, err := r.Read(dst[offset:]) - if nn <= 0 { - if err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - return dst[:offset], err - } - panic(fmt.Sprintf("BUG: bufio.Read() returned (%d, nil)", nn)) - } - offset += nn - if offset == dstLen { - return dst, nil - } - } -} - -// ErrBrokenChunk is returned when server receives a broken chunked body (Transfer-Encoding: chunked). -type ErrBrokenChunk struct { - error -} - -func readBodyChunked(r *bufio.Reader, maxBodySize int, dst []byte) ([]byte, error) { - if len(dst) > 0 { - panic("BUG: expected zero-length buffer") - } - - strCRLFLen := len(strCRLF) - for { - chunkSize, err := parseChunkSize(r) - if err != nil { - return dst, err - } - if maxBodySize > 0 && len(dst)+chunkSize > maxBodySize { - return dst, ErrBodyTooLarge - } - dst, err = appendBodyFixedSize(r, dst, chunkSize+strCRLFLen) - if err != nil { - return dst, err - } - if !bytes.Equal(dst[len(dst)-strCRLFLen:], strCRLF) { - return dst, ErrBrokenChunk{ - error: fmt.Errorf("cannot find crlf at the end of chunk"), - } - } - dst = dst[:len(dst)-strCRLFLen] - if chunkSize == 0 { - return dst, nil - } - } -} - -func parseChunkSize(r *bufio.Reader) (int, error) { - n, err := readHexInt(r) - if err != nil { - return -1, err - } - for { - c, err := r.ReadByte() - if err != nil { - return -1, ErrBrokenChunk{ - error: fmt.Errorf("cannot read '\r' char at the end of chunk size: %s", err), - } - } - // Skip any trailing whitespace after chunk size. - if c == ' ' { - continue - } - if c != '\r' { - return -1, ErrBrokenChunk{ - error: fmt.Errorf("unexpected char %q at the end of chunk size. Expected %q", c, '\r'), - } - } - break - } - c, err := r.ReadByte() - if err != nil { - return -1, ErrBrokenChunk{ - error: fmt.Errorf("cannot read '\n' char at the end of chunk size: %s", err), - } - } - if c != '\n' { - return -1, ErrBrokenChunk{ - error: fmt.Errorf("unexpected char %q at the end of chunk size. Expected %q", c, '\n'), - } - } - return n, nil -} - -func round2(n int) int { - if n <= 0 { - return 0 - } - n-- - x := uint(0) - for n > 0 { - n >>= 1 - x++ - } - return 1 << x -} diff --git a/vendor/github.com/valyala/fasthttp/lbclient.go b/vendor/github.com/valyala/fasthttp/lbclient.go deleted file mode 100644 index 12418b6b..00000000 --- a/vendor/github.com/valyala/fasthttp/lbclient.go +++ /dev/null @@ -1,183 +0,0 @@ -package fasthttp - -import ( - "sync" - "sync/atomic" - "time" -) - -// BalancingClient is the interface for clients, which may be passed -// to LBClient.Clients. -type BalancingClient interface { - DoDeadline(req *Request, resp *Response, deadline time.Time) error - PendingRequests() int -} - -// LBClient balances requests among available LBClient.Clients. -// -// It has the following features: -// -// - Balances load among available clients using 'least loaded' + 'round robin' -// hybrid technique. -// - Dynamically decreases load on unhealthy clients. -// -// It is forbidden copying LBClient instances. Create new instances instead. -// -// It is safe calling LBClient methods from concurrently running goroutines. -type LBClient struct { - noCopy noCopy - - // Clients must contain non-zero clients list. - // Incoming requests are balanced among these clients. - Clients []BalancingClient - - // HealthCheck is a callback called after each request. - // - // The request, response and the error returned by the client - // is passed to HealthCheck, so the callback may determine whether - // the client is healthy. - // - // Load on the current client is decreased if HealthCheck returns false. - // - // By default HealthCheck returns false if err != nil. - HealthCheck func(req *Request, resp *Response, err error) bool - - // Timeout is the request timeout used when calling LBClient.Do. - // - // DefaultLBClientTimeout is used by default. - Timeout time.Duration - - cs []*lbClient - - // nextIdx is for spreading requests among equally loaded clients - // in a round-robin fashion. - nextIdx uint32 - - once sync.Once -} - -// DefaultLBClientTimeout is the default request timeout used by LBClient -// when calling LBClient.Do. -// -// The timeout may be overridden via LBClient.Timeout. -const DefaultLBClientTimeout = time.Second - -// DoDeadline calls DoDeadline on the least loaded client -func (cc *LBClient) DoDeadline(req *Request, resp *Response, deadline time.Time) error { - return cc.get().DoDeadline(req, resp, deadline) -} - -// DoTimeout calculates deadline and calls DoDeadline on the least loaded client -func (cc *LBClient) DoTimeout(req *Request, resp *Response, timeout time.Duration) error { - deadline := time.Now().Add(timeout) - return cc.get().DoDeadline(req, resp, deadline) -} - -// Do calls calculates deadline using LBClient.Timeout and calls DoDeadline -// on the least loaded client. -func (cc *LBClient) Do(req *Request, resp *Response) error { - timeout := cc.Timeout - if timeout <= 0 { - timeout = DefaultLBClientTimeout - } - return cc.DoTimeout(req, resp, timeout) -} - -func (cc *LBClient) init() { - if len(cc.Clients) == 0 { - panic("BUG: LBClient.Clients cannot be empty") - } - for _, c := range cc.Clients { - cc.cs = append(cc.cs, &lbClient{ - c: c, - healthCheck: cc.HealthCheck, - }) - } - - // Randomize nextIdx in order to prevent initial servers' - // hammering from a cluster of identical LBClients. - cc.nextIdx = uint32(time.Now().UnixNano()) -} - -func (cc *LBClient) get() *lbClient { - cc.once.Do(cc.init) - - cs := cc.cs - idx := atomic.AddUint32(&cc.nextIdx, 1) - idx %= uint32(len(cs)) - - minC := cs[idx] - minN := minC.PendingRequests() - if minN == 0 { - return minC - } - for _, c := range cs[idx+1:] { - n := c.PendingRequests() - if n == 0 { - return c - } - if n < minN { - minC = c - minN = n - } - } - for _, c := range cs[:idx] { - n := c.PendingRequests() - if n == 0 { - return c - } - if n < minN { - minC = c - minN = n - } - } - return minC -} - -type lbClient struct { - c BalancingClient - healthCheck func(req *Request, resp *Response, err error) bool - penalty uint32 -} - -func (c *lbClient) DoDeadline(req *Request, resp *Response, deadline time.Time) error { - err := c.c.DoDeadline(req, resp, deadline) - if !c.isHealthy(req, resp, err) && c.incPenalty() { - // Penalize the client returning error, so the next requests - // are routed to another clients. - time.AfterFunc(penaltyDuration, c.decPenalty) - } - return err -} - -func (c *lbClient) PendingRequests() int { - n := c.c.PendingRequests() - m := atomic.LoadUint32(&c.penalty) - return n + int(m) -} - -func (c *lbClient) isHealthy(req *Request, resp *Response, err error) bool { - if c.healthCheck == nil { - return err == nil - } - return c.healthCheck(req, resp, err) -} - -func (c *lbClient) incPenalty() bool { - m := atomic.AddUint32(&c.penalty, 1) - if m > maxPenalty { - c.decPenalty() - return false - } - return true -} - -func (c *lbClient) decPenalty() { - atomic.AddUint32(&c.penalty, ^uint32(0)) -} - -const ( - maxPenalty = 300 - - penaltyDuration = 3 * time.Second -) diff --git a/vendor/github.com/valyala/fasthttp/nocopy.go b/vendor/github.com/valyala/fasthttp/nocopy.go deleted file mode 100644 index 8e9b89a4..00000000 --- a/vendor/github.com/valyala/fasthttp/nocopy.go +++ /dev/null @@ -1,11 +0,0 @@ -package fasthttp - -// Embed this type into a struct, which mustn't be copied, -// so `go vet` gives a warning if this struct is copied. -// -// See https://github.com/golang/go/issues/8005#issuecomment-190753527 for details. -// and also: https://stackoverflow.com/questions/52494458/nocopy-minimal-example -type noCopy struct{} - -func (*noCopy) Lock() {} -func (*noCopy) Unlock() {} diff --git a/vendor/github.com/valyala/fasthttp/peripconn.go b/vendor/github.com/valyala/fasthttp/peripconn.go deleted file mode 100644 index afd2a927..00000000 --- a/vendor/github.com/valyala/fasthttp/peripconn.go +++ /dev/null @@ -1,100 +0,0 @@ -package fasthttp - -import ( - "fmt" - "net" - "sync" -) - -type perIPConnCounter struct { - pool sync.Pool - lock sync.Mutex - m map[uint32]int -} - -func (cc *perIPConnCounter) Register(ip uint32) int { - cc.lock.Lock() - if cc.m == nil { - cc.m = make(map[uint32]int) - } - n := cc.m[ip] + 1 - cc.m[ip] = n - cc.lock.Unlock() - return n -} - -func (cc *perIPConnCounter) Unregister(ip uint32) { - cc.lock.Lock() - if cc.m == nil { - cc.lock.Unlock() - panic("BUG: perIPConnCounter.Register() wasn't called") - } - n := cc.m[ip] - 1 - if n < 0 { - cc.lock.Unlock() - panic(fmt.Sprintf("BUG: negative per-ip counter=%d for ip=%d", n, ip)) - } - cc.m[ip] = n - cc.lock.Unlock() -} - -type perIPConn struct { - net.Conn - - ip uint32 - perIPConnCounter *perIPConnCounter -} - -func acquirePerIPConn(conn net.Conn, ip uint32, counter *perIPConnCounter) *perIPConn { - v := counter.pool.Get() - if v == nil { - v = &perIPConn{ - perIPConnCounter: counter, - } - } - c := v.(*perIPConn) - c.Conn = conn - c.ip = ip - return c -} - -func releasePerIPConn(c *perIPConn) { - c.Conn = nil - c.perIPConnCounter.pool.Put(c) -} - -func (c *perIPConn) Close() error { - err := c.Conn.Close() - c.perIPConnCounter.Unregister(c.ip) - releasePerIPConn(c) - return err -} - -func getUint32IP(c net.Conn) uint32 { - return ip2uint32(getConnIP4(c)) -} - -func getConnIP4(c net.Conn) net.IP { - addr := c.RemoteAddr() - ipAddr, ok := addr.(*net.TCPAddr) - if !ok { - return net.IPv4zero - } - return ipAddr.IP.To4() -} - -func ip2uint32(ip net.IP) uint32 { - if len(ip) != 4 { - return 0 - } - return uint32(ip[0])<<24 | uint32(ip[1])<<16 | uint32(ip[2])<<8 | uint32(ip[3]) -} - -func uint322ip(ip uint32) net.IP { - b := make([]byte, 4) - b[0] = byte(ip >> 24) - b[1] = byte(ip >> 16) - b[2] = byte(ip >> 8) - b[3] = byte(ip) - return b -} diff --git a/vendor/github.com/valyala/fasthttp/server.go b/vendor/github.com/valyala/fasthttp/server.go deleted file mode 100644 index 5fcf20f5..00000000 --- a/vendor/github.com/valyala/fasthttp/server.go +++ /dev/null @@ -1,2501 +0,0 @@ -package fasthttp - -import ( - "bufio" - "context" - "crypto/tls" - "errors" - "fmt" - "io" - "log" - "mime/multipart" - "net" - "os" - "strings" - "sync" - "sync/atomic" - "time" -) - -var errNoCertOrKeyProvided = errors.New("Cert or key has not provided") - -var ( - // ErrAlreadyServing is returned when calling Serve on a Server - // that is already serving connections. - ErrAlreadyServing = errors.New("Server is already serving connections") -) - -// ServeConn serves HTTP requests from the given connection -// using the given handler. -// -// ServeConn returns nil if all requests from the c are successfully served. -// It returns non-nil error otherwise. -// -// Connection c must immediately propagate all the data passed to Write() -// to the client. Otherwise requests' processing may hang. -// -// ServeConn closes c before returning. -func ServeConn(c net.Conn, handler RequestHandler) error { - v := serverPool.Get() - if v == nil { - v = &Server{} - } - s := v.(*Server) - s.Handler = handler - err := s.ServeConn(c) - s.Handler = nil - serverPool.Put(v) - return err -} - -var serverPool sync.Pool - -// Serve serves incoming connections from the given listener -// using the given handler. -// -// Serve blocks until the given listener returns permanent error. -func Serve(ln net.Listener, handler RequestHandler) error { - s := &Server{ - Handler: handler, - } - return s.Serve(ln) -} - -// ServeTLS serves HTTPS requests from the given net.Listener -// using the given handler. -// -// certFile and keyFile are paths to TLS certificate and key files. -func ServeTLS(ln net.Listener, certFile, keyFile string, handler RequestHandler) error { - s := &Server{ - Handler: handler, - } - return s.ServeTLS(ln, certFile, keyFile) -} - -// ServeTLSEmbed serves HTTPS requests from the given net.Listener -// using the given handler. -// -// certData and keyData must contain valid TLS certificate and key data. -func ServeTLSEmbed(ln net.Listener, certData, keyData []byte, handler RequestHandler) error { - s := &Server{ - Handler: handler, - } - return s.ServeTLSEmbed(ln, certData, keyData) -} - -// ListenAndServe serves HTTP requests from the given TCP addr -// using the given handler. -func ListenAndServe(addr string, handler RequestHandler) error { - s := &Server{ - Handler: handler, - } - return s.ListenAndServe(addr) -} - -// ListenAndServeUNIX serves HTTP requests from the given UNIX addr -// using the given handler. -// -// The function deletes existing file at addr before starting serving. -// -// The server sets the given file mode for the UNIX addr. -func ListenAndServeUNIX(addr string, mode os.FileMode, handler RequestHandler) error { - s := &Server{ - Handler: handler, - } - return s.ListenAndServeUNIX(addr, mode) -} - -// ListenAndServeTLS serves HTTPS requests from the given TCP addr -// using the given handler. -// -// certFile and keyFile are paths to TLS certificate and key files. -func ListenAndServeTLS(addr, certFile, keyFile string, handler RequestHandler) error { - s := &Server{ - Handler: handler, - } - return s.ListenAndServeTLS(addr, certFile, keyFile) -} - -// ListenAndServeTLSEmbed serves HTTPS requests from the given TCP addr -// using the given handler. -// -// certData and keyData must contain valid TLS certificate and key data. -func ListenAndServeTLSEmbed(addr string, certData, keyData []byte, handler RequestHandler) error { - s := &Server{ - Handler: handler, - } - return s.ListenAndServeTLSEmbed(addr, certData, keyData) -} - -// RequestHandler must process incoming requests. -// -// RequestHandler must call ctx.TimeoutError() before returning -// if it keeps references to ctx and/or its' members after the return. -// Consider wrapping RequestHandler into TimeoutHandler if response time -// must be limited. -type RequestHandler func(ctx *RequestCtx) - -// ServeHandler must process tls.Config.NextProto negotiated requests. -type ServeHandler func(c net.Conn) error - -// Server implements HTTP server. -// -// Default Server settings should satisfy the majority of Server users. -// Adjust Server settings only if you really understand the consequences. -// -// It is forbidden copying Server instances. Create new Server instances -// instead. -// -// It is safe to call Server methods from concurrently running goroutines. -type Server struct { - noCopy noCopy - - // Handler for processing incoming requests. - // - // Take into account that no `panic` recovery is done by `fasthttp` (thus any `panic` will take down the entire server). - // Instead the user should use `recover` to handle these situations. - Handler RequestHandler - - // ErrorHandler for returning a response in case of an error while receiving or parsing the request. - // - // The following is a non-exhaustive list of errors that can be expected as argument: - // * io.EOF - // * io.ErrUnexpectedEOF - // * ErrGetOnly - // * ErrSmallBuffer - // * ErrBodyTooLarge - // * ErrBrokenChunks - ErrorHandler func(ctx *RequestCtx, err error) - - // Server name for sending in response headers. - // - // Default server name is used if left blank. - Name string - - // The maximum number of concurrent connections the server may serve. - // - // DefaultConcurrency is used if not set. - Concurrency int - - // Whether to disable keep-alive connections. - // - // The server will close all the incoming connections after sending - // the first response to client if this option is set to true. - // - // By default keep-alive connections are enabled. - DisableKeepalive bool - - // Per-connection buffer size for requests' reading. - // This also limits the maximum header size. - // - // Increase this buffer if your clients send multi-KB RequestURIs - // and/or multi-KB headers (for example, BIG cookies). - // - // Default buffer size is used if not set. - ReadBufferSize int - - // Per-connection buffer size for responses' writing. - // - // Default buffer size is used if not set. - WriteBufferSize int - - // Maximum duration for reading the full request (including body). - // - // This also limits the maximum duration for idle keep-alive - // connections. - // - // By default request read timeout is unlimited. - ReadTimeout time.Duration - - // Maximum duration for writing the full response (including body). - // - // By default response write timeout is unlimited. - WriteTimeout time.Duration - - // Maximum number of concurrent client connections allowed per IP. - // - // By default unlimited number of concurrent connections - // may be established to the server from a single IP address. - MaxConnsPerIP int - - // Maximum number of requests served per connection. - // - // The server closes connection after the last request. - // 'Connection: close' header is added to the last response. - // - // By default unlimited number of requests may be served per connection. - MaxRequestsPerConn int - - // Maximum keep-alive connection lifetime. - // - // The server closes keep-alive connection after its' lifetime - // expiration. - // - // See also ReadTimeout for limiting the duration of idle keep-alive - // connections. - // - // By default keep-alive connection lifetime is unlimited. - MaxKeepaliveDuration time.Duration - - // Whether to enable tcp keep-alive connections. - // - // Whether the operating system should send tcp keep-alive messages on the tcp connection. - // - // By default tcp keep-alive connections are disabled. - TCPKeepalive bool - - // Period between tcp keep-alive messages. - // - // TCP keep-alive period is determined by operation system by default. - TCPKeepalivePeriod time.Duration - - // Maximum request body size. - // - // The server rejects requests with bodies exceeding this limit. - // - // Request body size is limited by DefaultMaxRequestBodySize by default. - MaxRequestBodySize int - - // Aggressively reduces memory usage at the cost of higher CPU usage - // if set to true. - // - // Try enabling this option only if the server consumes too much memory - // serving mostly idle keep-alive connections. This may reduce memory - // usage by more than 50%. - // - // Aggressive memory usage reduction is disabled by default. - ReduceMemoryUsage bool - - // Rejects all non-GET requests if set to true. - // - // This option is useful as anti-DoS protection for servers - // accepting only GET requests. The request size is limited - // by ReadBufferSize if GetOnly is set. - // - // Server accepts all the requests by default. - GetOnly bool - - // Logs all errors, including the most frequent - // 'connection reset by peer', 'broken pipe' and 'connection timeout' - // errors. Such errors are common in production serving real-world - // clients. - // - // By default the most frequent errors such as - // 'connection reset by peer', 'broken pipe' and 'connection timeout' - // are suppressed in order to limit output log traffic. - LogAllErrors bool - - // Header names are passed as-is without normalization - // if this option is set. - // - // Disabled header names' normalization may be useful only for proxying - // incoming requests to other servers expecting case-sensitive - // header names. See https://github.com/valyala/fasthttp/issues/57 - // for details. - // - // By default request and response header names are normalized, i.e. - // The first letter and the first letters following dashes - // are uppercased, while all the other letters are lowercased. - // Examples: - // - // * HOST -> Host - // * content-type -> Content-Type - // * cONTENT-lenGTH -> Content-Length - DisableHeaderNamesNormalizing bool - - // SleepWhenConcurrencyLimitsExceeded is a duration to be slept of if - // the concurrency limit in exceeded (default [when is 0]: don't sleep - // and accept new connections immidiatelly). - SleepWhenConcurrencyLimitsExceeded time.Duration - - // NoDefaultServerHeader, when set to true, causes the default Server header - // to be excluded from the Response. - // - // The default Server header value is the value of the Name field or an - // internal default value in its absence. With this option set to true, - // the only time a Server header will be sent is if a non-zero length - // value is explicitly provided during a request. - NoDefaultServerHeader bool - - // NoDefaultContentType, when set to true, causes the default Content-Type - // header to be excluded from the Response. - // - // The default Content-Type header value is the internal default value. When - // set to true, the Content-Type will not be present. - NoDefaultContentType bool - - // ConnState specifies an optional callback function that is - // called when a client connection changes state. See the - // ConnState type and associated constants for details. - ConnState func(net.Conn, ConnState) - - // Logger, which is used by RequestCtx.Logger(). - // - // By default standard logger from log package is used. - Logger Logger - - tlsConfig *tls.Config - nextProtos map[string]ServeHandler - - concurrency uint32 - concurrencyCh chan struct{} - perIPConnCounter perIPConnCounter - serverName atomic.Value - - ctxPool sync.Pool - readerPool sync.Pool - writerPool sync.Pool - hijackConnPool sync.Pool - bytePool sync.Pool - - // We need to know our listener so we can close it in Shutdown(). - ln net.Listener - - mu sync.Mutex - open int32 - stop int32 - done chan struct{} -} - -// TimeoutHandler creates RequestHandler, which returns StatusRequestTimeout -// error with the given msg to the client if h didn't return during -// the given duration. -// -// The returned handler may return StatusTooManyRequests error with the given -// msg to the client if there are more than Server.Concurrency concurrent -// handlers h are running at the moment. -func TimeoutHandler(h RequestHandler, timeout time.Duration, msg string) RequestHandler { - if timeout <= 0 { - return h - } - - return func(ctx *RequestCtx) { - concurrencyCh := ctx.s.concurrencyCh - select { - case concurrencyCh <- struct{}{}: - default: - ctx.Error(msg, StatusTooManyRequests) - return - } - - ch := ctx.timeoutCh - if ch == nil { - ch = make(chan struct{}, 1) - ctx.timeoutCh = ch - } - go func() { - h(ctx) - ch <- struct{}{} - <-concurrencyCh - }() - ctx.timeoutTimer = initTimer(ctx.timeoutTimer, timeout) - select { - case <-ch: - case <-ctx.timeoutTimer.C: - ctx.TimeoutError(msg) - } - stopTimer(ctx.timeoutTimer) - } -} - -// CompressHandler returns RequestHandler that transparently compresses -// response body generated by h if the request contains 'gzip' or 'deflate' -// 'Accept-Encoding' header. -func CompressHandler(h RequestHandler) RequestHandler { - return CompressHandlerLevel(h, CompressDefaultCompression) -} - -// CompressHandlerLevel returns RequestHandler that transparently compresses -// response body generated by h if the request contains 'gzip' or 'deflate' -// 'Accept-Encoding' header. -// -// Level is the desired compression level: -// -// * CompressNoCompression -// * CompressBestSpeed -// * CompressBestCompression -// * CompressDefaultCompression -// * CompressHuffmanOnly -func CompressHandlerLevel(h RequestHandler, level int) RequestHandler { - return func(ctx *RequestCtx) { - h(ctx) - if ctx.Request.Header.HasAcceptEncodingBytes(strGzip) { - ctx.Response.gzipBody(level) - } else if ctx.Request.Header.HasAcceptEncodingBytes(strDeflate) { - ctx.Response.deflateBody(level) - } - } -} - -// RequestCtx contains incoming request and manages outgoing response. -// -// It is forbidden copying RequestCtx instances. -// -// RequestHandler should avoid holding references to incoming RequestCtx and/or -// its' members after the return. -// If holding RequestCtx references after the return is unavoidable -// (for instance, ctx is passed to a separate goroutine and ctx lifetime cannot -// be controlled), then the RequestHandler MUST call ctx.TimeoutError() -// before return. -// -// It is unsafe modifying/reading RequestCtx instance from concurrently -// running goroutines. The only exception is TimeoutError*, which may be called -// while other goroutines accessing RequestCtx. -type RequestCtx struct { - noCopy noCopy - - // Incoming request. - // - // Copying Request by value is forbidden. Use pointer to Request instead. - Request Request - - // Outgoing response. - // - // Copying Response by value is forbidden. Use pointer to Response instead. - Response Response - - userValues userData - - lastReadDuration time.Duration - - connID uint64 - connRequestNum uint64 - connTime time.Time - - time time.Time - - logger ctxLogger - s *Server - c net.Conn - fbr firstByteReader - - timeoutResponse *Response - timeoutCh chan struct{} - timeoutTimer *time.Timer - - hijackHandler HijackHandler -} - -// HijackHandler must process the hijacked connection c. -// -// The connection c is automatically closed after returning from HijackHandler. -// -// The connection c must not be used after returning from the handler. -type HijackHandler func(c net.Conn) - -// Hijack registers the given handler for connection hijacking. -// -// The handler is called after returning from RequestHandler -// and sending http response. The current connection is passed -// to the handler. The connection is automatically closed after -// returning from the handler. -// -// The server skips calling the handler in the following cases: -// -// * 'Connection: close' header exists in either request or response. -// * Unexpected error during response writing to the connection. -// -// The server stops processing requests from hijacked connections. -// Server limits such as Concurrency, ReadTimeout, WriteTimeout, etc. -// aren't applied to hijacked connections. -// -// The handler must not retain references to ctx members. -// -// Arbitrary 'Connection: Upgrade' protocols may be implemented -// with HijackHandler. For instance, -// -// * WebSocket ( https://en.wikipedia.org/wiki/WebSocket ) -// * HTTP/2.0 ( https://en.wikipedia.org/wiki/HTTP/2 ) -// -func (ctx *RequestCtx) Hijack(handler HijackHandler) { - ctx.hijackHandler = handler -} - -// Hijacked returns true after Hijack is called. -func (ctx *RequestCtx) Hijacked() bool { - return ctx.hijackHandler != nil -} - -// SetUserValue stores the given value (arbitrary object) -// under the given key in ctx. -// -// The value stored in ctx may be obtained by UserValue*. -// -// This functionality may be useful for passing arbitrary values between -// functions involved in request processing. -// -// All the values are removed from ctx after returning from the top -// RequestHandler. Additionally, Close method is called on each value -// implementing io.Closer before removing the value from ctx. -func (ctx *RequestCtx) SetUserValue(key string, value interface{}) { - ctx.userValues.Set(key, value) -} - -// SetUserValueBytes stores the given value (arbitrary object) -// under the given key in ctx. -// -// The value stored in ctx may be obtained by UserValue*. -// -// This functionality may be useful for passing arbitrary values between -// functions involved in request processing. -// -// All the values stored in ctx are deleted after returning from RequestHandler. -func (ctx *RequestCtx) SetUserValueBytes(key []byte, value interface{}) { - ctx.userValues.SetBytes(key, value) -} - -// UserValue returns the value stored via SetUserValue* under the given key. -func (ctx *RequestCtx) UserValue(key string) interface{} { - return ctx.userValues.Get(key) -} - -// UserValueBytes returns the value stored via SetUserValue* -// under the given key. -func (ctx *RequestCtx) UserValueBytes(key []byte) interface{} { - return ctx.userValues.GetBytes(key) -} - -// VisitUserValues calls visitor for each existing userValue. -// -// visitor must not retain references to key and value after returning. -// Make key and/or value copies if you need storing them after returning. -func (ctx *RequestCtx) VisitUserValues(visitor func([]byte, interface{})) { - for i, n := 0, len(ctx.userValues); i < n; i++ { - kv := &ctx.userValues[i] - visitor(kv.key, kv.value) - } -} - -type connTLSer interface { - Handshake() error - ConnectionState() tls.ConnectionState -} - -// IsTLS returns true if the underlying connection is tls.Conn. -// -// tls.Conn is an encrypted connection (aka SSL, HTTPS). -func (ctx *RequestCtx) IsTLS() bool { - // cast to (connTLSer) instead of (*tls.Conn), since it catches - // cases with overridden tls.Conn such as: - // - // type customConn struct { - // *tls.Conn - // - // // other custom fields here - // } - _, ok := ctx.c.(connTLSer) - return ok -} - -// TLSConnectionState returns TLS connection state. -// -// The function returns nil if the underlying connection isn't tls.Conn. -// -// The returned state may be used for verifying TLS version, client certificates, -// etc. -func (ctx *RequestCtx) TLSConnectionState() *tls.ConnectionState { - tlsConn, ok := ctx.c.(connTLSer) - if !ok { - return nil - } - state := tlsConn.ConnectionState() - return &state -} - -// Conn returns a reference to the underlying net.Conn. -// -// WARNING: Only use this method if you know what you are doing! -// -// Reading from or writing to the returned connection will end badly! -func (ctx *RequestCtx) Conn() net.Conn { - return ctx.c -} - -type firstByteReader struct { - c net.Conn - ch byte - byteRead bool -} - -func (r *firstByteReader) Read(b []byte) (int, error) { - if len(b) == 0 { - return 0, nil - } - nn := 0 - if !r.byteRead { - b[0] = r.ch - b = b[1:] - r.byteRead = true - nn = 1 - } - n, err := r.c.Read(b) - return n + nn, err -} - -// Logger is used for logging formatted messages. -type Logger interface { - // Printf must have the same semantics as log.Printf. - Printf(format string, args ...interface{}) -} - -var ctxLoggerLock sync.Mutex - -type ctxLogger struct { - ctx *RequestCtx - logger Logger -} - -func (cl *ctxLogger) Printf(format string, args ...interface{}) { - ctxLoggerLock.Lock() - msg := fmt.Sprintf(format, args...) - ctx := cl.ctx - cl.logger.Printf("%.3f %s - %s", time.Since(ctx.Time()).Seconds(), ctx.String(), msg) - ctxLoggerLock.Unlock() -} - -var zeroTCPAddr = &net.TCPAddr{ - IP: net.IPv4zero, -} - -// String returns unique string representation of the ctx. -// -// The returned value may be useful for logging. -func (ctx *RequestCtx) String() string { - return fmt.Sprintf("#%016X - %s<->%s - %s %s", ctx.ID(), ctx.LocalAddr(), ctx.RemoteAddr(), ctx.Request.Header.Method(), ctx.URI().FullURI()) -} - -// ID returns unique ID of the request. -func (ctx *RequestCtx) ID() uint64 { - return (ctx.connID << 32) | ctx.connRequestNum -} - -// ConnID returns unique connection ID. -// -// This ID may be used to match distinct requests to the same incoming -// connection. -func (ctx *RequestCtx) ConnID() uint64 { - return ctx.connID -} - -// Time returns RequestHandler call time. -func (ctx *RequestCtx) Time() time.Time { - return ctx.time -} - -// ConnTime returns the time the server started serving the connection -// the current request came from. -func (ctx *RequestCtx) ConnTime() time.Time { - return ctx.connTime -} - -// ConnRequestNum returns request sequence number -// for the current connection. -// -// Sequence starts with 1. -func (ctx *RequestCtx) ConnRequestNum() uint64 { - return ctx.connRequestNum -} - -// SetConnectionClose sets 'Connection: close' response header and closes -// connection after the RequestHandler returns. -func (ctx *RequestCtx) SetConnectionClose() { - ctx.Response.SetConnectionClose() -} - -// SetStatusCode sets response status code. -func (ctx *RequestCtx) SetStatusCode(statusCode int) { - ctx.Response.SetStatusCode(statusCode) -} - -// SetContentType sets response Content-Type. -func (ctx *RequestCtx) SetContentType(contentType string) { - ctx.Response.Header.SetContentType(contentType) -} - -// SetContentTypeBytes sets response Content-Type. -// -// It is safe modifying contentType buffer after function return. -func (ctx *RequestCtx) SetContentTypeBytes(contentType []byte) { - ctx.Response.Header.SetContentTypeBytes(contentType) -} - -// RequestURI returns RequestURI. -// -// This uri is valid until returning from RequestHandler. -func (ctx *RequestCtx) RequestURI() []byte { - return ctx.Request.Header.RequestURI() -} - -// URI returns requested uri. -// -// The uri is valid until returning from RequestHandler. -func (ctx *RequestCtx) URI() *URI { - return ctx.Request.URI() -} - -// Referer returns request referer. -// -// The referer is valid until returning from RequestHandler. -func (ctx *RequestCtx) Referer() []byte { - return ctx.Request.Header.Referer() -} - -// UserAgent returns User-Agent header value from the request. -func (ctx *RequestCtx) UserAgent() []byte { - return ctx.Request.Header.UserAgent() -} - -// Path returns requested path. -// -// The path is valid until returning from RequestHandler. -func (ctx *RequestCtx) Path() []byte { - return ctx.URI().Path() -} - -// Host returns requested host. -// -// The host is valid until returning from RequestHandler. -func (ctx *RequestCtx) Host() []byte { - return ctx.URI().Host() -} - -// QueryArgs returns query arguments from RequestURI. -// -// It doesn't return POST'ed arguments - use PostArgs() for this. -// -// Returned arguments are valid until returning from RequestHandler. -// -// See also PostArgs, FormValue and FormFile. -func (ctx *RequestCtx) QueryArgs() *Args { - return ctx.URI().QueryArgs() -} - -// PostArgs returns POST arguments. -// -// It doesn't return query arguments from RequestURI - use QueryArgs for this. -// -// Returned arguments are valid until returning from RequestHandler. -// -// See also QueryArgs, FormValue and FormFile. -func (ctx *RequestCtx) PostArgs() *Args { - return ctx.Request.PostArgs() -} - -// MultipartForm returns requests's multipart form. -// -// Returns ErrNoMultipartForm if request's content-type -// isn't 'multipart/form-data'. -// -// All uploaded temporary files are automatically deleted after -// returning from RequestHandler. Either move or copy uploaded files -// into new place if you want retaining them. -// -// Use SaveMultipartFile function for permanently saving uploaded file. -// -// The returned form is valid until returning from RequestHandler. -// -// See also FormFile and FormValue. -func (ctx *RequestCtx) MultipartForm() (*multipart.Form, error) { - return ctx.Request.MultipartForm() -} - -// FormFile returns uploaded file associated with the given multipart form key. -// -// The file is automatically deleted after returning from RequestHandler, -// so either move or copy uploaded file into new place if you want retaining it. -// -// Use SaveMultipartFile function for permanently saving uploaded file. -// -// The returned file header is valid until returning from RequestHandler. -func (ctx *RequestCtx) FormFile(key string) (*multipart.FileHeader, error) { - mf, err := ctx.MultipartForm() - if err != nil { - return nil, err - } - if mf.File == nil { - return nil, err - } - fhh := mf.File[key] - if fhh == nil { - return nil, ErrMissingFile - } - return fhh[0], nil -} - -// ErrMissingFile may be returned from FormFile when the is no uploaded file -// associated with the given multipart form key. -var ErrMissingFile = errors.New("there is no uploaded file associated with the given key") - -// SaveMultipartFile saves multipart file fh under the given filename path. -func SaveMultipartFile(fh *multipart.FileHeader, path string) error { - f, err := fh.Open() - if err != nil { - return err - } - - if ff, ok := f.(*os.File); ok { - // Windows can't rename files that are opened. - if err := f.Close(); err != nil { - return err - } - - // If renaming fails we try the normal copying method. - // Renaming could fail if the files are on different devices. - if os.Rename(ff.Name(), path) == nil { - return nil - } - - // Reopen f for the code below. - f, err = fh.Open() - if err != nil { - return err - } - } - - defer f.Close() - - ff, err := os.Create(path) - if err != nil { - return err - } - defer ff.Close() - _, err = copyZeroAlloc(ff, f) - return err -} - -// FormValue returns form value associated with the given key. -// -// The value is searched in the following places: -// -// * Query string. -// * POST or PUT body. -// -// There are more fine-grained methods for obtaining form values: -// -// * QueryArgs for obtaining values from query string. -// * PostArgs for obtaining values from POST or PUT body. -// * MultipartForm for obtaining values from multipart form. -// * FormFile for obtaining uploaded files. -// -// The returned value is valid until returning from RequestHandler. -func (ctx *RequestCtx) FormValue(key string) []byte { - v := ctx.QueryArgs().Peek(key) - if len(v) > 0 { - return v - } - v = ctx.PostArgs().Peek(key) - if len(v) > 0 { - return v - } - mf, err := ctx.MultipartForm() - if err == nil && mf.Value != nil { - vv := mf.Value[key] - if len(vv) > 0 { - return []byte(vv[0]) - } - } - return nil -} - -// IsGet returns true if request method is GET. -func (ctx *RequestCtx) IsGet() bool { - return ctx.Request.Header.IsGet() -} - -// IsPost returns true if request method is POST. -func (ctx *RequestCtx) IsPost() bool { - return ctx.Request.Header.IsPost() -} - -// IsPut returns true if request method is PUT. -func (ctx *RequestCtx) IsPut() bool { - return ctx.Request.Header.IsPut() -} - -// IsDelete returns true if request method is DELETE. -func (ctx *RequestCtx) IsDelete() bool { - return ctx.Request.Header.IsDelete() -} - -// IsConnect returns true if request method is CONNECT. -func (ctx *RequestCtx) IsConnect() bool { - return ctx.Request.Header.IsConnect() -} - -// IsOptions returns true if request method is OPTIONS. -func (ctx *RequestCtx) IsOptions() bool { - return ctx.Request.Header.IsOptions() -} - -// IsTrace returns true if request method is TRACE. -func (ctx *RequestCtx) IsTrace() bool { - return ctx.Request.Header.IsTrace() -} - -// IsPatch returns true if request method is PATCH. -func (ctx *RequestCtx) IsPatch() bool { - return ctx.Request.Header.IsPatch() -} - -// Method return request method. -// -// Returned value is valid until returning from RequestHandler. -func (ctx *RequestCtx) Method() []byte { - return ctx.Request.Header.Method() -} - -// IsHead returns true if request method is HEAD. -func (ctx *RequestCtx) IsHead() bool { - return ctx.Request.Header.IsHead() -} - -// RemoteAddr returns client address for the given request. -// -// Always returns non-nil result. -func (ctx *RequestCtx) RemoteAddr() net.Addr { - if ctx.c == nil { - return zeroTCPAddr - } - addr := ctx.c.RemoteAddr() - if addr == nil { - return zeroTCPAddr - } - return addr -} - -// LocalAddr returns server address for the given request. -// -// Always returns non-nil result. -func (ctx *RequestCtx) LocalAddr() net.Addr { - if ctx.c == nil { - return zeroTCPAddr - } - addr := ctx.c.LocalAddr() - if addr == nil { - return zeroTCPAddr - } - return addr -} - -// RemoteIP returns the client ip the request came from. -// -// Always returns non-nil result. -func (ctx *RequestCtx) RemoteIP() net.IP { - return addrToIP(ctx.RemoteAddr()) -} - -// LocalIP returns the server ip the request came to. -// -// Always returns non-nil result. -func (ctx *RequestCtx) LocalIP() net.IP { - return addrToIP(ctx.LocalAddr()) -} - -func addrToIP(addr net.Addr) net.IP { - x, ok := addr.(*net.TCPAddr) - if !ok { - return net.IPv4zero - } - return x.IP -} - -// Error sets response status code to the given value and sets response body -// to the given message. -func (ctx *RequestCtx) Error(msg string, statusCode int) { - ctx.Response.Reset() - ctx.SetStatusCode(statusCode) - ctx.SetContentTypeBytes(defaultContentType) - ctx.SetBodyString(msg) -} - -// Success sets response Content-Type and body to the given values. -func (ctx *RequestCtx) Success(contentType string, body []byte) { - ctx.SetContentType(contentType) - ctx.SetBody(body) -} - -// SuccessString sets response Content-Type and body to the given values. -func (ctx *RequestCtx) SuccessString(contentType, body string) { - ctx.SetContentType(contentType) - ctx.SetBodyString(body) -} - -// Redirect sets 'Location: uri' response header and sets the given statusCode. -// -// statusCode must have one of the following values: -// -// * StatusMovedPermanently (301) -// * StatusFound (302) -// * StatusSeeOther (303) -// * StatusTemporaryRedirect (307) -// * StatusPermanentRedirect (308) -// -// All other statusCode values are replaced by StatusFound (302). -// -// The redirect uri may be either absolute or relative to the current -// request uri. Fasthttp will always send an absolute uri back to the client. -// To send a relative uri you can use the following code: -// -// strLocation = []byte("Location") // Put this with your top level var () declarations. -// ctx.Response.Header.SetCanonical(strLocation, "/relative?uri") -// ctx.Response.SetStatusCode(fasthttp.StatusMovedPermanently) -// -func (ctx *RequestCtx) Redirect(uri string, statusCode int) { - u := AcquireURI() - ctx.URI().CopyTo(u) - u.Update(uri) - ctx.redirect(u.FullURI(), statusCode) - ReleaseURI(u) -} - -// RedirectBytes sets 'Location: uri' response header and sets -// the given statusCode. -// -// statusCode must have one of the following values: -// -// * StatusMovedPermanently (301) -// * StatusFound (302) -// * StatusSeeOther (303) -// * StatusTemporaryRedirect (307) -// * StatusPermanentRedirect (308) -// -// All other statusCode values are replaced by StatusFound (302). -// -// The redirect uri may be either absolute or relative to the current -// request uri. Fasthttp will always send an absolute uri back to the client. -// To send a relative uri you can use the following code: -// -// strLocation = []byte("Location") // Put this with your top level var () declarations. -// ctx.Response.Header.SetCanonical(strLocation, "/relative?uri") -// ctx.Response.SetStatusCode(fasthttp.StatusMovedPermanently) -// -func (ctx *RequestCtx) RedirectBytes(uri []byte, statusCode int) { - s := b2s(uri) - ctx.Redirect(s, statusCode) -} - -func (ctx *RequestCtx) redirect(uri []byte, statusCode int) { - ctx.Response.Header.SetCanonical(strLocation, uri) - statusCode = getRedirectStatusCode(statusCode) - ctx.Response.SetStatusCode(statusCode) -} - -func getRedirectStatusCode(statusCode int) int { - if statusCode == StatusMovedPermanently || statusCode == StatusFound || - statusCode == StatusSeeOther || statusCode == StatusTemporaryRedirect || - statusCode == StatusPermanentRedirect { - return statusCode - } - return StatusFound -} - -// SetBody sets response body to the given value. -// -// It is safe re-using body argument after the function returns. -func (ctx *RequestCtx) SetBody(body []byte) { - ctx.Response.SetBody(body) -} - -// SetBodyString sets response body to the given value. -func (ctx *RequestCtx) SetBodyString(body string) { - ctx.Response.SetBodyString(body) -} - -// ResetBody resets response body contents. -func (ctx *RequestCtx) ResetBody() { - ctx.Response.ResetBody() -} - -// SendFile sends local file contents from the given path as response body. -// -// This is a shortcut to ServeFile(ctx, path). -// -// SendFile logs all the errors via ctx.Logger. -// -// See also ServeFile, FSHandler and FS. -func (ctx *RequestCtx) SendFile(path string) { - ServeFile(ctx, path) -} - -// SendFileBytes sends local file contents from the given path as response body. -// -// This is a shortcut to ServeFileBytes(ctx, path). -// -// SendFileBytes logs all the errors via ctx.Logger. -// -// See also ServeFileBytes, FSHandler and FS. -func (ctx *RequestCtx) SendFileBytes(path []byte) { - ServeFileBytes(ctx, path) -} - -// IfModifiedSince returns true if lastModified exceeds 'If-Modified-Since' -// value from the request header. -// -// The function returns true also 'If-Modified-Since' request header is missing. -func (ctx *RequestCtx) IfModifiedSince(lastModified time.Time) bool { - ifModStr := ctx.Request.Header.peek(strIfModifiedSince) - if len(ifModStr) == 0 { - return true - } - ifMod, err := ParseHTTPDate(ifModStr) - if err != nil { - return true - } - lastModified = lastModified.Truncate(time.Second) - return ifMod.Before(lastModified) -} - -// NotModified resets response and sets '304 Not Modified' response status code. -func (ctx *RequestCtx) NotModified() { - ctx.Response.Reset() - ctx.SetStatusCode(StatusNotModified) -} - -// NotFound resets response and sets '404 Not Found' response status code. -func (ctx *RequestCtx) NotFound() { - ctx.Response.Reset() - ctx.SetStatusCode(StatusNotFound) - ctx.SetBodyString("404 Page not found") -} - -// Write writes p into response body. -func (ctx *RequestCtx) Write(p []byte) (int, error) { - ctx.Response.AppendBody(p) - return len(p), nil -} - -// WriteString appends s to response body. -func (ctx *RequestCtx) WriteString(s string) (int, error) { - ctx.Response.AppendBodyString(s) - return len(s), nil -} - -// PostBody returns POST request body. -// -// The returned value is valid until RequestHandler return. -func (ctx *RequestCtx) PostBody() []byte { - return ctx.Request.Body() -} - -// SetBodyStream sets response body stream and, optionally body size. -// -// bodyStream.Close() is called after finishing reading all body data -// if it implements io.Closer. -// -// If bodySize is >= 0, then bodySize bytes must be provided by bodyStream -// before returning io.EOF. -// -// If bodySize < 0, then bodyStream is read until io.EOF. -// -// See also SetBodyStreamWriter. -func (ctx *RequestCtx) SetBodyStream(bodyStream io.Reader, bodySize int) { - ctx.Response.SetBodyStream(bodyStream, bodySize) -} - -// SetBodyStreamWriter registers the given stream writer for populating -// response body. -// -// Access to RequestCtx and/or its' members is forbidden from sw. -// -// This function may be used in the following cases: -// -// * if response body is too big (more than 10MB). -// * if response body is streamed from slow external sources. -// * if response body must be streamed to the client in chunks. -// (aka `http server push`). -func (ctx *RequestCtx) SetBodyStreamWriter(sw StreamWriter) { - ctx.Response.SetBodyStreamWriter(sw) -} - -// IsBodyStream returns true if response body is set via SetBodyStream*. -func (ctx *RequestCtx) IsBodyStream() bool { - return ctx.Response.IsBodyStream() -} - -// Logger returns logger, which may be used for logging arbitrary -// request-specific messages inside RequestHandler. -// -// Each message logged via returned logger contains request-specific information -// such as request id, request duration, local address, remote address, -// request method and request url. -// -// It is safe re-using returned logger for logging multiple messages -// for the current request. -// -// The returned logger is valid until returning from RequestHandler. -func (ctx *RequestCtx) Logger() Logger { - if ctx.logger.ctx == nil { - ctx.logger.ctx = ctx - } - if ctx.logger.logger == nil { - ctx.logger.logger = ctx.s.logger() - } - return &ctx.logger -} - -// TimeoutError sets response status code to StatusRequestTimeout and sets -// body to the given msg. -// -// All response modifications after TimeoutError call are ignored. -// -// TimeoutError MUST be called before returning from RequestHandler if there are -// references to ctx and/or its members in other goroutines remain. -// -// Usage of this function is discouraged. Prefer eliminating ctx references -// from pending goroutines instead of using this function. -func (ctx *RequestCtx) TimeoutError(msg string) { - ctx.TimeoutErrorWithCode(msg, StatusRequestTimeout) -} - -// TimeoutErrorWithCode sets response body to msg and response status -// code to statusCode. -// -// All response modifications after TimeoutErrorWithCode call are ignored. -// -// TimeoutErrorWithCode MUST be called before returning from RequestHandler -// if there are references to ctx and/or its members in other goroutines remain. -// -// Usage of this function is discouraged. Prefer eliminating ctx references -// from pending goroutines instead of using this function. -func (ctx *RequestCtx) TimeoutErrorWithCode(msg string, statusCode int) { - var resp Response - resp.SetStatusCode(statusCode) - resp.SetBodyString(msg) - ctx.TimeoutErrorWithResponse(&resp) -} - -// TimeoutErrorWithResponse marks the ctx as timed out and sends the given -// response to the client. -// -// All ctx modifications after TimeoutErrorWithResponse call are ignored. -// -// TimeoutErrorWithResponse MUST be called before returning from RequestHandler -// if there are references to ctx and/or its members in other goroutines remain. -// -// Usage of this function is discouraged. Prefer eliminating ctx references -// from pending goroutines instead of using this function. -func (ctx *RequestCtx) TimeoutErrorWithResponse(resp *Response) { - respCopy := &Response{} - resp.CopyTo(respCopy) - ctx.timeoutResponse = respCopy -} - -// NextProto adds nph to be processed when key is negotiated when TLS -// connection is established. -// -// This function can only be called before the server is started. -func (s *Server) NextProto(key string, nph ServeHandler) { - if s.nextProtos == nil { - s.nextProtos = make(map[string]ServeHandler) - } - s.configTLS() - s.tlsConfig.NextProtos = append(s.tlsConfig.NextProtos, key) - s.nextProtos[key] = nph -} - -func (s *Server) getNextProto(c net.Conn) (proto string, err error) { - if tlsConn, ok := c.(connTLSer); ok { - err = tlsConn.Handshake() - if err == nil { - proto = tlsConn.ConnectionState().NegotiatedProtocol - } - } - return -} - -// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted -// connections. It's used by ListenAndServe, ListenAndServeTLS and -// ListenAndServeTLSEmbed so dead TCP connections (e.g. closing laptop mid-download) -// eventually go away. -type tcpKeepaliveListener struct { - *net.TCPListener - keepalivePeriod time.Duration -} - -func (ln tcpKeepaliveListener) Accept() (net.Conn, error) { - tc, err := ln.AcceptTCP() - if err != nil { - return nil, err - } - tc.SetKeepAlive(true) - if ln.keepalivePeriod > 0 { - tc.SetKeepAlivePeriod(ln.keepalivePeriod) - } - return tc, nil -} - -// ListenAndServe serves HTTP requests from the given TCP4 addr. -// -// Pass custom listener to Serve if you need listening on non-TCP4 media -// such as IPv6. -// -// Accepted connections are configured to enable TCP keep-alives. -func (s *Server) ListenAndServe(addr string) error { - ln, err := net.Listen("tcp4", addr) - if err != nil { - return err - } - if s.TCPKeepalive { - if tcpln, ok := ln.(*net.TCPListener); ok { - return s.Serve(tcpKeepaliveListener{ - TCPListener: tcpln, - keepalivePeriod: s.TCPKeepalivePeriod, - }) - } - } - return s.Serve(ln) -} - -// ListenAndServeUNIX serves HTTP requests from the given UNIX addr. -// -// The function deletes existing file at addr before starting serving. -// -// The server sets the given file mode for the UNIX addr. -func (s *Server) ListenAndServeUNIX(addr string, mode os.FileMode) error { - if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("unexpected error when trying to remove unix socket file %q: %s", addr, err) - } - ln, err := net.Listen("unix", addr) - if err != nil { - return err - } - if err = os.Chmod(addr, mode); err != nil { - return fmt.Errorf("cannot chmod %#o for %q: %s", mode, addr, err) - } - return s.Serve(ln) -} - -// ListenAndServeTLS serves HTTPS requests from the given TCP4 addr. -// -// certFile and keyFile are paths to TLS certificate and key files. -// -// Pass custom listener to Serve if you need listening on non-TCP4 media -// such as IPv6. -// -// If the certFile or keyFile has not been provided to the server structure, -// the function will use the previously added TLS configuration. -// -// Accepted connections are configured to enable TCP keep-alives. -func (s *Server) ListenAndServeTLS(addr, certFile, keyFile string) error { - ln, err := net.Listen("tcp4", addr) - if err != nil { - return err - } - if s.TCPKeepalive { - if tcpln, ok := ln.(*net.TCPListener); ok { - return s.ServeTLS(tcpKeepaliveListener{ - TCPListener: tcpln, - keepalivePeriod: s.TCPKeepalivePeriod, - }, certFile, keyFile) - } - } - return s.ServeTLS(ln, certFile, keyFile) -} - -// ListenAndServeTLSEmbed serves HTTPS requests from the given TCP4 addr. -// -// certData and keyData must contain valid TLS certificate and key data. -// -// Pass custom listener to Serve if you need listening on arbitrary media -// such as IPv6. -// -// If the certFile or keyFile has not been provided the server structure, -// the function will use previously added TLS configuration. -// -// Accepted connections are configured to enable TCP keep-alives. -func (s *Server) ListenAndServeTLSEmbed(addr string, certData, keyData []byte) error { - ln, err := net.Listen("tcp4", addr) - if err != nil { - return err - } - if s.TCPKeepalive { - if tcpln, ok := ln.(*net.TCPListener); ok { - return s.ServeTLSEmbed(tcpKeepaliveListener{ - TCPListener: tcpln, - keepalivePeriod: s.TCPKeepalivePeriod, - }, certData, keyData) - } - } - return s.ServeTLSEmbed(ln, certData, keyData) -} - -// ServeTLS serves HTTPS requests from the given listener. -// -// certFile and keyFile are paths to TLS certificate and key files. -// -// If the certFile or keyFile has not been provided the server structure, -// the function will use previously added TLS configuration. -func (s *Server) ServeTLS(ln net.Listener, certFile, keyFile string) error { - err := s.AppendCert(certFile, keyFile) - if err != nil && err != errNoCertOrKeyProvided { - return err - } - if s.tlsConfig == nil { - return errNoCertOrKeyProvided - } - s.tlsConfig.BuildNameToCertificate() - - return s.Serve( - tls.NewListener(ln, s.tlsConfig), - ) -} - -// ServeTLSEmbed serves HTTPS requests from the given listener. -// -// certData and keyData must contain valid TLS certificate and key data. -// -// If the certFile or keyFile has not been provided the server structure, -// the function will use previously added TLS configuration. -func (s *Server) ServeTLSEmbed(ln net.Listener, certData, keyData []byte) error { - err := s.AppendCertEmbed(certData, keyData) - if err != nil && err != errNoCertOrKeyProvided { - return err - } - if s.tlsConfig == nil { - return errNoCertOrKeyProvided - } - s.tlsConfig.BuildNameToCertificate() - - return s.Serve( - tls.NewListener(ln, s.tlsConfig), - ) -} - -// AppendCert appends certificate and keyfile to TLS Configuration. -// -// This function allows programmer to handle multiple domains -// in one server structure. See examples/multidomain -func (s *Server) AppendCert(certFile, keyFile string) error { - if len(certFile) == 0 && len(keyFile) == 0 { - return errNoCertOrKeyProvided - } - - cert, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - return fmt.Errorf("cannot load TLS key pair from certFile=%q and keyFile=%q: %s", certFile, keyFile, err) - } - - s.configTLS() - - s.tlsConfig.Certificates = append(s.tlsConfig.Certificates, cert) - return nil -} - -// AppendCertEmbed does the same as AppendCert but using in-memory data. -func (s *Server) AppendCertEmbed(certData, keyData []byte) error { - if len(certData) == 0 && len(keyData) == 0 { - return errNoCertOrKeyProvided - } - - cert, err := tls.X509KeyPair(certData, keyData) - if err != nil { - return fmt.Errorf("cannot load TLS key pair from the provided certData(%d) and keyData(%d): %s", - len(certData), len(keyData), err) - } - - s.configTLS() - - s.tlsConfig.Certificates = append(s.tlsConfig.Certificates, cert) - return nil -} - -func (s *Server) configTLS() { - if s.tlsConfig == nil { - s.tlsConfig = &tls.Config{ - PreferServerCipherSuites: true, - } - } -} - -// DefaultConcurrency is the maximum number of concurrent connections -// the Server may serve by default (i.e. if Server.Concurrency isn't set). -const DefaultConcurrency = 256 * 1024 - -// Serve serves incoming connections from the given listener. -// -// Serve blocks until the given listener returns permanent error. -func (s *Server) Serve(ln net.Listener) error { - var lastOverflowErrorTime time.Time - var lastPerIPErrorTime time.Time - var c net.Conn - var err error - - s.mu.Lock() - { - if s.ln != nil { - s.mu.Unlock() - return ErrAlreadyServing - } - - s.ln = ln - s.done = make(chan struct{}) - } - s.mu.Unlock() - - maxWorkersCount := s.getConcurrency() - s.concurrencyCh = make(chan struct{}, maxWorkersCount) - wp := &workerPool{ - WorkerFunc: s.serveConn, - MaxWorkersCount: maxWorkersCount, - LogAllErrors: s.LogAllErrors, - Logger: s.logger(), - connState: s.setState, - } - wp.Start() - - // Count our waiting to accept a connection as an open connection. - // This way we can't get into any weird state where just after accepting - // a connection Shutdown is called which reads open as 0 because it isn't - // incremented yet. - atomic.AddInt32(&s.open, 1) - defer atomic.AddInt32(&s.open, -1) - - for { - if c, err = acceptConn(s, ln, &lastPerIPErrorTime); err != nil { - wp.Stop() - if err == io.EOF { - return nil - } - return err - } - s.setState(c, StateNew) - atomic.AddInt32(&s.open, 1) - if !wp.Serve(c) { - atomic.AddInt32(&s.open, -1) - s.writeFastError(c, StatusServiceUnavailable, - "The connection cannot be served because Server.Concurrency limit exceeded") - c.Close() - s.setState(c, StateClosed) - if time.Since(lastOverflowErrorTime) > time.Minute { - s.logger().Printf("The incoming connection cannot be served, because %d concurrent connections are served. "+ - "Try increasing Server.Concurrency", maxWorkersCount) - lastOverflowErrorTime = time.Now() - } - - // The current server reached concurrency limit, - // so give other concurrently running servers a chance - // accepting incoming connections on the same address. - // - // There is a hope other servers didn't reach their - // concurrency limits yet :) - // - // See also: https://github.com/valyala/fasthttp/pull/485#discussion_r239994990 - if s.SleepWhenConcurrencyLimitsExceeded > 0 { - time.Sleep(s.SleepWhenConcurrencyLimitsExceeded) - } - } - c = nil - } -} - -// Shutdown gracefully shuts down the server without interrupting any active connections. -// Shutdown works by first closing all open listeners and then waiting indefinitely for all connections to return to idle and then shut down. -// -// When Shutdown is called, Serve, ListenAndServe, and ListenAndServeTLS immediately return nil. -// Make sure the program doesn't exit and waits instead for Shutdown to return. -// -// Shutdown does not close keepalive connections so its recommended to set ReadTimeout to something else than 0. -func (s *Server) Shutdown() error { - s.mu.Lock() - defer s.mu.Unlock() - - atomic.StoreInt32(&s.stop, 1) - defer atomic.StoreInt32(&s.stop, 0) - - if s.ln == nil { - return nil - } - - if err := s.ln.Close(); err != nil { - return err - } - - if s.done != nil { - close(s.done) - } - - // Closing the listener will make Serve() call Stop on the worker pool. - // Setting .stop to 1 will make serveConn() break out of its loop. - // Now we just have to wait until all workers are done. - for { - if open := atomic.LoadInt32(&s.open); open == 0 { - break - } - // This is not an optimal solution but using a sync.WaitGroup - // here causes data races as it's hard to prevent Add() to be called - // while Wait() is waiting. - time.Sleep(time.Millisecond * 100) - } - - s.ln = nil - return nil -} - -func acceptConn(s *Server, ln net.Listener, lastPerIPErrorTime *time.Time) (net.Conn, error) { - for { - c, err := ln.Accept() - if err != nil { - if c != nil { - panic("BUG: net.Listener returned non-nil conn and non-nil error") - } - if netErr, ok := err.(net.Error); ok && netErr.Temporary() { - s.logger().Printf("Temporary error when accepting new connections: %s", netErr) - time.Sleep(time.Second) - continue - } - if err != io.EOF && !strings.Contains(err.Error(), "use of closed network connection") { - s.logger().Printf("Permanent error when accepting new connections: %s", err) - return nil, err - } - return nil, io.EOF - } - if c == nil { - panic("BUG: net.Listener returned (nil, nil)") - } - if s.MaxConnsPerIP > 0 { - pic := wrapPerIPConn(s, c) - if pic == nil { - if time.Since(*lastPerIPErrorTime) > time.Minute { - s.logger().Printf("The number of connections from %s exceeds MaxConnsPerIP=%d", - getConnIP4(c), s.MaxConnsPerIP) - *lastPerIPErrorTime = time.Now() - } - continue - } - c = pic - } - return c, nil - } -} - -func wrapPerIPConn(s *Server, c net.Conn) net.Conn { - ip := getUint32IP(c) - if ip == 0 { - return c - } - n := s.perIPConnCounter.Register(ip) - if n > s.MaxConnsPerIP { - s.perIPConnCounter.Unregister(ip) - s.writeFastError(c, StatusTooManyRequests, "The number of connections from your ip exceeds MaxConnsPerIP") - c.Close() - return nil - } - return acquirePerIPConn(c, ip, &s.perIPConnCounter) -} - -var defaultLogger = Logger(log.New(os.Stderr, "", log.LstdFlags)) - -func (s *Server) logger() Logger { - if s.Logger != nil { - return s.Logger - } - return defaultLogger -} - -var ( - // ErrPerIPConnLimit may be returned from ServeConn if the number of connections - // per ip exceeds Server.MaxConnsPerIP. - ErrPerIPConnLimit = errors.New("too many connections per ip") - - // ErrConcurrencyLimit may be returned from ServeConn if the number - // of concurrently served connections exceeds Server.Concurrency. - ErrConcurrencyLimit = errors.New("cannot serve the connection because Server.Concurrency concurrent connections are served") - - // ErrKeepaliveTimeout is returned from ServeConn - // if the connection lifetime exceeds MaxKeepaliveDuration. - ErrKeepaliveTimeout = errors.New("exceeded MaxKeepaliveDuration") -) - -// ServeConn serves HTTP requests from the given connection. -// -// ServeConn returns nil if all requests from the c are successfully served. -// It returns non-nil error otherwise. -// -// Connection c must immediately propagate all the data passed to Write() -// to the client. Otherwise requests' processing may hang. -// -// ServeConn closes c before returning. -func (s *Server) ServeConn(c net.Conn) error { - if s.MaxConnsPerIP > 0 { - pic := wrapPerIPConn(s, c) - if pic == nil { - return ErrPerIPConnLimit - } - c = pic - } - - n := atomic.AddUint32(&s.concurrency, 1) - if n > uint32(s.getConcurrency()) { - atomic.AddUint32(&s.concurrency, ^uint32(0)) - s.writeFastError(c, StatusServiceUnavailable, "The connection cannot be served because Server.Concurrency limit exceeded") - c.Close() - return ErrConcurrencyLimit - } - - atomic.AddInt32(&s.open, 1) - - err := s.serveConn(c) - - atomic.AddUint32(&s.concurrency, ^uint32(0)) - - if err != errHijacked { - err1 := c.Close() - s.setState(c, StateClosed) - if err == nil { - err = err1 - } - } else { - err = nil - s.setState(c, StateHijacked) - } - return err -} - -var errHijacked = errors.New("connection has been hijacked") - -// GetCurrentConcurrency returns a number of currently served -// connections. -// -// This function is intended be used by monitoring systems -func (s *Server) GetCurrentConcurrency() uint32 { - return atomic.LoadUint32(&s.concurrency) -} - -// GetOpenConnectionsCount returns a number of opened connections. -// -// This function is intended be used by monitoring systems -func (s *Server) GetOpenConnectionsCount() int32 { - return atomic.LoadInt32(&s.open) - 1 -} - -func (s *Server) getConcurrency() int { - n := s.Concurrency - if n <= 0 { - n = DefaultConcurrency - } - return n -} - -var globalConnID uint64 - -func nextConnID() uint64 { - return atomic.AddUint64(&globalConnID, 1) -} - -// DefaultMaxRequestBodySize is the maximum request body size the server -// reads by default. -// -// See Server.MaxRequestBodySize for details. -const DefaultMaxRequestBodySize = 4 * 1024 * 1024 - -func (s *Server) serveConn(c net.Conn) error { - defer atomic.AddInt32(&s.open, -1) - - if proto, err := s.getNextProto(c); err != nil { - return err - } else { - handler, ok := s.nextProtos[proto] - if ok { - return handler(c) - } - } - - var serverName []byte - if !s.NoDefaultServerHeader { - serverName = s.getServerName() - } - connRequestNum := uint64(0) - connID := nextConnID() - currentTime := time.Now() - connTime := currentTime - maxRequestBodySize := s.MaxRequestBodySize - if maxRequestBodySize <= 0 { - maxRequestBodySize = DefaultMaxRequestBodySize - } - - ctx := s.acquireCtx(c) - ctx.connTime = connTime - isTLS := ctx.IsTLS() - var ( - br *bufio.Reader - bw *bufio.Writer - - err error - timeoutResponse *Response - hijackHandler HijackHandler - - lastReadDeadlineTime time.Time - lastWriteDeadlineTime time.Time - - connectionClose bool - isHTTP11 bool - ) - for { - connRequestNum++ - ctx.time = currentTime - - if s.ReadTimeout > 0 || s.MaxKeepaliveDuration > 0 { - lastReadDeadlineTime = s.updateReadDeadline(c, ctx, lastReadDeadlineTime) - if lastReadDeadlineTime.IsZero() { - err = ErrKeepaliveTimeout - break - } - } - - if !(s.ReduceMemoryUsage || ctx.lastReadDuration > time.Second) || br != nil { - if br == nil { - br = acquireReader(ctx) - } - } else { - br, err = acquireByteReader(&ctx) - } - ctx.Request.isTLS = isTLS - ctx.Response.Header.noDefaultContentType = s.NoDefaultContentType - - if err == nil { - if s.DisableHeaderNamesNormalizing { - ctx.Request.Header.DisableNormalizing() - ctx.Response.Header.DisableNormalizing() - } - // reading Headers and Body - err = ctx.Request.readLimitBody(br, maxRequestBodySize, s.GetOnly) - if err == nil { - // If we read any bytes off the wire, we're active. - s.setState(c, StateActive) - } - if (s.ReduceMemoryUsage && br.Buffered() == 0) || err != nil { - releaseReader(s, br) - br = nil - } - } - - currentTime = time.Now() - ctx.lastReadDuration = currentTime.Sub(ctx.time) - - if err != nil { - if err == io.EOF { - err = nil - } else if connRequestNum > 1 && err == errNothingRead { - // This is not the first request and we haven't read a single byte - // of a new request yet. This means it's just a keep-alive connection - // closing down either because the remote closed it or because - // or a read timeout on our side. Either way just close the connection - // and don't return any error response. - err = nil - } else { - bw = s.writeErrorResponse(bw, ctx, serverName, err) - } - break - } - - // 'Expect: 100-continue' request handling. - // See http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html for details. - if !ctx.Request.Header.ignoreBody() && ctx.Request.MayContinue() { - // Send 'HTTP/1.1 100 Continue' response. - if bw == nil { - bw = acquireWriter(ctx) - } - bw.Write(strResponseContinue) - err = bw.Flush() - if err != nil { - break - } - if s.ReduceMemoryUsage { - releaseWriter(s, bw) - bw = nil - } - - // Read request body. - if br == nil { - br = acquireReader(ctx) - } - err = ctx.Request.ContinueReadBody(br, maxRequestBodySize) - if (s.ReduceMemoryUsage && br.Buffered() == 0) || err != nil { - releaseReader(s, br) - br = nil - } - if err != nil { - bw = s.writeErrorResponse(bw, ctx, serverName, err) - break - } - } - - connectionClose = s.DisableKeepalive || ctx.Request.Header.ConnectionClose() - isHTTP11 = ctx.Request.Header.IsHTTP11() - - if serverName != nil { - ctx.Response.Header.SetServerBytes(serverName) - } - ctx.connID = connID - ctx.connRequestNum = connRequestNum - ctx.time = currentTime - s.Handler(ctx) - - timeoutResponse = ctx.timeoutResponse - if timeoutResponse != nil { - ctx = s.acquireCtx(c) - timeoutResponse.CopyTo(&ctx.Response) - if br != nil { - // Close connection, since br may be attached to the old ctx via ctx.fbr. - ctx.SetConnectionClose() - } - } - - if !ctx.IsGet() && ctx.IsHead() { - ctx.Response.SkipBody = true - } - ctx.Request.Reset() - - hijackHandler = ctx.hijackHandler - ctx.hijackHandler = nil - - ctx.userValues.Reset() - - if s.MaxRequestsPerConn > 0 && connRequestNum >= uint64(s.MaxRequestsPerConn) { - ctx.SetConnectionClose() - } - - if s.WriteTimeout > 0 || s.MaxKeepaliveDuration > 0 { - lastWriteDeadlineTime = s.updateWriteDeadline(c, ctx, lastWriteDeadlineTime) - } - - connectionClose = connectionClose || ctx.Response.ConnectionClose() - if connectionClose { - ctx.Response.Header.SetCanonical(strConnection, strClose) - } else if !isHTTP11 { - // Set 'Connection: keep-alive' response header for non-HTTP/1.1 request. - // There is no need in setting this header for http/1.1, since in http/1.1 - // connections are keep-alive by default. - ctx.Response.Header.SetCanonical(strConnection, strKeepAlive) - } - - if serverName != nil && len(ctx.Response.Header.Server()) == 0 { - ctx.Response.Header.SetServerBytes(serverName) - } - - if bw == nil { - bw = acquireWriter(ctx) - } - if err = writeResponse(ctx, bw); err != nil { - break - } - - // Only flush the writer if we don't have another request in the pipeline. - // This is a big of an ugly optimization for https://www.techempower.com/benchmarks/ - // This benchmark will send 16 pipelined requests. It is faster to pack as many responses - // in a TCP packet and send it back at once than waiting for a flush every request. - // In real world circumstances this behaviour could be argued as being wrong. - if br == nil || br.Buffered() == 0 || connectionClose { - err = bw.Flush() - if err != nil { - break - } - } - if connectionClose { - break - } - if s.ReduceMemoryUsage { - releaseWriter(s, bw) - bw = nil - } - - if hijackHandler != nil { - var hjr io.Reader = c - if br != nil { - hjr = br - br = nil - - // br may point to ctx.fbr, so do not return ctx into pool. - ctx = s.acquireCtx(c) - } - if bw != nil { - err = bw.Flush() - if err != nil { - break - } - releaseWriter(s, bw) - bw = nil - } - c.SetReadDeadline(zeroTime) - c.SetWriteDeadline(zeroTime) - go hijackConnHandler(hjr, c, s, hijackHandler) - hijackHandler = nil - err = errHijacked - break - } - - currentTime = time.Now() - s.setState(c, StateIdle) - - if atomic.LoadInt32(&s.stop) == 1 { - err = nil - break - } - } - - if br != nil { - releaseReader(s, br) - } - if bw != nil { - releaseWriter(s, bw) - } - s.releaseCtx(ctx) - return err -} - -func (s *Server) setState(nc net.Conn, state ConnState) { - if hook := s.ConnState; hook != nil { - hook(nc, state) - } -} - -func (s *Server) updateReadDeadline(c net.Conn, ctx *RequestCtx, lastDeadlineTime time.Time) time.Time { - readTimeout := s.ReadTimeout - currentTime := ctx.time - if s.MaxKeepaliveDuration > 0 { - connTimeout := s.MaxKeepaliveDuration - currentTime.Sub(ctx.connTime) - if connTimeout <= 0 { - return zeroTime - } - if connTimeout < readTimeout { - readTimeout = connTimeout - } - } - - // Optimization: update read deadline only if more than 25% - // of the last read deadline exceeded. - // See https://github.com/golang/go/issues/15133 for details. - if currentTime.Sub(lastDeadlineTime) > (readTimeout >> 2) { - if err := c.SetReadDeadline(currentTime.Add(readTimeout)); err != nil { - panic(fmt.Sprintf("BUG: error in SetReadDeadline(%s): %s", readTimeout, err)) - } - lastDeadlineTime = currentTime - } - return lastDeadlineTime -} - -func (s *Server) updateWriteDeadline(c net.Conn, ctx *RequestCtx, lastDeadlineTime time.Time) time.Time { - writeTimeout := s.WriteTimeout - if s.MaxKeepaliveDuration > 0 { - connTimeout := s.MaxKeepaliveDuration - time.Since(ctx.connTime) - if connTimeout <= 0 { - // MaxKeepAliveDuration exceeded, but let's try sending response anyway - // in 100ms with 'Connection: close' header. - ctx.SetConnectionClose() - connTimeout = 100 * time.Millisecond - } - if connTimeout < writeTimeout { - writeTimeout = connTimeout - } - } - - // Optimization: update write deadline only if more than 25% - // of the last write deadline exceeded. - // See https://github.com/golang/go/issues/15133 for details. - currentTime := time.Now() - if currentTime.Sub(lastDeadlineTime) > (writeTimeout >> 2) { - if err := c.SetWriteDeadline(currentTime.Add(writeTimeout)); err != nil { - panic(fmt.Sprintf("BUG: error in SetWriteDeadline(%s): %s", writeTimeout, err)) - } - lastDeadlineTime = currentTime - } - return lastDeadlineTime -} - -func hijackConnHandler(r io.Reader, c net.Conn, s *Server, h HijackHandler) { - hjc := s.acquireHijackConn(r, c) - h(hjc) - - if br, ok := r.(*bufio.Reader); ok { - releaseReader(s, br) - } - c.Close() - s.releaseHijackConn(hjc) -} - -func (s *Server) acquireHijackConn(r io.Reader, c net.Conn) *hijackConn { - v := s.hijackConnPool.Get() - if v == nil { - hjc := &hijackConn{ - Conn: c, - r: r, - } - return hjc - } - hjc := v.(*hijackConn) - hjc.Conn = c - hjc.r = r - return hjc -} - -func (s *Server) releaseHijackConn(hjc *hijackConn) { - hjc.Conn = nil - hjc.r = nil - s.hijackConnPool.Put(hjc) -} - -type hijackConn struct { - net.Conn - r io.Reader -} - -func (c hijackConn) Read(p []byte) (int, error) { - return c.r.Read(p) -} - -func (c hijackConn) Close() error { - // hijacked conn is closed in hijackConnHandler. - return nil -} - -// LastTimeoutErrorResponse returns the last timeout response set -// via TimeoutError* call. -// -// This function is intended for custom server implementations. -func (ctx *RequestCtx) LastTimeoutErrorResponse() *Response { - return ctx.timeoutResponse -} - -func writeResponse(ctx *RequestCtx, w *bufio.Writer) error { - if ctx.timeoutResponse != nil { - panic("BUG: cannot write timed out response") - } - err := ctx.Response.Write(w) - ctx.Response.Reset() - return err -} - -const ( - defaultReadBufferSize = 4096 - defaultWriteBufferSize = 4096 -) - -func acquireByteReader(ctxP **RequestCtx) (*bufio.Reader, error) { - ctx := *ctxP - s := ctx.s - c := ctx.c - t := ctx.time - s.releaseCtx(ctx) - - // Make GC happy, so it could garbage collect ctx - // while we waiting for the next request. - ctx = nil - *ctxP = nil - - v := s.bytePool.Get() - if v == nil { - v = make([]byte, 1) - } - b := v.([]byte) - n, err := c.Read(b) - ch := b[0] - s.bytePool.Put(v) - ctx = s.acquireCtx(c) - ctx.time = t - *ctxP = ctx - if err != nil { - // Treat all errors as EOF on unsuccessful read - // of the first request byte. - return nil, io.EOF - } - if n != 1 { - panic("BUG: Reader must return at least one byte") - } - - ctx.fbr.c = c - ctx.fbr.ch = ch - ctx.fbr.byteRead = false - r := acquireReader(ctx) - r.Reset(&ctx.fbr) - return r, nil -} - -func acquireReader(ctx *RequestCtx) *bufio.Reader { - v := ctx.s.readerPool.Get() - if v == nil { - n := ctx.s.ReadBufferSize - if n <= 0 { - n = defaultReadBufferSize - } - return bufio.NewReaderSize(ctx.c, n) - } - r := v.(*bufio.Reader) - r.Reset(ctx.c) - return r -} - -func releaseReader(s *Server, r *bufio.Reader) { - s.readerPool.Put(r) -} - -func acquireWriter(ctx *RequestCtx) *bufio.Writer { - v := ctx.s.writerPool.Get() - if v == nil { - n := ctx.s.WriteBufferSize - if n <= 0 { - n = defaultWriteBufferSize - } - return bufio.NewWriterSize(ctx.c, n) - } - w := v.(*bufio.Writer) - w.Reset(ctx.c) - return w -} - -func releaseWriter(s *Server, w *bufio.Writer) { - s.writerPool.Put(w) -} - -func (s *Server) acquireCtx(c net.Conn) (ctx *RequestCtx) { - v := s.ctxPool.Get() - if v == nil { - ctx = &RequestCtx{ - s: s, - } - keepBodyBuffer := !s.ReduceMemoryUsage - ctx.Request.keepBodyBuffer = keepBodyBuffer - ctx.Response.keepBodyBuffer = keepBodyBuffer - } else { - ctx = v.(*RequestCtx) - } - ctx.c = c - return -} - -// Init2 prepares ctx for passing to RequestHandler. -// -// conn is used only for determining local and remote addresses. -// -// This function is intended for custom Server implementations. -// See https://github.com/valyala/httpteleport for details. -func (ctx *RequestCtx) Init2(conn net.Conn, logger Logger, reduceMemoryUsage bool) { - ctx.c = conn - ctx.logger.logger = logger - ctx.connID = nextConnID() - ctx.s = fakeServer - ctx.connRequestNum = 0 - ctx.connTime = time.Now() - ctx.time = ctx.connTime - - keepBodyBuffer := !reduceMemoryUsage - ctx.Request.keepBodyBuffer = keepBodyBuffer - ctx.Response.keepBodyBuffer = keepBodyBuffer -} - -// Init prepares ctx for passing to RequestHandler. -// -// remoteAddr and logger are optional. They are used by RequestCtx.Logger(). -// -// This function is intended for custom Server implementations. -func (ctx *RequestCtx) Init(req *Request, remoteAddr net.Addr, logger Logger) { - if remoteAddr == nil { - remoteAddr = zeroTCPAddr - } - c := &fakeAddrer{ - laddr: zeroTCPAddr, - raddr: remoteAddr, - } - if logger == nil { - logger = defaultLogger - } - ctx.Init2(c, logger, true) - req.CopyTo(&ctx.Request) -} - -// Deadline returns the time when work done on behalf of this context -// should be canceled. Deadline returns ok==false when no deadline is -// set. Successive calls to Deadline return the same results. -// -// This method always returns 0, false and is only present to make -// RequestCtx implement the context interface. -func (ctx *RequestCtx) Deadline() (deadline time.Time, ok bool) { - return -} - -// Done returns a channel that's closed when work done on behalf of this -// context should be canceled. Done may return nil if this context can -// never be canceled. Successive calls to Done return the same value. -func (ctx *RequestCtx) Done() <-chan struct{} { - return ctx.s.done -} - -// Err returns a non-nil error value after Done is closed, -// successive calls to Err return the same error. -// If Done is not yet closed, Err returns nil. -// If Done is closed, Err returns a non-nil error explaining why: -// Canceled if the context was canceled (via server Shutdown) -// or DeadlineExceeded if the context's deadline passed. -func (ctx *RequestCtx) Err() error { - select { - case <-ctx.s.done: - return context.Canceled - default: - return nil - } -} - -// Value returns the value associated with this context for key, or nil -// if no value is associated with key. Successive calls to Value with -// the same key returns the same result. -// -// This method is present to make RequestCtx implement the context interface. -// This method is the same as calling ctx.UserValue(key) -func (ctx *RequestCtx) Value(key interface{}) interface{} { - if keyString, ok := key.(string); ok { - return ctx.UserValue(keyString) - } - return nil -} - -var fakeServer = &Server{ - // Initialize concurrencyCh for TimeoutHandler - concurrencyCh: make(chan struct{}, DefaultConcurrency), -} - -type fakeAddrer struct { - net.Conn - laddr net.Addr - raddr net.Addr -} - -func (fa *fakeAddrer) RemoteAddr() net.Addr { - return fa.raddr -} - -func (fa *fakeAddrer) LocalAddr() net.Addr { - return fa.laddr -} - -func (fa *fakeAddrer) Read(p []byte) (int, error) { - panic("BUG: unexpected Read call") -} - -func (fa *fakeAddrer) Write(p []byte) (int, error) { - panic("BUG: unexpected Write call") -} - -func (fa *fakeAddrer) Close() error { - panic("BUG: unexpected Close call") -} - -func (s *Server) releaseCtx(ctx *RequestCtx) { - if ctx.timeoutResponse != nil { - panic("BUG: cannot release timed out RequestCtx") - } - ctx.c = nil - ctx.fbr.c = nil - s.ctxPool.Put(ctx) -} - -func (s *Server) getServerName() []byte { - v := s.serverName.Load() - var serverName []byte - if v == nil { - serverName = []byte(s.Name) - if len(serverName) == 0 { - serverName = defaultServerName - } - s.serverName.Store(serverName) - } else { - serverName = v.([]byte) - } - return serverName -} - -func (s *Server) writeFastError(w io.Writer, statusCode int, msg string) { - w.Write(statusLine(statusCode)) - - server := "" - if !s.NoDefaultServerHeader { - server = fmt.Sprintf("Server: %s\r\n", s.getServerName()) - } - - fmt.Fprintf(w, "Connection: close\r\n"+ - server+ - "Date: %s\r\n"+ - "Content-Type: text/plain\r\n"+ - "Content-Length: %d\r\n"+ - "\r\n"+ - "%s", - serverDate.Load(), len(msg), msg) -} - -func defaultErrorHandler(ctx *RequestCtx, err error) { - if _, ok := err.(*ErrSmallBuffer); ok { - ctx.Error("Too big request header", StatusRequestHeaderFieldsTooLarge) - } else { - ctx.Error("Error when parsing request", StatusBadRequest) - } -} - -func (s *Server) writeErrorResponse(bw *bufio.Writer, ctx *RequestCtx, serverName []byte, err error) *bufio.Writer { - errorHandler := defaultErrorHandler - if s.ErrorHandler != nil { - errorHandler = s.ErrorHandler - } - - errorHandler(ctx, err) - - if serverName != nil { - ctx.Response.Header.SetServerBytes(serverName) - } - ctx.SetConnectionClose() - if bw == nil { - bw = acquireWriter(ctx) - } - writeResponse(ctx, bw) - bw.Flush() - return bw -} - -// A ConnState represents the state of a client connection to a server. -// It's used by the optional Server.ConnState hook. -type ConnState int - -const ( - // StateNew represents a new connection that is expected to - // send a request immediately. Connections begin at this - // state and then transition to either StateActive or - // StateClosed. - StateNew ConnState = iota - - // StateActive represents a connection that has read 1 or more - // bytes of a request. The Server.ConnState hook for - // StateActive fires before the request has entered a handler - // and doesn't fire again until the request has been - // handled. After the request is handled, the state - // transitions to StateClosed, StateHijacked, or StateIdle. - // For HTTP/2, StateActive fires on the transition from zero - // to one active request, and only transitions away once all - // active requests are complete. That means that ConnState - // cannot be used to do per-request work; ConnState only notes - // the overall state of the connection. - StateActive - - // StateIdle represents a connection that has finished - // handling a request and is in the keep-alive state, waiting - // for a new request. Connections transition from StateIdle - // to either StateActive or StateClosed. - StateIdle - - // StateHijacked represents a hijacked connection. - // This is a terminal state. It does not transition to StateClosed. - StateHijacked - - // StateClosed represents a closed connection. - // This is a terminal state. Hijacked connections do not - // transition to StateClosed. - StateClosed -) - -var stateName = map[ConnState]string{ - StateNew: "new", - StateActive: "active", - StateIdle: "idle", - StateHijacked: "hijacked", - StateClosed: "closed", -} - -func (c ConnState) String() string { - return stateName[c] -} diff --git a/vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.key b/vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.key deleted file mode 100644 index 00a79a3b..00000000 --- a/vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQD4IQusAs8PJdnG -3mURt/AXtgC+ceqLOatJ49JJE1VPTkMAy+oE1f1XvkMrYsHqmDf6GWVzgVXryL4U -wq2/nJSm56ddhN55nI8oSN3dtywUB8/ShelEN73nlN77PeD9tl6NksPwWaKrqxq0 -FlabRPZSQCfmgZbhDV8Sa8mfCkFU0G0lit6kLGceCKMvmW+9Bz7ebsYmVdmVMxmf -IJStFD44lWFTdUc65WISKEdW2ELcUefb0zOLw+0PCbXFGJH5x5ktksW8+BBk2Hkg -GeQRL/qPCccthbScO0VgNj3zJ3ZZL0ObSDAbvNDG85joeNjDNq5DT/BAZ0bOSbEF -sh+f9BAzAgMBAAECggEBAJWv2cq7Jw6MVwSRxYca38xuD6TUNBopgBvjREixURW2 -sNUaLuMb9Omp7fuOaE2N5rcJ+xnjPGIxh/oeN5MQctz9gwn3zf6vY+15h97pUb4D -uGvYPRDaT8YVGS+X9NMZ4ZCmqW2lpWzKnCFoGHcy8yZLbcaxBsRdvKzwOYGoPiFb -K2QuhXZ/1UPmqK9i2DFKtj40X6vBszTNboFxOVpXrPu0FJwLVSDf2hSZ4fMM0DH3 -YqwKcYf5te+hxGKgrqRA3tn0NCWii0in6QIwXMC+kMw1ebg/tZKqyDLMNptAK8J+ -DVw9m5X1seUHS5ehU/g2jrQrtK5WYn7MrFK4lBzlRwECgYEA/d1TeANYECDWRRDk -B0aaRZs87Rwl/J9PsvbsKvtU/bX+OfSOUjOa9iQBqn0LmU8GqusEET/QVUfocVwV -Bggf/5qDLxz100Rj0ags/yE/kNr0Bb31kkkKHFMnCT06YasR7qKllwrAlPJvQv9x -IzBKq+T/Dx08Wep9bCRSFhzRCnsCgYEA+jdeZXTDr/Vz+D2B3nAw1frqYFfGnEVY -wqmoK3VXMDkGuxsloO2rN+SyiUo3JNiQNPDub/t7175GH5pmKtZOlftePANsUjBj -wZ1D0rI5Bxu/71ibIUYIRVmXsTEQkh/ozoh3jXCZ9+bLgYiYx7789IUZZSokFQ3D -FICUT9KJ36kCgYAGoq9Y1rWJjmIrYfqj2guUQC+CfxbbGIrrwZqAsRsSmpwvhZ3m -tiSZxG0quKQB+NfSxdvQW5ulbwC7Xc3K35F+i9pb8+TVBdeaFkw+yu6vaZmxQLrX -fQM/pEjD7A7HmMIaO7QaU5SfEAsqdCTP56Y8AftMuNXn/8IRfo2KuGwaWwKBgFpU -ILzJoVdlad9E/Rw7LjYhZfkv1uBVXIyxyKcfrkEXZSmozDXDdxsvcZCEfVHM6Ipk -K/+7LuMcqp4AFEAEq8wTOdq6daFaHLkpt/FZK6M4TlruhtpFOPkoNc3e45eM83OT -6mziKINJC1CQ6m65sQHpBtjxlKMRG8rL/D6wx9s5AoGBAMRlqNPMwglT3hvDmsAt -9Lf9pdmhERUlHhD8bj8mDaBj2Aqv7f6VRJaYZqP403pKKQexuqcn80mtjkSAPFkN -Cj7BVt/RXm5uoxDTnfi26RF9F6yNDEJ7UU9+peBr99aazF/fTgW/1GcMkQnum8uV -c257YgaWmjK9uB0Y2r2VxS0G ------END PRIVATE KEY----- diff --git a/vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.pem b/vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.pem deleted file mode 100644 index 93e77cd9..00000000 --- a/vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.pem +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICujCCAaKgAwIBAgIJAMbXnKZ/cikUMA0GCSqGSIb3DQEBCwUAMBUxEzARBgNV -BAMTCnVidW50dS5uYW4wHhcNMTUwMjA0MDgwMTM5WhcNMjUwMjAxMDgwMTM5WjAV -MRMwEQYDVQQDEwp1YnVudHUubmFuMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEA+CELrALPDyXZxt5lEbfwF7YAvnHqizmrSePSSRNVT05DAMvqBNX9V75D -K2LB6pg3+hllc4FV68i+FMKtv5yUpuenXYTeeZyPKEjd3bcsFAfP0oXpRDe955Te -+z3g/bZejZLD8Fmiq6satBZWm0T2UkAn5oGW4Q1fEmvJnwpBVNBtJYrepCxnHgij -L5lvvQc+3m7GJlXZlTMZnyCUrRQ+OJVhU3VHOuViEihHVthC3FHn29Mzi8PtDwm1 -xRiR+ceZLZLFvPgQZNh5IBnkES/6jwnHLYW0nDtFYDY98yd2WS9Dm0gwG7zQxvOY -6HjYwzauQ0/wQGdGzkmxBbIfn/QQMwIDAQABow0wCzAJBgNVHRMEAjAAMA0GCSqG -SIb3DQEBCwUAA4IBAQBQjKm/4KN/iTgXbLTL3i7zaxYXFLXsnT1tF+ay4VA8aj98 -L3JwRTciZ3A5iy/W4VSCt3eASwOaPWHKqDBB5RTtL73LoAqsWmO3APOGQAbixcQ2 -45GXi05OKeyiYRi1Nvq7Unv9jUkRDHUYVPZVSAjCpsXzPhFkmZoTRxmx5l0ZF7Li -K91lI5h+eFq0dwZwrmlPambyh1vQUi70VHv8DNToVU29kel7YLbxGbuqETfhrcy6 -X+Mha6RYITkAn5FqsZcKMsc9eYGEF4l3XV+oS7q6xfTxktYJMFTI18J0lQ2Lv/CI -whdMnYGntDQBE/iFCrJEGNsKGc38796GBOb5j+zd ------END CERTIFICATE----- diff --git a/vendor/github.com/valyala/fasthttp/stackless/doc.go b/vendor/github.com/valyala/fasthttp/stackless/doc.go deleted file mode 100644 index 8c0cc497..00000000 --- a/vendor/github.com/valyala/fasthttp/stackless/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package stackless provides functionality that may save stack space -// for high number of concurrently running goroutines. -package stackless diff --git a/vendor/github.com/valyala/fasthttp/stackless/func.go b/vendor/github.com/valyala/fasthttp/stackless/func.go deleted file mode 100644 index 9a49bcc2..00000000 --- a/vendor/github.com/valyala/fasthttp/stackless/func.go +++ /dev/null @@ -1,79 +0,0 @@ -package stackless - -import ( - "runtime" - "sync" -) - -// NewFunc returns stackless wrapper for the function f. -// -// Unlike f, the returned stackless wrapper doesn't use stack space -// on the goroutine that calls it. -// The wrapper may save a lot of stack space if the following conditions -// are met: -// -// - f doesn't contain blocking calls on network, I/O or channels; -// - f uses a lot of stack space; -// - the wrapper is called from high number of concurrent goroutines. -// -// The stackless wrapper returns false if the call cannot be processed -// at the moment due to high load. -func NewFunc(f func(ctx interface{})) func(ctx interface{}) bool { - if f == nil { - panic("BUG: f cannot be nil") - } - - funcWorkCh := make(chan *funcWork, runtime.GOMAXPROCS(-1)*2048) - onceInit := func() { - n := runtime.GOMAXPROCS(-1) - for i := 0; i < n; i++ { - go funcWorker(funcWorkCh, f) - } - } - var once sync.Once - - return func(ctx interface{}) bool { - once.Do(onceInit) - fw := getFuncWork() - fw.ctx = ctx - - select { - case funcWorkCh <- fw: - default: - putFuncWork(fw) - return false - } - <-fw.done - putFuncWork(fw) - return true - } -} - -func funcWorker(funcWorkCh <-chan *funcWork, f func(ctx interface{})) { - for fw := range funcWorkCh { - f(fw.ctx) - fw.done <- struct{}{} - } -} - -func getFuncWork() *funcWork { - v := funcWorkPool.Get() - if v == nil { - v = &funcWork{ - done: make(chan struct{}, 1), - } - } - return v.(*funcWork) -} - -func putFuncWork(fw *funcWork) { - fw.ctx = nil - funcWorkPool.Put(fw) -} - -var funcWorkPool sync.Pool - -type funcWork struct { - ctx interface{} - done chan struct{} -} diff --git a/vendor/github.com/valyala/fasthttp/stackless/writer.go b/vendor/github.com/valyala/fasthttp/stackless/writer.go deleted file mode 100644 index c2053f9a..00000000 --- a/vendor/github.com/valyala/fasthttp/stackless/writer.go +++ /dev/null @@ -1,139 +0,0 @@ -package stackless - -import ( - "errors" - "fmt" - "io" - - "github.com/valyala/bytebufferpool" -) - -// Writer is an interface stackless writer must conform to. -// -// The interface contains common subset for Writers from compress/* packages. -type Writer interface { - Write(p []byte) (int, error) - Flush() error - Close() error - Reset(w io.Writer) -} - -// NewWriterFunc must return new writer that will be wrapped into -// stackless writer. -type NewWriterFunc func(w io.Writer) Writer - -// NewWriter creates a stackless writer around a writer returned -// from newWriter. -// -// The returned writer writes data to dstW. -// -// Writers that use a lot of stack space may be wrapped into stackless writer, -// thus saving stack space for high number of concurrently running goroutines. -func NewWriter(dstW io.Writer, newWriter NewWriterFunc) Writer { - w := &writer{ - dstW: dstW, - } - w.zw = newWriter(&w.xw) - return w -} - -type writer struct { - dstW io.Writer - zw Writer - xw xWriter - - err error - n int - - p []byte - op op -} - -type op int - -const ( - opWrite op = iota - opFlush - opClose - opReset -) - -func (w *writer) Write(p []byte) (int, error) { - w.p = p - err := w.do(opWrite) - w.p = nil - return w.n, err -} - -func (w *writer) Flush() error { - return w.do(opFlush) -} - -func (w *writer) Close() error { - return w.do(opClose) -} - -func (w *writer) Reset(dstW io.Writer) { - w.xw.Reset() - w.do(opReset) - w.dstW = dstW -} - -func (w *writer) do(op op) error { - w.op = op - if !stacklessWriterFunc(w) { - return errHighLoad - } - err := w.err - if err != nil { - return err - } - if w.xw.bb != nil && len(w.xw.bb.B) > 0 { - _, err = w.dstW.Write(w.xw.bb.B) - } - w.xw.Reset() - - return err -} - -var errHighLoad = errors.New("cannot compress data due to high load") - -var stacklessWriterFunc = NewFunc(writerFunc) - -func writerFunc(ctx interface{}) { - w := ctx.(*writer) - switch w.op { - case opWrite: - w.n, w.err = w.zw.Write(w.p) - case opFlush: - w.err = w.zw.Flush() - case opClose: - w.err = w.zw.Close() - case opReset: - w.zw.Reset(&w.xw) - w.err = nil - default: - panic(fmt.Sprintf("BUG: unexpected op: %d", w.op)) - } -} - -type xWriter struct { - bb *bytebufferpool.ByteBuffer -} - -func (w *xWriter) Write(p []byte) (int, error) { - if w.bb == nil { - w.bb = bufferPool.Get() - } - w.bb.Write(p) - return len(p), nil -} - -func (w *xWriter) Reset() { - if w.bb != nil { - bufferPool.Put(w.bb) - w.bb = nil - } -} - -var bufferPool bytebufferpool.Pool diff --git a/vendor/github.com/valyala/fasthttp/status.go b/vendor/github.com/valyala/fasthttp/status.go deleted file mode 100644 index 6687efb4..00000000 --- a/vendor/github.com/valyala/fasthttp/status.go +++ /dev/null @@ -1,176 +0,0 @@ -package fasthttp - -import ( - "fmt" - "sync/atomic" -) - -// HTTP status codes were stolen from net/http. -const ( - StatusContinue = 100 // RFC 7231, 6.2.1 - StatusSwitchingProtocols = 101 // RFC 7231, 6.2.2 - StatusProcessing = 102 // RFC 2518, 10.1 - - StatusOK = 200 // RFC 7231, 6.3.1 - StatusCreated = 201 // RFC 7231, 6.3.2 - StatusAccepted = 202 // RFC 7231, 6.3.3 - StatusNonAuthoritativeInfo = 203 // RFC 7231, 6.3.4 - StatusNoContent = 204 // RFC 7231, 6.3.5 - StatusResetContent = 205 // RFC 7231, 6.3.6 - StatusPartialContent = 206 // RFC 7233, 4.1 - StatusMultiStatus = 207 // RFC 4918, 11.1 - StatusAlreadyReported = 208 // RFC 5842, 7.1 - StatusIMUsed = 226 // RFC 3229, 10.4.1 - - StatusMultipleChoices = 300 // RFC 7231, 6.4.1 - StatusMovedPermanently = 301 // RFC 7231, 6.4.2 - StatusFound = 302 // RFC 7231, 6.4.3 - StatusSeeOther = 303 // RFC 7231, 6.4.4 - StatusNotModified = 304 // RFC 7232, 4.1 - StatusUseProxy = 305 // RFC 7231, 6.4.5 - _ = 306 // RFC 7231, 6.4.6 (Unused) - StatusTemporaryRedirect = 307 // RFC 7231, 6.4.7 - StatusPermanentRedirect = 308 // RFC 7538, 3 - - StatusBadRequest = 400 // RFC 7231, 6.5.1 - StatusUnauthorized = 401 // RFC 7235, 3.1 - StatusPaymentRequired = 402 // RFC 7231, 6.5.2 - StatusForbidden = 403 // RFC 7231, 6.5.3 - StatusNotFound = 404 // RFC 7231, 6.5.4 - StatusMethodNotAllowed = 405 // RFC 7231, 6.5.5 - StatusNotAcceptable = 406 // RFC 7231, 6.5.6 - StatusProxyAuthRequired = 407 // RFC 7235, 3.2 - StatusRequestTimeout = 408 // RFC 7231, 6.5.7 - StatusConflict = 409 // RFC 7231, 6.5.8 - StatusGone = 410 // RFC 7231, 6.5.9 - StatusLengthRequired = 411 // RFC 7231, 6.5.10 - StatusPreconditionFailed = 412 // RFC 7232, 4.2 - StatusRequestEntityTooLarge = 413 // RFC 7231, 6.5.11 - StatusRequestURITooLong = 414 // RFC 7231, 6.5.12 - StatusUnsupportedMediaType = 415 // RFC 7231, 6.5.13 - StatusRequestedRangeNotSatisfiable = 416 // RFC 7233, 4.4 - StatusExpectationFailed = 417 // RFC 7231, 6.5.14 - StatusTeapot = 418 // RFC 7168, 2.3.3 - StatusUnprocessableEntity = 422 // RFC 4918, 11.2 - StatusLocked = 423 // RFC 4918, 11.3 - StatusFailedDependency = 424 // RFC 4918, 11.4 - StatusUpgradeRequired = 426 // RFC 7231, 6.5.15 - StatusPreconditionRequired = 428 // RFC 6585, 3 - StatusTooManyRequests = 429 // RFC 6585, 4 - StatusRequestHeaderFieldsTooLarge = 431 // RFC 6585, 5 - StatusUnavailableForLegalReasons = 451 // RFC 7725, 3 - - StatusInternalServerError = 500 // RFC 7231, 6.6.1 - StatusNotImplemented = 501 // RFC 7231, 6.6.2 - StatusBadGateway = 502 // RFC 7231, 6.6.3 - StatusServiceUnavailable = 503 // RFC 7231, 6.6.4 - StatusGatewayTimeout = 504 // RFC 7231, 6.6.5 - StatusHTTPVersionNotSupported = 505 // RFC 7231, 6.6.6 - StatusVariantAlsoNegotiates = 506 // RFC 2295, 8.1 - StatusInsufficientStorage = 507 // RFC 4918, 11.5 - StatusLoopDetected = 508 // RFC 5842, 7.2 - StatusNotExtended = 510 // RFC 2774, 7 - StatusNetworkAuthenticationRequired = 511 // RFC 6585, 6 -) - -var ( - statusLines atomic.Value - - statusMessages = map[int]string{ - StatusContinue: "Continue", - StatusSwitchingProtocols: "Switching Protocols", - StatusProcessing: "Processing", - - StatusOK: "OK", - StatusCreated: "Created", - StatusAccepted: "Accepted", - StatusNonAuthoritativeInfo: "Non-Authoritative Information", - StatusNoContent: "No Content", - StatusResetContent: "Reset Content", - StatusPartialContent: "Partial Content", - StatusMultiStatus: "Multi-Status", - StatusAlreadyReported: "Already Reported", - StatusIMUsed: "IM Used", - - StatusMultipleChoices: "Multiple Choices", - StatusMovedPermanently: "Moved Permanently", - StatusFound: "Found", - StatusSeeOther: "See Other", - StatusNotModified: "Not Modified", - StatusUseProxy: "Use Proxy", - StatusTemporaryRedirect: "Temporary Redirect", - StatusPermanentRedirect: "Permanent Redirect", - - StatusBadRequest: "Bad Request", - StatusUnauthorized: "Unauthorized", - StatusPaymentRequired: "Payment Required", - StatusForbidden: "Forbidden", - StatusNotFound: "Not Found", - StatusMethodNotAllowed: "Method Not Allowed", - StatusNotAcceptable: "Not Acceptable", - StatusProxyAuthRequired: "Proxy Authentication Required", - StatusRequestTimeout: "Request Timeout", - StatusConflict: "Conflict", - StatusGone: "Gone", - StatusLengthRequired: "Length Required", - StatusPreconditionFailed: "Precondition Failed", - StatusRequestEntityTooLarge: "Request Entity Too Large", - StatusRequestURITooLong: "Request URI Too Long", - StatusUnsupportedMediaType: "Unsupported Media Type", - StatusRequestedRangeNotSatisfiable: "Requested Range Not Satisfiable", - StatusExpectationFailed: "Expectation Failed", - StatusTeapot: "I'm a teapot", - StatusUnprocessableEntity: "Unprocessable Entity", - StatusLocked: "Locked", - StatusFailedDependency: "Failed Dependency", - StatusUpgradeRequired: "Upgrade Required", - StatusPreconditionRequired: "Precondition Required", - StatusTooManyRequests: "Too Many Requests", - StatusRequestHeaderFieldsTooLarge: "Request Header Fields Too Large", - StatusUnavailableForLegalReasons: "Unavailable For Legal Reasons", - - StatusInternalServerError: "Internal Server Error", - StatusNotImplemented: "Not Implemented", - StatusBadGateway: "Bad Gateway", - StatusServiceUnavailable: "Service Unavailable", - StatusGatewayTimeout: "Gateway Timeout", - StatusHTTPVersionNotSupported: "HTTP Version Not Supported", - StatusVariantAlsoNegotiates: "Variant Also Negotiates", - StatusInsufficientStorage: "Insufficient Storage", - StatusLoopDetected: "Loop Detected", - StatusNotExtended: "Not Extended", - StatusNetworkAuthenticationRequired: "Network Authentication Required", - } -) - -// StatusMessage returns HTTP status message for the given status code. -func StatusMessage(statusCode int) string { - s := statusMessages[statusCode] - if s == "" { - s = "Unknown Status Code" - } - return s -} - -func init() { - statusLines.Store(make(map[int][]byte)) -} - -func statusLine(statusCode int) []byte { - m := statusLines.Load().(map[int][]byte) - h := m[statusCode] - if h != nil { - return h - } - - statusText := StatusMessage(statusCode) - - h = []byte(fmt.Sprintf("HTTP/1.1 %d %s\r\n", statusCode, statusText)) - newM := make(map[int][]byte, len(m)+1) - for k, v := range m { - newM[k] = v - } - newM[statusCode] = h - statusLines.Store(newM) - return h -} diff --git a/vendor/github.com/valyala/fasthttp/stream.go b/vendor/github.com/valyala/fasthttp/stream.go deleted file mode 100644 index aa23b1af..00000000 --- a/vendor/github.com/valyala/fasthttp/stream.go +++ /dev/null @@ -1,54 +0,0 @@ -package fasthttp - -import ( - "bufio" - "io" - "sync" - - "github.com/valyala/fasthttp/fasthttputil" -) - -// StreamWriter must write data to w. -// -// Usually StreamWriter writes data to w in a loop (aka 'data streaming'). -// -// StreamWriter must return immediately if w returns error. -// -// Since the written data is buffered, do not forget calling w.Flush -// when the data must be propagated to reader. -type StreamWriter func(w *bufio.Writer) - -// NewStreamReader returns a reader, which replays all the data generated by sw. -// -// The returned reader may be passed to Response.SetBodyStream. -// -// Close must be called on the returned reader after all the required data -// has been read. Otherwise goroutine leak may occur. -// -// See also Response.SetBodyStreamWriter. -func NewStreamReader(sw StreamWriter) io.ReadCloser { - pc := fasthttputil.NewPipeConns() - pw := pc.Conn1() - pr := pc.Conn2() - - var bw *bufio.Writer - v := streamWriterBufPool.Get() - if v == nil { - bw = bufio.NewWriter(pw) - } else { - bw = v.(*bufio.Writer) - bw.Reset(pw) - } - - go func() { - sw(bw) - bw.Flush() - pw.Close() - - streamWriterBufPool.Put(bw) - }() - - return pr -} - -var streamWriterBufPool sync.Pool diff --git a/vendor/github.com/valyala/fasthttp/strings.go b/vendor/github.com/valyala/fasthttp/strings.go deleted file mode 100644 index 6d832310..00000000 --- a/vendor/github.com/valyala/fasthttp/strings.go +++ /dev/null @@ -1,80 +0,0 @@ -package fasthttp - -var ( - defaultServerName = []byte("fasthttp") - defaultUserAgent = []byte("fasthttp") - defaultContentType = []byte("text/plain; charset=utf-8") -) - -var ( - strSlash = []byte("/") - strSlashSlash = []byte("//") - strSlashDotDot = []byte("/..") - strSlashDotSlash = []byte("/./") - strSlashDotDotSlash = []byte("/../") - strCRLF = []byte("\r\n") - strHTTP = []byte("http") - strHTTPS = []byte("https") - strHTTP11 = []byte("HTTP/1.1") - strColonSlashSlash = []byte("://") - strColonSpace = []byte(": ") - strGMT = []byte("GMT") - - strResponseContinue = []byte("HTTP/1.1 100 Continue\r\n\r\n") - - strGet = []byte("GET") - strHead = []byte("HEAD") - strPost = []byte("POST") - strPut = []byte("PUT") - strDelete = []byte("DELETE") - strConnect = []byte("CONNECT") - strOptions = []byte("OPTIONS") - strTrace = []byte("TRACE") - strPatch = []byte("PATCH") - - strExpect = []byte("Expect") - strConnection = []byte("Connection") - strContentLength = []byte("Content-Length") - strContentType = []byte("Content-Type") - strDate = []byte("Date") - strHost = []byte("Host") - strReferer = []byte("Referer") - strServer = []byte("Server") - strTransferEncoding = []byte("Transfer-Encoding") - strContentEncoding = []byte("Content-Encoding") - strAcceptEncoding = []byte("Accept-Encoding") - strUserAgent = []byte("User-Agent") - strCookie = []byte("Cookie") - strSetCookie = []byte("Set-Cookie") - strLocation = []byte("Location") - strIfModifiedSince = []byte("If-Modified-Since") - strLastModified = []byte("Last-Modified") - strAcceptRanges = []byte("Accept-Ranges") - strRange = []byte("Range") - strContentRange = []byte("Content-Range") - - strCookieExpires = []byte("expires") - strCookieDomain = []byte("domain") - strCookiePath = []byte("path") - strCookieHTTPOnly = []byte("HttpOnly") - strCookieSecure = []byte("secure") - strCookieMaxAge = []byte("max-age") - strCookieSameSite = []byte("SameSite") - strCookieSameSiteLax = []byte("Lax") - strCookieSameSiteStrict = []byte("Strict") - - strClose = []byte("close") - strGzip = []byte("gzip") - strDeflate = []byte("deflate") - strKeepAlive = []byte("keep-alive") - strUpgrade = []byte("Upgrade") - strChunked = []byte("chunked") - strIdentity = []byte("identity") - str100Continue = []byte("100-continue") - strPostArgsContentType = []byte("application/x-www-form-urlencoded") - strMultipartFormData = []byte("multipart/form-data") - strBoundary = []byte("boundary") - strBytes = []byte("bytes") - strTextSlash = []byte("text/") - strApplicationSlash = []byte("application/") -) diff --git a/vendor/github.com/valyala/fasthttp/tcpdialer.go b/vendor/github.com/valyala/fasthttp/tcpdialer.go deleted file mode 100644 index 6a5cd3a1..00000000 --- a/vendor/github.com/valyala/fasthttp/tcpdialer.go +++ /dev/null @@ -1,448 +0,0 @@ -package fasthttp - -import ( - "errors" - "net" - "strconv" - "sync" - "sync/atomic" - "time" -) - -// Dial dials the given TCP addr using tcp4. -// -// This function has the following additional features comparing to net.Dial: -// -// * It reduces load on DNS resolver by caching resolved TCP addressed -// for DefaultDNSCacheDuration. -// * It dials all the resolved TCP addresses in round-robin manner until -// connection is established. This may be useful if certain addresses -// are temporarily unreachable. -// * It returns ErrDialTimeout if connection cannot be established during -// DefaultDialTimeout seconds. Use DialTimeout for customizing dial timeout. -// -// This dialer is intended for custom code wrapping before passing -// to Client.Dial or HostClient.Dial. -// -// For instance, per-host counters and/or limits may be implemented -// by such wrappers. -// -// The addr passed to the function must contain port. Example addr values: -// -// * foobar.baz:443 -// * foo.bar:80 -// * aaa.com:8080 -func Dial(addr string) (net.Conn, error) { - return defaultDialer.Dial(addr) -} - -// DialTimeout dials the given TCP addr using tcp4 using the given timeout. -// -// This function has the following additional features comparing to net.Dial: -// -// * It reduces load on DNS resolver by caching resolved TCP addressed -// for DefaultDNSCacheDuration. -// * It dials all the resolved TCP addresses in round-robin manner until -// connection is established. This may be useful if certain addresses -// are temporarily unreachable. -// -// This dialer is intended for custom code wrapping before passing -// to Client.Dial or HostClient.Dial. -// -// For instance, per-host counters and/or limits may be implemented -// by such wrappers. -// -// The addr passed to the function must contain port. Example addr values: -// -// * foobar.baz:443 -// * foo.bar:80 -// * aaa.com:8080 -func DialTimeout(addr string, timeout time.Duration) (net.Conn, error) { - return defaultDialer.DialTimeout(addr, timeout) -} - -// DialDualStack dials the given TCP addr using both tcp4 and tcp6. -// -// This function has the following additional features comparing to net.Dial: -// -// * It reduces load on DNS resolver by caching resolved TCP addressed -// for DefaultDNSCacheDuration. -// * It dials all the resolved TCP addresses in round-robin manner until -// connection is established. This may be useful if certain addresses -// are temporarily unreachable. -// * It returns ErrDialTimeout if connection cannot be established during -// DefaultDialTimeout seconds. Use DialDualStackTimeout for custom dial -// timeout. -// -// This dialer is intended for custom code wrapping before passing -// to Client.Dial or HostClient.Dial. -// -// For instance, per-host counters and/or limits may be implemented -// by such wrappers. -// -// The addr passed to the function must contain port. Example addr values: -// -// * foobar.baz:443 -// * foo.bar:80 -// * aaa.com:8080 -func DialDualStack(addr string) (net.Conn, error) { - return defaultDialer.DialDualStack(addr) -} - -// DialDualStackTimeout dials the given TCP addr using both tcp4 and tcp6 -// using the given timeout. -// -// This function has the following additional features comparing to net.Dial: -// -// * It reduces load on DNS resolver by caching resolved TCP addressed -// for DefaultDNSCacheDuration. -// * It dials all the resolved TCP addresses in round-robin manner until -// connection is established. This may be useful if certain addresses -// are temporarily unreachable. -// -// This dialer is intended for custom code wrapping before passing -// to Client.Dial or HostClient.Dial. -// -// For instance, per-host counters and/or limits may be implemented -// by such wrappers. -// -// The addr passed to the function must contain port. Example addr values: -// -// * foobar.baz:443 -// * foo.bar:80 -// * aaa.com:8080 -func DialDualStackTimeout(addr string, timeout time.Duration) (net.Conn, error) { - return defaultDialer.DialDualStackTimeout(addr, timeout) -} - -var ( - defaultDialer = &TCPDialer{Concurrency: 1000} -) - -// TCPDialer contains options to control a group of Dial calls. -type TCPDialer struct { - // Concurrency controls the maximum number of concurrent Dails - // that can be performed using this object. - // Setting this to 0 means unlimited. - // - // WARNING: This can only be changed before the first Dial. - // Changes made after the first Dial will not affect anything. - Concurrency int - - tcpAddrsLock sync.Mutex - tcpAddrsMap map[string]*tcpAddrEntry - - concurrencyCh chan struct{} - - once sync.Once -} - -// Dial dials the given TCP addr using tcp4. -// -// This function has the following additional features comparing to net.Dial: -// -// * It reduces load on DNS resolver by caching resolved TCP addressed -// for DefaultDNSCacheDuration. -// * It dials all the resolved TCP addresses in round-robin manner until -// connection is established. This may be useful if certain addresses -// are temporarily unreachable. -// * It returns ErrDialTimeout if connection cannot be established during -// DefaultDialTimeout seconds. Use DialTimeout for customizing dial timeout. -// -// This dialer is intended for custom code wrapping before passing -// to Client.Dial or HostClient.Dial. -// -// For instance, per-host counters and/or limits may be implemented -// by such wrappers. -// -// The addr passed to the function must contain port. Example addr values: -// -// * foobar.baz:443 -// * foo.bar:80 -// * aaa.com:8080 -func (d *TCPDialer) Dial(addr string) (net.Conn, error) { - return d.dial(addr, false, DefaultDialTimeout) -} - -// DialTimeout dials the given TCP addr using tcp4 using the given timeout. -// -// This function has the following additional features comparing to net.Dial: -// -// * It reduces load on DNS resolver by caching resolved TCP addressed -// for DefaultDNSCacheDuration. -// * It dials all the resolved TCP addresses in round-robin manner until -// connection is established. This may be useful if certain addresses -// are temporarily unreachable. -// -// This dialer is intended for custom code wrapping before passing -// to Client.Dial or HostClient.Dial. -// -// For instance, per-host counters and/or limits may be implemented -// by such wrappers. -// -// The addr passed to the function must contain port. Example addr values: -// -// * foobar.baz:443 -// * foo.bar:80 -// * aaa.com:8080 -func (d *TCPDialer) DialTimeout(addr string, timeout time.Duration) (net.Conn, error) { - return d.dial(addr, false, timeout) -} - -// DialDualStack dials the given TCP addr using both tcp4 and tcp6. -// -// This function has the following additional features comparing to net.Dial: -// -// * It reduces load on DNS resolver by caching resolved TCP addressed -// for DefaultDNSCacheDuration. -// * It dials all the resolved TCP addresses in round-robin manner until -// connection is established. This may be useful if certain addresses -// are temporarily unreachable. -// * It returns ErrDialTimeout if connection cannot be established during -// DefaultDialTimeout seconds. Use DialDualStackTimeout for custom dial -// timeout. -// -// This dialer is intended for custom code wrapping before passing -// to Client.Dial or HostClient.Dial. -// -// For instance, per-host counters and/or limits may be implemented -// by such wrappers. -// -// The addr passed to the function must contain port. Example addr values: -// -// * foobar.baz:443 -// * foo.bar:80 -// * aaa.com:8080 -func (d *TCPDialer) DialDualStack(addr string) (net.Conn, error) { - return d.dial(addr, true, DefaultDialTimeout) -} - -// DialDualStackTimeout dials the given TCP addr using both tcp4 and tcp6 -// using the given timeout. -// -// This function has the following additional features comparing to net.Dial: -// -// * It reduces load on DNS resolver by caching resolved TCP addressed -// for DefaultDNSCacheDuration. -// * It dials all the resolved TCP addresses in round-robin manner until -// connection is established. This may be useful if certain addresses -// are temporarily unreachable. -// -// This dialer is intended for custom code wrapping before passing -// to Client.Dial or HostClient.Dial. -// -// For instance, per-host counters and/or limits may be implemented -// by such wrappers. -// -// The addr passed to the function must contain port. Example addr values: -// -// * foobar.baz:443 -// * foo.bar:80 -// * aaa.com:8080 -func (d *TCPDialer) DialDualStackTimeout(addr string, timeout time.Duration) (net.Conn, error) { - return d.dial(addr, true, timeout) -} - -func (d *TCPDialer) dial(addr string, dualStack bool, timeout time.Duration) (net.Conn, error) { - d.once.Do(func() { - if d.Concurrency > 0 { - d.concurrencyCh = make(chan struct{}, d.Concurrency) - } - d.tcpAddrsMap = make(map[string]*tcpAddrEntry) - go d.tcpAddrsClean() - }) - - addrs, idx, err := d.getTCPAddrs(addr, dualStack) - if err != nil { - return nil, err - } - network := "tcp4" - if dualStack { - network = "tcp" - } - - var conn net.Conn - n := uint32(len(addrs)) - deadline := time.Now().Add(timeout) - for n > 0 { - conn, err = tryDial(network, &addrs[idx%n], deadline, d.concurrencyCh) - if err == nil { - return conn, nil - } - if err == ErrDialTimeout { - return nil, err - } - idx++ - n-- - } - return nil, err -} - -func tryDial(network string, addr *net.TCPAddr, deadline time.Time, concurrencyCh chan struct{}) (net.Conn, error) { - timeout := -time.Since(deadline) - if timeout <= 0 { - return nil, ErrDialTimeout - } - - if concurrencyCh != nil { - select { - case concurrencyCh <- struct{}{}: - default: - tc := AcquireTimer(timeout) - isTimeout := false - select { - case concurrencyCh <- struct{}{}: - case <-tc.C: - isTimeout = true - } - ReleaseTimer(tc) - if isTimeout { - return nil, ErrDialTimeout - } - } - } - - chv := dialResultChanPool.Get() - if chv == nil { - chv = make(chan dialResult, 1) - } - ch := chv.(chan dialResult) - go func() { - var dr dialResult - dr.conn, dr.err = net.DialTCP(network, nil, addr) - ch <- dr - if concurrencyCh != nil { - <-concurrencyCh - } - }() - - var ( - conn net.Conn - err error - ) - - tc := AcquireTimer(timeout) - select { - case dr := <-ch: - conn = dr.conn - err = dr.err - dialResultChanPool.Put(ch) - case <-tc.C: - err = ErrDialTimeout - } - ReleaseTimer(tc) - - return conn, err -} - -var dialResultChanPool sync.Pool - -type dialResult struct { - conn net.Conn - err error -} - -// ErrDialTimeout is returned when TCP dialing is timed out. -var ErrDialTimeout = errors.New("dialing to the given TCP address timed out") - -// DefaultDialTimeout is timeout used by Dial and DialDualStack -// for establishing TCP connections. -const DefaultDialTimeout = 3 * time.Second - -type tcpAddrEntry struct { - addrs []net.TCPAddr - addrsIdx uint32 - - resolveTime time.Time - pending bool -} - -// DefaultDNSCacheDuration is the duration for caching resolved TCP addresses -// by Dial* functions. -const DefaultDNSCacheDuration = time.Minute - -func (d *TCPDialer) tcpAddrsClean() { - expireDuration := 2 * DefaultDNSCacheDuration - for { - time.Sleep(time.Second) - t := time.Now() - - d.tcpAddrsLock.Lock() - for k, e := range d.tcpAddrsMap { - if t.Sub(e.resolveTime) > expireDuration { - delete(d.tcpAddrsMap, k) - } - } - d.tcpAddrsLock.Unlock() - } -} - -func (d *TCPDialer) getTCPAddrs(addr string, dualStack bool) ([]net.TCPAddr, uint32, error) { - d.tcpAddrsLock.Lock() - e := d.tcpAddrsMap[addr] - if e != nil && !e.pending && time.Since(e.resolveTime) > DefaultDNSCacheDuration { - e.pending = true - e = nil - } - d.tcpAddrsLock.Unlock() - - if e == nil { - addrs, err := resolveTCPAddrs(addr, dualStack) - if err != nil { - d.tcpAddrsLock.Lock() - e = d.tcpAddrsMap[addr] - if e != nil && e.pending { - e.pending = false - } - d.tcpAddrsLock.Unlock() - return nil, 0, err - } - - e = &tcpAddrEntry{ - addrs: addrs, - resolveTime: time.Now(), - } - - d.tcpAddrsLock.Lock() - d.tcpAddrsMap[addr] = e - d.tcpAddrsLock.Unlock() - } - - idx := atomic.AddUint32(&e.addrsIdx, 1) - return e.addrs, idx, nil -} - -func resolveTCPAddrs(addr string, dualStack bool) ([]net.TCPAddr, error) { - host, portS, err := net.SplitHostPort(addr) - if err != nil { - return nil, err - } - port, err := strconv.Atoi(portS) - if err != nil { - return nil, err - } - - ips, err := net.LookupIP(host) - if err != nil { - return nil, err - } - - n := len(ips) - addrs := make([]net.TCPAddr, 0, n) - for i := 0; i < n; i++ { - ip := ips[i] - if !dualStack && ip.To4() == nil { - continue - } - addrs = append(addrs, net.TCPAddr{ - IP: ip, - Port: port, - }) - } - if len(addrs) == 0 { - return nil, errNoDNSEntries - } - return addrs, nil -} - -var errNoDNSEntries = errors.New("couldn't find DNS entries for the given domain. Try using DialDualStack") diff --git a/vendor/github.com/valyala/fasthttp/timer.go b/vendor/github.com/valyala/fasthttp/timer.go deleted file mode 100644 index 4e919384..00000000 --- a/vendor/github.com/valyala/fasthttp/timer.go +++ /dev/null @@ -1,54 +0,0 @@ -package fasthttp - -import ( - "sync" - "time" -) - -func initTimer(t *time.Timer, timeout time.Duration) *time.Timer { - if t == nil { - return time.NewTimer(timeout) - } - if t.Reset(timeout) { - panic("BUG: active timer trapped into initTimer()") - } - return t -} - -func stopTimer(t *time.Timer) { - if !t.Stop() { - // Collect possibly added time from the channel - // if timer has been stopped and nobody collected its' value. - select { - case <-t.C: - default: - } - } -} - -// AcquireTimer returns a time.Timer from the pool and updates it to -// send the current time on its channel after at least timeout. -// -// The returned Timer may be returned to the pool with ReleaseTimer -// when no longer needed. This allows reducing GC load. -func AcquireTimer(timeout time.Duration) *time.Timer { - v := timerPool.Get() - if v == nil { - return time.NewTimer(timeout) - } - t := v.(*time.Timer) - initTimer(t, timeout) - return t -} - -// ReleaseTimer returns the time.Timer acquired via AcquireTimer to the pool -// and prevents the Timer from firing. -// -// Do not access the released time.Timer or read from it's channel otherwise -// data races may occur. -func ReleaseTimer(t *time.Timer) { - stopTimer(t) - timerPool.Put(t) -} - -var timerPool sync.Pool diff --git a/vendor/github.com/valyala/fasthttp/uri.go b/vendor/github.com/valyala/fasthttp/uri.go deleted file mode 100644 index d536f593..00000000 --- a/vendor/github.com/valyala/fasthttp/uri.go +++ /dev/null @@ -1,525 +0,0 @@ -package fasthttp - -import ( - "bytes" - "io" - "sync" -) - -// AcquireURI returns an empty URI instance from the pool. -// -// Release the URI with ReleaseURI after the URI is no longer needed. -// This allows reducing GC load. -func AcquireURI() *URI { - return uriPool.Get().(*URI) -} - -// ReleaseURI releases the URI acquired via AcquireURI. -// -// The released URI mustn't be used after releasing it, otherwise data races -// may occur. -func ReleaseURI(u *URI) { - u.Reset() - uriPool.Put(u) -} - -var uriPool = &sync.Pool{ - New: func() interface{} { - return &URI{} - }, -} - -// URI represents URI :) . -// -// It is forbidden copying URI instances. Create new instance and use CopyTo -// instead. -// -// URI instance MUST NOT be used from concurrently running goroutines. -type URI struct { - noCopy noCopy - - pathOriginal []byte - scheme []byte - path []byte - queryString []byte - hash []byte - host []byte - - queryArgs Args - parsedQueryArgs bool - - fullURI []byte - requestURI []byte - - h *RequestHeader -} - -// CopyTo copies uri contents to dst. -func (u *URI) CopyTo(dst *URI) { - dst.Reset() - dst.pathOriginal = append(dst.pathOriginal[:0], u.pathOriginal...) - dst.scheme = append(dst.scheme[:0], u.scheme...) - dst.path = append(dst.path[:0], u.path...) - dst.queryString = append(dst.queryString[:0], u.queryString...) - dst.hash = append(dst.hash[:0], u.hash...) - dst.host = append(dst.host[:0], u.host...) - - u.queryArgs.CopyTo(&dst.queryArgs) - dst.parsedQueryArgs = u.parsedQueryArgs - - // fullURI and requestURI shouldn't be copied, since they are created - // from scratch on each FullURI() and RequestURI() call. - dst.h = u.h -} - -// Hash returns URI hash, i.e. qwe of http://aaa.com/foo/bar?baz=123#qwe . -// -// The returned value is valid until the next URI method call. -func (u *URI) Hash() []byte { - return u.hash -} - -// SetHash sets URI hash. -func (u *URI) SetHash(hash string) { - u.hash = append(u.hash[:0], hash...) -} - -// SetHashBytes sets URI hash. -func (u *URI) SetHashBytes(hash []byte) { - u.hash = append(u.hash[:0], hash...) -} - -// QueryString returns URI query string, -// i.e. baz=123 of http://aaa.com/foo/bar?baz=123#qwe . -// -// The returned value is valid until the next URI method call. -func (u *URI) QueryString() []byte { - return u.queryString -} - -// SetQueryString sets URI query string. -func (u *URI) SetQueryString(queryString string) { - u.queryString = append(u.queryString[:0], queryString...) - u.parsedQueryArgs = false -} - -// SetQueryStringBytes sets URI query string. -func (u *URI) SetQueryStringBytes(queryString []byte) { - u.queryString = append(u.queryString[:0], queryString...) - u.parsedQueryArgs = false -} - -// Path returns URI path, i.e. /foo/bar of http://aaa.com/foo/bar?baz=123#qwe . -// -// The returned path is always urldecoded and normalized, -// i.e. '//f%20obar/baz/../zzz' becomes '/f obar/zzz'. -// -// The returned value is valid until the next URI method call. -func (u *URI) Path() []byte { - path := u.path - if len(path) == 0 { - path = strSlash - } - return path -} - -// SetPath sets URI path. -func (u *URI) SetPath(path string) { - u.pathOriginal = append(u.pathOriginal[:0], path...) - u.path = normalizePath(u.path, u.pathOriginal) -} - -// SetPathBytes sets URI path. -func (u *URI) SetPathBytes(path []byte) { - u.pathOriginal = append(u.pathOriginal[:0], path...) - u.path = normalizePath(u.path, u.pathOriginal) -} - -// PathOriginal returns the original path from requestURI passed to URI.Parse(). -// -// The returned value is valid until the next URI method call. -func (u *URI) PathOriginal() []byte { - return u.pathOriginal -} - -// Scheme returns URI scheme, i.e. http of http://aaa.com/foo/bar?baz=123#qwe . -// -// Returned scheme is always lowercased. -// -// The returned value is valid until the next URI method call. -func (u *URI) Scheme() []byte { - scheme := u.scheme - if len(scheme) == 0 { - scheme = strHTTP - } - return scheme -} - -// SetScheme sets URI scheme, i.e. http, https, ftp, etc. -func (u *URI) SetScheme(scheme string) { - u.scheme = append(u.scheme[:0], scheme...) - lowercaseBytes(u.scheme) -} - -// SetSchemeBytes sets URI scheme, i.e. http, https, ftp, etc. -func (u *URI) SetSchemeBytes(scheme []byte) { - u.scheme = append(u.scheme[:0], scheme...) - lowercaseBytes(u.scheme) -} - -// Reset clears uri. -func (u *URI) Reset() { - u.pathOriginal = u.pathOriginal[:0] - u.scheme = u.scheme[:0] - u.path = u.path[:0] - u.queryString = u.queryString[:0] - u.hash = u.hash[:0] - - u.host = u.host[:0] - u.queryArgs.Reset() - u.parsedQueryArgs = false - - // There is no need in u.fullURI = u.fullURI[:0], since full uri - // is calculated on each call to FullURI(). - - // There is no need in u.requestURI = u.requestURI[:0], since requestURI - // is calculated on each call to RequestURI(). - - u.h = nil -} - -// Host returns host part, i.e. aaa.com of http://aaa.com/foo/bar?baz=123#qwe . -// -// Host is always lowercased. -func (u *URI) Host() []byte { - if len(u.host) == 0 && u.h != nil { - u.host = append(u.host[:0], u.h.Host()...) - lowercaseBytes(u.host) - u.h = nil - } - return u.host -} - -// SetHost sets host for the uri. -func (u *URI) SetHost(host string) { - u.host = append(u.host[:0], host...) - lowercaseBytes(u.host) -} - -// SetHostBytes sets host for the uri. -func (u *URI) SetHostBytes(host []byte) { - u.host = append(u.host[:0], host...) - lowercaseBytes(u.host) -} - -// Parse initializes URI from the given host and uri. -// -// host may be nil. In this case uri must contain fully qualified uri, -// i.e. with scheme and host. http is assumed if scheme is omitted. -// -// uri may contain e.g. RequestURI without scheme and host if host is non-empty. -func (u *URI) Parse(host, uri []byte) { - u.parse(host, uri, nil) -} - -func (u *URI) parseQuick(uri []byte, h *RequestHeader, isTLS bool) { - u.parse(nil, uri, h) - if isTLS { - u.scheme = append(u.scheme[:0], strHTTPS...) - } -} - -func (u *URI) parse(host, uri []byte, h *RequestHeader) { - u.Reset() - u.h = h - - scheme, host, uri := splitHostURI(host, uri) - u.scheme = append(u.scheme, scheme...) - lowercaseBytes(u.scheme) - u.host = append(u.host, host...) - lowercaseBytes(u.host) - - b := uri - queryIndex := bytes.IndexByte(b, '?') - fragmentIndex := bytes.IndexByte(b, '#') - // Ignore query in fragment part - if fragmentIndex >= 0 && queryIndex > fragmentIndex { - queryIndex = -1 - } - - if queryIndex < 0 && fragmentIndex < 0 { - u.pathOriginal = append(u.pathOriginal, b...) - u.path = normalizePath(u.path, u.pathOriginal) - return - } - - if queryIndex >= 0 { - // Path is everything up to the start of the query - u.pathOriginal = append(u.pathOriginal, b[:queryIndex]...) - u.path = normalizePath(u.path, u.pathOriginal) - - if fragmentIndex < 0 { - u.queryString = append(u.queryString, b[queryIndex+1:]...) - } else { - u.queryString = append(u.queryString, b[queryIndex+1:fragmentIndex]...) - u.hash = append(u.hash, b[fragmentIndex+1:]...) - } - return - } - - // fragmentIndex >= 0 && queryIndex < 0 - // Path is up to the start of fragment - u.pathOriginal = append(u.pathOriginal, b[:fragmentIndex]...) - u.path = normalizePath(u.path, u.pathOriginal) - u.hash = append(u.hash, b[fragmentIndex+1:]...) -} - -func normalizePath(dst, src []byte) []byte { - dst = dst[:0] - dst = addLeadingSlash(dst, src) - dst = decodeArgAppendNoPlus(dst, src) - - // remove duplicate slashes - b := dst - bSize := len(b) - for { - n := bytes.Index(b, strSlashSlash) - if n < 0 { - break - } - b = b[n:] - copy(b, b[1:]) - b = b[:len(b)-1] - bSize-- - } - dst = dst[:bSize] - - // remove /./ parts - b = dst - for { - n := bytes.Index(b, strSlashDotSlash) - if n < 0 { - break - } - nn := n + len(strSlashDotSlash) - 1 - copy(b[n:], b[nn:]) - b = b[:len(b)-nn+n] - } - - // remove /foo/../ parts - for { - n := bytes.Index(b, strSlashDotDotSlash) - if n < 0 { - break - } - nn := bytes.LastIndexByte(b[:n], '/') - if nn < 0 { - nn = 0 - } - n += len(strSlashDotDotSlash) - 1 - copy(b[nn:], b[n:]) - b = b[:len(b)-n+nn] - } - - // remove trailing /foo/.. - n := bytes.LastIndex(b, strSlashDotDot) - if n >= 0 && n+len(strSlashDotDot) == len(b) { - nn := bytes.LastIndexByte(b[:n], '/') - if nn < 0 { - return strSlash - } - b = b[:nn+1] - } - - return b -} - -// RequestURI returns RequestURI - i.e. URI without Scheme and Host. -func (u *URI) RequestURI() []byte { - dst := appendQuotedPath(u.requestURI[:0], u.Path()) - if u.queryArgs.Len() > 0 { - dst = append(dst, '?') - dst = u.queryArgs.AppendBytes(dst) - } else if len(u.queryString) > 0 { - dst = append(dst, '?') - dst = append(dst, u.queryString...) - } - if len(u.hash) > 0 { - dst = append(dst, '#') - dst = append(dst, u.hash...) - } - u.requestURI = dst - return u.requestURI -} - -// LastPathSegment returns the last part of uri path after '/'. -// -// Examples: -// -// * For /foo/bar/baz.html path returns baz.html. -// * For /foo/bar/ returns empty byte slice. -// * For /foobar.js returns foobar.js. -func (u *URI) LastPathSegment() []byte { - path := u.Path() - n := bytes.LastIndexByte(path, '/') - if n < 0 { - return path - } - return path[n+1:] -} - -// Update updates uri. -// -// The following newURI types are accepted: -// -// * Absolute, i.e. http://foobar.com/aaa/bb?cc . In this case the original -// uri is replaced by newURI. -// * Absolute without scheme, i.e. //foobar.com/aaa/bb?cc. In this case -// the original scheme is preserved. -// * Missing host, i.e. /aaa/bb?cc . In this case only RequestURI part -// of the original uri is replaced. -// * Relative path, i.e. xx?yy=abc . In this case the original RequestURI -// is updated according to the new relative path. -func (u *URI) Update(newURI string) { - u.UpdateBytes(s2b(newURI)) -} - -// UpdateBytes updates uri. -// -// The following newURI types are accepted: -// -// * Absolute, i.e. http://foobar.com/aaa/bb?cc . In this case the original -// uri is replaced by newURI. -// * Absolute without scheme, i.e. //foobar.com/aaa/bb?cc. In this case -// the original scheme is preserved. -// * Missing host, i.e. /aaa/bb?cc . In this case only RequestURI part -// of the original uri is replaced. -// * Relative path, i.e. xx?yy=abc . In this case the original RequestURI -// is updated according to the new relative path. -func (u *URI) UpdateBytes(newURI []byte) { - u.requestURI = u.updateBytes(newURI, u.requestURI) -} - -func (u *URI) updateBytes(newURI, buf []byte) []byte { - if len(newURI) == 0 { - return buf - } - - n := bytes.Index(newURI, strSlashSlash) - if n >= 0 { - // absolute uri - var b [32]byte - schemeOriginal := b[:0] - if len(u.scheme) > 0 { - schemeOriginal = append([]byte(nil), u.scheme...) - } - u.Parse(nil, newURI) - if len(schemeOriginal) > 0 && len(u.scheme) == 0 { - u.scheme = append(u.scheme[:0], schemeOriginal...) - } - return buf - } - - if newURI[0] == '/' { - // uri without host - buf = u.appendSchemeHost(buf[:0]) - buf = append(buf, newURI...) - u.Parse(nil, buf) - return buf - } - - // relative path - switch newURI[0] { - case '?': - // query string only update - u.SetQueryStringBytes(newURI[1:]) - return append(buf[:0], u.FullURI()...) - case '#': - // update only hash - u.SetHashBytes(newURI[1:]) - return append(buf[:0], u.FullURI()...) - default: - // update the last path part after the slash - path := u.Path() - n = bytes.LastIndexByte(path, '/') - if n < 0 { - panic("BUG: path must contain at least one slash") - } - buf = u.appendSchemeHost(buf[:0]) - buf = appendQuotedPath(buf, path[:n+1]) - buf = append(buf, newURI...) - u.Parse(nil, buf) - return buf - } -} - -// FullURI returns full uri in the form {Scheme}://{Host}{RequestURI}#{Hash}. -func (u *URI) FullURI() []byte { - u.fullURI = u.AppendBytes(u.fullURI[:0]) - return u.fullURI -} - -// AppendBytes appends full uri to dst and returns the extended dst. -func (u *URI) AppendBytes(dst []byte) []byte { - dst = u.appendSchemeHost(dst) - return append(dst, u.RequestURI()...) -} - -func (u *URI) appendSchemeHost(dst []byte) []byte { - dst = append(dst, u.Scheme()...) - dst = append(dst, strColonSlashSlash...) - return append(dst, u.Host()...) -} - -// WriteTo writes full uri to w. -// -// WriteTo implements io.WriterTo interface. -func (u *URI) WriteTo(w io.Writer) (int64, error) { - n, err := w.Write(u.FullURI()) - return int64(n), err -} - -// String returns full uri. -func (u *URI) String() string { - return string(u.FullURI()) -} - -func splitHostURI(host, uri []byte) ([]byte, []byte, []byte) { - n := bytes.Index(uri, strSlashSlash) - if n < 0 { - return strHTTP, host, uri - } - scheme := uri[:n] - if bytes.IndexByte(scheme, '/') >= 0 { - return strHTTP, host, uri - } - if len(scheme) > 0 && scheme[len(scheme)-1] == ':' { - scheme = scheme[:len(scheme)-1] - } - n += len(strSlashSlash) - uri = uri[n:] - n = bytes.IndexByte(uri, '/') - if n < 0 { - // A hack for bogus urls like foobar.com?a=b without - // slash after host. - if n = bytes.IndexByte(uri, '?'); n >= 0 { - return scheme, uri[:n], uri[n:] - } - return scheme, uri, strSlash - } - return scheme, uri[:n], uri[n:] -} - -// QueryArgs returns query args. -func (u *URI) QueryArgs() *Args { - u.parseQueryArgs() - return &u.queryArgs -} - -func (u *URI) parseQueryArgs() { - if u.parsedQueryArgs { - return - } - u.queryArgs.ParseBytes(u.queryString) - u.parsedQueryArgs = true -} diff --git a/vendor/github.com/valyala/fasthttp/uri_unix.go b/vendor/github.com/valyala/fasthttp/uri_unix.go deleted file mode 100644 index 1e307332..00000000 --- a/vendor/github.com/valyala/fasthttp/uri_unix.go +++ /dev/null @@ -1,12 +0,0 @@ -// +build !windows - -package fasthttp - -func addLeadingSlash(dst, src []byte) []byte { - // add leading slash for unix paths - if len(src) == 0 || src[0] != '/' { - dst = append(dst, '/') - } - - return dst -} diff --git a/vendor/github.com/valyala/fasthttp/uri_windows.go b/vendor/github.com/valyala/fasthttp/uri_windows.go deleted file mode 100644 index 95917a6b..00000000 --- a/vendor/github.com/valyala/fasthttp/uri_windows.go +++ /dev/null @@ -1,12 +0,0 @@ -// +build windows - -package fasthttp - -func addLeadingSlash(dst, src []byte) []byte { - // zero length and "C:/" case - if len(src) == 0 || (len(src) > 2 && src[1] != ':') { - dst = append(dst, '/') - } - - return dst -} diff --git a/vendor/github.com/valyala/fasthttp/userdata.go b/vendor/github.com/valyala/fasthttp/userdata.go deleted file mode 100644 index bd3e28aa..00000000 --- a/vendor/github.com/valyala/fasthttp/userdata.go +++ /dev/null @@ -1,71 +0,0 @@ -package fasthttp - -import ( - "io" -) - -type userDataKV struct { - key []byte - value interface{} -} - -type userData []userDataKV - -func (d *userData) Set(key string, value interface{}) { - args := *d - n := len(args) - for i := 0; i < n; i++ { - kv := &args[i] - if string(kv.key) == key { - kv.value = value - return - } - } - - c := cap(args) - if c > n { - args = args[:n+1] - kv := &args[n] - kv.key = append(kv.key[:0], key...) - kv.value = value - *d = args - return - } - - kv := userDataKV{} - kv.key = append(kv.key[:0], key...) - kv.value = value - *d = append(args, kv) -} - -func (d *userData) SetBytes(key []byte, value interface{}) { - d.Set(b2s(key), value) -} - -func (d *userData) Get(key string) interface{} { - args := *d - n := len(args) - for i := 0; i < n; i++ { - kv := &args[i] - if string(kv.key) == key { - return kv.value - } - } - return nil -} - -func (d *userData) GetBytes(key []byte) interface{} { - return d.Get(b2s(key)) -} - -func (d *userData) Reset() { - args := *d - n := len(args) - for i := 0; i < n; i++ { - v := args[i].value - if vc, ok := v.(io.Closer); ok { - vc.Close() - } - } - *d = (*d)[:0] -} diff --git a/vendor/github.com/valyala/fasthttp/workerpool.go b/vendor/github.com/valyala/fasthttp/workerpool.go deleted file mode 100644 index bfd297c3..00000000 --- a/vendor/github.com/valyala/fasthttp/workerpool.go +++ /dev/null @@ -1,237 +0,0 @@ -package fasthttp - -import ( - "net" - "runtime" - "strings" - "sync" - "time" -) - -// workerPool serves incoming connections via a pool of workers -// in FILO order, i.e. the most recently stopped worker will serve the next -// incoming connection. -// -// Such a scheme keeps CPU caches hot (in theory). -type workerPool struct { - // Function for serving server connections. - // It must leave c unclosed. - WorkerFunc ServeHandler - - MaxWorkersCount int - - LogAllErrors bool - - MaxIdleWorkerDuration time.Duration - - Logger Logger - - lock sync.Mutex - workersCount int - mustStop bool - - ready []*workerChan - - stopCh chan struct{} - - workerChanPool sync.Pool - - connState func(net.Conn, ConnState) -} - -type workerChan struct { - lastUseTime time.Time - ch chan net.Conn -} - -func (wp *workerPool) Start() { - if wp.stopCh != nil { - panic("BUG: workerPool already started") - } - wp.stopCh = make(chan struct{}) - stopCh := wp.stopCh - go func() { - var scratch []*workerChan - for { - wp.clean(&scratch) - select { - case <-stopCh: - return - default: - time.Sleep(wp.getMaxIdleWorkerDuration()) - } - } - }() -} - -func (wp *workerPool) Stop() { - if wp.stopCh == nil { - panic("BUG: workerPool wasn't started") - } - close(wp.stopCh) - wp.stopCh = nil - - // Stop all the workers waiting for incoming connections. - // Do not wait for busy workers - they will stop after - // serving the connection and noticing wp.mustStop = true. - wp.lock.Lock() - ready := wp.ready - for i, ch := range ready { - ch.ch <- nil - ready[i] = nil - } - wp.ready = ready[:0] - wp.mustStop = true - wp.lock.Unlock() -} - -func (wp *workerPool) getMaxIdleWorkerDuration() time.Duration { - if wp.MaxIdleWorkerDuration <= 0 { - return 10 * time.Second - } - return wp.MaxIdleWorkerDuration -} - -func (wp *workerPool) clean(scratch *[]*workerChan) { - maxIdleWorkerDuration := wp.getMaxIdleWorkerDuration() - - // Clean least recently used workers if they didn't serve connections - // for more than maxIdleWorkerDuration. - currentTime := time.Now() - - wp.lock.Lock() - ready := wp.ready - n := len(ready) - i := 0 - for i < n && currentTime.Sub(ready[i].lastUseTime) > maxIdleWorkerDuration { - i++ - } - *scratch = append((*scratch)[:0], ready[:i]...) - if i > 0 { - m := copy(ready, ready[i:]) - for i = m; i < n; i++ { - ready[i] = nil - } - wp.ready = ready[:m] - } - wp.lock.Unlock() - - // Notify obsolete workers to stop. - // This notification must be outside the wp.lock, since ch.ch - // may be blocking and may consume a lot of time if many workers - // are located on non-local CPUs. - tmp := *scratch - for i, ch := range tmp { - ch.ch <- nil - tmp[i] = nil - } -} - -func (wp *workerPool) Serve(c net.Conn) bool { - ch := wp.getCh() - if ch == nil { - return false - } - ch.ch <- c - return true -} - -var workerChanCap = func() int { - // Use blocking workerChan if GOMAXPROCS=1. - // This immediately switches Serve to WorkerFunc, which results - // in higher performance (under go1.5 at least). - if runtime.GOMAXPROCS(0) == 1 { - return 0 - } - - // Use non-blocking workerChan if GOMAXPROCS>1, - // since otherwise the Serve caller (Acceptor) may lag accepting - // new connections if WorkerFunc is CPU-bound. - return 1 -}() - -func (wp *workerPool) getCh() *workerChan { - var ch *workerChan - createWorker := false - - wp.lock.Lock() - ready := wp.ready - n := len(ready) - 1 - if n < 0 { - if wp.workersCount < wp.MaxWorkersCount { - createWorker = true - wp.workersCount++ - } - } else { - ch = ready[n] - ready[n] = nil - wp.ready = ready[:n] - } - wp.lock.Unlock() - - if ch == nil { - if !createWorker { - return nil - } - vch := wp.workerChanPool.Get() - if vch == nil { - vch = &workerChan{ - ch: make(chan net.Conn, workerChanCap), - } - } - ch = vch.(*workerChan) - go func() { - wp.workerFunc(ch) - wp.workerChanPool.Put(vch) - }() - } - return ch -} - -func (wp *workerPool) release(ch *workerChan) bool { - ch.lastUseTime = time.Now() - wp.lock.Lock() - if wp.mustStop { - wp.lock.Unlock() - return false - } - wp.ready = append(wp.ready, ch) - wp.lock.Unlock() - return true -} - -func (wp *workerPool) workerFunc(ch *workerChan) { - var c net.Conn - - var err error - for c = range ch.ch { - if c == nil { - break - } - - if err = wp.WorkerFunc(c); err != nil && err != errHijacked { - errStr := err.Error() - if wp.LogAllErrors || !(strings.Contains(errStr, "broken pipe") || - strings.Contains(errStr, "reset by peer") || - strings.Contains(errStr, "request headers: small read buffer") || - strings.Contains(errStr, "i/o timeout")) { - wp.Logger.Printf("error when serving connection %q<->%q: %s", c.LocalAddr(), c.RemoteAddr(), err) - } - } - if err == errHijacked { - wp.connState(c, StateHijacked) - } else { - c.Close() - wp.connState(c, StateClosed) - } - c = nil - - if !wp.release(ch) { - break - } - } - - wp.lock.Lock() - wp.workersCount-- - wp.lock.Unlock() -} diff --git a/vendor/golang.org/x/net/netutil/listen.go b/vendor/golang.org/x/net/netutil/listen.go new file mode 100644 index 00000000..cee46e33 --- /dev/null +++ b/vendor/golang.org/x/net/netutil/listen.go @@ -0,0 +1,74 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package netutil provides network utility functions, complementing the more +// common ones in the net package. +package netutil // import "golang.org/x/net/netutil" + +import ( + "net" + "sync" +) + +// LimitListener returns a Listener that accepts at most n simultaneous +// connections from the provided Listener. +func LimitListener(l net.Listener, n int) net.Listener { + return &limitListener{ + Listener: l, + sem: make(chan struct{}, n), + done: make(chan struct{}), + } +} + +type limitListener struct { + net.Listener + sem chan struct{} + closeOnce sync.Once // ensures the done chan is only closed once + done chan struct{} // no values sent; closed when Close is called +} + +// acquire acquires the limiting semaphore. Returns true if successfully +// accquired, false if the listener is closed and the semaphore is not +// acquired. +func (l *limitListener) acquire() bool { + select { + case <-l.done: + return false + case l.sem <- struct{}{}: + return true + } +} +func (l *limitListener) release() { <-l.sem } + +func (l *limitListener) Accept() (net.Conn, error) { + acquired := l.acquire() + // If the semaphore isn't acquired because the listener was closed, expect + // that this call to accept won't block, but immediately return an error. + c, err := l.Listener.Accept() + if err != nil { + if acquired { + l.release() + } + return nil, err + } + return &limitListenerConn{Conn: c, release: l.release}, nil +} + +func (l *limitListener) Close() error { + err := l.Listener.Close() + l.closeOnce.Do(func() { close(l.done) }) + return err +} + +type limitListenerConn struct { + net.Conn + releaseOnce sync.Once + release func() +} + +func (l *limitListenerConn) Close() error { + err := l.Conn.Close() + l.releaseOnce.Do(l.release) + return err +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 8a65991a..432ac3a5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -83,12 +83,6 @@ github.com/honeybadger-io/honeybadger-go github.com/jmespath/go-jmespath # github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 github.com/kardianos/osext -# github.com/klauspost/compress v1.4.1 -github.com/klauspost/compress/flate -github.com/klauspost/compress/gzip -github.com/klauspost/compress/zlib -# github.com/klauspost/cpuid v1.2.0 -github.com/klauspost/cpuid # github.com/mat/besticon v3.9.0+incompatible github.com/mat/besticon/ico # github.com/matoous/go-nanoid v0.0.0-20181114085210-eab626deece6 @@ -132,12 +126,6 @@ github.com/shirou/gopsutil/internal/common github.com/stretchr/testify/assert github.com/stretchr/testify/require github.com/stretchr/testify/suite -# github.com/valyala/bytebufferpool v1.0.0 -github.com/valyala/bytebufferpool -# github.com/valyala/fasthttp v1.2.0 -github.com/valyala/fasthttp -github.com/valyala/fasthttp/fasthttputil -github.com/valyala/fasthttp/stackless # go.opencensus.io v0.18.0 go.opencensus.io/trace go.opencensus.io/plugin/ochttp @@ -159,6 +147,7 @@ golang.org/x/image/vp8 golang.org/x/image/vp8l golang.org/x/image/bmp # golang.org/x/net v0.0.0-20190110200230-915654e7eabc +golang.org/x/net/netutil golang.org/x/net/context golang.org/x/net/trace golang.org/x/net/context/ctxhttp