From ceec679c81d3f51dc086c10159b34ecd2a371ef9 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Thu, 25 Oct 2018 17:31:31 +0600 Subject: [PATCH] Back to net/http because of fasthttps incompatibiliti with almost everything --- Gopkg.lock | 41 - Gopkg.toml | 4 - README.md | 2 +- etag.go | 7 +- mutex.go | 15 - processing_options.go | 11 +- server.go | 144 +- vendor/github.com/klauspost/compress/LICENSE | 27 - .../klauspost/compress/flate/copy.go | 32 - .../klauspost/compress/flate/crc32_amd64.go | 41 - .../klauspost/compress/flate/crc32_amd64.s | 213 -- .../klauspost/compress/flate/crc32_noasm.go | 35 - .../klauspost/compress/flate/deflate.go | 1353 ---------- .../klauspost/compress/flate/dict_decoder.go | 184 -- .../klauspost/compress/flate/gen.go | 265 -- .../compress/flate/huffman_bit_writer.go | 701 ----- .../klauspost/compress/flate/huffman_code.go | 344 --- .../klauspost/compress/flate/inflate.go | 880 ------ .../klauspost/compress/flate/reverse_bits.go | 48 - .../klauspost/compress/flate/snappy.go | 900 ------- .../klauspost/compress/flate/token.go | 115 - .../klauspost/compress/gzip/gunzip.go | 344 --- .../klauspost/compress/gzip/gzip.go | 251 -- .../klauspost/compress/snappy/AUTHORS | 15 - .../klauspost/compress/snappy/CONTRIBUTORS | 37 - .../klauspost/compress/snappy/LICENSE | 27 - .../klauspost/compress/zlib/reader.go | 183 -- .../klauspost/compress/zlib/writer.go | 201 -- vendor/github.com/klauspost/cpuid/.gitignore | 24 - vendor/github.com/klauspost/cpuid/.travis.yml | 24 - vendor/github.com/klauspost/cpuid/LICENSE | 22 - vendor/github.com/klauspost/cpuid/README.md | 145 - vendor/github.com/klauspost/cpuid/cpuid.go | 1030 ------- vendor/github.com/klauspost/cpuid/cpuid_386.s | 42 - .../github.com/klauspost/cpuid/cpuid_amd64.s | 42 - .../klauspost/cpuid/detect_intel.go | 17 - .../github.com/klauspost/cpuid/detect_ref.go | 23 - vendor/github.com/klauspost/cpuid/generate.go | 4 - .../github.com/klauspost/cpuid/private-gen.go | 476 ---- .../valyala/bytebufferpool/.travis.yml | 15 - .../github.com/valyala/bytebufferpool/LICENSE | 22 - .../valyala/bytebufferpool/README.md | 21 - .../valyala/bytebufferpool/bytebuffer.go | 111 - .../github.com/valyala/bytebufferpool/doc.go | 7 - .../github.com/valyala/bytebufferpool/pool.go | 151 -- vendor/github.com/valyala/fasthttp/.gitignore | 3 - .../github.com/valyala/fasthttp/.travis.yml | 36 - vendor/github.com/valyala/fasthttp/LICENSE | 25 - vendor/github.com/valyala/fasthttp/README.md | 584 ---- vendor/github.com/valyala/fasthttp/TODO | 4 - vendor/github.com/valyala/fasthttp/args.go | 522 ---- .../github.com/valyala/fasthttp/bytesconv.go | 456 ---- .../valyala/fasthttp/bytesconv_32.go | 8 - .../valyala/fasthttp/bytesconv_64.go | 8 - vendor/github.com/valyala/fasthttp/client.go | 2188 --------------- .../github.com/valyala/fasthttp/coarseTime.go | 13 - .../github.com/valyala/fasthttp/compress.go | 438 --- vendor/github.com/valyala/fasthttp/cookie.go | 474 ---- vendor/github.com/valyala/fasthttp/doc.go | 37 - .../valyala/fasthttp/fasthttputil/doc.go | 2 - .../valyala/fasthttp/fasthttputil/ecdsa.key | 5 - .../valyala/fasthttp/fasthttputil/ecdsa.pem | 10 - .../fasthttputil/inmemory_listener.go | 84 - .../fasthttp/fasthttputil/pipeconns.go | 283 -- .../valyala/fasthttp/fasthttputil/rsa.key | 28 - .../valyala/fasthttp/fasthttputil/rsa.pem | 17 - vendor/github.com/valyala/fasthttp/fs.go | 1271 --------- vendor/github.com/valyala/fasthttp/go.mod | 9 - vendor/github.com/valyala/fasthttp/go.sum | 10 - vendor/github.com/valyala/fasthttp/header.go | 2147 --------------- vendor/github.com/valyala/fasthttp/http.go | 1720 ------------ .../github.com/valyala/fasthttp/lbclient.go | 183 -- vendor/github.com/valyala/fasthttp/nocopy.go | 9 - .../github.com/valyala/fasthttp/peripconn.go | 100 - .../valyala/fasthttp/reuseport/LICENSE | 21 - vendor/github.com/valyala/fasthttp/server.go | 2361 ----------------- .../valyala/fasthttp/ssl-cert-snakeoil.key | 28 - .../valyala/fasthttp/ssl-cert-snakeoil.pem | 17 - .../valyala/fasthttp/stackless/doc.go | 3 - .../valyala/fasthttp/stackless/func.go | 79 - .../valyala/fasthttp/stackless/writer.go | 139 - vendor/github.com/valyala/fasthttp/status.go | 176 -- vendor/github.com/valyala/fasthttp/stream.go | 54 - vendor/github.com/valyala/fasthttp/strings.go | 78 - .../github.com/valyala/fasthttp/tcpdialer.go | 369 --- vendor/github.com/valyala/fasthttp/timer.go | 44 - vendor/github.com/valyala/fasthttp/uri.go | 525 ---- .../github.com/valyala/fasthttp/uri_unix.go | 12 - .../valyala/fasthttp/uri_windows.go | 12 - .../github.com/valyala/fasthttp/userdata.go | 71 - .../github.com/valyala/fasthttp/workerpool.go | 237 -- 91 files changed, 99 insertions(+), 23397 deletions(-) delete mode 100644 mutex.go delete mode 100644 vendor/github.com/klauspost/compress/LICENSE delete mode 100644 vendor/github.com/klauspost/compress/flate/copy.go delete mode 100644 vendor/github.com/klauspost/compress/flate/crc32_amd64.go delete mode 100644 vendor/github.com/klauspost/compress/flate/crc32_amd64.s delete mode 100644 vendor/github.com/klauspost/compress/flate/crc32_noasm.go delete mode 100644 vendor/github.com/klauspost/compress/flate/deflate.go delete mode 100644 vendor/github.com/klauspost/compress/flate/dict_decoder.go delete mode 100644 vendor/github.com/klauspost/compress/flate/gen.go delete mode 100644 vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go delete mode 100644 vendor/github.com/klauspost/compress/flate/huffman_code.go delete mode 100644 vendor/github.com/klauspost/compress/flate/inflate.go delete mode 100644 vendor/github.com/klauspost/compress/flate/reverse_bits.go delete mode 100644 vendor/github.com/klauspost/compress/flate/snappy.go delete mode 100644 vendor/github.com/klauspost/compress/flate/token.go delete mode 100644 vendor/github.com/klauspost/compress/gzip/gunzip.go delete mode 100644 vendor/github.com/klauspost/compress/gzip/gzip.go delete mode 100644 vendor/github.com/klauspost/compress/snappy/AUTHORS delete mode 100644 vendor/github.com/klauspost/compress/snappy/CONTRIBUTORS delete mode 100644 vendor/github.com/klauspost/compress/snappy/LICENSE delete mode 100644 vendor/github.com/klauspost/compress/zlib/reader.go delete mode 100644 vendor/github.com/klauspost/compress/zlib/writer.go delete mode 100644 vendor/github.com/klauspost/cpuid/.gitignore delete mode 100644 vendor/github.com/klauspost/cpuid/.travis.yml delete mode 100644 vendor/github.com/klauspost/cpuid/LICENSE delete mode 100644 vendor/github.com/klauspost/cpuid/README.md delete mode 100644 vendor/github.com/klauspost/cpuid/cpuid.go delete mode 100644 vendor/github.com/klauspost/cpuid/cpuid_386.s delete mode 100644 vendor/github.com/klauspost/cpuid/cpuid_amd64.s delete mode 100644 vendor/github.com/klauspost/cpuid/detect_intel.go delete mode 100644 vendor/github.com/klauspost/cpuid/detect_ref.go delete mode 100644 vendor/github.com/klauspost/cpuid/generate.go delete mode 100644 vendor/github.com/klauspost/cpuid/private-gen.go delete mode 100644 vendor/github.com/valyala/bytebufferpool/.travis.yml delete mode 100644 vendor/github.com/valyala/bytebufferpool/LICENSE delete mode 100644 vendor/github.com/valyala/bytebufferpool/README.md delete mode 100644 vendor/github.com/valyala/bytebufferpool/bytebuffer.go delete mode 100644 vendor/github.com/valyala/bytebufferpool/doc.go delete mode 100644 vendor/github.com/valyala/bytebufferpool/pool.go delete mode 100644 vendor/github.com/valyala/fasthttp/.gitignore delete mode 100644 vendor/github.com/valyala/fasthttp/.travis.yml delete mode 100644 vendor/github.com/valyala/fasthttp/LICENSE delete mode 100644 vendor/github.com/valyala/fasthttp/README.md delete mode 100644 vendor/github.com/valyala/fasthttp/TODO delete mode 100644 vendor/github.com/valyala/fasthttp/args.go delete mode 100644 vendor/github.com/valyala/fasthttp/bytesconv.go delete mode 100644 vendor/github.com/valyala/fasthttp/bytesconv_32.go delete mode 100644 vendor/github.com/valyala/fasthttp/bytesconv_64.go delete mode 100644 vendor/github.com/valyala/fasthttp/client.go delete mode 100644 vendor/github.com/valyala/fasthttp/coarseTime.go delete mode 100644 vendor/github.com/valyala/fasthttp/compress.go delete mode 100644 vendor/github.com/valyala/fasthttp/cookie.go delete mode 100644 vendor/github.com/valyala/fasthttp/doc.go delete mode 100644 vendor/github.com/valyala/fasthttp/fasthttputil/doc.go delete mode 100644 vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.key delete mode 100644 vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.pem delete mode 100644 vendor/github.com/valyala/fasthttp/fasthttputil/inmemory_listener.go delete mode 100644 vendor/github.com/valyala/fasthttp/fasthttputil/pipeconns.go delete mode 100644 vendor/github.com/valyala/fasthttp/fasthttputil/rsa.key delete mode 100644 vendor/github.com/valyala/fasthttp/fasthttputil/rsa.pem delete mode 100644 vendor/github.com/valyala/fasthttp/fs.go delete mode 100644 vendor/github.com/valyala/fasthttp/go.mod delete mode 100644 vendor/github.com/valyala/fasthttp/go.sum delete mode 100644 vendor/github.com/valyala/fasthttp/header.go delete mode 100644 vendor/github.com/valyala/fasthttp/http.go delete mode 100644 vendor/github.com/valyala/fasthttp/lbclient.go delete mode 100644 vendor/github.com/valyala/fasthttp/nocopy.go delete mode 100644 vendor/github.com/valyala/fasthttp/peripconn.go delete mode 100644 vendor/github.com/valyala/fasthttp/reuseport/LICENSE delete mode 100644 vendor/github.com/valyala/fasthttp/server.go delete mode 100644 vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.key delete mode 100644 vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.pem delete mode 100644 vendor/github.com/valyala/fasthttp/stackless/doc.go delete mode 100644 vendor/github.com/valyala/fasthttp/stackless/func.go delete mode 100644 vendor/github.com/valyala/fasthttp/stackless/writer.go delete mode 100644 vendor/github.com/valyala/fasthttp/status.go delete mode 100644 vendor/github.com/valyala/fasthttp/stream.go delete mode 100644 vendor/github.com/valyala/fasthttp/strings.go delete mode 100644 vendor/github.com/valyala/fasthttp/tcpdialer.go delete mode 100644 vendor/github.com/valyala/fasthttp/timer.go delete mode 100644 vendor/github.com/valyala/fasthttp/uri.go delete mode 100644 vendor/github.com/valyala/fasthttp/uri_unix.go delete mode 100644 vendor/github.com/valyala/fasthttp/uri_windows.go delete mode 100644 vendor/github.com/valyala/fasthttp/userdata.go delete mode 100644 vendor/github.com/valyala/fasthttp/workerpool.go diff --git a/Gopkg.lock b/Gopkg.lock index 40d0780b..2f8c4f92 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -56,26 +56,6 @@ pruneopts = "UT" revision = "0b12d6b5" -[[projects]] - digest = "1:cdbed6a7b574b13bbabcd8c36c5be56d6444789bab21fcd37d39655eab0b2141" - name = "github.com/klauspost/compress" - packages = [ - "flate", - "gzip", - "zlib", - ] - pruneopts = "UT" - revision = "b939724e787a27c0005cabe3f78e7ed7987ac74f" - version = "v1.4.0" - -[[projects]] - digest = "1:4ea0668d490ca32a38366453a486e2e8c60fbdaf1f2607c96b4a093d8a5c8de7" - name = "github.com/klauspost/cpuid" - packages = ["."] - pruneopts = "UT" - revision = "ae7887de9fa5d2db4eaa8174a7eff2c1ac00f2da" - version = "v1.1" - [[projects]] branch = "master" digest = "1:97e35d2efabea52337e58f96e8b798fd461bf42d2519beb50f485029c6ad6aa5" @@ -84,26 +64,6 @@ pruneopts = "UT" revision = "3de1538a83bcb1749aa54482113e5cd06e4fb938" -[[projects]] - digest = "1:c468422f334a6b46a19448ad59aaffdfc0a36b08fdcc1c749a0b29b6453d7e59" - name = "github.com/valyala/bytebufferpool" - packages = ["."] - pruneopts = "UT" - revision = "e746df99fe4a3986f4d4f79e13c1e0117ce9c2f7" - version = "v1.0.0" - -[[projects]] - branch = "master" - digest = "1:7c1085da40cd79011bff52dde0c8e489d3947bd296bc8361aa71285b44b79737" - name = "github.com/valyala/fasthttp" - packages = [ - ".", - "fasthttputil", - "stackless", - ] - pruneopts = "UT" - revision = "8dfc881b9ec613747a4b7bb8ebe7504b3a18a409" - [[projects]] digest = "1:72b36febaabad58e1864de2b43de05689ce27a2c9a582a61a25e71a31ba23d0b" name = "golang.org/x/image" @@ -131,7 +91,6 @@ "github.com/aws/aws-sdk-go/aws/session", "github.com/aws/aws-sdk-go/service/s3", "github.com/matoous/go-nanoid", - "github.com/valyala/fasthttp", "golang.org/x/image/webp", "golang.org/x/net/netutil", ] diff --git a/Gopkg.toml b/Gopkg.toml index 7b95e1e3..0430054a 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -33,10 +33,6 @@ branch = "master" name = "github.com/matoous/go-nanoid" -[[constraint]] - branch = "master" - name = "github.com/valyala/fasthttp" - [prune] go-tests = true unused-packages = true diff --git a/README.md b/README.md index c3c32631..e68c076a 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ imgproxy only includes the must-have features for image processing, fine-tuning imgproxy uses probably the most efficient image processing library there is, called `libvips`. It is screaming fast and has a very low memory footprint; with it, we can handle the processing for a massive amount of images on the fly. -imgproxy also uses [fasthttp](https://github.com/valyala/fasthttp) for the best HTTP networking speed and less memory usage. +imgproxy also uses native Go's `net/http` routing for the best HTTP networking support. #### Security diff --git a/etag.go b/etag.go index acb6aa0a..6c25dc97 100644 --- a/etag.go +++ b/etag.go @@ -28,7 +28,7 @@ var eTagCalcPool = sync.Pool{ }, } -func calcETag(ctx context.Context) []byte { +func calcETag(ctx context.Context) string { c := eTagCalcPool.Get().(*eTagCalc) defer eTagCalcPool.Put(c) @@ -42,8 +42,5 @@ func calcETag(ctx context.Context) []byte { c.enc.Encode(conf) c.enc.Encode(getProcessingOptions(ctx)) - etag := make([]byte, 64) - hex.Encode(etag, c.hash.Sum(nil)) - - return etag + return hex.EncodeToString(c.hash.Sum(nil)) } diff --git a/mutex.go b/mutex.go deleted file mode 100644 index 3e902a4f..00000000 --- a/mutex.go +++ /dev/null @@ -1,15 +0,0 @@ -package main - -type mutex chan struct{} - -func newMutex(size int) mutex { - return make(mutex, size) -} - -func (m mutex) Lock() { - m <- struct{}{} -} - -func (m mutex) Unock() { - <-m -} diff --git a/processing_options.go b/processing_options.go index ea5918ba..77e2feee 100644 --- a/processing_options.go +++ b/processing_options.go @@ -11,12 +11,11 @@ import ( "encoding/base64" "errors" "fmt" + "net/http" "net/url" "regexp" "strconv" "strings" - - "github.com/valyala/fasthttp" ) type urlOptions map[string][]string @@ -685,13 +684,13 @@ func parsePathSimple(parts []string, acceptHeader string) (string, *processingOp return string(url), po, nil } -func parsePath(ctx context.Context, rctx *fasthttp.RequestCtx) (context.Context, error) { - path := string(rctx.Path()) +func parsePath(ctx context.Context, r *http.Request) (context.Context, error) { + path := r.URL.Path parts := strings.Split(strings.TrimPrefix(path, "/"), "/") var acceptHeader string - if h := rctx.Request.Header.Peek("Accept"); len(h) > 0 { - acceptHeader = string(h) + if h, ok := r.Header["Accept"]; ok { + acceptHeader = h[0] } if len(parts) < 3 { diff --git a/server.go b/server.go index 5bf68263..0904055d 100644 --- a/server.go +++ b/server.go @@ -6,13 +6,19 @@ import ( "crypto/subtle" "fmt" "log" + "net" "net/http" + "strconv" + "strings" + "sync" "time" nanoid "github.com/matoous/go-nanoid" - "github.com/valyala/fasthttp" + "golang.org/x/net/netutil" ) +const healthPath = "/health" + var ( mimes = map[imageType]string{ imageTypeJPEG: "image/jpeg", @@ -22,27 +28,40 @@ var ( authHeaderMust []byte - healthPath = []byte("/health") - - serverMutex mutex + imgproxyIsRunningMsg = []byte("imgproxy is running") errInvalidMethod = newError(422, "Invalid request method", "Method doesn't allowed") errInvalidSecret = newError(403, "Invalid secret", "Forbidden") ) -func startServer() *fasthttp.Server { - serverMutex = newMutex(conf.Concurrency) +var responseBufPool = sync.Pool{ + New: func() interface{} { + return new(bytes.Buffer) + }, +} - s := &fasthttp.Server{ - Name: "imgproxy", - Handler: serveHTTP, - Concurrency: conf.MaxClients, - ReadTimeout: time.Duration(conf.ReadTimeout) * time.Second, +type httpHandler struct { + sem chan struct{} +} + +func newHTTPHandler() *httpHandler { + return &httpHandler{make(chan struct{}, conf.Concurrency)} +} + +func startServer() *http.Server { + l, err := net.Listen("tcp", conf.Bind) + if err != nil { + log.Fatal(err) + } + s := &http.Server{ + Handler: newHTTPHandler(), + ReadTimeout: time.Duration(conf.ReadTimeout) * time.Second, + MaxHeaderBytes: 1 << 20, } go func() { log.Printf("Starting server at %s\n", conf.Bind) - if err := s.ListenAndServe(conf.Bind); err != nil { + if err := s.Serve(netutil.LimitListener(l, conf.MaxClients)); err != nil && err != http.ErrServerClosed { log.Fatalln(err) } }() @@ -50,9 +69,13 @@ func startServer() *fasthttp.Server { return s } -func shutdownServer(s *fasthttp.Server) { +func shutdownServer(s *http.Server) { log.Println("Shutting down the server...") - s.Shutdown() + + ctx, close := context.WithTimeout(context.Background(), 5*time.Second) + defer close() + + s.Shutdown(ctx) } func logResponse(status int, msg string) { @@ -69,42 +92,49 @@ func logResponse(status int, msg string) { log.Printf("|\033[7;%dm %d \033[0m| %s\n", color, status, msg) } -func writeCORS(rctx *fasthttp.RequestCtx) { +func writeCORS(rw http.ResponseWriter) { if len(conf.AllowOrigin) > 0 { - rctx.Request.Header.Set("Access-Control-Allow-Origin", conf.AllowOrigin) - rctx.Request.Header.Set("Access-Control-Allow-Methods", "GET, OPTIONs") + rw.Header().Set("Access-Control-Allow-Origin", conf.AllowOrigin) + rw.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONs") } } -func respondWithImage(ctx context.Context, reqID string, rctx *fasthttp.RequestCtx, data []byte) { - rctx.SetStatusCode(200) - +func respondWithImage(ctx context.Context, reqID string, r *http.Request, rw http.ResponseWriter, data []byte) { po := getProcessingOptions(ctx) - rctx.SetContentType(mimes[po.Format]) - rctx.Response.Header.Set("Cache-Control", fmt.Sprintf("max-age=%d, public", conf.TTL)) - rctx.Response.Header.Set("Expires", time.Now().Add(time.Second*time.Duration(conf.TTL)).Format(http.TimeFormat)) + rw.Header().Set("Expires", time.Now().Add(time.Second*time.Duration(conf.TTL)).Format(http.TimeFormat)) + rw.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d, public", conf.TTL)) + rw.Header().Set("Content-Type", mimes[po.Format]) - if conf.GZipCompression > 0 && rctx.Request.Header.HasAcceptEncodingBytes([]byte("gzip")) { - rctx.Response.Header.Set("Content-Encoding", "gzip") - gzipData(data, rctx) - } else { - rctx.SetBody(data) + dataToRespond := data + + if conf.GZipCompression > 0 && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + rw.Header().Set("Content-Encoding", "gzip") + + buf := responseBufPool.Get().(*bytes.Buffer) + defer responseBufPool.Put(buf) + + gzipData(data, buf) + dataToRespond = buf.Bytes() } + rw.Header().Set("Content-Length", strconv.Itoa(len(dataToRespond))) + rw.WriteHeader(200) + rw.Write(dataToRespond) + logResponse(200, fmt.Sprintf("[%s] Processed in %s: %s; %+v", reqID, getTimerSince(ctx), getImageURL(ctx), po)) } -func respondWithError(reqID string, rctx *fasthttp.RequestCtx, err imgproxyError) { +func respondWithError(reqID string, rw http.ResponseWriter, err imgproxyError) { logResponse(err.StatusCode, fmt.Sprintf("[%s] %s", reqID, err.Message)) - rctx.SetStatusCode(err.StatusCode) - rctx.SetBodyString(err.PublicMessage) + rw.WriteHeader(err.StatusCode) + rw.Write([]byte(err.PublicMessage)) } -func respondWithOptions(reqID string, rctx *fasthttp.RequestCtx) { +func respondWithOptions(reqID string, rw http.ResponseWriter) { logResponse(200, fmt.Sprintf("[%s] Respond with options", reqID)) - rctx.SetStatusCode(200) + rw.WriteHeader(200) } func prepareAuthHeaderMust() []byte { @@ -115,60 +145,68 @@ func prepareAuthHeaderMust() []byte { return authHeaderMust } -func checkSecret(rctx *fasthttp.RequestCtx) bool { +func checkSecret(r *http.Request) bool { if len(conf.Secret) == 0 { return true } return subtle.ConstantTimeCompare( - rctx.Request.Header.Peek("Authorization"), + []byte(r.Header.Get("Authorization")), prepareAuthHeaderMust(), ) == 1 } -func serveHTTP(rctx *fasthttp.RequestCtx) { +func (h *httpHandler) lock() { + h.sem <- struct{}{} +} + +func (h *httpHandler) unlock() { + <-h.sem +} + +func (h *httpHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { reqID, _ := nanoid.Nanoid() defer func() { if r := recover(); r != nil { if err, ok := r.(imgproxyError); ok { - respondWithError(reqID, rctx, err) + respondWithError(reqID, rw, err) } else { - respondWithError(reqID, rctx, newUnexpectedError(r.(error), 4)) + respondWithError(reqID, rw, newUnexpectedError(r.(error), 4)) } } }() - log.Printf("[%s] %s: %s\n", reqID, rctx.Method(), rctx.Path()) + log.Printf("[%s] %s: %s\n", reqID, r.Method, r.URL.RequestURI()) - writeCORS(rctx) + writeCORS(rw) - if rctx.Request.Header.IsOptions() { - respondWithOptions(reqID, rctx) + if r.Method == http.MethodOptions { + respondWithOptions(reqID, rw) return } - if !rctx.IsGet() { + if r.Method != http.MethodGet { panic(errInvalidMethod) } - if !checkSecret(rctx) { + if !checkSecret(r) { panic(errInvalidSecret) } - serverMutex.Lock() - defer serverMutex.Unock() + h.lock() + defer h.unlock() - if bytes.Equal(rctx.Path(), healthPath) { - rctx.SetStatusCode(200) - rctx.SetBodyString("imgproxy is running") + if r.URL.RequestURI() == healthPath { + rw.WriteHeader(200) + rw.Write(imgproxyIsRunningMsg) return } ctx, timeoutCancel := startTimer(time.Duration(conf.WriteTimeout) * time.Second) defer timeoutCancel() - ctx, err := parsePath(ctx, rctx) + ctx, err := parsePath(ctx, r) if err != nil { panic(newError(404, err.Error(), "Invalid image url")) } @@ -183,9 +221,9 @@ func serveHTTP(rctx *fasthttp.RequestCtx) { if conf.ETagEnabled { eTag := calcETag(ctx) - rctx.Response.Header.SetBytesV("ETag", eTag) + rw.Header().Set("ETag", eTag) - if bytes.Equal(eTag, rctx.Request.Header.Peek("If-None-Match")) { + if eTag == r.Header.Get("If-None-Match") { panic(errNotModified) } } @@ -199,5 +237,5 @@ func serveHTTP(rctx *fasthttp.RequestCtx) { checkTimeout(ctx) - respondWithImage(ctx, reqID, rctx, imageData) + respondWithImage(ctx, reqID, r, rw, imageData) } diff --git a/vendor/github.com/klauspost/compress/LICENSE b/vendor/github.com/klauspost/compress/LICENSE deleted file mode 100644 index 74487567..00000000 --- a/vendor/github.com/klauspost/compress/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/klauspost/compress/flate/copy.go b/vendor/github.com/klauspost/compress/flate/copy.go deleted file mode 100644 index a3200a8f..00000000 --- a/vendor/github.com/klauspost/compress/flate/copy.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -// forwardCopy is like the built-in copy function except that it always goes -// forward from the start, even if the dst and src overlap. -// It is equivalent to: -// for i := 0; i < n; i++ { -// mem[dst+i] = mem[src+i] -// } -func forwardCopy(mem []byte, dst, src, n int) { - if dst <= src { - copy(mem[dst:dst+n], mem[src:src+n]) - return - } - for { - if dst >= src+n { - copy(mem[dst:dst+n], mem[src:src+n]) - return - } - // There is some forward overlap. The destination - // will be filled with a repeated pattern of mem[src:src+k]. - // We copy one instance of the pattern here, then repeat. - // Each time around this loop k will double. - k := dst - src - copy(mem[dst:dst+k], mem[src:src+k]) - n -= k - dst += k - } -} diff --git a/vendor/github.com/klauspost/compress/flate/crc32_amd64.go b/vendor/github.com/klauspost/compress/flate/crc32_amd64.go deleted file mode 100644 index 70a6095e..00000000 --- a/vendor/github.com/klauspost/compress/flate/crc32_amd64.go +++ /dev/null @@ -1,41 +0,0 @@ -//+build !noasm -//+build !appengine - -// Copyright 2015, Klaus Post, see LICENSE for details. - -package flate - -import ( - "github.com/klauspost/cpuid" -) - -// crc32sse returns a hash for the first 4 bytes of the slice -// len(a) must be >= 4. -//go:noescape -func crc32sse(a []byte) uint32 - -// crc32sseAll calculates hashes for each 4-byte set in a. -// dst must be east len(a) - 4 in size. -// The size is not checked by the assembly. -//go:noescape -func crc32sseAll(a []byte, dst []uint32) - -// matchLenSSE4 returns the number of matching bytes in a and b -// up to length 'max'. Both slices must be at least 'max' -// bytes in size. -// -// TODO: drop the "SSE4" name, since it doesn't use any SSE instructions. -// -//go:noescape -func matchLenSSE4(a, b []byte, max int) int - -// histogram accumulates a histogram of b in h. -// h must be at least 256 entries in length, -// and must be cleared before calling this function. -//go:noescape -func histogram(b []byte, h []int32) - -// Detect SSE 4.2 feature. -func init() { - useSSE42 = cpuid.CPU.SSE42() -} diff --git a/vendor/github.com/klauspost/compress/flate/crc32_amd64.s b/vendor/github.com/klauspost/compress/flate/crc32_amd64.s deleted file mode 100644 index 2fb2079b..00000000 --- a/vendor/github.com/klauspost/compress/flate/crc32_amd64.s +++ /dev/null @@ -1,213 +0,0 @@ -//+build !noasm -//+build !appengine - -// Copyright 2015, Klaus Post, see LICENSE for details. - -// func crc32sse(a []byte) uint32 -TEXT ·crc32sse(SB), 4, $0 - MOVQ a+0(FP), R10 - XORQ BX, BX - - // CRC32 dword (R10), EBX - BYTE $0xF2; BYTE $0x41; BYTE $0x0f - BYTE $0x38; BYTE $0xf1; BYTE $0x1a - - MOVL BX, ret+24(FP) - RET - -// func crc32sseAll(a []byte, dst []uint32) -TEXT ·crc32sseAll(SB), 4, $0 - MOVQ a+0(FP), R8 // R8: src - MOVQ a_len+8(FP), R10 // input length - MOVQ dst+24(FP), R9 // R9: dst - SUBQ $4, R10 - JS end - JZ one_crc - MOVQ R10, R13 - SHRQ $2, R10 // len/4 - ANDQ $3, R13 // len&3 - XORQ BX, BX - ADDQ $1, R13 - TESTQ R10, R10 - JZ rem_loop - -crc_loop: - MOVQ (R8), R11 - XORQ BX, BX - XORQ DX, DX - XORQ DI, DI - MOVQ R11, R12 - SHRQ $8, R11 - MOVQ R12, AX - MOVQ R11, CX - SHRQ $16, R12 - SHRQ $16, R11 - MOVQ R12, SI - - // CRC32 EAX, EBX - BYTE $0xF2; BYTE $0x0f - BYTE $0x38; BYTE $0xf1; BYTE $0xd8 - - // CRC32 ECX, EDX - BYTE $0xF2; BYTE $0x0f - BYTE $0x38; BYTE $0xf1; BYTE $0xd1 - - // CRC32 ESI, EDI - BYTE $0xF2; BYTE $0x0f - BYTE $0x38; BYTE $0xf1; BYTE $0xfe - MOVL BX, (R9) - MOVL DX, 4(R9) - MOVL DI, 8(R9) - - XORQ BX, BX - MOVL R11, AX - - // CRC32 EAX, EBX - BYTE $0xF2; BYTE $0x0f - BYTE $0x38; BYTE $0xf1; BYTE $0xd8 - MOVL BX, 12(R9) - - ADDQ $16, R9 - ADDQ $4, R8 - XORQ BX, BX - SUBQ $1, R10 - JNZ crc_loop - -rem_loop: - MOVL (R8), AX - - // CRC32 EAX, EBX - BYTE $0xF2; BYTE $0x0f - BYTE $0x38; BYTE $0xf1; BYTE $0xd8 - - MOVL BX, (R9) - ADDQ $4, R9 - ADDQ $1, R8 - XORQ BX, BX - SUBQ $1, R13 - JNZ rem_loop - -end: - RET - -one_crc: - MOVQ $1, R13 - XORQ BX, BX - JMP rem_loop - -// func matchLenSSE4(a, b []byte, max int) int -TEXT ·matchLenSSE4(SB), 4, $0 - MOVQ a_base+0(FP), SI - MOVQ b_base+24(FP), DI - MOVQ DI, DX - MOVQ max+48(FP), CX - -cmp8: - // As long as we are 8 or more bytes before the end of max, we can load and - // compare 8 bytes at a time. If those 8 bytes are equal, repeat. - CMPQ CX, $8 - JLT cmp1 - MOVQ (SI), AX - MOVQ (DI), BX - CMPQ AX, BX - JNE bsf - ADDQ $8, SI - ADDQ $8, DI - SUBQ $8, CX - JMP cmp8 - -bsf: - // If those 8 bytes were not equal, XOR the two 8 byte values, and return - // the index of the first byte that differs. The BSF instruction finds the - // least significant 1 bit, the amd64 architecture is little-endian, and - // the shift by 3 converts a bit index to a byte index. - XORQ AX, BX - BSFQ BX, BX - SHRQ $3, BX - ADDQ BX, DI - - // Subtract off &b[0] to convert from &b[ret] to ret, and return. - SUBQ DX, DI - MOVQ DI, ret+56(FP) - RET - -cmp1: - // In the slices' tail, compare 1 byte at a time. - CMPQ CX, $0 - JEQ matchLenEnd - MOVB (SI), AX - MOVB (DI), BX - CMPB AX, BX - JNE matchLenEnd - ADDQ $1, SI - ADDQ $1, DI - SUBQ $1, CX - JMP cmp1 - -matchLenEnd: - // Subtract off &b[0] to convert from &b[ret] to ret, and return. - SUBQ DX, DI - MOVQ DI, ret+56(FP) - RET - -// func histogram(b []byte, h []int32) -TEXT ·histogram(SB), 4, $0 - MOVQ b+0(FP), SI // SI: &b - MOVQ b_len+8(FP), R9 // R9: len(b) - MOVQ h+24(FP), DI // DI: Histogram - MOVQ R9, R8 - SHRQ $3, R8 - JZ hist1 - XORQ R11, R11 - -loop_hist8: - MOVQ (SI), R10 - - MOVB R10, R11 - INCL (DI)(R11*4) - SHRQ $8, R10 - - MOVB R10, R11 - INCL (DI)(R11*4) - SHRQ $8, R10 - - MOVB R10, R11 - INCL (DI)(R11*4) - SHRQ $8, R10 - - MOVB R10, R11 - INCL (DI)(R11*4) - SHRQ $8, R10 - - MOVB R10, R11 - INCL (DI)(R11*4) - SHRQ $8, R10 - - MOVB R10, R11 - INCL (DI)(R11*4) - SHRQ $8, R10 - - MOVB R10, R11 - INCL (DI)(R11*4) - SHRQ $8, R10 - - INCL (DI)(R10*4) - - ADDQ $8, SI - DECQ R8 - JNZ loop_hist8 - -hist1: - ANDQ $7, R9 - JZ end_hist - XORQ R10, R10 - -loop_hist1: - MOVB (SI), R10 - INCL (DI)(R10*4) - INCQ SI - DECQ R9 - JNZ loop_hist1 - -end_hist: - RET diff --git a/vendor/github.com/klauspost/compress/flate/crc32_noasm.go b/vendor/github.com/klauspost/compress/flate/crc32_noasm.go deleted file mode 100644 index bd98bd59..00000000 --- a/vendor/github.com/klauspost/compress/flate/crc32_noasm.go +++ /dev/null @@ -1,35 +0,0 @@ -//+build !amd64 noasm appengine - -// Copyright 2015, Klaus Post, see LICENSE for details. - -package flate - -func init() { - useSSE42 = false -} - -// crc32sse should never be called. -func crc32sse(a []byte) uint32 { - panic("no assembler") -} - -// crc32sseAll should never be called. -func crc32sseAll(a []byte, dst []uint32) { - panic("no assembler") -} - -// matchLenSSE4 should never be called. -func matchLenSSE4(a, b []byte, max int) int { - panic("no assembler") - return 0 -} - -// histogram accumulates a histogram of b in h. -// -// len(h) must be >= 256, and h's elements must be all zeroes. -func histogram(b []byte, h []int32) { - h = h[:256] - for _, t := range b { - h[t]++ - } -} diff --git a/vendor/github.com/klauspost/compress/flate/deflate.go b/vendor/github.com/klauspost/compress/flate/deflate.go deleted file mode 100644 index 9e6e7ff0..00000000 --- a/vendor/github.com/klauspost/compress/flate/deflate.go +++ /dev/null @@ -1,1353 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Copyright (c) 2015 Klaus Post -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import ( - "fmt" - "io" - "math" -) - -const ( - NoCompression = 0 - BestSpeed = 1 - BestCompression = 9 - DefaultCompression = -1 - - // HuffmanOnly disables Lempel-Ziv match searching and only performs Huffman - // entropy encoding. This mode is useful in compressing data that has - // already been compressed with an LZ style algorithm (e.g. Snappy or LZ4) - // that lacks an entropy encoder. Compression gains are achieved when - // certain bytes in the input stream occur more frequently than others. - // - // Note that HuffmanOnly produces a compressed output that is - // RFC 1951 compliant. That is, any valid DEFLATE decompressor will - // continue to be able to decompress this output. - HuffmanOnly = -2 - ConstantCompression = HuffmanOnly // compatibility alias. - - logWindowSize = 15 - windowSize = 1 << logWindowSize - windowMask = windowSize - 1 - logMaxOffsetSize = 15 // Standard DEFLATE - minMatchLength = 4 // The smallest match that the compressor looks for - maxMatchLength = 258 // The longest match for the compressor - minOffsetSize = 1 // The shortest offset that makes any sense - - // The maximum number of tokens we put into a single flat block, just too - // stop things from getting too large. - maxFlateBlockTokens = 1 << 14 - maxStoreBlockSize = 65535 - hashBits = 17 // After 17 performance degrades - hashSize = 1 << hashBits - hashMask = (1 << hashBits) - 1 - hashShift = (hashBits + minMatchLength - 1) / minMatchLength - maxHashOffset = 1 << 24 - - skipNever = math.MaxInt32 -) - -var useSSE42 bool - -type compressionLevel struct { - good, lazy, nice, chain, fastSkipHashing, level int -} - -// Compression levels have been rebalanced from zlib deflate defaults -// to give a bigger spread in speed and compression. -// See https://blog.klauspost.com/rebalancing-deflate-compression-levels/ -var levels = []compressionLevel{ - {}, // 0 - // Level 1-4 uses specialized algorithm - values not used - {0, 0, 0, 0, 0, 1}, - {0, 0, 0, 0, 0, 2}, - {0, 0, 0, 0, 0, 3}, - {0, 0, 0, 0, 0, 4}, - // For levels 5-6 we don't bother trying with lazy matches. - // Lazy matching is at least 30% slower, with 1.5% increase. - {6, 0, 12, 8, 12, 5}, - {8, 0, 24, 16, 16, 6}, - // Levels 7-9 use increasingly more lazy matching - // and increasingly stringent conditions for "good enough". - {8, 8, 24, 16, skipNever, 7}, - {10, 16, 24, 64, skipNever, 8}, - {32, 258, 258, 4096, skipNever, 9}, -} - -type compressor struct { - compressionLevel - - w *huffmanBitWriter - bulkHasher func([]byte, []uint32) - - // compression algorithm - fill func(*compressor, []byte) int // copy data to window - step func(*compressor) // process window - sync bool // requesting flush - - // Input hash chains - // hashHead[hashValue] contains the largest inputIndex with the specified hash value - // If hashHead[hashValue] is within the current window, then - // hashPrev[hashHead[hashValue] & windowMask] contains the previous index - // with the same hash value. - chainHead int - hashHead [hashSize]uint32 - hashPrev [windowSize]uint32 - hashOffset int - - // input window: unprocessed data is window[index:windowEnd] - index int - window []byte - windowEnd int - blockStart int // window index where current tokens start - byteAvailable bool // if true, still need to process window[index-1]. - - // queued output tokens - tokens tokens - - // deflate state - length int - offset int - hash uint32 - maxInsertIndex int - err error - ii uint16 // position of last match, intended to overflow to reset. - - snap snappyEnc - hashMatch [maxMatchLength + minMatchLength]uint32 -} - -func (d *compressor) fillDeflate(b []byte) int { - if d.index >= 2*windowSize-(minMatchLength+maxMatchLength) { - // shift the window by windowSize - copy(d.window[:], d.window[windowSize:2*windowSize]) - d.index -= windowSize - d.windowEnd -= windowSize - if d.blockStart >= windowSize { - d.blockStart -= windowSize - } else { - d.blockStart = math.MaxInt32 - } - d.hashOffset += windowSize - if d.hashOffset > maxHashOffset { - delta := d.hashOffset - 1 - d.hashOffset -= delta - d.chainHead -= delta - // Iterate over slices instead of arrays to avoid copying - // the entire table onto the stack (Issue #18625). - for i, v := range d.hashPrev[:] { - if int(v) > delta { - d.hashPrev[i] = uint32(int(v) - delta) - } else { - d.hashPrev[i] = 0 - } - } - for i, v := range d.hashHead[:] { - if int(v) > delta { - d.hashHead[i] = uint32(int(v) - delta) - } else { - d.hashHead[i] = 0 - } - } - } - } - n := copy(d.window[d.windowEnd:], b) - d.windowEnd += n - return n -} - -func (d *compressor) writeBlock(tok tokens, index int, eof bool) error { - if index > 0 || eof { - var window []byte - if d.blockStart <= index { - window = d.window[d.blockStart:index] - } - d.blockStart = index - d.w.writeBlock(tok.tokens[:tok.n], eof, window) - return d.w.err - } - return nil -} - -// writeBlockSkip writes the current block and uses the number of tokens -// to determine if the block should be stored on no matches, or -// only huffman encoded. -func (d *compressor) writeBlockSkip(tok tokens, index int, eof bool) error { - if index > 0 || eof { - if d.blockStart <= index { - window := d.window[d.blockStart:index] - // If we removed less than a 64th of all literals - // we huffman compress the block. - if int(tok.n) > len(window)-int(tok.n>>6) { - d.w.writeBlockHuff(eof, window) - } else { - // Write a dynamic huffman block. - d.w.writeBlockDynamic(tok.tokens[:tok.n], eof, window) - } - } else { - d.w.writeBlock(tok.tokens[:tok.n], eof, nil) - } - d.blockStart = index - return d.w.err - } - return nil -} - -// fillWindow will fill the current window with the supplied -// dictionary and calculate all hashes. -// This is much faster than doing a full encode. -// Should only be used after a start/reset. -func (d *compressor) fillWindow(b []byte) { - // Do not fill window if we are in store-only mode, - // use constant or Snappy compression. - switch d.compressionLevel.level { - case 0, 1, 2: - return - } - // If we are given too much, cut it. - if len(b) > windowSize { - b = b[len(b)-windowSize:] - } - // Add all to window. - n := copy(d.window[d.windowEnd:], b) - - // Calculate 256 hashes at the time (more L1 cache hits) - loops := (n + 256 - minMatchLength) / 256 - for j := 0; j < loops; j++ { - startindex := j * 256 - end := startindex + 256 + minMatchLength - 1 - if end > n { - end = n - } - tocheck := d.window[startindex:end] - dstSize := len(tocheck) - minMatchLength + 1 - - if dstSize <= 0 { - continue - } - - dst := d.hashMatch[:dstSize] - d.bulkHasher(tocheck, dst) - var newH uint32 - for i, val := range dst { - di := i + startindex - newH = val & hashMask - // Get previous value with the same hash. - // Our chain should point to the previous value. - d.hashPrev[di&windowMask] = d.hashHead[newH] - // Set the head of the hash chain to us. - d.hashHead[newH] = uint32(di + d.hashOffset) - } - d.hash = newH - } - // Update window information. - d.windowEnd += n - d.index = n -} - -// Try to find a match starting at index whose length is greater than prevSize. -// We only look at chainCount possibilities before giving up. -// pos = d.index, prevHead = d.chainHead-d.hashOffset, prevLength=minMatchLength-1, lookahead -func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) { - minMatchLook := maxMatchLength - if lookahead < minMatchLook { - minMatchLook = lookahead - } - - win := d.window[0 : pos+minMatchLook] - - // We quit when we get a match that's at least nice long - nice := len(win) - pos - if d.nice < nice { - nice = d.nice - } - - // If we've got a match that's good enough, only look in 1/4 the chain. - tries := d.chain - length = prevLength - if length >= d.good { - tries >>= 2 - } - - wEnd := win[pos+length] - wPos := win[pos:] - minIndex := pos - windowSize - - for i := prevHead; tries > 0; tries-- { - if wEnd == win[i+length] { - n := matchLen(win[i:], wPos, minMatchLook) - - if n > length && (n > minMatchLength || pos-i <= 4096) { - length = n - offset = pos - i - ok = true - if n >= nice { - // The match is good enough that we don't try to find a better one. - break - } - wEnd = win[pos+n] - } - } - if i == minIndex { - // hashPrev[i & windowMask] has already been overwritten, so stop now. - break - } - i = int(d.hashPrev[i&windowMask]) - d.hashOffset - if i < minIndex || i < 0 { - break - } - } - return -} - -// Try to find a match starting at index whose length is greater than prevSize. -// We only look at chainCount possibilities before giving up. -// pos = d.index, prevHead = d.chainHead-d.hashOffset, prevLength=minMatchLength-1, lookahead -func (d *compressor) findMatchSSE(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) { - minMatchLook := maxMatchLength - if lookahead < minMatchLook { - minMatchLook = lookahead - } - - win := d.window[0 : pos+minMatchLook] - - // We quit when we get a match that's at least nice long - nice := len(win) - pos - if d.nice < nice { - nice = d.nice - } - - // If we've got a match that's good enough, only look in 1/4 the chain. - tries := d.chain - length = prevLength - if length >= d.good { - tries >>= 2 - } - - wEnd := win[pos+length] - wPos := win[pos:] - minIndex := pos - windowSize - - for i := prevHead; tries > 0; tries-- { - if wEnd == win[i+length] { - n := matchLenSSE4(win[i:], wPos, minMatchLook) - - if n > length && (n > minMatchLength || pos-i <= 4096) { - length = n - offset = pos - i - ok = true - if n >= nice { - // The match is good enough that we don't try to find a better one. - break - } - wEnd = win[pos+n] - } - } - if i == minIndex { - // hashPrev[i & windowMask] has already been overwritten, so stop now. - break - } - i = int(d.hashPrev[i&windowMask]) - d.hashOffset - if i < minIndex || i < 0 { - break - } - } - return -} - -func (d *compressor) writeStoredBlock(buf []byte) error { - if d.w.writeStoredHeader(len(buf), false); d.w.err != nil { - return d.w.err - } - d.w.writeBytes(buf) - return d.w.err -} - -const hashmul = 0x1e35a7bd - -// hash4 returns a hash representation of the first 4 bytes -// of the supplied slice. -// The caller must ensure that len(b) >= 4. -func hash4(b []byte) uint32 { - return ((uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24) * hashmul) >> (32 - hashBits) -} - -// bulkHash4 will compute hashes using the same -// algorithm as hash4 -func bulkHash4(b []byte, dst []uint32) { - if len(b) < minMatchLength { - return - } - hb := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 - dst[0] = (hb * hashmul) >> (32 - hashBits) - end := len(b) - minMatchLength + 1 - for i := 1; i < end; i++ { - hb = (hb << 8) | uint32(b[i+3]) - dst[i] = (hb * hashmul) >> (32 - hashBits) - } -} - -// matchLen returns the number of matching bytes in a and b -// up to length 'max'. Both slices must be at least 'max' -// bytes in size. -func matchLen(a, b []byte, max int) int { - a = a[:max] - b = b[:len(a)] - for i, av := range a { - if b[i] != av { - return i - } - } - return max -} - -func (d *compressor) initDeflate() { - d.window = make([]byte, 2*windowSize) - d.hashOffset = 1 - d.length = minMatchLength - 1 - d.offset = 0 - d.byteAvailable = false - d.index = 0 - d.hash = 0 - d.chainHead = -1 - d.bulkHasher = bulkHash4 - if useSSE42 { - d.bulkHasher = crc32sseAll - } -} - -// Assumes that d.fastSkipHashing != skipNever, -// otherwise use deflateLazy -func (d *compressor) deflate() { - - // Sanity enables additional runtime tests. - // It's intended to be used during development - // to supplement the currently ad-hoc unit tests. - const sanity = false - - if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync { - return - } - - d.maxInsertIndex = d.windowEnd - (minMatchLength - 1) - if d.index < d.maxInsertIndex { - d.hash = hash4(d.window[d.index : d.index+minMatchLength]) - } - - for { - if sanity && d.index > d.windowEnd { - panic("index > windowEnd") - } - lookahead := d.windowEnd - d.index - if lookahead < minMatchLength+maxMatchLength { - if !d.sync { - return - } - if sanity && d.index > d.windowEnd { - panic("index > windowEnd") - } - if lookahead == 0 { - if d.tokens.n > 0 { - if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - return - } - } - if d.index < d.maxInsertIndex { - // Update the hash - d.hash = hash4(d.window[d.index : d.index+minMatchLength]) - ch := d.hashHead[d.hash&hashMask] - d.chainHead = int(ch) - d.hashPrev[d.index&windowMask] = ch - d.hashHead[d.hash&hashMask] = uint32(d.index + d.hashOffset) - } - d.length = minMatchLength - 1 - d.offset = 0 - minIndex := d.index - windowSize - if minIndex < 0 { - minIndex = 0 - } - - if d.chainHead-d.hashOffset >= minIndex && lookahead > minMatchLength-1 { - if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok { - d.length = newLength - d.offset = newOffset - } - } - if d.length >= minMatchLength { - d.ii = 0 - // There was a match at the previous step, and the current match is - // not better. Output the previous match. - // "d.length-3" should NOT be "d.length-minMatchLength", since the format always assume 3 - d.tokens.tokens[d.tokens.n] = matchToken(uint32(d.length-3), uint32(d.offset-minOffsetSize)) - d.tokens.n++ - // Insert in the hash table all strings up to the end of the match. - // index and index-1 are already inserted. If there is not enough - // lookahead, the last two strings are not inserted into the hash - // table. - if d.length <= d.fastSkipHashing { - var newIndex int - newIndex = d.index + d.length - // Calculate missing hashes - end := newIndex - if end > d.maxInsertIndex { - end = d.maxInsertIndex - } - end += minMatchLength - 1 - startindex := d.index + 1 - if startindex > d.maxInsertIndex { - startindex = d.maxInsertIndex - } - tocheck := d.window[startindex:end] - dstSize := len(tocheck) - minMatchLength + 1 - if dstSize > 0 { - dst := d.hashMatch[:dstSize] - bulkHash4(tocheck, dst) - var newH uint32 - for i, val := range dst { - di := i + startindex - newH = val & hashMask - // Get previous value with the same hash. - // Our chain should point to the previous value. - d.hashPrev[di&windowMask] = d.hashHead[newH] - // Set the head of the hash chain to us. - d.hashHead[newH] = uint32(di + d.hashOffset) - } - d.hash = newH - } - d.index = newIndex - } else { - // For matches this long, we don't bother inserting each individual - // item into the table. - d.index += d.length - if d.index < d.maxInsertIndex { - d.hash = hash4(d.window[d.index : d.index+minMatchLength]) - } - } - if d.tokens.n == maxFlateBlockTokens { - // The block includes the current character - if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - } else { - d.ii++ - end := d.index + int(d.ii>>uint(d.fastSkipHashing)) + 1 - if end > d.windowEnd { - end = d.windowEnd - } - for i := d.index; i < end; i++ { - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[i])) - d.tokens.n++ - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlockSkip(d.tokens, i+1, false); d.err != nil { - return - } - d.tokens.n = 0 - } - } - d.index = end - } - } -} - -// deflateLazy is the same as deflate, but with d.fastSkipHashing == skipNever, -// meaning it always has lazy matching on. -func (d *compressor) deflateLazy() { - // Sanity enables additional runtime tests. - // It's intended to be used during development - // to supplement the currently ad-hoc unit tests. - const sanity = false - - if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync { - return - } - - d.maxInsertIndex = d.windowEnd - (minMatchLength - 1) - if d.index < d.maxInsertIndex { - d.hash = hash4(d.window[d.index : d.index+minMatchLength]) - } - - for { - if sanity && d.index > d.windowEnd { - panic("index > windowEnd") - } - lookahead := d.windowEnd - d.index - if lookahead < minMatchLength+maxMatchLength { - if !d.sync { - return - } - if sanity && d.index > d.windowEnd { - panic("index > windowEnd") - } - if lookahead == 0 { - // Flush current output block if any. - if d.byteAvailable { - // There is still one pending token that needs to be flushed - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) - d.tokens.n++ - d.byteAvailable = false - } - if d.tokens.n > 0 { - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - return - } - } - if d.index < d.maxInsertIndex { - // Update the hash - d.hash = hash4(d.window[d.index : d.index+minMatchLength]) - ch := d.hashHead[d.hash&hashMask] - d.chainHead = int(ch) - d.hashPrev[d.index&windowMask] = ch - d.hashHead[d.hash&hashMask] = uint32(d.index + d.hashOffset) - } - prevLength := d.length - prevOffset := d.offset - d.length = minMatchLength - 1 - d.offset = 0 - minIndex := d.index - windowSize - if minIndex < 0 { - minIndex = 0 - } - - if d.chainHead-d.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy { - if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok { - d.length = newLength - d.offset = newOffset - } - } - if prevLength >= minMatchLength && d.length <= prevLength { - // There was a match at the previous step, and the current match is - // not better. Output the previous match. - d.tokens.tokens[d.tokens.n] = matchToken(uint32(prevLength-3), uint32(prevOffset-minOffsetSize)) - d.tokens.n++ - - // Insert in the hash table all strings up to the end of the match. - // index and index-1 are already inserted. If there is not enough - // lookahead, the last two strings are not inserted into the hash - // table. - var newIndex int - newIndex = d.index + prevLength - 1 - // Calculate missing hashes - end := newIndex - if end > d.maxInsertIndex { - end = d.maxInsertIndex - } - end += minMatchLength - 1 - startindex := d.index + 1 - if startindex > d.maxInsertIndex { - startindex = d.maxInsertIndex - } - tocheck := d.window[startindex:end] - dstSize := len(tocheck) - minMatchLength + 1 - if dstSize > 0 { - dst := d.hashMatch[:dstSize] - bulkHash4(tocheck, dst) - var newH uint32 - for i, val := range dst { - di := i + startindex - newH = val & hashMask - // Get previous value with the same hash. - // Our chain should point to the previous value. - d.hashPrev[di&windowMask] = d.hashHead[newH] - // Set the head of the hash chain to us. - d.hashHead[newH] = uint32(di + d.hashOffset) - } - d.hash = newH - } - - d.index = newIndex - d.byteAvailable = false - d.length = minMatchLength - 1 - if d.tokens.n == maxFlateBlockTokens { - // The block includes the current character - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - } else { - // Reset, if we got a match this run. - if d.length >= minMatchLength { - d.ii = 0 - } - // We have a byte waiting. Emit it. - if d.byteAvailable { - d.ii++ - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) - d.tokens.n++ - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - d.index++ - - // If we have a long run of no matches, skip additional bytes - // Resets when d.ii overflows after 64KB. - if d.ii > 31 { - n := int(d.ii >> 5) - for j := 0; j < n; j++ { - if d.index >= d.windowEnd-1 { - break - } - - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) - d.tokens.n++ - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - d.index++ - } - // Flush last byte - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) - d.tokens.n++ - d.byteAvailable = false - // d.length = minMatchLength - 1 // not needed, since d.ii is reset above, so it should never be > minMatchLength - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - } - } else { - d.index++ - d.byteAvailable = true - } - } - } -} - -// Assumes that d.fastSkipHashing != skipNever, -// otherwise use deflateLazySSE -func (d *compressor) deflateSSE() { - - // Sanity enables additional runtime tests. - // It's intended to be used during development - // to supplement the currently ad-hoc unit tests. - const sanity = false - - if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync { - return - } - - d.maxInsertIndex = d.windowEnd - (minMatchLength - 1) - if d.index < d.maxInsertIndex { - d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask - } - - for { - if sanity && d.index > d.windowEnd { - panic("index > windowEnd") - } - lookahead := d.windowEnd - d.index - if lookahead < minMatchLength+maxMatchLength { - if !d.sync { - return - } - if sanity && d.index > d.windowEnd { - panic("index > windowEnd") - } - if lookahead == 0 { - if d.tokens.n > 0 { - if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - return - } - } - if d.index < d.maxInsertIndex { - // Update the hash - d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask - ch := d.hashHead[d.hash] - d.chainHead = int(ch) - d.hashPrev[d.index&windowMask] = ch - d.hashHead[d.hash] = uint32(d.index + d.hashOffset) - } - d.length = minMatchLength - 1 - d.offset = 0 - minIndex := d.index - windowSize - if minIndex < 0 { - minIndex = 0 - } - - if d.chainHead-d.hashOffset >= minIndex && lookahead > minMatchLength-1 { - if newLength, newOffset, ok := d.findMatchSSE(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok { - d.length = newLength - d.offset = newOffset - } - } - if d.length >= minMatchLength { - d.ii = 0 - // There was a match at the previous step, and the current match is - // not better. Output the previous match. - // "d.length-3" should NOT be "d.length-minMatchLength", since the format always assume 3 - d.tokens.tokens[d.tokens.n] = matchToken(uint32(d.length-3), uint32(d.offset-minOffsetSize)) - d.tokens.n++ - // Insert in the hash table all strings up to the end of the match. - // index and index-1 are already inserted. If there is not enough - // lookahead, the last two strings are not inserted into the hash - // table. - if d.length <= d.fastSkipHashing { - var newIndex int - newIndex = d.index + d.length - // Calculate missing hashes - end := newIndex - if end > d.maxInsertIndex { - end = d.maxInsertIndex - } - end += minMatchLength - 1 - startindex := d.index + 1 - if startindex > d.maxInsertIndex { - startindex = d.maxInsertIndex - } - tocheck := d.window[startindex:end] - dstSize := len(tocheck) - minMatchLength + 1 - if dstSize > 0 { - dst := d.hashMatch[:dstSize] - - crc32sseAll(tocheck, dst) - var newH uint32 - for i, val := range dst { - di := i + startindex - newH = val & hashMask - // Get previous value with the same hash. - // Our chain should point to the previous value. - d.hashPrev[di&windowMask] = d.hashHead[newH] - // Set the head of the hash chain to us. - d.hashHead[newH] = uint32(di + d.hashOffset) - } - d.hash = newH - } - d.index = newIndex - } else { - // For matches this long, we don't bother inserting each individual - // item into the table. - d.index += d.length - if d.index < d.maxInsertIndex { - d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask - } - } - if d.tokens.n == maxFlateBlockTokens { - // The block includes the current character - if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - } else { - d.ii++ - end := d.index + int(d.ii>>5) + 1 - if end > d.windowEnd { - end = d.windowEnd - } - for i := d.index; i < end; i++ { - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[i])) - d.tokens.n++ - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlockSkip(d.tokens, i+1, false); d.err != nil { - return - } - d.tokens.n = 0 - } - } - d.index = end - } - } -} - -// deflateLazy is the same as deflate, but with d.fastSkipHashing == skipNever, -// meaning it always has lazy matching on. -func (d *compressor) deflateLazySSE() { - // Sanity enables additional runtime tests. - // It's intended to be used during development - // to supplement the currently ad-hoc unit tests. - const sanity = false - - if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync { - return - } - - d.maxInsertIndex = d.windowEnd - (minMatchLength - 1) - if d.index < d.maxInsertIndex { - d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask - } - - for { - if sanity && d.index > d.windowEnd { - panic("index > windowEnd") - } - lookahead := d.windowEnd - d.index - if lookahead < minMatchLength+maxMatchLength { - if !d.sync { - return - } - if sanity && d.index > d.windowEnd { - panic("index > windowEnd") - } - if lookahead == 0 { - // Flush current output block if any. - if d.byteAvailable { - // There is still one pending token that needs to be flushed - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) - d.tokens.n++ - d.byteAvailable = false - } - if d.tokens.n > 0 { - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - return - } - } - if d.index < d.maxInsertIndex { - // Update the hash - d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask - ch := d.hashHead[d.hash] - d.chainHead = int(ch) - d.hashPrev[d.index&windowMask] = ch - d.hashHead[d.hash] = uint32(d.index + d.hashOffset) - } - prevLength := d.length - prevOffset := d.offset - d.length = minMatchLength - 1 - d.offset = 0 - minIndex := d.index - windowSize - if minIndex < 0 { - minIndex = 0 - } - - if d.chainHead-d.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy { - if newLength, newOffset, ok := d.findMatchSSE(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok { - d.length = newLength - d.offset = newOffset - } - } - if prevLength >= minMatchLength && d.length <= prevLength { - // There was a match at the previous step, and the current match is - // not better. Output the previous match. - d.tokens.tokens[d.tokens.n] = matchToken(uint32(prevLength-3), uint32(prevOffset-minOffsetSize)) - d.tokens.n++ - - // Insert in the hash table all strings up to the end of the match. - // index and index-1 are already inserted. If there is not enough - // lookahead, the last two strings are not inserted into the hash - // table. - var newIndex int - newIndex = d.index + prevLength - 1 - // Calculate missing hashes - end := newIndex - if end > d.maxInsertIndex { - end = d.maxInsertIndex - } - end += minMatchLength - 1 - startindex := d.index + 1 - if startindex > d.maxInsertIndex { - startindex = d.maxInsertIndex - } - tocheck := d.window[startindex:end] - dstSize := len(tocheck) - minMatchLength + 1 - if dstSize > 0 { - dst := d.hashMatch[:dstSize] - crc32sseAll(tocheck, dst) - var newH uint32 - for i, val := range dst { - di := i + startindex - newH = val & hashMask - // Get previous value with the same hash. - // Our chain should point to the previous value. - d.hashPrev[di&windowMask] = d.hashHead[newH] - // Set the head of the hash chain to us. - d.hashHead[newH] = uint32(di + d.hashOffset) - } - d.hash = newH - } - - d.index = newIndex - d.byteAvailable = false - d.length = minMatchLength - 1 - if d.tokens.n == maxFlateBlockTokens { - // The block includes the current character - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - } else { - // Reset, if we got a match this run. - if d.length >= minMatchLength { - d.ii = 0 - } - // We have a byte waiting. Emit it. - if d.byteAvailable { - d.ii++ - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) - d.tokens.n++ - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - d.index++ - - // If we have a long run of no matches, skip additional bytes - // Resets when d.ii overflows after 64KB. - if d.ii > 31 { - n := int(d.ii >> 6) - for j := 0; j < n; j++ { - if d.index >= d.windowEnd-1 { - break - } - - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) - d.tokens.n++ - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - d.index++ - } - // Flush last byte - d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) - d.tokens.n++ - d.byteAvailable = false - // d.length = minMatchLength - 1 // not needed, since d.ii is reset above, so it should never be > minMatchLength - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { - return - } - d.tokens.n = 0 - } - } - } else { - d.index++ - d.byteAvailable = true - } - } - } -} - -func (d *compressor) store() { - if d.windowEnd > 0 && (d.windowEnd == maxStoreBlockSize || d.sync) { - d.err = d.writeStoredBlock(d.window[:d.windowEnd]) - d.windowEnd = 0 - } -} - -// fillWindow will fill the buffer with data for huffman-only compression. -// The number of bytes copied is returned. -func (d *compressor) fillBlock(b []byte) int { - n := copy(d.window[d.windowEnd:], b) - d.windowEnd += n - return n -} - -// storeHuff will compress and store the currently added data, -// if enough has been accumulated or we at the end of the stream. -// Any error that occurred will be in d.err -func (d *compressor) storeHuff() { - if d.windowEnd < len(d.window) && !d.sync || d.windowEnd == 0 { - return - } - d.w.writeBlockHuff(false, d.window[:d.windowEnd]) - d.err = d.w.err - d.windowEnd = 0 -} - -// storeHuff will compress and store the currently added data, -// if enough has been accumulated or we at the end of the stream. -// Any error that occurred will be in d.err -func (d *compressor) storeSnappy() { - // We only compress if we have maxStoreBlockSize. - if d.windowEnd < maxStoreBlockSize { - if !d.sync { - return - } - // Handle extremely small sizes. - if d.windowEnd < 128 { - if d.windowEnd == 0 { - return - } - if d.windowEnd <= 32 { - d.err = d.writeStoredBlock(d.window[:d.windowEnd]) - d.tokens.n = 0 - d.windowEnd = 0 - } else { - d.w.writeBlockHuff(false, d.window[:d.windowEnd]) - d.err = d.w.err - } - d.tokens.n = 0 - d.windowEnd = 0 - d.snap.Reset() - return - } - } - - d.snap.Encode(&d.tokens, d.window[:d.windowEnd]) - // If we made zero matches, store the block as is. - if int(d.tokens.n) == d.windowEnd { - d.err = d.writeStoredBlock(d.window[:d.windowEnd]) - // If we removed less than 1/16th, huffman compress the block. - } else if int(d.tokens.n) > d.windowEnd-(d.windowEnd>>4) { - d.w.writeBlockHuff(false, d.window[:d.windowEnd]) - d.err = d.w.err - } else { - d.w.writeBlockDynamic(d.tokens.tokens[:d.tokens.n], false, d.window[:d.windowEnd]) - d.err = d.w.err - } - d.tokens.n = 0 - d.windowEnd = 0 -} - -// write will add input byte to the stream. -// Unless an error occurs all bytes will be consumed. -func (d *compressor) write(b []byte) (n int, err error) { - if d.err != nil { - return 0, d.err - } - n = len(b) - for len(b) > 0 { - d.step(d) - b = b[d.fill(d, b):] - if d.err != nil { - return 0, d.err - } - } - return n, d.err -} - -func (d *compressor) syncFlush() error { - d.sync = true - if d.err != nil { - return d.err - } - d.step(d) - if d.err == nil { - d.w.writeStoredHeader(0, false) - d.w.flush() - d.err = d.w.err - } - d.sync = false - return d.err -} - -func (d *compressor) init(w io.Writer, level int) (err error) { - d.w = newHuffmanBitWriter(w) - - switch { - case level == NoCompression: - d.window = make([]byte, maxStoreBlockSize) - d.fill = (*compressor).fillBlock - d.step = (*compressor).store - case level == ConstantCompression: - d.window = make([]byte, maxStoreBlockSize) - d.fill = (*compressor).fillBlock - d.step = (*compressor).storeHuff - case level >= 1 && level <= 4: - d.snap = newSnappy(level) - d.window = make([]byte, maxStoreBlockSize) - d.fill = (*compressor).fillBlock - d.step = (*compressor).storeSnappy - case level == DefaultCompression: - level = 5 - fallthrough - case 5 <= level && level <= 9: - d.compressionLevel = levels[level] - d.initDeflate() - d.fill = (*compressor).fillDeflate - if d.fastSkipHashing == skipNever { - if useSSE42 { - d.step = (*compressor).deflateLazySSE - } else { - d.step = (*compressor).deflateLazy - } - } else { - if useSSE42 { - d.step = (*compressor).deflateSSE - } else { - d.step = (*compressor).deflate - - } - } - default: - return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level) - } - return nil -} - -// reset the state of the compressor. -func (d *compressor) reset(w io.Writer) { - d.w.reset(w) - d.sync = false - d.err = nil - // We only need to reset a few things for Snappy. - if d.snap != nil { - d.snap.Reset() - d.windowEnd = 0 - d.tokens.n = 0 - return - } - switch d.compressionLevel.chain { - case 0: - // level was NoCompression or ConstantCompresssion. - d.windowEnd = 0 - default: - d.chainHead = -1 - for i := range d.hashHead { - d.hashHead[i] = 0 - } - for i := range d.hashPrev { - d.hashPrev[i] = 0 - } - d.hashOffset = 1 - d.index, d.windowEnd = 0, 0 - d.blockStart, d.byteAvailable = 0, false - d.tokens.n = 0 - d.length = minMatchLength - 1 - d.offset = 0 - d.hash = 0 - d.ii = 0 - d.maxInsertIndex = 0 - } -} - -func (d *compressor) close() error { - if d.err != nil { - return d.err - } - d.sync = true - d.step(d) - if d.err != nil { - return d.err - } - if d.w.writeStoredHeader(0, true); d.w.err != nil { - return d.w.err - } - d.w.flush() - return d.w.err -} - -// NewWriter returns a new Writer compressing data at the given level. -// Following zlib, levels range from 1 (BestSpeed) to 9 (BestCompression); -// higher levels typically run slower but compress more. -// Level 0 (NoCompression) does not attempt any compression; it only adds the -// necessary DEFLATE framing. -// Level -1 (DefaultCompression) uses the default compression level. -// Level -2 (ConstantCompression) will use Huffman compression only, giving -// a very fast compression for all types of input, but sacrificing considerable -// compression efficiency. -// -// If level is in the range [-2, 9] then the error returned will be nil. -// Otherwise the error returned will be non-nil. -func NewWriter(w io.Writer, level int) (*Writer, error) { - var dw Writer - if err := dw.d.init(w, level); err != nil { - return nil, err - } - return &dw, nil -} - -// NewWriterDict is like NewWriter but initializes the new -// Writer with a preset dictionary. The returned Writer behaves -// as if the dictionary had been written to it without producing -// any compressed output. The compressed data written to w -// can only be decompressed by a Reader initialized with the -// same dictionary. -func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) { - dw := &dictWriter{w} - zw, err := NewWriter(dw, level) - if err != nil { - return nil, err - } - zw.d.fillWindow(dict) - zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method. - return zw, err -} - -type dictWriter struct { - w io.Writer -} - -func (w *dictWriter) Write(b []byte) (n int, err error) { - return w.w.Write(b) -} - -// A Writer takes data written to it and writes the compressed -// form of that data to an underlying writer (see NewWriter). -type Writer struct { - d compressor - dict []byte -} - -// Write writes data to w, which will eventually write the -// compressed form of data to its underlying writer. -func (w *Writer) Write(data []byte) (n int, err error) { - return w.d.write(data) -} - -// Flush flushes any pending data to the underlying writer. -// It is useful mainly in compressed network protocols, to ensure that -// a remote reader has enough data to reconstruct a packet. -// Flush does not return until the data has been written. -// Calling Flush when there is no pending data still causes the Writer -// to emit a sync marker of at least 4 bytes. -// If the underlying writer returns an error, Flush returns that error. -// -// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH. -func (w *Writer) Flush() error { - // For more about flushing: - // http://www.bolet.org/~pornin/deflate-flush.html - return w.d.syncFlush() -} - -// Close flushes and closes the writer. -func (w *Writer) Close() error { - return w.d.close() -} - -// Reset discards the writer's state and makes it equivalent to -// the result of NewWriter or NewWriterDict called with dst -// and w's level and dictionary. -func (w *Writer) Reset(dst io.Writer) { - if dw, ok := w.d.w.writer.(*dictWriter); ok { - // w was created with NewWriterDict - dw.w = dst - w.d.reset(dw) - w.d.fillWindow(w.dict) - } else { - // w was created with NewWriter - w.d.reset(dst) - } -} - -// ResetDict discards the writer's state and makes it equivalent to -// the result of NewWriter or NewWriterDict called with dst -// and w's level, but sets a specific dictionary. -func (w *Writer) ResetDict(dst io.Writer, dict []byte) { - w.dict = dict - w.d.reset(dst) - w.d.fillWindow(w.dict) -} diff --git a/vendor/github.com/klauspost/compress/flate/dict_decoder.go b/vendor/github.com/klauspost/compress/flate/dict_decoder.go deleted file mode 100644 index 71c75a06..00000000 --- a/vendor/github.com/klauspost/compress/flate/dict_decoder.go +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -// dictDecoder implements the LZ77 sliding dictionary as used in decompression. -// LZ77 decompresses data through sequences of two forms of commands: -// -// * Literal insertions: Runs of one or more symbols are inserted into the data -// stream as is. This is accomplished through the writeByte method for a -// single symbol, or combinations of writeSlice/writeMark for multiple symbols. -// Any valid stream must start with a literal insertion if no preset dictionary -// is used. -// -// * Backward copies: Runs of one or more symbols are copied from previously -// emitted data. Backward copies come as the tuple (dist, length) where dist -// determines how far back in the stream to copy from and length determines how -// many bytes to copy. Note that it is valid for the length to be greater than -// the distance. Since LZ77 uses forward copies, that situation is used to -// perform a form of run-length encoding on repeated runs of symbols. -// The writeCopy and tryWriteCopy are used to implement this command. -// -// For performance reasons, this implementation performs little to no sanity -// checks about the arguments. As such, the invariants documented for each -// method call must be respected. -type dictDecoder struct { - hist []byte // Sliding window history - - // Invariant: 0 <= rdPos <= wrPos <= len(hist) - wrPos int // Current output position in buffer - rdPos int // Have emitted hist[:rdPos] already - full bool // Has a full window length been written yet? -} - -// init initializes dictDecoder to have a sliding window dictionary of the given -// size. If a preset dict is provided, it will initialize the dictionary with -// the contents of dict. -func (dd *dictDecoder) init(size int, dict []byte) { - *dd = dictDecoder{hist: dd.hist} - - if cap(dd.hist) < size { - dd.hist = make([]byte, size) - } - dd.hist = dd.hist[:size] - - if len(dict) > len(dd.hist) { - dict = dict[len(dict)-len(dd.hist):] - } - dd.wrPos = copy(dd.hist, dict) - if dd.wrPos == len(dd.hist) { - dd.wrPos = 0 - dd.full = true - } - dd.rdPos = dd.wrPos -} - -// histSize reports the total amount of historical data in the dictionary. -func (dd *dictDecoder) histSize() int { - if dd.full { - return len(dd.hist) - } - return dd.wrPos -} - -// availRead reports the number of bytes that can be flushed by readFlush. -func (dd *dictDecoder) availRead() int { - return dd.wrPos - dd.rdPos -} - -// availWrite reports the available amount of output buffer space. -func (dd *dictDecoder) availWrite() int { - return len(dd.hist) - dd.wrPos -} - -// writeSlice returns a slice of the available buffer to write data to. -// -// This invariant will be kept: len(s) <= availWrite() -func (dd *dictDecoder) writeSlice() []byte { - return dd.hist[dd.wrPos:] -} - -// writeMark advances the writer pointer by cnt. -// -// This invariant must be kept: 0 <= cnt <= availWrite() -func (dd *dictDecoder) writeMark(cnt int) { - dd.wrPos += cnt -} - -// writeByte writes a single byte to the dictionary. -// -// This invariant must be kept: 0 < availWrite() -func (dd *dictDecoder) writeByte(c byte) { - dd.hist[dd.wrPos] = c - dd.wrPos++ -} - -// writeCopy copies a string at a given (dist, length) to the output. -// This returns the number of bytes copied and may be less than the requested -// length if the available space in the output buffer is too small. -// -// This invariant must be kept: 0 < dist <= histSize() -func (dd *dictDecoder) writeCopy(dist, length int) int { - dstBase := dd.wrPos - dstPos := dstBase - srcPos := dstPos - dist - endPos := dstPos + length - if endPos > len(dd.hist) { - endPos = len(dd.hist) - } - - // Copy non-overlapping section after destination position. - // - // This section is non-overlapping in that the copy length for this section - // is always less than or equal to the backwards distance. This can occur - // if a distance refers to data that wraps-around in the buffer. - // Thus, a backwards copy is performed here; that is, the exact bytes in - // the source prior to the copy is placed in the destination. - if srcPos < 0 { - srcPos += len(dd.hist) - dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:]) - srcPos = 0 - } - - // Copy possibly overlapping section before destination position. - // - // This section can overlap if the copy length for this section is larger - // than the backwards distance. This is allowed by LZ77 so that repeated - // strings can be succinctly represented using (dist, length) pairs. - // Thus, a forwards copy is performed here; that is, the bytes copied is - // possibly dependent on the resulting bytes in the destination as the copy - // progresses along. This is functionally equivalent to the following: - // - // for i := 0; i < endPos-dstPos; i++ { - // dd.hist[dstPos+i] = dd.hist[srcPos+i] - // } - // dstPos = endPos - // - for dstPos < endPos { - dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos]) - } - - dd.wrPos = dstPos - return dstPos - dstBase -} - -// tryWriteCopy tries to copy a string at a given (distance, length) to the -// output. This specialized version is optimized for short distances. -// -// This method is designed to be inlined for performance reasons. -// -// This invariant must be kept: 0 < dist <= histSize() -func (dd *dictDecoder) tryWriteCopy(dist, length int) int { - dstPos := dd.wrPos - endPos := dstPos + length - if dstPos < dist || endPos > len(dd.hist) { - return 0 - } - dstBase := dstPos - srcPos := dstPos - dist - - // Copy possibly overlapping section before destination position. -loop: - dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos]) - if dstPos < endPos { - goto loop // Avoid for-loop so that this function can be inlined - } - - dd.wrPos = dstPos - return dstPos - dstBase -} - -// readFlush returns a slice of the historical buffer that is ready to be -// emitted to the user. The data returned by readFlush must be fully consumed -// before calling any other dictDecoder methods. -func (dd *dictDecoder) readFlush() []byte { - toRead := dd.hist[dd.rdPos:dd.wrPos] - dd.rdPos = dd.wrPos - if dd.wrPos == len(dd.hist) { - dd.wrPos, dd.rdPos = 0, 0 - dd.full = true - } - return toRead -} diff --git a/vendor/github.com/klauspost/compress/flate/gen.go b/vendor/github.com/klauspost/compress/flate/gen.go deleted file mode 100644 index 154c89a4..00000000 --- a/vendor/github.com/klauspost/compress/flate/gen.go +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -// This program generates fixedhuff.go -// Invoke as -// -// go run gen.go -output fixedhuff.go - -package main - -import ( - "bytes" - "flag" - "fmt" - "go/format" - "io/ioutil" - "log" -) - -var filename = flag.String("output", "fixedhuff.go", "output file name") - -const maxCodeLen = 16 - -// Note: the definition of the huffmanDecoder struct is copied from -// inflate.go, as it is private to the implementation. - -// chunk & 15 is number of bits -// chunk >> 4 is value, including table link - -const ( - huffmanChunkBits = 9 - huffmanNumChunks = 1 << huffmanChunkBits - huffmanCountMask = 15 - huffmanValueShift = 4 -) - -type huffmanDecoder struct { - min int // the minimum code length - chunks [huffmanNumChunks]uint32 // chunks as described above - links [][]uint32 // overflow links - linkMask uint32 // mask the width of the link table -} - -// Initialize Huffman decoding tables from array of code lengths. -// Following this function, h is guaranteed to be initialized into a complete -// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a -// degenerate case where the tree has only a single symbol with length 1. Empty -// trees are permitted. -func (h *huffmanDecoder) init(bits []int) bool { - // Sanity enables additional runtime tests during Huffman - // table construction. It's intended to be used during - // development to supplement the currently ad-hoc unit tests. - const sanity = false - - if h.min != 0 { - *h = huffmanDecoder{} - } - - // Count number of codes of each length, - // compute min and max length. - var count [maxCodeLen]int - var min, max int - for _, n := range bits { - if n == 0 { - continue - } - if min == 0 || n < min { - min = n - } - if n > max { - max = n - } - count[n]++ - } - - // Empty tree. The decompressor.huffSym function will fail later if the tree - // is used. Technically, an empty tree is only valid for the HDIST tree and - // not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree - // is guaranteed to fail since it will attempt to use the tree to decode the - // codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is - // guaranteed to fail later since the compressed data section must be - // composed of at least one symbol (the end-of-block marker). - if max == 0 { - return true - } - - code := 0 - var nextcode [maxCodeLen]int - for i := min; i <= max; i++ { - code <<= 1 - nextcode[i] = code - code += count[i] - } - - // Check that the coding is complete (i.e., that we've - // assigned all 2-to-the-max possible bit sequences). - // Exception: To be compatible with zlib, we also need to - // accept degenerate single-code codings. See also - // TestDegenerateHuffmanCoding. - if code != 1< huffmanChunkBits { - numLinks := 1 << (uint(max) - huffmanChunkBits) - h.linkMask = uint32(numLinks - 1) - - // create link tables - link := nextcode[huffmanChunkBits+1] >> 1 - h.links = make([][]uint32, huffmanNumChunks-link) - for j := uint(link); j < huffmanNumChunks; j++ { - reverse := int(reverseByte[j>>8]) | int(reverseByte[j&0xff])<<8 - reverse >>= uint(16 - huffmanChunkBits) - off := j - uint(link) - if sanity && h.chunks[reverse] != 0 { - panic("impossible: overwriting existing chunk") - } - h.chunks[reverse] = uint32(off<>8]) | int(reverseByte[code&0xff])<<8 - reverse >>= uint(16 - n) - if n <= huffmanChunkBits { - for off := reverse; off < len(h.chunks); off += 1 << uint(n) { - // We should never need to overwrite - // an existing chunk. Also, 0 is - // never a valid chunk, because the - // lower 4 "count" bits should be - // between 1 and 15. - if sanity && h.chunks[off] != 0 { - panic("impossible: overwriting existing chunk") - } - h.chunks[off] = chunk - } - } else { - j := reverse & (huffmanNumChunks - 1) - if sanity && h.chunks[j]&huffmanCountMask != huffmanChunkBits+1 { - // Longer codes should have been - // associated with a link table above. - panic("impossible: not an indirect chunk") - } - value := h.chunks[j] >> huffmanValueShift - linktab := h.links[value] - reverse >>= huffmanChunkBits - for off := reverse; off < len(linktab); off += 1 << uint(n-huffmanChunkBits) { - if sanity && linktab[off] != 0 { - panic("impossible: overwriting existing chunk") - } - linktab[off] = chunk - } - } - } - - if sanity { - // Above we've sanity checked that we never overwrote - // an existing entry. Here we additionally check that - // we filled the tables completely. - for i, chunk := range h.chunks { - if chunk == 0 { - // As an exception, in the degenerate - // single-code case, we allow odd - // chunks to be missing. - if code == 1 && i%2 == 1 { - continue - } - panic("impossible: missing chunk") - } - } - for _, linktab := range h.links { - for _, chunk := range linktab { - if chunk == 0 { - panic("impossible: missing chunk") - } - } - } - } - - return true -} - -func main() { - flag.Parse() - - var h huffmanDecoder - var bits [288]int - initReverseByte() - for i := 0; i < 144; i++ { - bits[i] = 8 - } - for i := 144; i < 256; i++ { - bits[i] = 9 - } - for i := 256; i < 280; i++ { - bits[i] = 7 - } - for i := 280; i < 288; i++ { - bits[i] = 8 - } - h.init(bits[:]) - if h.links != nil { - log.Fatal("Unexpected links table in fixed Huffman decoder") - } - - var buf bytes.Buffer - - fmt.Fprintf(&buf, `// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file.`+"\n\n") - - fmt.Fprintln(&buf, "package flate") - fmt.Fprintln(&buf) - fmt.Fprintln(&buf, "// autogenerated by go run gen.go -output fixedhuff.go, DO NOT EDIT") - fmt.Fprintln(&buf) - fmt.Fprintln(&buf, "var fixedHuffmanDecoder = huffmanDecoder{") - fmt.Fprintf(&buf, "\t%d,\n", h.min) - fmt.Fprintln(&buf, "\t[huffmanNumChunks]uint32{") - for i := 0; i < huffmanNumChunks; i++ { - if i&7 == 0 { - fmt.Fprintf(&buf, "\t\t") - } else { - fmt.Fprintf(&buf, " ") - } - fmt.Fprintf(&buf, "0x%04x,", h.chunks[i]) - if i&7 == 7 { - fmt.Fprintln(&buf) - } - } - fmt.Fprintln(&buf, "\t},") - fmt.Fprintln(&buf, "\tnil, 0,") - fmt.Fprintln(&buf, "}") - - data, err := format.Source(buf.Bytes()) - if err != nil { - log.Fatal(err) - } - err = ioutil.WriteFile(*filename, data, 0644) - if err != nil { - log.Fatal(err) - } -} - -var reverseByte [256]byte - -func initReverseByte() { - for x := 0; x < 256; x++ { - var result byte - for i := uint(0); i < 8; i++ { - result |= byte(((x >> i) & 1) << (7 - i)) - } - reverseByte[x] = result - } -} diff --git a/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go b/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go deleted file mode 100644 index f9b2a699..00000000 --- a/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go +++ /dev/null @@ -1,701 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import ( - "io" -) - -const ( - // The largest offset code. - offsetCodeCount = 30 - - // The special code used to mark the end of a block. - endBlockMarker = 256 - - // The first length code. - lengthCodesStart = 257 - - // The number of codegen codes. - codegenCodeCount = 19 - badCode = 255 - - // bufferFlushSize indicates the buffer size - // after which bytes are flushed to the writer. - // Should preferably be a multiple of 6, since - // we accumulate 6 bytes between writes to the buffer. - bufferFlushSize = 240 - - // bufferSize is the actual output byte buffer size. - // It must have additional headroom for a flush - // which can contain up to 8 bytes. - bufferSize = bufferFlushSize + 8 -) - -// The number of extra bits needed by length code X - LENGTH_CODES_START. -var lengthExtraBits = []int8{ - /* 257 */ 0, 0, 0, - /* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, - /* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, - /* 280 */ 4, 5, 5, 5, 5, 0, -} - -// The length indicated by length code X - LENGTH_CODES_START. -var lengthBase = []uint32{ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, - 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, - 64, 80, 96, 112, 128, 160, 192, 224, 255, -} - -// offset code word extra bits. -var offsetExtraBits = []int8{ - 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, - 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, - 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, - /* extended window */ - 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, -} - -var offsetBase = []uint32{ - /* normal deflate */ - 0x000000, 0x000001, 0x000002, 0x000003, 0x000004, - 0x000006, 0x000008, 0x00000c, 0x000010, 0x000018, - 0x000020, 0x000030, 0x000040, 0x000060, 0x000080, - 0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300, - 0x000400, 0x000600, 0x000800, 0x000c00, 0x001000, - 0x001800, 0x002000, 0x003000, 0x004000, 0x006000, - - /* extended window */ - 0x008000, 0x00c000, 0x010000, 0x018000, 0x020000, - 0x030000, 0x040000, 0x060000, 0x080000, 0x0c0000, - 0x100000, 0x180000, 0x200000, 0x300000, -} - -// The odd order in which the codegen code sizes are written. -var codegenOrder = []uint32{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} - -type huffmanBitWriter struct { - // writer is the underlying writer. - // Do not use it directly; use the write method, which ensures - // that Write errors are sticky. - writer io.Writer - - // Data waiting to be written is bytes[0:nbytes] - // and then the low nbits of bits. - bits uint64 - nbits uint - bytes [bufferSize]byte - codegenFreq [codegenCodeCount]int32 - nbytes int - literalFreq []int32 - offsetFreq []int32 - codegen []uint8 - literalEncoding *huffmanEncoder - offsetEncoding *huffmanEncoder - codegenEncoding *huffmanEncoder - err error -} - -func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter { - return &huffmanBitWriter{ - writer: w, - literalFreq: make([]int32, maxNumLit), - offsetFreq: make([]int32, offsetCodeCount), - codegen: make([]uint8, maxNumLit+offsetCodeCount+1), - literalEncoding: newHuffmanEncoder(maxNumLit), - codegenEncoding: newHuffmanEncoder(codegenCodeCount), - offsetEncoding: newHuffmanEncoder(offsetCodeCount), - } -} - -func (w *huffmanBitWriter) reset(writer io.Writer) { - w.writer = writer - w.bits, w.nbits, w.nbytes, w.err = 0, 0, 0, nil - w.bytes = [bufferSize]byte{} -} - -func (w *huffmanBitWriter) flush() { - if w.err != nil { - w.nbits = 0 - return - } - n := w.nbytes - for w.nbits != 0 { - w.bytes[n] = byte(w.bits) - w.bits >>= 8 - if w.nbits > 8 { // Avoid underflow - w.nbits -= 8 - } else { - w.nbits = 0 - } - n++ - } - w.bits = 0 - w.write(w.bytes[:n]) - w.nbytes = 0 -} - -func (w *huffmanBitWriter) write(b []byte) { - if w.err != nil { - return - } - _, w.err = w.writer.Write(b) -} - -func (w *huffmanBitWriter) writeBits(b int32, nb uint) { - if w.err != nil { - return - } - w.bits |= uint64(b) << w.nbits - w.nbits += nb - if w.nbits >= 48 { - bits := w.bits - w.bits >>= 48 - w.nbits -= 48 - n := w.nbytes - bytes := w.bytes[n : n+6] - bytes[0] = byte(bits) - bytes[1] = byte(bits >> 8) - bytes[2] = byte(bits >> 16) - bytes[3] = byte(bits >> 24) - bytes[4] = byte(bits >> 32) - bytes[5] = byte(bits >> 40) - n += 6 - if n >= bufferFlushSize { - w.write(w.bytes[:n]) - n = 0 - } - w.nbytes = n - } -} - -func (w *huffmanBitWriter) writeBytes(bytes []byte) { - if w.err != nil { - return - } - n := w.nbytes - if w.nbits&7 != 0 { - w.err = InternalError("writeBytes with unfinished bits") - return - } - for w.nbits != 0 { - w.bytes[n] = byte(w.bits) - w.bits >>= 8 - w.nbits -= 8 - n++ - } - if n != 0 { - w.write(w.bytes[:n]) - } - w.nbytes = 0 - w.write(bytes) -} - -// RFC 1951 3.2.7 specifies a special run-length encoding for specifying -// the literal and offset lengths arrays (which are concatenated into a single -// array). This method generates that run-length encoding. -// -// The result is written into the codegen array, and the frequencies -// of each code is written into the codegenFreq array. -// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional -// information. Code badCode is an end marker -// -// numLiterals The number of literals in literalEncoding -// numOffsets The number of offsets in offsetEncoding -// litenc, offenc The literal and offset encoder to use -func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litEnc, offEnc *huffmanEncoder) { - for i := range w.codegenFreq { - w.codegenFreq[i] = 0 - } - // Note that we are using codegen both as a temporary variable for holding - // a copy of the frequencies, and as the place where we put the result. - // This is fine because the output is always shorter than the input used - // so far. - codegen := w.codegen // cache - // Copy the concatenated code sizes to codegen. Put a marker at the end. - cgnl := codegen[:numLiterals] - for i := range cgnl { - cgnl[i] = uint8(litEnc.codes[i].len) - } - - cgnl = codegen[numLiterals : numLiterals+numOffsets] - for i := range cgnl { - cgnl[i] = uint8(offEnc.codes[i].len) - } - codegen[numLiterals+numOffsets] = badCode - - size := codegen[0] - count := 1 - outIndex := 0 - for inIndex := 1; size != badCode; inIndex++ { - // INVARIANT: We have seen "count" copies of size that have not yet - // had output generated for them. - nextSize := codegen[inIndex] - if nextSize == size { - count++ - continue - } - // We need to generate codegen indicating "count" of size. - if size != 0 { - codegen[outIndex] = size - outIndex++ - w.codegenFreq[size]++ - count-- - for count >= 3 { - n := 6 - if n > count { - n = count - } - codegen[outIndex] = 16 - outIndex++ - codegen[outIndex] = uint8(n - 3) - outIndex++ - w.codegenFreq[16]++ - count -= n - } - } else { - for count >= 11 { - n := 138 - if n > count { - n = count - } - codegen[outIndex] = 18 - outIndex++ - codegen[outIndex] = uint8(n - 11) - outIndex++ - w.codegenFreq[18]++ - count -= n - } - if count >= 3 { - // count >= 3 && count <= 10 - codegen[outIndex] = 17 - outIndex++ - codegen[outIndex] = uint8(count - 3) - outIndex++ - w.codegenFreq[17]++ - count = 0 - } - } - count-- - for ; count >= 0; count-- { - codegen[outIndex] = size - outIndex++ - w.codegenFreq[size]++ - } - // Set up invariant for next time through the loop. - size = nextSize - count = 1 - } - // Marker indicating the end of the codegen. - codegen[outIndex] = badCode -} - -// dynamicSize returns the size of dynamically encoded data in bits. -func (w *huffmanBitWriter) dynamicSize(litEnc, offEnc *huffmanEncoder, extraBits int) (size, numCodegens int) { - numCodegens = len(w.codegenFreq) - for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 { - numCodegens-- - } - header := 3 + 5 + 5 + 4 + (3 * numCodegens) + - w.codegenEncoding.bitLength(w.codegenFreq[:]) + - int(w.codegenFreq[16])*2 + - int(w.codegenFreq[17])*3 + - int(w.codegenFreq[18])*7 - size = header + - litEnc.bitLength(w.literalFreq) + - offEnc.bitLength(w.offsetFreq) + - extraBits - - return size, numCodegens -} - -// fixedSize returns the size of dynamically encoded data in bits. -func (w *huffmanBitWriter) fixedSize(extraBits int) int { - return 3 + - fixedLiteralEncoding.bitLength(w.literalFreq) + - fixedOffsetEncoding.bitLength(w.offsetFreq) + - extraBits -} - -// storedSize calculates the stored size, including header. -// The function returns the size in bits and whether the block -// fits inside a single block. -func (w *huffmanBitWriter) storedSize(in []byte) (int, bool) { - if in == nil { - return 0, false - } - if len(in) <= maxStoreBlockSize { - return (len(in) + 5) * 8, true - } - return 0, false -} - -func (w *huffmanBitWriter) writeCode(c hcode) { - if w.err != nil { - return - } - w.bits |= uint64(c.code) << w.nbits - w.nbits += uint(c.len) - if w.nbits >= 48 { - bits := w.bits - w.bits >>= 48 - w.nbits -= 48 - n := w.nbytes - bytes := w.bytes[n : n+6] - bytes[0] = byte(bits) - bytes[1] = byte(bits >> 8) - bytes[2] = byte(bits >> 16) - bytes[3] = byte(bits >> 24) - bytes[4] = byte(bits >> 32) - bytes[5] = byte(bits >> 40) - n += 6 - if n >= bufferFlushSize { - w.write(w.bytes[:n]) - n = 0 - } - w.nbytes = n - } -} - -// Write the header of a dynamic Huffman block to the output stream. -// -// numLiterals The number of literals specified in codegen -// numOffsets The number of offsets specified in codegen -// numCodegens The number of codegens used in codegen -func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) { - if w.err != nil { - return - } - var firstBits int32 = 4 - if isEof { - firstBits = 5 - } - w.writeBits(firstBits, 3) - w.writeBits(int32(numLiterals-257), 5) - w.writeBits(int32(numOffsets-1), 5) - w.writeBits(int32(numCodegens-4), 4) - - for i := 0; i < numCodegens; i++ { - value := uint(w.codegenEncoding.codes[codegenOrder[i]].len) - w.writeBits(int32(value), 3) - } - - i := 0 - for { - var codeWord int = int(w.codegen[i]) - i++ - if codeWord == badCode { - break - } - w.writeCode(w.codegenEncoding.codes[uint32(codeWord)]) - - switch codeWord { - case 16: - w.writeBits(int32(w.codegen[i]), 2) - i++ - break - case 17: - w.writeBits(int32(w.codegen[i]), 3) - i++ - break - case 18: - w.writeBits(int32(w.codegen[i]), 7) - i++ - break - } - } -} - -func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) { - if w.err != nil { - return - } - var flag int32 - if isEof { - flag = 1 - } - w.writeBits(flag, 3) - w.flush() - w.writeBits(int32(length), 16) - w.writeBits(int32(^uint16(length)), 16) -} - -func (w *huffmanBitWriter) writeFixedHeader(isEof bool) { - if w.err != nil { - return - } - // Indicate that we are a fixed Huffman block - var value int32 = 2 - if isEof { - value = 3 - } - w.writeBits(value, 3) -} - -// writeBlock will write a block of tokens with the smallest encoding. -// The original input can be supplied, and if the huffman encoded data -// is larger than the original bytes, the data will be written as a -// stored block. -// If the input is nil, the tokens will always be Huffman encoded. -func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) { - if w.err != nil { - return - } - - tokens = append(tokens, endBlockMarker) - numLiterals, numOffsets := w.indexTokens(tokens) - - var extraBits int - storedSize, storable := w.storedSize(input) - if storable { - // We only bother calculating the costs of the extra bits required by - // the length of offset fields (which will be the same for both fixed - // and dynamic encoding), if we need to compare those two encodings - // against stored encoding. - for lengthCode := lengthCodesStart + 8; lengthCode < numLiterals; lengthCode++ { - // First eight length codes have extra size = 0. - extraBits += int(w.literalFreq[lengthCode]) * int(lengthExtraBits[lengthCode-lengthCodesStart]) - } - for offsetCode := 4; offsetCode < numOffsets; offsetCode++ { - // First four offset codes have extra size = 0. - extraBits += int(w.offsetFreq[offsetCode]) * int(offsetExtraBits[offsetCode]) - } - } - - // Figure out smallest code. - // Fixed Huffman baseline. - var literalEncoding = fixedLiteralEncoding - var offsetEncoding = fixedOffsetEncoding - var size = w.fixedSize(extraBits) - - // Dynamic Huffman? - var numCodegens int - - // Generate codegen and codegenFrequencies, which indicates how to encode - // the literalEncoding and the offsetEncoding. - w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding) - w.codegenEncoding.generate(w.codegenFreq[:], 7) - dynamicSize, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits) - - if dynamicSize < size { - size = dynamicSize - literalEncoding = w.literalEncoding - offsetEncoding = w.offsetEncoding - } - - // Stored bytes? - if storable && storedSize < size { - w.writeStoredHeader(len(input), eof) - w.writeBytes(input) - return - } - - // Huffman. - if literalEncoding == fixedLiteralEncoding { - w.writeFixedHeader(eof) - } else { - w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) - } - - // Write the tokens. - w.writeTokens(tokens, literalEncoding.codes, offsetEncoding.codes) -} - -// writeBlockDynamic encodes a block using a dynamic Huffman table. -// This should be used if the symbols used have a disproportionate -// histogram distribution. -// If input is supplied and the compression savings are below 1/16th of the -// input size the block is stored. -func (w *huffmanBitWriter) writeBlockDynamic(tokens []token, eof bool, input []byte) { - if w.err != nil { - return - } - - tokens = append(tokens, endBlockMarker) - numLiterals, numOffsets := w.indexTokens(tokens) - - // Generate codegen and codegenFrequencies, which indicates how to encode - // the literalEncoding and the offsetEncoding. - w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding) - w.codegenEncoding.generate(w.codegenFreq[:], 7) - size, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, 0) - - // Store bytes, if we don't get a reasonable improvement. - if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) { - w.writeStoredHeader(len(input), eof) - w.writeBytes(input) - return - } - - // Write Huffman table. - w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) - - // Write the tokens. - w.writeTokens(tokens, w.literalEncoding.codes, w.offsetEncoding.codes) -} - -// indexTokens indexes a slice of tokens, and updates -// literalFreq and offsetFreq, and generates literalEncoding -// and offsetEncoding. -// The number of literal and offset tokens is returned. -func (w *huffmanBitWriter) indexTokens(tokens []token) (numLiterals, numOffsets int) { - for i := range w.literalFreq { - w.literalFreq[i] = 0 - } - for i := range w.offsetFreq { - w.offsetFreq[i] = 0 - } - - for _, t := range tokens { - if t < matchType { - w.literalFreq[t.literal()]++ - continue - } - length := t.length() - offset := t.offset() - w.literalFreq[lengthCodesStart+lengthCode(length)]++ - w.offsetFreq[offsetCode(offset)]++ - } - - // get the number of literals - numLiterals = len(w.literalFreq) - for w.literalFreq[numLiterals-1] == 0 { - numLiterals-- - } - // get the number of offsets - numOffsets = len(w.offsetFreq) - for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 { - numOffsets-- - } - if numOffsets == 0 { - // We haven't found a single match. If we want to go with the dynamic encoding, - // we should count at least one offset to be sure that the offset huffman tree could be encoded. - w.offsetFreq[0] = 1 - numOffsets = 1 - } - w.literalEncoding.generate(w.literalFreq, 15) - w.offsetEncoding.generate(w.offsetFreq, 15) - return -} - -// writeTokens writes a slice of tokens to the output. -// codes for literal and offset encoding must be supplied. -func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) { - if w.err != nil { - return - } - for _, t := range tokens { - if t < matchType { - w.writeCode(leCodes[t.literal()]) - continue - } - // Write the length - length := t.length() - lengthCode := lengthCode(length) - w.writeCode(leCodes[lengthCode+lengthCodesStart]) - extraLengthBits := uint(lengthExtraBits[lengthCode]) - if extraLengthBits > 0 { - extraLength := int32(length - lengthBase[lengthCode]) - w.writeBits(extraLength, extraLengthBits) - } - // Write the offset - offset := t.offset() - offsetCode := offsetCode(offset) - w.writeCode(oeCodes[offsetCode]) - extraOffsetBits := uint(offsetExtraBits[offsetCode]) - if extraOffsetBits > 0 { - extraOffset := int32(offset - offsetBase[offsetCode]) - w.writeBits(extraOffset, extraOffsetBits) - } - } -} - -// huffOffset is a static offset encoder used for huffman only encoding. -// It can be reused since we will not be encoding offset values. -var huffOffset *huffmanEncoder - -func init() { - w := newHuffmanBitWriter(nil) - w.offsetFreq[0] = 1 - huffOffset = newHuffmanEncoder(offsetCodeCount) - huffOffset.generate(w.offsetFreq, 15) -} - -// writeBlockHuff encodes a block of bytes as either -// Huffman encoded literals or uncompressed bytes if the -// results only gains very little from compression. -func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte) { - if w.err != nil { - return - } - - // Clear histogram - for i := range w.literalFreq { - w.literalFreq[i] = 0 - } - - // Add everything as literals - histogram(input, w.literalFreq) - - w.literalFreq[endBlockMarker] = 1 - - const numLiterals = endBlockMarker + 1 - const numOffsets = 1 - - w.literalEncoding.generate(w.literalFreq, 15) - - // Figure out smallest code. - // Always use dynamic Huffman or Store - var numCodegens int - - // Generate codegen and codegenFrequencies, which indicates how to encode - // the literalEncoding and the offsetEncoding. - w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, huffOffset) - w.codegenEncoding.generate(w.codegenFreq[:], 7) - size, numCodegens := w.dynamicSize(w.literalEncoding, huffOffset, 0) - - // Store bytes, if we don't get a reasonable improvement. - if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) { - w.writeStoredHeader(len(input), eof) - w.writeBytes(input) - return - } - - // Huffman. - w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) - encoding := w.literalEncoding.codes[:257] - n := w.nbytes - for _, t := range input { - // Bitwriting inlined, ~30% speedup - c := encoding[t] - w.bits |= uint64(c.code) << w.nbits - w.nbits += uint(c.len) - if w.nbits < 48 { - continue - } - // Store 6 bytes - bits := w.bits - w.bits >>= 48 - w.nbits -= 48 - bytes := w.bytes[n : n+6] - bytes[0] = byte(bits) - bytes[1] = byte(bits >> 8) - bytes[2] = byte(bits >> 16) - bytes[3] = byte(bits >> 24) - bytes[4] = byte(bits >> 32) - bytes[5] = byte(bits >> 40) - n += 6 - if n < bufferFlushSize { - continue - } - w.write(w.bytes[:n]) - if w.err != nil { - return // Return early in the event of write failures - } - n = 0 - } - w.nbytes = n - w.writeCode(encoding[endBlockMarker]) -} diff --git a/vendor/github.com/klauspost/compress/flate/huffman_code.go b/vendor/github.com/klauspost/compress/flate/huffman_code.go deleted file mode 100644 index bdcbd823..00000000 --- a/vendor/github.com/klauspost/compress/flate/huffman_code.go +++ /dev/null @@ -1,344 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import ( - "math" - "sort" -) - -// hcode is a huffman code with a bit code and bit length. -type hcode struct { - code, len uint16 -} - -type huffmanEncoder struct { - codes []hcode - freqcache []literalNode - bitCount [17]int32 - lns byLiteral // stored to avoid repeated allocation in generate - lfs byFreq // stored to avoid repeated allocation in generate -} - -type literalNode struct { - literal uint16 - freq int32 -} - -// A levelInfo describes the state of the constructed tree for a given depth. -type levelInfo struct { - // Our level. for better printing - level int32 - - // The frequency of the last node at this level - lastFreq int32 - - // The frequency of the next character to add to this level - nextCharFreq int32 - - // The frequency of the next pair (from level below) to add to this level. - // Only valid if the "needed" value of the next lower level is 0. - nextPairFreq int32 - - // The number of chains remaining to generate for this level before moving - // up to the next level - needed int32 -} - -// set sets the code and length of an hcode. -func (h *hcode) set(code uint16, length uint16) { - h.len = length - h.code = code -} - -func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} } - -func newHuffmanEncoder(size int) *huffmanEncoder { - return &huffmanEncoder{codes: make([]hcode, size)} -} - -// Generates a HuffmanCode corresponding to the fixed literal table -func generateFixedLiteralEncoding() *huffmanEncoder { - h := newHuffmanEncoder(maxNumLit) - codes := h.codes - var ch uint16 - for ch = 0; ch < maxNumLit; ch++ { - var bits uint16 - var size uint16 - switch { - case ch < 144: - // size 8, 000110000 .. 10111111 - bits = ch + 48 - size = 8 - break - case ch < 256: - // size 9, 110010000 .. 111111111 - bits = ch + 400 - 144 - size = 9 - break - case ch < 280: - // size 7, 0000000 .. 0010111 - bits = ch - 256 - size = 7 - break - default: - // size 8, 11000000 .. 11000111 - bits = ch + 192 - 280 - size = 8 - } - codes[ch] = hcode{code: reverseBits(bits, byte(size)), len: size} - } - return h -} - -func generateFixedOffsetEncoding() *huffmanEncoder { - h := newHuffmanEncoder(30) - codes := h.codes - for ch := range codes { - codes[ch] = hcode{code: reverseBits(uint16(ch), 5), len: 5} - } - return h -} - -var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding() -var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding() - -func (h *huffmanEncoder) bitLength(freq []int32) int { - var total int - for i, f := range freq { - if f != 0 { - total += int(f) * int(h.codes[i].len) - } - } - return total -} - -const maxBitsLimit = 16 - -// Return the number of literals assigned to each bit size in the Huffman encoding -// -// This method is only called when list.length >= 3 -// The cases of 0, 1, and 2 literals are handled by special case code. -// -// list An array of the literals with non-zero frequencies -// and their associated frequencies. The array is in order of increasing -// frequency, and has as its last element a special element with frequency -// MaxInt32 -// maxBits The maximum number of bits that should be used to encode any literal. -// Must be less than 16. -// return An integer array in which array[i] indicates the number of literals -// that should be encoded in i bits. -func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 { - if maxBits >= maxBitsLimit { - panic("flate: maxBits too large") - } - n := int32(len(list)) - list = list[0 : n+1] - list[n] = maxNode() - - // The tree can't have greater depth than n - 1, no matter what. This - // saves a little bit of work in some small cases - if maxBits > n-1 { - maxBits = n - 1 - } - - // Create information about each of the levels. - // A bogus "Level 0" whose sole purpose is so that - // level1.prev.needed==0. This makes level1.nextPairFreq - // be a legitimate value that never gets chosen. - var levels [maxBitsLimit]levelInfo - // leafCounts[i] counts the number of literals at the left - // of ancestors of the rightmost node at level i. - // leafCounts[i][j] is the number of literals at the left - // of the level j ancestor. - var leafCounts [maxBitsLimit][maxBitsLimit]int32 - - for level := int32(1); level <= maxBits; level++ { - // For every level, the first two items are the first two characters. - // We initialize the levels as if we had already figured this out. - levels[level] = levelInfo{ - level: level, - lastFreq: list[1].freq, - nextCharFreq: list[2].freq, - nextPairFreq: list[0].freq + list[1].freq, - } - leafCounts[level][level] = 2 - if level == 1 { - levels[level].nextPairFreq = math.MaxInt32 - } - } - - // We need a total of 2*n - 2 items at top level and have already generated 2. - levels[maxBits].needed = 2*n - 4 - - level := maxBits - for { - l := &levels[level] - if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 { - // We've run out of both leafs and pairs. - // End all calculations for this level. - // To make sure we never come back to this level or any lower level, - // set nextPairFreq impossibly large. - l.needed = 0 - levels[level+1].nextPairFreq = math.MaxInt32 - level++ - continue - } - - prevFreq := l.lastFreq - if l.nextCharFreq < l.nextPairFreq { - // The next item on this row is a leaf node. - n := leafCounts[level][level] + 1 - l.lastFreq = l.nextCharFreq - // Lower leafCounts are the same of the previous node. - leafCounts[level][level] = n - l.nextCharFreq = list[n].freq - } else { - // The next item on this row is a pair from the previous row. - // nextPairFreq isn't valid until we generate two - // more values in the level below - l.lastFreq = l.nextPairFreq - // Take leaf counts from the lower level, except counts[level] remains the same. - copy(leafCounts[level][:level], leafCounts[level-1][:level]) - levels[l.level-1].needed = 2 - } - - if l.needed--; l.needed == 0 { - // We've done everything we need to do for this level. - // Continue calculating one level up. Fill in nextPairFreq - // of that level with the sum of the two nodes we've just calculated on - // this level. - if l.level == maxBits { - // All done! - break - } - levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq - level++ - } else { - // If we stole from below, move down temporarily to replenish it. - for levels[level-1].needed > 0 { - level-- - } - } - } - - // Somethings is wrong if at the end, the top level is null or hasn't used - // all of the leaves. - if leafCounts[maxBits][maxBits] != n { - panic("leafCounts[maxBits][maxBits] != n") - } - - bitCount := h.bitCount[:maxBits+1] - bits := 1 - counts := &leafCounts[maxBits] - for level := maxBits; level > 0; level-- { - // chain.leafCount gives the number of literals requiring at least "bits" - // bits to encode. - bitCount[bits] = counts[level] - counts[level-1] - bits++ - } - return bitCount -} - -// Look at the leaves and assign them a bit count and an encoding as specified -// in RFC 1951 3.2.2 -func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) { - code := uint16(0) - for n, bits := range bitCount { - code <<= 1 - if n == 0 || bits == 0 { - continue - } - // The literals list[len(list)-bits] .. list[len(list)-bits] - // are encoded using "bits" bits, and get the values - // code, code + 1, .... The code values are - // assigned in literal order (not frequency order). - chunk := list[len(list)-int(bits):] - - h.lns.sort(chunk) - for _, node := range chunk { - h.codes[node.literal] = hcode{code: reverseBits(code, uint8(n)), len: uint16(n)} - code++ - } - list = list[0 : len(list)-int(bits)] - } -} - -// Update this Huffman Code object to be the minimum code for the specified frequency count. -// -// freq An array of frequencies, in which frequency[i] gives the frequency of literal i. -// maxBits The maximum number of bits to use for any literal. -func (h *huffmanEncoder) generate(freq []int32, maxBits int32) { - if h.freqcache == nil { - // Allocate a reusable buffer with the longest possible frequency table. - // Possible lengths are codegenCodeCount, offsetCodeCount and maxNumLit. - // The largest of these is maxNumLit, so we allocate for that case. - h.freqcache = make([]literalNode, maxNumLit+1) - } - list := h.freqcache[:len(freq)+1] - // Number of non-zero literals - count := 0 - // Set list to be the set of all non-zero literals and their frequencies - for i, f := range freq { - if f != 0 { - list[count] = literalNode{uint16(i), f} - count++ - } else { - list[count] = literalNode{} - h.codes[i].len = 0 - } - } - list[len(freq)] = literalNode{} - - list = list[:count] - if count <= 2 { - // Handle the small cases here, because they are awkward for the general case code. With - // two or fewer literals, everything has bit length 1. - for i, node := range list { - // "list" is in order of increasing literal value. - h.codes[node.literal].set(uint16(i), 1) - } - return - } - h.lfs.sort(list) - - // Get the number of literals for each bit count - bitCount := h.bitCounts(list, maxBits) - // And do the assignment - h.assignEncodingAndSize(bitCount, list) -} - -type byLiteral []literalNode - -func (s *byLiteral) sort(a []literalNode) { - *s = byLiteral(a) - sort.Sort(s) -} - -func (s byLiteral) Len() int { return len(s) } - -func (s byLiteral) Less(i, j int) bool { - return s[i].literal < s[j].literal -} - -func (s byLiteral) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -type byFreq []literalNode - -func (s *byFreq) sort(a []literalNode) { - *s = byFreq(a) - sort.Sort(s) -} - -func (s byFreq) Len() int { return len(s) } - -func (s byFreq) Less(i, j int) bool { - if s[i].freq == s[j].freq { - return s[i].literal < s[j].literal - } - return s[i].freq < s[j].freq -} - -func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/vendor/github.com/klauspost/compress/flate/inflate.go b/vendor/github.com/klauspost/compress/flate/inflate.go deleted file mode 100644 index 800d0ce9..00000000 --- a/vendor/github.com/klauspost/compress/flate/inflate.go +++ /dev/null @@ -1,880 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package flate implements the DEFLATE compressed data format, described in -// RFC 1951. The gzip and zlib packages implement access to DEFLATE-based file -// formats. -package flate - -import ( - "bufio" - "io" - "math/bits" - "strconv" - "sync" -) - -const ( - maxCodeLen = 16 // max length of Huffman code - maxCodeLenMask = 15 // mask for max length of Huffman code - // The next three numbers come from the RFC section 3.2.7, with the - // additional proviso in section 3.2.5 which implies that distance codes - // 30 and 31 should never occur in compressed data. - maxNumLit = 286 - maxNumDist = 30 - numCodes = 19 // number of codes in Huffman meta-code -) - -// Initialize the fixedHuffmanDecoder only once upon first use. -var fixedOnce sync.Once -var fixedHuffmanDecoder huffmanDecoder - -// A CorruptInputError reports the presence of corrupt input at a given offset. -type CorruptInputError int64 - -func (e CorruptInputError) Error() string { - return "flate: corrupt input before offset " + strconv.FormatInt(int64(e), 10) -} - -// An InternalError reports an error in the flate code itself. -type InternalError string - -func (e InternalError) Error() string { return "flate: internal error: " + string(e) } - -// A ReadError reports an error encountered while reading input. -// -// Deprecated: No longer returned. -type ReadError struct { - Offset int64 // byte offset where error occurred - Err error // error returned by underlying Read -} - -func (e *ReadError) Error() string { - return "flate: read error at offset " + strconv.FormatInt(e.Offset, 10) + ": " + e.Err.Error() -} - -// A WriteError reports an error encountered while writing output. -// -// Deprecated: No longer returned. -type WriteError struct { - Offset int64 // byte offset where error occurred - Err error // error returned by underlying Write -} - -func (e *WriteError) Error() string { - return "flate: write error at offset " + strconv.FormatInt(e.Offset, 10) + ": " + e.Err.Error() -} - -// Resetter resets a ReadCloser returned by NewReader or NewReaderDict to -// to switch to a new underlying Reader. This permits reusing a ReadCloser -// instead of allocating a new one. -type Resetter interface { - // Reset discards any buffered data and resets the Resetter as if it was - // newly initialized with the given reader. - Reset(r io.Reader, dict []byte) error -} - -// The data structure for decoding Huffman tables is based on that of -// zlib. There is a lookup table of a fixed bit width (huffmanChunkBits), -// For codes smaller than the table width, there are multiple entries -// (each combination of trailing bits has the same value). For codes -// larger than the table width, the table contains a link to an overflow -// table. The width of each entry in the link table is the maximum code -// size minus the chunk width. -// -// Note that you can do a lookup in the table even without all bits -// filled. Since the extra bits are zero, and the DEFLATE Huffman codes -// have the property that shorter codes come before longer ones, the -// bit length estimate in the result is a lower bound on the actual -// number of bits. -// -// See the following: -// http://www.gzip.org/algorithm.txt - -// chunk & 15 is number of bits -// chunk >> 4 is value, including table link - -const ( - huffmanChunkBits = 9 - huffmanNumChunks = 1 << huffmanChunkBits - huffmanCountMask = 15 - huffmanValueShift = 4 -) - -type huffmanDecoder struct { - min int // the minimum code length - chunks *[huffmanNumChunks]uint32 // chunks as described above - links [][]uint32 // overflow links - linkMask uint32 // mask the width of the link table -} - -// Initialize Huffman decoding tables from array of code lengths. -// Following this function, h is guaranteed to be initialized into a complete -// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a -// degenerate case where the tree has only a single symbol with length 1. Empty -// trees are permitted. -func (h *huffmanDecoder) init(lengths []int) bool { - // Sanity enables additional runtime tests during Huffman - // table construction. It's intended to be used during - // development to supplement the currently ad-hoc unit tests. - const sanity = false - - if h.chunks == nil { - h.chunks = &[huffmanNumChunks]uint32{} - } - if h.min != 0 { - *h = huffmanDecoder{chunks: h.chunks, links: h.links} - } - - // Count number of codes of each length, - // compute min and max length. - var count [maxCodeLen]int - var min, max int - for _, n := range lengths { - if n == 0 { - continue - } - if min == 0 || n < min { - min = n - } - if n > max { - max = n - } - count[n&maxCodeLenMask]++ - } - - // Empty tree. The decompressor.huffSym function will fail later if the tree - // is used. Technically, an empty tree is only valid for the HDIST tree and - // not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree - // is guaranteed to fail since it will attempt to use the tree to decode the - // codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is - // guaranteed to fail later since the compressed data section must be - // composed of at least one symbol (the end-of-block marker). - if max == 0 { - return true - } - - code := 0 - var nextcode [maxCodeLen]int - for i := min; i <= max; i++ { - code <<= 1 - nextcode[i&maxCodeLenMask] = code - code += count[i&maxCodeLenMask] - } - - // Check that the coding is complete (i.e., that we've - // assigned all 2-to-the-max possible bit sequences). - // Exception: To be compatible with zlib, we also need to - // accept degenerate single-code codings. See also - // TestDegenerateHuffmanCoding. - if code != 1< huffmanChunkBits { - numLinks := 1 << (uint(max) - huffmanChunkBits) - h.linkMask = uint32(numLinks - 1) - - // create link tables - link := nextcode[huffmanChunkBits+1] >> 1 - if cap(h.links) < huffmanNumChunks-link { - h.links = make([][]uint32, huffmanNumChunks-link) - } else { - h.links = h.links[:huffmanNumChunks-link] - } - for j := uint(link); j < huffmanNumChunks; j++ { - reverse := int(bits.Reverse16(uint16(j))) - reverse >>= uint(16 - huffmanChunkBits) - off := j - uint(link) - if sanity && h.chunks[reverse] != 0 { - panic("impossible: overwriting existing chunk") - } - h.chunks[reverse] = uint32(off<>= uint(16 - n) - if n <= huffmanChunkBits { - for off := reverse; off < len(h.chunks); off += 1 << uint(n) { - // We should never need to overwrite - // an existing chunk. Also, 0 is - // never a valid chunk, because the - // lower 4 "count" bits should be - // between 1 and 15. - if sanity && h.chunks[off] != 0 { - panic("impossible: overwriting existing chunk") - } - h.chunks[off] = chunk - } - } else { - j := reverse & (huffmanNumChunks - 1) - if sanity && h.chunks[j]&huffmanCountMask != huffmanChunkBits+1 { - // Longer codes should have been - // associated with a link table above. - panic("impossible: not an indirect chunk") - } - value := h.chunks[j] >> huffmanValueShift - linktab := h.links[value] - reverse >>= huffmanChunkBits - for off := reverse; off < len(linktab); off += 1 << uint(n-huffmanChunkBits) { - if sanity && linktab[off] != 0 { - panic("impossible: overwriting existing chunk") - } - linktab[off] = chunk - } - } - } - - if sanity { - // Above we've sanity checked that we never overwrote - // an existing entry. Here we additionally check that - // we filled the tables completely. - for i, chunk := range h.chunks { - if chunk == 0 { - // As an exception, in the degenerate - // single-code case, we allow odd - // chunks to be missing. - if code == 1 && i%2 == 1 { - continue - } - panic("impossible: missing chunk") - } - } - for _, linktab := range h.links { - for _, chunk := range linktab { - if chunk == 0 { - panic("impossible: missing chunk") - } - } - } - } - - return true -} - -// The actual read interface needed by NewReader. -// If the passed in io.Reader does not also have ReadByte, -// the NewReader will introduce its own buffering. -type Reader interface { - io.Reader - io.ByteReader -} - -// Decompress state. -type decompressor struct { - // Input source. - r Reader - roffset int64 - - // Input bits, in top of b. - b uint32 - nb uint - - // Huffman decoders for literal/length, distance. - h1, h2 huffmanDecoder - - // Length arrays used to define Huffman codes. - bits *[maxNumLit + maxNumDist]int - codebits *[numCodes]int - - // Output history, buffer. - dict dictDecoder - - // Temporary buffer (avoids repeated allocation). - buf [4]byte - - // Next step in the decompression, - // and decompression state. - step func(*decompressor) - stepState int - final bool - err error - toRead []byte - hl, hd *huffmanDecoder - copyLen int - copyDist int -} - -func (f *decompressor) nextBlock() { - for f.nb < 1+2 { - if f.err = f.moreBits(); f.err != nil { - return - } - } - f.final = f.b&1 == 1 - f.b >>= 1 - typ := f.b & 3 - f.b >>= 2 - f.nb -= 1 + 2 - switch typ { - case 0: - f.dataBlock() - case 1: - // compressed, fixed Huffman tables - f.hl = &fixedHuffmanDecoder - f.hd = nil - f.huffmanBlock() - case 2: - // compressed, dynamic Huffman tables - if f.err = f.readHuffman(); f.err != nil { - break - } - f.hl = &f.h1 - f.hd = &f.h2 - f.huffmanBlock() - default: - // 3 is reserved. - f.err = CorruptInputError(f.roffset) - } -} - -func (f *decompressor) Read(b []byte) (int, error) { - for { - if len(f.toRead) > 0 { - n := copy(b, f.toRead) - f.toRead = f.toRead[n:] - if len(f.toRead) == 0 { - return n, f.err - } - return n, nil - } - if f.err != nil { - return 0, f.err - } - f.step(f) - if f.err != nil && len(f.toRead) == 0 { - f.toRead = f.dict.readFlush() // Flush what's left in case of error - } - } -} - -// Support the io.WriteTo interface for io.Copy and friends. -func (f *decompressor) WriteTo(w io.Writer) (int64, error) { - total := int64(0) - flushed := false - for { - if len(f.toRead) > 0 { - n, err := w.Write(f.toRead) - total += int64(n) - if err != nil { - f.err = err - return total, err - } - if n != len(f.toRead) { - return total, io.ErrShortWrite - } - f.toRead = f.toRead[:0] - } - if f.err != nil && flushed { - if f.err == io.EOF { - return total, nil - } - return total, f.err - } - if f.err == nil { - f.step(f) - } - if len(f.toRead) == 0 && f.err != nil && !flushed { - f.toRead = f.dict.readFlush() // Flush what's left in case of error - flushed = true - } - } -} - -func (f *decompressor) Close() error { - if f.err == io.EOF { - return nil - } - return f.err -} - -// RFC 1951 section 3.2.7. -// Compression with dynamic Huffman codes - -var codeOrder = [...]int{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} - -func (f *decompressor) readHuffman() error { - // HLIT[5], HDIST[5], HCLEN[4]. - for f.nb < 5+5+4 { - if err := f.moreBits(); err != nil { - return err - } - } - nlit := int(f.b&0x1F) + 257 - if nlit > maxNumLit { - return CorruptInputError(f.roffset) - } - f.b >>= 5 - ndist := int(f.b&0x1F) + 1 - if ndist > maxNumDist { - return CorruptInputError(f.roffset) - } - f.b >>= 5 - nclen := int(f.b&0xF) + 4 - // numCodes is 19, so nclen is always valid. - f.b >>= 4 - f.nb -= 5 + 5 + 4 - - // (HCLEN+4)*3 bits: code lengths in the magic codeOrder order. - for i := 0; i < nclen; i++ { - for f.nb < 3 { - if err := f.moreBits(); err != nil { - return err - } - } - f.codebits[codeOrder[i]] = int(f.b & 0x7) - f.b >>= 3 - f.nb -= 3 - } - for i := nclen; i < len(codeOrder); i++ { - f.codebits[codeOrder[i]] = 0 - } - if !f.h1.init(f.codebits[0:]) { - return CorruptInputError(f.roffset) - } - - // HLIT + 257 code lengths, HDIST + 1 code lengths, - // using the code length Huffman code. - for i, n := 0, nlit+ndist; i < n; { - x, err := f.huffSym(&f.h1) - if err != nil { - return err - } - if x < 16 { - // Actual length. - f.bits[i] = x - i++ - continue - } - // Repeat previous length or zero. - var rep int - var nb uint - var b int - switch x { - default: - return InternalError("unexpected length code") - case 16: - rep = 3 - nb = 2 - if i == 0 { - return CorruptInputError(f.roffset) - } - b = f.bits[i-1] - case 17: - rep = 3 - nb = 3 - b = 0 - case 18: - rep = 11 - nb = 7 - b = 0 - } - for f.nb < nb { - if err := f.moreBits(); err != nil { - return err - } - } - rep += int(f.b & uint32(1<>= nb - f.nb -= nb - if i+rep > n { - return CorruptInputError(f.roffset) - } - for j := 0; j < rep; j++ { - f.bits[i] = b - i++ - } - } - - if !f.h1.init(f.bits[0:nlit]) || !f.h2.init(f.bits[nlit:nlit+ndist]) { - return CorruptInputError(f.roffset) - } - - // As an optimization, we can initialize the min bits to read at a time - // for the HLIT tree to the length of the EOB marker since we know that - // every block must terminate with one. This preserves the property that - // we never read any extra bytes after the end of the DEFLATE stream. - if f.h1.min < f.bits[endBlockMarker] { - f.h1.min = f.bits[endBlockMarker] - } - - return nil -} - -// Decode a single Huffman block from f. -// hl and hd are the Huffman states for the lit/length values -// and the distance values, respectively. If hd == nil, using the -// fixed distance encoding associated with fixed Huffman blocks. -func (f *decompressor) huffmanBlock() { - const ( - stateInit = iota // Zero value must be stateInit - stateDict - ) - - switch f.stepState { - case stateInit: - goto readLiteral - case stateDict: - goto copyHistory - } - -readLiteral: - // Read literal and/or (length, distance) according to RFC section 3.2.3. - { - v, err := f.huffSym(f.hl) - if err != nil { - f.err = err - return - } - var n uint // number of bits extra - var length int - switch { - case v < 256: - f.dict.writeByte(byte(v)) - if f.dict.availWrite() == 0 { - f.toRead = f.dict.readFlush() - f.step = (*decompressor).huffmanBlock - f.stepState = stateInit - return - } - goto readLiteral - case v == 256: - f.finishBlock() - return - // otherwise, reference to older data - case v < 265: - length = v - (257 - 3) - n = 0 - case v < 269: - length = v*2 - (265*2 - 11) - n = 1 - case v < 273: - length = v*4 - (269*4 - 19) - n = 2 - case v < 277: - length = v*8 - (273*8 - 35) - n = 3 - case v < 281: - length = v*16 - (277*16 - 67) - n = 4 - case v < 285: - length = v*32 - (281*32 - 131) - n = 5 - case v < maxNumLit: - length = 258 - n = 0 - default: - f.err = CorruptInputError(f.roffset) - return - } - if n > 0 { - for f.nb < n { - if err = f.moreBits(); err != nil { - f.err = err - return - } - } - length += int(f.b & uint32(1<>= n - f.nb -= n - } - - var dist int - if f.hd == nil { - for f.nb < 5 { - if err = f.moreBits(); err != nil { - f.err = err - return - } - } - dist = int(bits.Reverse8(uint8(f.b & 0x1F << 3))) - f.b >>= 5 - f.nb -= 5 - } else { - if dist, err = f.huffSym(f.hd); err != nil { - f.err = err - return - } - } - - switch { - case dist < 4: - dist++ - case dist < maxNumDist: - nb := uint(dist-2) >> 1 - // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << nb - for f.nb < nb { - if err = f.moreBits(); err != nil { - f.err = err - return - } - } - extra |= int(f.b & uint32(1<>= nb - f.nb -= nb - dist = 1<<(nb+1) + 1 + extra - default: - f.err = CorruptInputError(f.roffset) - return - } - - // No check on length; encoding can be prescient. - if dist > f.dict.histSize() { - f.err = CorruptInputError(f.roffset) - return - } - - f.copyLen, f.copyDist = length, dist - goto copyHistory - } - -copyHistory: - // Perform a backwards copy according to RFC section 3.2.3. - { - cnt := f.dict.tryWriteCopy(f.copyDist, f.copyLen) - if cnt == 0 { - cnt = f.dict.writeCopy(f.copyDist, f.copyLen) - } - f.copyLen -= cnt - - if f.dict.availWrite() == 0 || f.copyLen > 0 { - f.toRead = f.dict.readFlush() - f.step = (*decompressor).huffmanBlock // We need to continue this work - f.stepState = stateDict - return - } - goto readLiteral - } -} - -// Copy a single uncompressed data block from input to output. -func (f *decompressor) dataBlock() { - // Uncompressed. - // Discard current half-byte. - f.nb = 0 - f.b = 0 - - // Length then ones-complement of length. - nr, err := io.ReadFull(f.r, f.buf[0:4]) - f.roffset += int64(nr) - if err != nil { - f.err = noEOF(err) - return - } - n := int(f.buf[0]) | int(f.buf[1])<<8 - nn := int(f.buf[2]) | int(f.buf[3])<<8 - if uint16(nn) != uint16(^n) { - f.err = CorruptInputError(f.roffset) - return - } - - if n == 0 { - f.toRead = f.dict.readFlush() - f.finishBlock() - return - } - - f.copyLen = n - f.copyData() -} - -// copyData copies f.copyLen bytes from the underlying reader into f.hist. -// It pauses for reads when f.hist is full. -func (f *decompressor) copyData() { - buf := f.dict.writeSlice() - if len(buf) > f.copyLen { - buf = buf[:f.copyLen] - } - - cnt, err := io.ReadFull(f.r, buf) - f.roffset += int64(cnt) - f.copyLen -= cnt - f.dict.writeMark(cnt) - if err != nil { - f.err = noEOF(err) - return - } - - if f.dict.availWrite() == 0 || f.copyLen > 0 { - f.toRead = f.dict.readFlush() - f.step = (*decompressor).copyData - return - } - f.finishBlock() -} - -func (f *decompressor) finishBlock() { - if f.final { - if f.dict.availRead() > 0 { - f.toRead = f.dict.readFlush() - } - f.err = io.EOF - } - f.step = (*decompressor).nextBlock -} - -// noEOF returns err, unless err == io.EOF, in which case it returns io.ErrUnexpectedEOF. -func noEOF(e error) error { - if e == io.EOF { - return io.ErrUnexpectedEOF - } - return e -} - -func (f *decompressor) moreBits() error { - c, err := f.r.ReadByte() - if err != nil { - return noEOF(err) - } - f.roffset++ - f.b |= uint32(c) << f.nb - f.nb += 8 - return nil -} - -// Read the next Huffman-encoded symbol from f according to h. -func (f *decompressor) huffSym(h *huffmanDecoder) (int, error) { - // Since a huffmanDecoder can be empty or be composed of a degenerate tree - // with single element, huffSym must error on these two edge cases. In both - // cases, the chunks slice will be 0 for the invalid sequence, leading it - // satisfy the n == 0 check below. - n := uint(h.min) - // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, - // but is smart enough to keep local variables in registers, so use nb and b, - // inline call to moreBits and reassign b,nb back to f on return. - nb, b := f.nb, f.b - for { - for nb < n { - c, err := f.r.ReadByte() - if err != nil { - f.b = b - f.nb = nb - return 0, noEOF(err) - } - f.roffset++ - b |= uint32(c) << (nb & 31) - nb += 8 - } - chunk := h.chunks[b&(huffmanNumChunks-1)] - n = uint(chunk & huffmanCountMask) - if n > huffmanChunkBits { - chunk = h.links[chunk>>huffmanValueShift][(b>>huffmanChunkBits)&h.linkMask] - n = uint(chunk & huffmanCountMask) - } - if n <= nb { - if n == 0 { - f.b = b - f.nb = nb - f.err = CorruptInputError(f.roffset) - return 0, f.err - } - f.b = b >> (n & 31) - f.nb = nb - n - return int(chunk >> huffmanValueShift), nil - } - } -} - -func makeReader(r io.Reader) Reader { - if rr, ok := r.(Reader); ok { - return rr - } - return bufio.NewReader(r) -} - -func fixedHuffmanDecoderInit() { - fixedOnce.Do(func() { - // These come from the RFC section 3.2.6. - var bits [288]int - for i := 0; i < 144; i++ { - bits[i] = 8 - } - for i := 144; i < 256; i++ { - bits[i] = 9 - } - for i := 256; i < 280; i++ { - bits[i] = 7 - } - for i := 280; i < 288; i++ { - bits[i] = 8 - } - fixedHuffmanDecoder.init(bits[:]) - }) -} - -func (f *decompressor) Reset(r io.Reader, dict []byte) error { - *f = decompressor{ - r: makeReader(r), - bits: f.bits, - codebits: f.codebits, - h1: f.h1, - h2: f.h2, - dict: f.dict, - step: (*decompressor).nextBlock, - } - f.dict.init(maxMatchOffset, dict) - return nil -} - -// NewReader returns a new ReadCloser that can be used -// to read the uncompressed version of r. -// If r does not also implement io.ByteReader, -// the decompressor may read more data than necessary from r. -// It is the caller's responsibility to call Close on the ReadCloser -// when finished reading. -// -// The ReadCloser returned by NewReader also implements Resetter. -func NewReader(r io.Reader) io.ReadCloser { - fixedHuffmanDecoderInit() - - var f decompressor - f.r = makeReader(r) - f.bits = new([maxNumLit + maxNumDist]int) - f.codebits = new([numCodes]int) - f.step = (*decompressor).nextBlock - f.dict.init(maxMatchOffset, nil) - return &f -} - -// NewReaderDict is like NewReader but initializes the reader -// with a preset dictionary. The returned Reader behaves as if -// the uncompressed data stream started with the given dictionary, -// which has already been read. NewReaderDict is typically used -// to read data compressed by NewWriterDict. -// -// The ReadCloser returned by NewReader also implements Resetter. -func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser { - fixedHuffmanDecoderInit() - - var f decompressor - f.r = makeReader(r) - f.bits = new([maxNumLit + maxNumDist]int) - f.codebits = new([numCodes]int) - f.step = (*decompressor).nextBlock - f.dict.init(maxMatchOffset, dict) - return &f -} diff --git a/vendor/github.com/klauspost/compress/flate/reverse_bits.go b/vendor/github.com/klauspost/compress/flate/reverse_bits.go deleted file mode 100644 index c1a02720..00000000 --- a/vendor/github.com/klauspost/compress/flate/reverse_bits.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -var reverseByte = [256]byte{ - 0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, - 0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0, - 0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8, - 0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8, - 0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4, - 0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4, - 0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec, - 0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc, - 0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2, - 0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2, - 0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea, - 0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa, - 0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6, - 0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6, - 0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee, - 0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe, - 0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1, - 0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1, - 0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9, - 0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9, - 0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5, - 0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5, - 0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed, - 0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd, - 0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3, - 0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3, - 0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb, - 0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb, - 0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7, - 0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7, - 0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef, - 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff, -} - -func reverseUint16(v uint16) uint16 { - return uint16(reverseByte[v>>8]) | uint16(reverseByte[v&0xFF])<<8 -} - -func reverseBits(number uint16, bitLength byte) uint16 { - return reverseUint16(number << uint8(16-bitLength)) -} diff --git a/vendor/github.com/klauspost/compress/flate/snappy.go b/vendor/github.com/klauspost/compress/flate/snappy.go deleted file mode 100644 index d853320a..00000000 --- a/vendor/github.com/klauspost/compress/flate/snappy.go +++ /dev/null @@ -1,900 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Modified for deflate by Klaus Post (c) 2015. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -// emitLiteral writes a literal chunk and returns the number of bytes written. -func emitLiteral(dst *tokens, lit []byte) { - ol := int(dst.n) - for i, v := range lit { - dst.tokens[(i+ol)&maxStoreBlockSize] = token(v) - } - dst.n += uint16(len(lit)) -} - -// emitCopy writes a copy chunk and returns the number of bytes written. -func emitCopy(dst *tokens, offset, length int) { - dst.tokens[dst.n] = matchToken(uint32(length-3), uint32(offset-minOffsetSize)) - dst.n++ -} - -type snappyEnc interface { - Encode(dst *tokens, src []byte) - Reset() -} - -func newSnappy(level int) snappyEnc { - switch level { - case 1: - return &snappyL1{} - case 2: - return &snappyL2{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}} - case 3: - return &snappyL3{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}} - case 4: - return &snappyL4{snappyL3{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}}} - default: - panic("invalid level specified") - } -} - -const ( - tableBits = 14 // Bits used in the table - tableSize = 1 << tableBits // Size of the table - tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. - tableShift = 32 - tableBits // Right-shift to get the tableBits most significant bits of a uint32. - baseMatchOffset = 1 // The smallest match offset - baseMatchLength = 3 // The smallest match length per the RFC section 3.2.5 - maxMatchOffset = 1 << 15 // The largest match offset -) - -func load32(b []byte, i int) uint32 { - b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line. - return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 -} - -func load64(b []byte, i int) uint64 { - b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line. - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 -} - -func hash(u uint32) uint32 { - return (u * 0x1e35a7bd) >> tableShift -} - -// snappyL1 encapsulates level 1 compression -type snappyL1 struct{} - -func (e *snappyL1) Reset() {} - -func (e *snappyL1) Encode(dst *tokens, src []byte) { - const ( - inputMargin = 16 - 1 - minNonLiteralBlockSize = 1 + 1 + inputMargin - ) - - // This check isn't in the Snappy implementation, but there, the caller - // instead of the callee handles this case. - if len(src) < minNonLiteralBlockSize { - // We do not fill the token table. - // This will be picked up by caller. - dst.n = uint16(len(src)) - return - } - - // Initialize the hash table. - // - // The table element type is uint16, as s < sLimit and sLimit < len(src) - // and len(src) <= maxStoreBlockSize and maxStoreBlockSize == 65535. - var table [tableSize]uint16 - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := len(src) - inputMargin - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := 0 - - // The encoded form must start with a literal, as there are no previous - // bytes to copy, so we start looking for hash matches at s == 1. - s := 1 - nextHash := hash(load32(src, s)) - - for { - // Copied from the C++ snappy implementation: - // - // Heuristic match skipping: If 32 bytes are scanned with no matches - // found, start looking only at every other byte. If 32 more bytes are - // scanned (or skipped), look at every third byte, etc.. When a match - // is found, immediately go back to looking at every byte. This is a - // small loss (~5% performance, ~0.1% density) for compressible data - // due to more bookkeeping, but for non-compressible data (such as - // JPEG) it's a huge win since the compressor quickly "realizes" the - // data is incompressible and doesn't bother looking for matches - // everywhere. - // - // The "skip" variable keeps track of how many bytes there are since - // the last match; dividing it by 32 (ie. right-shifting by five) gives - // the number of bytes to move ahead for each iteration. - skip := 32 - - nextS := s - candidate := 0 - for { - s = nextS - bytesBetweenHashLookups := skip >> 5 - nextS = s + bytesBetweenHashLookups - skip += bytesBetweenHashLookups - if nextS > sLimit { - goto emitRemainder - } - candidate = int(table[nextHash&tableMask]) - table[nextHash&tableMask] = uint16(s) - nextHash = hash(load32(src, nextS)) - if s-candidate <= maxMatchOffset && load32(src, s) == load32(src, candidate) { - break - } - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - emitLiteral(dst, src[nextEmit:s]) - - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - base := s - - // Extend the 4-byte match as long as possible. - // - // This is an inlined version of Snappy's: - // s = extendMatch(src, candidate+4, s+4) - s += 4 - s1 := base + maxMatchLength - if s1 > len(src) { - s1 = len(src) - } - a := src[s:s1] - b := src[candidate+4:] - b = b[:len(a)] - l := len(a) - for i := range a { - if a[i] != b[i] { - l = i - break - } - } - s += l - - // matchToken is flate's equivalent of Snappy's emitCopy. - dst.tokens[dst.n] = matchToken(uint32(s-base-baseMatchLength), uint32(base-candidate-baseMatchOffset)) - dst.n++ - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-1 and at s. If - // another emitCopy is not our next move, also calculate nextHash - // at s+1. At least on GOARCH=amd64, these three hash calculations - // are faster as one load64 call (with some shifts) instead of - // three load32 calls. - x := load64(src, s-1) - prevHash := hash(uint32(x >> 0)) - table[prevHash&tableMask] = uint16(s - 1) - currHash := hash(uint32(x >> 8)) - candidate = int(table[currHash&tableMask]) - table[currHash&tableMask] = uint16(s) - if s-candidate > maxMatchOffset || uint32(x>>8) != load32(src, candidate) { - nextHash = hash(uint32(x >> 16)) - s++ - break - } - } - } - -emitRemainder: - if nextEmit < len(src) { - emitLiteral(dst, src[nextEmit:]) - } -} - -type tableEntry struct { - val uint32 - offset int32 -} - -func load3232(b []byte, i int32) uint32 { - b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line. - return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 -} - -func load6432(b []byte, i int32) uint64 { - b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line. - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 -} - -// snappyGen maintains the table for matches, -// and the previous byte block for level 2. -// This is the generic implementation. -type snappyGen struct { - prev []byte - cur int32 -} - -// snappyGen maintains the table for matches, -// and the previous byte block for level 2. -// This is the generic implementation. -type snappyL2 struct { - snappyGen - table [tableSize]tableEntry -} - -// EncodeL2 uses a similar algorithm to level 1, but is capable -// of matching across blocks giving better compression at a small slowdown. -func (e *snappyL2) Encode(dst *tokens, src []byte) { - const ( - inputMargin = 8 - 1 - minNonLiteralBlockSize = 1 + 1 + inputMargin - ) - - // Protect against e.cur wraparound. - if e.cur > 1<<30 { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } - e.cur = maxStoreBlockSize - } - - // This check isn't in the Snappy implementation, but there, the caller - // instead of the callee handles this case. - if len(src) < minNonLiteralBlockSize { - // We do not fill the token table. - // This will be picked up by caller. - dst.n = uint16(len(src)) - e.cur += maxStoreBlockSize - e.prev = e.prev[:0] - return - } - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := int32(len(src) - inputMargin) - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := int32(0) - s := int32(0) - cv := load3232(src, s) - nextHash := hash(cv) - - for { - // Copied from the C++ snappy implementation: - // - // Heuristic match skipping: If 32 bytes are scanned with no matches - // found, start looking only at every other byte. If 32 more bytes are - // scanned (or skipped), look at every third byte, etc.. When a match - // is found, immediately go back to looking at every byte. This is a - // small loss (~5% performance, ~0.1% density) for compressible data - // due to more bookkeeping, but for non-compressible data (such as - // JPEG) it's a huge win since the compressor quickly "realizes" the - // data is incompressible and doesn't bother looking for matches - // everywhere. - // - // The "skip" variable keeps track of how many bytes there are since - // the last match; dividing it by 32 (ie. right-shifting by five) gives - // the number of bytes to move ahead for each iteration. - skip := int32(32) - - nextS := s - var candidate tableEntry - for { - s = nextS - bytesBetweenHashLookups := skip >> 5 - nextS = s + bytesBetweenHashLookups - skip += bytesBetweenHashLookups - if nextS > sLimit { - goto emitRemainder - } - candidate = e.table[nextHash&tableMask] - now := load3232(src, nextS) - e.table[nextHash&tableMask] = tableEntry{offset: s + e.cur, val: cv} - nextHash = hash(now) - - offset := s - (candidate.offset - e.cur) - if offset > maxMatchOffset || cv != candidate.val { - // Out of range or not matched. - cv = now - continue - } - break - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - emitLiteral(dst, src[nextEmit:s]) - - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - - // Extend the 4-byte match as long as possible. - // - s += 4 - t := candidate.offset - e.cur + 4 - l := e.matchlen(s, t, src) - - // matchToken is flate's equivalent of Snappy's emitCopy. (length,offset) - dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset)) - dst.n++ - s += l - nextEmit = s - if s >= sLimit { - t += l - // Index first pair after match end. - if int(t+4) < len(src) && t > 0 { - cv := load3232(src, t) - e.table[hash(cv)&tableMask] = tableEntry{offset: t + e.cur, val: cv} - } - goto emitRemainder - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-1 and at s. If - // another emitCopy is not our next move, also calculate nextHash - // at s+1. At least on GOARCH=amd64, these three hash calculations - // are faster as one load64 call (with some shifts) instead of - // three load32 calls. - x := load6432(src, s-1) - prevHash := hash(uint32(x)) - e.table[prevHash&tableMask] = tableEntry{offset: e.cur + s - 1, val: uint32(x)} - x >>= 8 - currHash := hash(uint32(x)) - candidate = e.table[currHash&tableMask] - e.table[currHash&tableMask] = tableEntry{offset: e.cur + s, val: uint32(x)} - - offset := s - (candidate.offset - e.cur) - if offset > maxMatchOffset || uint32(x) != candidate.val { - cv = uint32(x >> 8) - nextHash = hash(cv) - s++ - break - } - } - } - -emitRemainder: - if int(nextEmit) < len(src) { - emitLiteral(dst, src[nextEmit:]) - } - e.cur += int32(len(src)) - e.prev = e.prev[:len(src)] - copy(e.prev, src) -} - -type tableEntryPrev struct { - Cur tableEntry - Prev tableEntry -} - -// snappyL3 -type snappyL3 struct { - snappyGen - table [tableSize]tableEntryPrev -} - -// Encode uses a similar algorithm to level 2, will check up to two candidates. -func (e *snappyL3) Encode(dst *tokens, src []byte) { - const ( - inputMargin = 8 - 1 - minNonLiteralBlockSize = 1 + 1 + inputMargin - ) - - // Protect against e.cur wraparound. - if e.cur > 1<<30 { - for i := range e.table[:] { - e.table[i] = tableEntryPrev{} - } - e.snappyGen = snappyGen{cur: maxStoreBlockSize, prev: e.prev[:0]} - } - - // This check isn't in the Snappy implementation, but there, the caller - // instead of the callee handles this case. - if len(src) < minNonLiteralBlockSize { - // We do not fill the token table. - // This will be picked up by caller. - dst.n = uint16(len(src)) - e.cur += maxStoreBlockSize - e.prev = e.prev[:0] - return - } - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := int32(len(src) - inputMargin) - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := int32(0) - s := int32(0) - cv := load3232(src, s) - nextHash := hash(cv) - - for { - // Copied from the C++ snappy implementation: - // - // Heuristic match skipping: If 32 bytes are scanned with no matches - // found, start looking only at every other byte. If 32 more bytes are - // scanned (or skipped), look at every third byte, etc.. When a match - // is found, immediately go back to looking at every byte. This is a - // small loss (~5% performance, ~0.1% density) for compressible data - // due to more bookkeeping, but for non-compressible data (such as - // JPEG) it's a huge win since the compressor quickly "realizes" the - // data is incompressible and doesn't bother looking for matches - // everywhere. - // - // The "skip" variable keeps track of how many bytes there are since - // the last match; dividing it by 32 (ie. right-shifting by five) gives - // the number of bytes to move ahead for each iteration. - skip := int32(32) - - nextS := s - var candidate tableEntry - for { - s = nextS - bytesBetweenHashLookups := skip >> 5 - nextS = s + bytesBetweenHashLookups - skip += bytesBetweenHashLookups - if nextS > sLimit { - goto emitRemainder - } - candidates := e.table[nextHash&tableMask] - now := load3232(src, nextS) - e.table[nextHash&tableMask] = tableEntryPrev{Prev: candidates.Cur, Cur: tableEntry{offset: s + e.cur, val: cv}} - nextHash = hash(now) - - // Check both candidates - candidate = candidates.Cur - if cv == candidate.val { - offset := s - (candidate.offset - e.cur) - if offset <= maxMatchOffset { - break - } - } else { - // We only check if value mismatches. - // Offset will always be invalid in other cases. - candidate = candidates.Prev - if cv == candidate.val { - offset := s - (candidate.offset - e.cur) - if offset <= maxMatchOffset { - break - } - } - } - cv = now - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - emitLiteral(dst, src[nextEmit:s]) - - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - - // Extend the 4-byte match as long as possible. - // - s += 4 - t := candidate.offset - e.cur + 4 - l := e.matchlen(s, t, src) - - // matchToken is flate's equivalent of Snappy's emitCopy. (length,offset) - dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset)) - dst.n++ - s += l - nextEmit = s - if s >= sLimit { - t += l - // Index first pair after match end. - if int(t+4) < len(src) && t > 0 { - cv := load3232(src, t) - nextHash = hash(cv) - e.table[nextHash&tableMask] = tableEntryPrev{ - Prev: e.table[nextHash&tableMask].Cur, - Cur: tableEntry{offset: e.cur + t, val: cv}, - } - } - goto emitRemainder - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-3 to s. If - // another emitCopy is not our next move, also calculate nextHash - // at s+1. At least on GOARCH=amd64, these three hash calculations - // are faster as one load64 call (with some shifts) instead of - // three load32 calls. - x := load6432(src, s-3) - prevHash := hash(uint32(x)) - e.table[prevHash&tableMask] = tableEntryPrev{ - Prev: e.table[prevHash&tableMask].Cur, - Cur: tableEntry{offset: e.cur + s - 3, val: uint32(x)}, - } - x >>= 8 - prevHash = hash(uint32(x)) - - e.table[prevHash&tableMask] = tableEntryPrev{ - Prev: e.table[prevHash&tableMask].Cur, - Cur: tableEntry{offset: e.cur + s - 2, val: uint32(x)}, - } - x >>= 8 - prevHash = hash(uint32(x)) - - e.table[prevHash&tableMask] = tableEntryPrev{ - Prev: e.table[prevHash&tableMask].Cur, - Cur: tableEntry{offset: e.cur + s - 1, val: uint32(x)}, - } - x >>= 8 - currHash := hash(uint32(x)) - candidates := e.table[currHash&tableMask] - cv = uint32(x) - e.table[currHash&tableMask] = tableEntryPrev{ - Prev: candidates.Cur, - Cur: tableEntry{offset: s + e.cur, val: cv}, - } - - // Check both candidates - candidate = candidates.Cur - if cv == candidate.val { - offset := s - (candidate.offset - e.cur) - if offset <= maxMatchOffset { - continue - } - } else { - // We only check if value mismatches. - // Offset will always be invalid in other cases. - candidate = candidates.Prev - if cv == candidate.val { - offset := s - (candidate.offset - e.cur) - if offset <= maxMatchOffset { - continue - } - } - } - cv = uint32(x >> 8) - nextHash = hash(cv) - s++ - break - } - } - -emitRemainder: - if int(nextEmit) < len(src) { - emitLiteral(dst, src[nextEmit:]) - } - e.cur += int32(len(src)) - e.prev = e.prev[:len(src)] - copy(e.prev, src) -} - -// snappyL4 -type snappyL4 struct { - snappyL3 -} - -// Encode uses a similar algorithm to level 3, -// but will check up to two candidates if first isn't long enough. -func (e *snappyL4) Encode(dst *tokens, src []byte) { - const ( - inputMargin = 8 - 3 - minNonLiteralBlockSize = 1 + 1 + inputMargin - matchLenGood = 12 - ) - - // Protect against e.cur wraparound. - if e.cur > 1<<30 { - for i := range e.table[:] { - e.table[i] = tableEntryPrev{} - } - e.snappyGen = snappyGen{cur: maxStoreBlockSize, prev: e.prev[:0]} - } - - // This check isn't in the Snappy implementation, but there, the caller - // instead of the callee handles this case. - if len(src) < minNonLiteralBlockSize { - // We do not fill the token table. - // This will be picked up by caller. - dst.n = uint16(len(src)) - e.cur += maxStoreBlockSize - e.prev = e.prev[:0] - return - } - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := int32(len(src) - inputMargin) - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := int32(0) - s := int32(0) - cv := load3232(src, s) - nextHash := hash(cv) - - for { - // Copied from the C++ snappy implementation: - // - // Heuristic match skipping: If 32 bytes are scanned with no matches - // found, start looking only at every other byte. If 32 more bytes are - // scanned (or skipped), look at every third byte, etc.. When a match - // is found, immediately go back to looking at every byte. This is a - // small loss (~5% performance, ~0.1% density) for compressible data - // due to more bookkeeping, but for non-compressible data (such as - // JPEG) it's a huge win since the compressor quickly "realizes" the - // data is incompressible and doesn't bother looking for matches - // everywhere. - // - // The "skip" variable keeps track of how many bytes there are since - // the last match; dividing it by 32 (ie. right-shifting by five) gives - // the number of bytes to move ahead for each iteration. - skip := int32(32) - - nextS := s - var candidate tableEntry - var candidateAlt tableEntry - for { - s = nextS - bytesBetweenHashLookups := skip >> 5 - nextS = s + bytesBetweenHashLookups - skip += bytesBetweenHashLookups - if nextS > sLimit { - goto emitRemainder - } - candidates := e.table[nextHash&tableMask] - now := load3232(src, nextS) - e.table[nextHash&tableMask] = tableEntryPrev{Prev: candidates.Cur, Cur: tableEntry{offset: s + e.cur, val: cv}} - nextHash = hash(now) - - // Check both candidates - candidate = candidates.Cur - if cv == candidate.val { - offset := s - (candidate.offset - e.cur) - if offset < maxMatchOffset { - offset = s - (candidates.Prev.offset - e.cur) - if cv == candidates.Prev.val && offset < maxMatchOffset { - candidateAlt = candidates.Prev - } - break - } - } else { - // We only check if value mismatches. - // Offset will always be invalid in other cases. - candidate = candidates.Prev - if cv == candidate.val { - offset := s - (candidate.offset - e.cur) - if offset < maxMatchOffset { - break - } - } - } - cv = now - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - emitLiteral(dst, src[nextEmit:s]) - - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - - // Extend the 4-byte match as long as possible. - // - s += 4 - t := candidate.offset - e.cur + 4 - l := e.matchlen(s, t, src) - // Try alternative candidate if match length < matchLenGood. - if l < matchLenGood-4 && candidateAlt.offset != 0 { - t2 := candidateAlt.offset - e.cur + 4 - l2 := e.matchlen(s, t2, src) - if l2 > l { - l = l2 - t = t2 - } - } - // matchToken is flate's equivalent of Snappy's emitCopy. (length,offset) - dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset)) - dst.n++ - s += l - nextEmit = s - if s >= sLimit { - t += l - // Index first pair after match end. - if int(t+4) < len(src) && t > 0 { - cv := load3232(src, t) - nextHash = hash(cv) - e.table[nextHash&tableMask] = tableEntryPrev{ - Prev: e.table[nextHash&tableMask].Cur, - Cur: tableEntry{offset: e.cur + t, val: cv}, - } - } - goto emitRemainder - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-3 to s. If - // another emitCopy is not our next move, also calculate nextHash - // at s+1. At least on GOARCH=amd64, these three hash calculations - // are faster as one load64 call (with some shifts) instead of - // three load32 calls. - x := load6432(src, s-3) - prevHash := hash(uint32(x)) - e.table[prevHash&tableMask] = tableEntryPrev{ - Prev: e.table[prevHash&tableMask].Cur, - Cur: tableEntry{offset: e.cur + s - 3, val: uint32(x)}, - } - x >>= 8 - prevHash = hash(uint32(x)) - - e.table[prevHash&tableMask] = tableEntryPrev{ - Prev: e.table[prevHash&tableMask].Cur, - Cur: tableEntry{offset: e.cur + s - 2, val: uint32(x)}, - } - x >>= 8 - prevHash = hash(uint32(x)) - - e.table[prevHash&tableMask] = tableEntryPrev{ - Prev: e.table[prevHash&tableMask].Cur, - Cur: tableEntry{offset: e.cur + s - 1, val: uint32(x)}, - } - x >>= 8 - currHash := hash(uint32(x)) - candidates := e.table[currHash&tableMask] - cv = uint32(x) - e.table[currHash&tableMask] = tableEntryPrev{ - Prev: candidates.Cur, - Cur: tableEntry{offset: s + e.cur, val: cv}, - } - - // Check both candidates - candidate = candidates.Cur - candidateAlt = tableEntry{} - if cv == candidate.val { - offset := s - (candidate.offset - e.cur) - if offset <= maxMatchOffset { - offset = s - (candidates.Prev.offset - e.cur) - if cv == candidates.Prev.val && offset <= maxMatchOffset { - candidateAlt = candidates.Prev - } - continue - } - } else { - // We only check if value mismatches. - // Offset will always be invalid in other cases. - candidate = candidates.Prev - if cv == candidate.val { - offset := s - (candidate.offset - e.cur) - if offset <= maxMatchOffset { - continue - } - } - } - cv = uint32(x >> 8) - nextHash = hash(cv) - s++ - break - } - } - -emitRemainder: - if int(nextEmit) < len(src) { - emitLiteral(dst, src[nextEmit:]) - } - e.cur += int32(len(src)) - e.prev = e.prev[:len(src)] - copy(e.prev, src) -} - -func (e *snappyGen) matchlen(s, t int32, src []byte) int32 { - s1 := int(s) + maxMatchLength - 4 - if s1 > len(src) { - s1 = len(src) - } - - // If we are inside the current block - if t >= 0 { - b := src[t:] - a := src[s:s1] - b = b[:len(a)] - // Extend the match to be as long as possible. - for i := range a { - if a[i] != b[i] { - return int32(i) - } - } - return int32(len(a)) - } - - // We found a match in the previous block. - tp := int32(len(e.prev)) + t - if tp < 0 { - return 0 - } - - // Extend the match to be as long as possible. - a := src[s:s1] - b := e.prev[tp:] - if len(b) > len(a) { - b = b[:len(a)] - } - a = a[:len(b)] - for i := range b { - if a[i] != b[i] { - return int32(i) - } - } - - // If we reached our limit, we matched everything we are - // allowed to in the previous block and we return. - n := int32(len(b)) - if int(s+n) == s1 { - return n - } - - // Continue looking for more matches in the current block. - a = src[s+n : s1] - b = src[:len(a)] - for i := range a { - if a[i] != b[i] { - return int32(i) + n - } - } - return int32(len(a)) + n -} - -// Reset the encoding table. -func (e *snappyGen) Reset() { - e.prev = e.prev[:0] - e.cur += maxMatchOffset -} diff --git a/vendor/github.com/klauspost/compress/flate/token.go b/vendor/github.com/klauspost/compress/flate/token.go deleted file mode 100644 index 4f275ea6..00000000 --- a/vendor/github.com/klauspost/compress/flate/token.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import "fmt" - -const ( - // 2 bits: type 0 = literal 1=EOF 2=Match 3=Unused - // 8 bits: xlength = length - MIN_MATCH_LENGTH - // 22 bits xoffset = offset - MIN_OFFSET_SIZE, or literal - lengthShift = 22 - offsetMask = 1< pair into a match token. -func matchToken(xlength uint32, xoffset uint32) token { - return token(matchType + xlength< maxMatchLength || xoffset > maxMatchOffset { - panic(fmt.Sprintf("Invalid match: len: %d, offset: %d\n", xlength, xoffset)) - return token(matchType) - } - return token(matchType + xlength<> lengthShift) } - -func lengthCode(len uint32) uint32 { return lengthCodes[len] } - -// Returns the offset code corresponding to a specific offset -func offsetCode(off uint32) uint32 { - if off < uint32(len(offsetCodes)) { - return offsetCodes[off] - } else if off>>7 < uint32(len(offsetCodes)) { - return offsetCodes[off>>7] + 14 - } else { - return offsetCodes[off>>14] + 28 - } -} diff --git a/vendor/github.com/klauspost/compress/gzip/gunzip.go b/vendor/github.com/klauspost/compress/gzip/gunzip.go deleted file mode 100644 index 568b5d4f..00000000 --- a/vendor/github.com/klauspost/compress/gzip/gunzip.go +++ /dev/null @@ -1,344 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package gzip implements reading and writing of gzip format compressed files, -// as specified in RFC 1952. -package gzip - -import ( - "bufio" - "encoding/binary" - "errors" - "hash/crc32" - "io" - "time" - - "github.com/klauspost/compress/flate" -) - -const ( - gzipID1 = 0x1f - gzipID2 = 0x8b - gzipDeflate = 8 - flagText = 1 << 0 - flagHdrCrc = 1 << 1 - flagExtra = 1 << 2 - flagName = 1 << 3 - flagComment = 1 << 4 -) - -var ( - // ErrChecksum is returned when reading GZIP data that has an invalid checksum. - ErrChecksum = errors.New("gzip: invalid checksum") - // ErrHeader is returned when reading GZIP data that has an invalid header. - ErrHeader = errors.New("gzip: invalid header") -) - -var le = binary.LittleEndian - -// noEOF converts io.EOF to io.ErrUnexpectedEOF. -func noEOF(err error) error { - if err == io.EOF { - return io.ErrUnexpectedEOF - } - return err -} - -// The gzip file stores a header giving metadata about the compressed file. -// That header is exposed as the fields of the Writer and Reader structs. -// -// Strings must be UTF-8 encoded and may only contain Unicode code points -// U+0001 through U+00FF, due to limitations of the GZIP file format. -type Header struct { - Comment string // comment - Extra []byte // "extra data" - ModTime time.Time // modification time - Name string // file name - OS byte // operating system type -} - -// A Reader is an io.Reader that can be read to retrieve -// uncompressed data from a gzip-format compressed file. -// -// In general, a gzip file can be a concatenation of gzip files, -// each with its own header. Reads from the Reader -// return the concatenation of the uncompressed data of each. -// Only the first header is recorded in the Reader fields. -// -// Gzip files store a length and checksum of the uncompressed data. -// The Reader will return a ErrChecksum when Read -// reaches the end of the uncompressed data if it does not -// have the expected length or checksum. Clients should treat data -// returned by Read as tentative until they receive the io.EOF -// marking the end of the data. -type Reader struct { - Header // valid after NewReader or Reader.Reset - r flate.Reader - decompressor io.ReadCloser - digest uint32 // CRC-32, IEEE polynomial (section 8) - size uint32 // Uncompressed size (section 2.3.1) - buf [512]byte - err error - multistream bool -} - -// NewReader creates a new Reader reading the given reader. -// If r does not also implement io.ByteReader, -// the decompressor may read more data than necessary from r. -// -// It is the caller's responsibility to call Close on the Reader when done. -// -// The Reader.Header fields will be valid in the Reader returned. -func NewReader(r io.Reader) (*Reader, error) { - z := new(Reader) - if err := z.Reset(r); err != nil { - return nil, err - } - return z, nil -} - -// Reset discards the Reader z's state and makes it equivalent to the -// result of its original state from NewReader, but reading from r instead. -// This permits reusing a Reader rather than allocating a new one. -func (z *Reader) Reset(r io.Reader) error { - *z = Reader{ - decompressor: z.decompressor, - multistream: true, - } - if rr, ok := r.(flate.Reader); ok { - z.r = rr - } else { - z.r = bufio.NewReader(r) - } - z.Header, z.err = z.readHeader() - return z.err -} - -// Multistream controls whether the reader supports multistream files. -// -// If enabled (the default), the Reader expects the input to be a sequence -// of individually gzipped data streams, each with its own header and -// trailer, ending at EOF. The effect is that the concatenation of a sequence -// of gzipped files is treated as equivalent to the gzip of the concatenation -// of the sequence. This is standard behavior for gzip readers. -// -// Calling Multistream(false) disables this behavior; disabling the behavior -// can be useful when reading file formats that distinguish individual gzip -// data streams or mix gzip data streams with other data streams. -// In this mode, when the Reader reaches the end of the data stream, -// Read returns io.EOF. If the underlying reader implements io.ByteReader, -// it will be left positioned just after the gzip stream. -// To start the next stream, call z.Reset(r) followed by z.Multistream(false). -// If there is no next stream, z.Reset(r) will return io.EOF. -func (z *Reader) Multistream(ok bool) { - z.multistream = ok -} - -// readString reads a NUL-terminated string from z.r. -// It treats the bytes read as being encoded as ISO 8859-1 (Latin-1) and -// will output a string encoded using UTF-8. -// This method always updates z.digest with the data read. -func (z *Reader) readString() (string, error) { - var err error - needConv := false - for i := 0; ; i++ { - if i >= len(z.buf) { - return "", ErrHeader - } - z.buf[i], err = z.r.ReadByte() - if err != nil { - return "", err - } - if z.buf[i] > 0x7f { - needConv = true - } - if z.buf[i] == 0 { - // Digest covers the NUL terminator. - z.digest = crc32.Update(z.digest, crc32.IEEETable, z.buf[:i+1]) - - // Strings are ISO 8859-1, Latin-1 (RFC 1952, section 2.3.1). - if needConv { - s := make([]rune, 0, i) - for _, v := range z.buf[:i] { - s = append(s, rune(v)) - } - return string(s), nil - } - return string(z.buf[:i]), nil - } - } -} - -// readHeader reads the GZIP header according to section 2.3.1. -// This method does not set z.err. -func (z *Reader) readHeader() (hdr Header, err error) { - if _, err = io.ReadFull(z.r, z.buf[:10]); err != nil { - // RFC 1952, section 2.2, says the following: - // A gzip file consists of a series of "members" (compressed data sets). - // - // Other than this, the specification does not clarify whether a - // "series" is defined as "one or more" or "zero or more". To err on the - // side of caution, Go interprets this to mean "zero or more". - // Thus, it is okay to return io.EOF here. - return hdr, err - } - if z.buf[0] != gzipID1 || z.buf[1] != gzipID2 || z.buf[2] != gzipDeflate { - return hdr, ErrHeader - } - flg := z.buf[3] - hdr.ModTime = time.Unix(int64(le.Uint32(z.buf[4:8])), 0) - // z.buf[8] is XFL and is currently ignored. - hdr.OS = z.buf[9] - z.digest = crc32.ChecksumIEEE(z.buf[:10]) - - if flg&flagExtra != 0 { - if _, err = io.ReadFull(z.r, z.buf[:2]); err != nil { - return hdr, noEOF(err) - } - z.digest = crc32.Update(z.digest, crc32.IEEETable, z.buf[:2]) - data := make([]byte, le.Uint16(z.buf[:2])) - if _, err = io.ReadFull(z.r, data); err != nil { - return hdr, noEOF(err) - } - z.digest = crc32.Update(z.digest, crc32.IEEETable, data) - hdr.Extra = data - } - - var s string - if flg&flagName != 0 { - if s, err = z.readString(); err != nil { - return hdr, err - } - hdr.Name = s - } - - if flg&flagComment != 0 { - if s, err = z.readString(); err != nil { - return hdr, err - } - hdr.Comment = s - } - - if flg&flagHdrCrc != 0 { - if _, err = io.ReadFull(z.r, z.buf[:2]); err != nil { - return hdr, noEOF(err) - } - digest := le.Uint16(z.buf[:2]) - if digest != uint16(z.digest) { - return hdr, ErrHeader - } - } - - z.digest = 0 - if z.decompressor == nil { - z.decompressor = flate.NewReader(z.r) - } else { - z.decompressor.(flate.Resetter).Reset(z.r, nil) - } - return hdr, nil -} - -// Read implements io.Reader, reading uncompressed bytes from its underlying Reader. -func (z *Reader) Read(p []byte) (n int, err error) { - if z.err != nil { - return 0, z.err - } - - n, z.err = z.decompressor.Read(p) - z.digest = crc32.Update(z.digest, crc32.IEEETable, p[:n]) - z.size += uint32(n) - if z.err != io.EOF { - // In the normal case we return here. - return n, z.err - } - - // Finished file; check checksum and size. - if _, err := io.ReadFull(z.r, z.buf[:8]); err != nil { - z.err = noEOF(err) - return n, z.err - } - digest := le.Uint32(z.buf[:4]) - size := le.Uint32(z.buf[4:8]) - if digest != z.digest || size != z.size { - z.err = ErrChecksum - return n, z.err - } - z.digest, z.size = 0, 0 - - // File is ok; check if there is another. - if !z.multistream { - return n, io.EOF - } - z.err = nil // Remove io.EOF - - if _, z.err = z.readHeader(); z.err != nil { - return n, z.err - } - - // Read from next file, if necessary. - if n > 0 { - return n, nil - } - return z.Read(p) -} - -// Support the io.WriteTo interface for io.Copy and friends. -func (z *Reader) WriteTo(w io.Writer) (int64, error) { - total := int64(0) - crcWriter := crc32.NewIEEE() - for { - if z.err != nil { - if z.err == io.EOF { - return total, nil - } - return total, z.err - } - - // We write both to output and digest. - mw := io.MultiWriter(w, crcWriter) - n, err := z.decompressor.(io.WriterTo).WriteTo(mw) - total += n - z.size += uint32(n) - if err != nil { - z.err = err - return total, z.err - } - - // Finished file; check checksum + size. - if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - z.err = err - return total, err - } - z.digest = crcWriter.Sum32() - digest := le.Uint32(z.buf[:4]) - size := le.Uint32(z.buf[4:8]) - if digest != z.digest || size != z.size { - z.err = ErrChecksum - return total, z.err - } - z.digest, z.size = 0, 0 - - // File is ok; check if there is another. - if !z.multistream { - return total, nil - } - crcWriter.Reset() - z.err = nil // Remove io.EOF - - if _, z.err = z.readHeader(); z.err != nil { - if z.err == io.EOF { - return total, nil - } - return total, z.err - } - } -} - -// Close closes the Reader. It does not close the underlying io.Reader. -// In order for the GZIP checksum to be verified, the reader must be -// fully consumed until the io.EOF. -func (z *Reader) Close() error { return z.decompressor.Close() } diff --git a/vendor/github.com/klauspost/compress/gzip/gzip.go b/vendor/github.com/klauspost/compress/gzip/gzip.go deleted file mode 100644 index 7da7ee74..00000000 --- a/vendor/github.com/klauspost/compress/gzip/gzip.go +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package gzip - -import ( - "errors" - "fmt" - "hash/crc32" - "io" - - "github.com/klauspost/compress/flate" -) - -// These constants are copied from the flate package, so that code that imports -// "compress/gzip" does not also have to import "compress/flate". -const ( - NoCompression = flate.NoCompression - BestSpeed = flate.BestSpeed - BestCompression = flate.BestCompression - DefaultCompression = flate.DefaultCompression - ConstantCompression = flate.ConstantCompression - HuffmanOnly = flate.HuffmanOnly -) - -// A Writer is an io.WriteCloser. -// Writes to a Writer are compressed and written to w. -type Writer struct { - Header // written at first call to Write, Flush, or Close - w io.Writer - level int - wroteHeader bool - compressor *flate.Writer - digest uint32 // CRC-32, IEEE polynomial (section 8) - size uint32 // Uncompressed size (section 2.3.1) - closed bool - buf [10]byte - err error -} - -// NewWriter returns a new Writer. -// Writes to the returned writer are compressed and written to w. -// -// It is the caller's responsibility to call Close on the WriteCloser when done. -// Writes may be buffered and not flushed until Close. -// -// Callers that wish to set the fields in Writer.Header must do so before -// the first call to Write, Flush, or Close. -func NewWriter(w io.Writer) *Writer { - z, _ := NewWriterLevel(w, DefaultCompression) - return z -} - -// NewWriterLevel is like NewWriter but specifies the compression level instead -// of assuming DefaultCompression. -// -// The compression level can be DefaultCompression, NoCompression, or any -// integer value between BestSpeed and BestCompression inclusive. The error -// returned will be nil if the level is valid. -func NewWriterLevel(w io.Writer, level int) (*Writer, error) { - if level < HuffmanOnly || level > BestCompression { - return nil, fmt.Errorf("gzip: invalid compression level: %d", level) - } - z := new(Writer) - z.init(w, level) - return z, nil -} - -func (z *Writer) init(w io.Writer, level int) { - compressor := z.compressor - if compressor != nil { - compressor.Reset(w) - } - *z = Writer{ - Header: Header{ - OS: 255, // unknown - }, - w: w, - level: level, - compressor: compressor, - } -} - -// Reset discards the Writer z's state and makes it equivalent to the -// result of its original state from NewWriter or NewWriterLevel, but -// writing to w instead. This permits reusing a Writer rather than -// allocating a new one. -func (z *Writer) Reset(w io.Writer) { - z.init(w, z.level) -} - -// writeBytes writes a length-prefixed byte slice to z.w. -func (z *Writer) writeBytes(b []byte) error { - if len(b) > 0xffff { - return errors.New("gzip.Write: Extra data is too large") - } - le.PutUint16(z.buf[:2], uint16(len(b))) - _, err := z.w.Write(z.buf[:2]) - if err != nil { - return err - } - _, err = z.w.Write(b) - return err -} - -// writeString writes a UTF-8 string s in GZIP's format to z.w. -// GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1). -func (z *Writer) writeString(s string) (err error) { - // GZIP stores Latin-1 strings; error if non-Latin-1; convert if non-ASCII. - needconv := false - for _, v := range s { - if v == 0 || v > 0xff { - return errors.New("gzip.Write: non-Latin-1 header string") - } - if v > 0x7f { - needconv = true - } - } - if needconv { - b := make([]byte, 0, len(s)) - for _, v := range s { - b = append(b, byte(v)) - } - _, err = z.w.Write(b) - } else { - _, err = io.WriteString(z.w, s) - } - if err != nil { - return err - } - // GZIP strings are NUL-terminated. - z.buf[0] = 0 - _, err = z.w.Write(z.buf[:1]) - return err -} - -// Write writes a compressed form of p to the underlying io.Writer. The -// compressed bytes are not necessarily flushed until the Writer is closed. -func (z *Writer) Write(p []byte) (int, error) { - if z.err != nil { - return 0, z.err - } - var n int - // Write the GZIP header lazily. - if !z.wroteHeader { - z.wroteHeader = true - z.buf[0] = gzipID1 - z.buf[1] = gzipID2 - z.buf[2] = gzipDeflate - z.buf[3] = 0 - if z.Extra != nil { - z.buf[3] |= 0x04 - } - if z.Name != "" { - z.buf[3] |= 0x08 - } - if z.Comment != "" { - z.buf[3] |= 0x10 - } - le.PutUint32(z.buf[4:8], uint32(z.ModTime.Unix())) - if z.level == BestCompression { - z.buf[8] = 2 - } else if z.level == BestSpeed { - z.buf[8] = 4 - } else { - z.buf[8] = 0 - } - z.buf[9] = z.OS - n, z.err = z.w.Write(z.buf[:10]) - if z.err != nil { - return n, z.err - } - if z.Extra != nil { - z.err = z.writeBytes(z.Extra) - if z.err != nil { - return n, z.err - } - } - if z.Name != "" { - z.err = z.writeString(z.Name) - if z.err != nil { - return n, z.err - } - } - if z.Comment != "" { - z.err = z.writeString(z.Comment) - if z.err != nil { - return n, z.err - } - } - if z.compressor == nil { - z.compressor, _ = flate.NewWriter(z.w, z.level) - } - } - z.size += uint32(len(p)) - z.digest = crc32.Update(z.digest, crc32.IEEETable, p) - n, z.err = z.compressor.Write(p) - return n, z.err -} - -// Flush flushes any pending compressed data to the underlying writer. -// -// It is useful mainly in compressed network protocols, to ensure that -// a remote reader has enough data to reconstruct a packet. Flush does -// not return until the data has been written. If the underlying -// writer returns an error, Flush returns that error. -// -// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH. -func (z *Writer) Flush() error { - if z.err != nil { - return z.err - } - if z.closed { - return nil - } - if !z.wroteHeader { - z.Write(nil) - if z.err != nil { - return z.err - } - } - z.err = z.compressor.Flush() - return z.err -} - -// Close closes the Writer, flushing any unwritten data to the underlying -// io.Writer, but does not close the underlying io.Writer. -func (z *Writer) Close() error { - if z.err != nil { - return z.err - } - if z.closed { - return nil - } - z.closed = true - if !z.wroteHeader { - z.Write(nil) - if z.err != nil { - return z.err - } - } - z.err = z.compressor.Close() - if z.err != nil { - return z.err - } - le.PutUint32(z.buf[:4], z.digest) - le.PutUint32(z.buf[4:8], z.size) - _, z.err = z.w.Write(z.buf[:8]) - return z.err -} diff --git a/vendor/github.com/klauspost/compress/snappy/AUTHORS b/vendor/github.com/klauspost/compress/snappy/AUTHORS deleted file mode 100644 index fd1c6f67..00000000 --- a/vendor/github.com/klauspost/compress/snappy/AUTHORS +++ /dev/null @@ -1,15 +0,0 @@ -# This is the official list of Snappy-Go authors for copyright purposes. -# This file is distinct from the CONTRIBUTORS files. -# See the latter for an explanation. - -# Names should be added to this file as -# Name or Organization -# The email address is not required for organizations. - -# Please keep the list sorted. - -Damian Gryski -Google Inc. -Jan Mercl <0xjnml@gmail.com> -Rodolfo Carvalho -Sebastien Binet diff --git a/vendor/github.com/klauspost/compress/snappy/CONTRIBUTORS b/vendor/github.com/klauspost/compress/snappy/CONTRIBUTORS deleted file mode 100644 index a29b1339..00000000 --- a/vendor/github.com/klauspost/compress/snappy/CONTRIBUTORS +++ /dev/null @@ -1,37 +0,0 @@ -# This is the official list of people who can contribute -# (and typically have contributed) code to the Snappy-Go repository. -# The AUTHORS file lists the copyright holders; this file -# lists people. For example, Google employees are listed here -# but not in AUTHORS, because Google holds the copyright. -# -# The submission process automatically checks to make sure -# that people submitting code are listed in this file (by email address). -# -# Names should be added to this file only after verifying that -# the individual or the individual's organization has agreed to -# the appropriate Contributor License Agreement, found here: -# -# http://code.google.com/legal/individual-cla-v1.0.html -# http://code.google.com/legal/corporate-cla-v1.0.html -# -# The agreement for individuals can be filled out on the web. -# -# When adding J Random Contributor's name to this file, -# either J's name or J's organization's name should be -# added to the AUTHORS file, depending on whether the -# individual or corporate CLA was used. - -# Names should be added to this file like so: -# Name - -# Please keep the list sorted. - -Damian Gryski -Jan Mercl <0xjnml@gmail.com> -Kai Backman -Marc-Antoine Ruel -Nigel Tao -Rob Pike -Rodolfo Carvalho -Russ Cox -Sebastien Binet diff --git a/vendor/github.com/klauspost/compress/snappy/LICENSE b/vendor/github.com/klauspost/compress/snappy/LICENSE deleted file mode 100644 index 7e69e1a2..00000000 --- a/vendor/github.com/klauspost/compress/snappy/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/klauspost/compress/zlib/reader.go b/vendor/github.com/klauspost/compress/zlib/reader.go deleted file mode 100644 index d9091e83..00000000 --- a/vendor/github.com/klauspost/compress/zlib/reader.go +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -/* -Package zlib implements reading and writing of zlib format compressed data, -as specified in RFC 1950. - -The implementation provides filters that uncompress during reading -and compress during writing. For example, to write compressed data -to a buffer: - - var b bytes.Buffer - w := zlib.NewWriter(&b) - w.Write([]byte("hello, world\n")) - w.Close() - -and to read that data back: - - r, err := zlib.NewReader(&b) - io.Copy(os.Stdout, r) - r.Close() -*/ -package zlib - -import ( - "bufio" - "errors" - "hash" - "hash/adler32" - "io" - - "github.com/klauspost/compress/flate" -) - -const zlibDeflate = 8 - -var ( - // ErrChecksum is returned when reading ZLIB data that has an invalid checksum. - ErrChecksum = errors.New("zlib: invalid checksum") - // ErrDictionary is returned when reading ZLIB data that has an invalid dictionary. - ErrDictionary = errors.New("zlib: invalid dictionary") - // ErrHeader is returned when reading ZLIB data that has an invalid header. - ErrHeader = errors.New("zlib: invalid header") -) - -type reader struct { - r flate.Reader - decompressor io.ReadCloser - digest hash.Hash32 - err error - scratch [4]byte -} - -// Resetter resets a ReadCloser returned by NewReader or NewReaderDict to -// to switch to a new underlying Reader. This permits reusing a ReadCloser -// instead of allocating a new one. -type Resetter interface { - // Reset discards any buffered data and resets the Resetter as if it was - // newly initialized with the given reader. - Reset(r io.Reader, dict []byte) error -} - -// NewReader creates a new ReadCloser. -// Reads from the returned ReadCloser read and decompress data from r. -// If r does not implement io.ByteReader, the decompressor may read more -// data than necessary from r. -// It is the caller's responsibility to call Close on the ReadCloser when done. -// -// The ReadCloser returned by NewReader also implements Resetter. -func NewReader(r io.Reader) (io.ReadCloser, error) { - return NewReaderDict(r, nil) -} - -// NewReaderDict is like NewReader but uses a preset dictionary. -// NewReaderDict ignores the dictionary if the compressed data does not refer to it. -// If the compressed data refers to a different dictionary, NewReaderDict returns ErrDictionary. -// -// The ReadCloser returned by NewReaderDict also implements Resetter. -func NewReaderDict(r io.Reader, dict []byte) (io.ReadCloser, error) { - z := new(reader) - err := z.Reset(r, dict) - if err != nil { - return nil, err - } - return z, nil -} - -func (z *reader) Read(p []byte) (int, error) { - if z.err != nil { - return 0, z.err - } - - var n int - n, z.err = z.decompressor.Read(p) - z.digest.Write(p[0:n]) - if z.err != io.EOF { - // In the normal case we return here. - return n, z.err - } - - // Finished file; check checksum. - if _, err := io.ReadFull(z.r, z.scratch[0:4]); err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - z.err = err - return n, z.err - } - // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). - checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3]) - if checksum != z.digest.Sum32() { - z.err = ErrChecksum - return n, z.err - } - return n, io.EOF -} - -// Calling Close does not close the wrapped io.Reader originally passed to NewReader. -// In order for the ZLIB checksum to be verified, the reader must be -// fully consumed until the io.EOF. -func (z *reader) Close() error { - if z.err != nil && z.err != io.EOF { - return z.err - } - z.err = z.decompressor.Close() - return z.err -} - -func (z *reader) Reset(r io.Reader, dict []byte) error { - *z = reader{decompressor: z.decompressor, digest: z.digest} - if fr, ok := r.(flate.Reader); ok { - z.r = fr - } else { - z.r = bufio.NewReader(r) - } - - // Read the header (RFC 1950 section 2.2.). - _, z.err = io.ReadFull(z.r, z.scratch[0:2]) - if z.err != nil { - if z.err == io.EOF { - z.err = io.ErrUnexpectedEOF - } - return z.err - } - h := uint(z.scratch[0])<<8 | uint(z.scratch[1]) - if (z.scratch[0]&0x0f != zlibDeflate) || (h%31 != 0) { - z.err = ErrHeader - return z.err - } - haveDict := z.scratch[1]&0x20 != 0 - if haveDict { - _, z.err = io.ReadFull(z.r, z.scratch[0:4]) - if z.err != nil { - if z.err == io.EOF { - z.err = io.ErrUnexpectedEOF - } - return z.err - } - checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3]) - if checksum != adler32.Checksum(dict) { - z.err = ErrDictionary - return z.err - } - } - - if z.decompressor == nil { - if haveDict { - z.decompressor = flate.NewReaderDict(z.r, dict) - } else { - z.decompressor = flate.NewReader(z.r) - } - } else { - z.decompressor.(flate.Resetter).Reset(z.r, dict) - } - - if z.digest != nil { - z.digest.Reset() - } else { - z.digest = adler32.New() - } - return nil -} diff --git a/vendor/github.com/klauspost/compress/zlib/writer.go b/vendor/github.com/klauspost/compress/zlib/writer.go deleted file mode 100644 index 605816ba..00000000 --- a/vendor/github.com/klauspost/compress/zlib/writer.go +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package zlib - -import ( - "fmt" - "hash" - "hash/adler32" - "io" - - "github.com/klauspost/compress/flate" -) - -// These constants are copied from the flate package, so that code that imports -// "compress/zlib" does not also have to import "compress/flate". -const ( - NoCompression = flate.NoCompression - BestSpeed = flate.BestSpeed - BestCompression = flate.BestCompression - DefaultCompression = flate.DefaultCompression - ConstantCompression = flate.ConstantCompression - HuffmanOnly = flate.HuffmanOnly -) - -// A Writer takes data written to it and writes the compressed -// form of that data to an underlying writer (see NewWriter). -type Writer struct { - w io.Writer - level int - dict []byte - compressor *flate.Writer - digest hash.Hash32 - err error - scratch [4]byte - wroteHeader bool -} - -// NewWriter creates a new Writer. -// Writes to the returned Writer are compressed and written to w. -// -// It is the caller's responsibility to call Close on the WriteCloser when done. -// Writes may be buffered and not flushed until Close. -func NewWriter(w io.Writer) *Writer { - z, _ := NewWriterLevelDict(w, DefaultCompression, nil) - return z -} - -// NewWriterLevel is like NewWriter but specifies the compression level instead -// of assuming DefaultCompression. -// -// The compression level can be DefaultCompression, NoCompression, HuffmanOnly -// or any integer value between BestSpeed and BestCompression inclusive. -// The error returned will be nil if the level is valid. -func NewWriterLevel(w io.Writer, level int) (*Writer, error) { - return NewWriterLevelDict(w, level, nil) -} - -// NewWriterLevelDict is like NewWriterLevel but specifies a dictionary to -// compress with. -// -// The dictionary may be nil. If not, its contents should not be modified until -// the Writer is closed. -func NewWriterLevelDict(w io.Writer, level int, dict []byte) (*Writer, error) { - if level < HuffmanOnly || level > BestCompression { - return nil, fmt.Errorf("zlib: invalid compression level: %d", level) - } - return &Writer{ - w: w, - level: level, - dict: dict, - }, nil -} - -// Reset clears the state of the Writer z such that it is equivalent to its -// initial state from NewWriterLevel or NewWriterLevelDict, but instead writing -// to w. -func (z *Writer) Reset(w io.Writer) { - z.w = w - // z.level and z.dict left unchanged. - if z.compressor != nil { - z.compressor.Reset(w) - } - if z.digest != nil { - z.digest.Reset() - } - z.err = nil - z.scratch = [4]byte{} - z.wroteHeader = false -} - -// writeHeader writes the ZLIB header. -func (z *Writer) writeHeader() (err error) { - z.wroteHeader = true - // ZLIB has a two-byte header (as documented in RFC 1950). - // The first four bits is the CINFO (compression info), which is 7 for the default deflate window size. - // The next four bits is the CM (compression method), which is 8 for deflate. - z.scratch[0] = 0x78 - // The next two bits is the FLEVEL (compression level). The four values are: - // 0=fastest, 1=fast, 2=default, 3=best. - // The next bit, FDICT, is set if a dictionary is given. - // The final five FCHECK bits form a mod-31 checksum. - switch z.level { - case -2, 0, 1: - z.scratch[1] = 0 << 6 - case 2, 3, 4, 5: - z.scratch[1] = 1 << 6 - case 6, -1: - z.scratch[1] = 2 << 6 - case 7, 8, 9: - z.scratch[1] = 3 << 6 - default: - panic("unreachable") - } - if z.dict != nil { - z.scratch[1] |= 1 << 5 - } - z.scratch[1] += uint8(31 - (uint16(z.scratch[0])<<8+uint16(z.scratch[1]))%31) - if _, err = z.w.Write(z.scratch[0:2]); err != nil { - return err - } - if z.dict != nil { - // The next four bytes are the Adler-32 checksum of the dictionary. - checksum := adler32.Checksum(z.dict) - z.scratch[0] = uint8(checksum >> 24) - z.scratch[1] = uint8(checksum >> 16) - z.scratch[2] = uint8(checksum >> 8) - z.scratch[3] = uint8(checksum >> 0) - if _, err = z.w.Write(z.scratch[0:4]); err != nil { - return err - } - } - if z.compressor == nil { - // Initialize deflater unless the Writer is being reused - // after a Reset call. - z.compressor, err = flate.NewWriterDict(z.w, z.level, z.dict) - if err != nil { - return err - } - z.digest = adler32.New() - } - return nil -} - -// Write writes a compressed form of p to the underlying io.Writer. The -// compressed bytes are not necessarily flushed until the Writer is closed or -// explicitly flushed. -func (z *Writer) Write(p []byte) (n int, err error) { - if !z.wroteHeader { - z.err = z.writeHeader() - } - if z.err != nil { - return 0, z.err - } - if len(p) == 0 { - return 0, nil - } - n, err = z.compressor.Write(p) - if err != nil { - z.err = err - return - } - z.digest.Write(p) - return -} - -// Flush flushes the Writer to its underlying io.Writer. -func (z *Writer) Flush() error { - if !z.wroteHeader { - z.err = z.writeHeader() - } - if z.err != nil { - return z.err - } - z.err = z.compressor.Flush() - return z.err -} - -// Close closes the Writer, flushing any unwritten data to the underlying -// io.Writer, but does not close the underlying io.Writer. -func (z *Writer) Close() error { - if !z.wroteHeader { - z.err = z.writeHeader() - } - if z.err != nil { - return z.err - } - z.err = z.compressor.Close() - if z.err != nil { - return z.err - } - checksum := z.digest.Sum32() - // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). - z.scratch[0] = uint8(checksum >> 24) - z.scratch[1] = uint8(checksum >> 16) - z.scratch[2] = uint8(checksum >> 8) - z.scratch[3] = uint8(checksum >> 0) - _, z.err = z.w.Write(z.scratch[0:4]) - return z.err -} diff --git a/vendor/github.com/klauspost/cpuid/.gitignore b/vendor/github.com/klauspost/cpuid/.gitignore deleted file mode 100644 index daf913b1..00000000 --- a/vendor/github.com/klauspost/cpuid/.gitignore +++ /dev/null @@ -1,24 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe -*.test -*.prof diff --git a/vendor/github.com/klauspost/cpuid/.travis.yml b/vendor/github.com/klauspost/cpuid/.travis.yml deleted file mode 100644 index 2332ea6d..00000000 --- a/vendor/github.com/klauspost/cpuid/.travis.yml +++ /dev/null @@ -1,24 +0,0 @@ -language: go - -sudo: false - -os: - - linux - - osx -go: - - 1.5.x - - 1.6.x - - 1.7.x - - 1.8.x - - master - -script: - - go vet ./... - - go test -v ./... - - go test -race ./... - - diff <(gofmt -d .) <("") - -matrix: - allow_failures: - - go: 'master' - fast_finish: true diff --git a/vendor/github.com/klauspost/cpuid/LICENSE b/vendor/github.com/klauspost/cpuid/LICENSE deleted file mode 100644 index 5cec7ee9..00000000 --- a/vendor/github.com/klauspost/cpuid/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 Klaus Post - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/vendor/github.com/klauspost/cpuid/README.md b/vendor/github.com/klauspost/cpuid/README.md deleted file mode 100644 index b2b6bee8..00000000 --- a/vendor/github.com/klauspost/cpuid/README.md +++ /dev/null @@ -1,145 +0,0 @@ -# cpuid -Package cpuid provides information about the CPU running the current program. - -CPU features are detected on startup, and kept for fast access through the life of the application. -Currently x86 / x64 (AMD64) is supported, and no external C (cgo) code is used, which should make the library very easy to use. - -You can access the CPU information by accessing the shared CPU variable of the cpuid library. - -Package home: https://github.com/klauspost/cpuid - -[![GoDoc][1]][2] [![Build Status][3]][4] - -[1]: https://godoc.org/github.com/klauspost/cpuid?status.svg -[2]: https://godoc.org/github.com/klauspost/cpuid -[3]: https://travis-ci.org/klauspost/cpuid.svg -[4]: https://travis-ci.org/klauspost/cpuid - -# features -## CPU Instructions -* **CMOV** (i686 CMOV) -* **NX** (NX (No-Execute) bit) -* **AMD3DNOW** (AMD 3DNOW) -* **AMD3DNOWEXT** (AMD 3DNowExt) -* **MMX** (standard MMX) -* **MMXEXT** (SSE integer functions or AMD MMX ext) -* **SSE** (SSE functions) -* **SSE2** (P4 SSE functions) -* **SSE3** (Prescott SSE3 functions) -* **SSSE3** (Conroe SSSE3 functions) -* **SSE4** (Penryn SSE4.1 functions) -* **SSE4A** (AMD Barcelona microarchitecture SSE4a instructions) -* **SSE42** (Nehalem SSE4.2 functions) -* **AVX** (AVX functions) -* **AVX2** (AVX2 functions) -* **FMA3** (Intel FMA 3) -* **FMA4** (Bulldozer FMA4 functions) -* **XOP** (Bulldozer XOP functions) -* **F16C** (Half-precision floating-point conversion) -* **BMI1** (Bit Manipulation Instruction Set 1) -* **BMI2** (Bit Manipulation Instruction Set 2) -* **TBM** (AMD Trailing Bit Manipulation) -* **LZCNT** (LZCNT instruction) -* **POPCNT** (POPCNT instruction) -* **AESNI** (Advanced Encryption Standard New Instructions) -* **CLMUL** (Carry-less Multiplication) -* **HTT** (Hyperthreading (enabled)) -* **HLE** (Hardware Lock Elision) -* **RTM** (Restricted Transactional Memory) -* **RDRAND** (RDRAND instruction is available) -* **RDSEED** (RDSEED instruction is available) -* **ADX** (Intel ADX (Multi-Precision Add-Carry Instruction Extensions)) -* **SHA** (Intel SHA Extensions) -* **AVX512F** (AVX-512 Foundation) -* **AVX512DQ** (AVX-512 Doubleword and Quadword Instructions) -* **AVX512IFMA** (AVX-512 Integer Fused Multiply-Add Instructions) -* **AVX512PF** (AVX-512 Prefetch Instructions) -* **AVX512ER** (AVX-512 Exponential and Reciprocal Instructions) -* **AVX512CD** (AVX-512 Conflict Detection Instructions) -* **AVX512BW** (AVX-512 Byte and Word Instructions) -* **AVX512VL** (AVX-512 Vector Length Extensions) -* **AVX512VBMI** (AVX-512 Vector Bit Manipulation Instructions) -* **MPX** (Intel MPX (Memory Protection Extensions)) -* **ERMS** (Enhanced REP MOVSB/STOSB) -* **RDTSCP** (RDTSCP Instruction) -* **CX16** (CMPXCHG16B Instruction) -* **SGX** (Software Guard Extensions, with activation details) - -## Performance -* **RDTSCP()** Returns current cycle count. Can be used for benchmarking. -* **SSE2SLOW** (SSE2 is supported, but usually not faster) -* **SSE3SLOW** (SSE3 is supported, but usually not faster) -* **ATOM** (Atom processor, some SSSE3 instructions are slower) -* **Cache line** (Probable size of a cache line). -* **L1, L2, L3 Cache size** on newer Intel/AMD CPUs. - -## Cpu Vendor/VM -* **Intel** -* **AMD** -* **VIA** -* **Transmeta** -* **NSC** -* **KVM** (Kernel-based Virtual Machine) -* **MSVM** (Microsoft Hyper-V or Windows Virtual PC) -* **VMware** -* **XenHVM** - -# installing - -```go get github.com/klauspost/cpuid``` - -# example - -```Go -package main - -import ( - "fmt" - "github.com/klauspost/cpuid" -) - -func main() { - // Print basic CPU information: - fmt.Println("Name:", cpuid.CPU.BrandName) - fmt.Println("PhysicalCores:", cpuid.CPU.PhysicalCores) - fmt.Println("ThreadsPerCore:", cpuid.CPU.ThreadsPerCore) - fmt.Println("LogicalCores:", cpuid.CPU.LogicalCores) - fmt.Println("Family", cpuid.CPU.Family, "Model:", cpuid.CPU.Model) - fmt.Println("Features:", cpuid.CPU.Features) - fmt.Println("Cacheline bytes:", cpuid.CPU.CacheLine) - fmt.Println("L1 Data Cache:", cpuid.CPU.Cache.L1D, "bytes") - fmt.Println("L1 Instruction Cache:", cpuid.CPU.Cache.L1D, "bytes") - fmt.Println("L2 Cache:", cpuid.CPU.Cache.L2, "bytes") - fmt.Println("L3 Cache:", cpuid.CPU.Cache.L3, "bytes") - - // Test if we have a specific feature: - if cpuid.CPU.SSE() { - fmt.Println("We have Streaming SIMD Extensions") - } -} -``` - -Sample output: -``` ->go run main.go -Name: Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz -PhysicalCores: 2 -ThreadsPerCore: 2 -LogicalCores: 4 -Family 6 Model: 42 -Features: CMOV,MMX,MMXEXT,SSE,SSE2,SSE3,SSSE3,SSE4.1,SSE4.2,AVX,AESNI,CLMUL -Cacheline bytes: 64 -We have Streaming SIMD Extensions -``` - -# private package - -In the "private" folder you can find an autogenerated version of the library you can include in your own packages. - -For this purpose all exports are removed, and functions and constants are lowercased. - -This is not a recommended way of using the library, but provided for convenience, if it is difficult for you to use external packages. - -# license - -This code is published under an MIT license. See LICENSE file for more information. diff --git a/vendor/github.com/klauspost/cpuid/cpuid.go b/vendor/github.com/klauspost/cpuid/cpuid.go deleted file mode 100644 index 44e50946..00000000 --- a/vendor/github.com/klauspost/cpuid/cpuid.go +++ /dev/null @@ -1,1030 +0,0 @@ -// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. - -// Package cpuid provides information about the CPU running the current program. -// -// CPU features are detected on startup, and kept for fast access through the life of the application. -// Currently x86 / x64 (AMD64) is supported. -// -// You can access the CPU information by accessing the shared CPU variable of the cpuid library. -// -// Package home: https://github.com/klauspost/cpuid -package cpuid - -import "strings" - -// Vendor is a representation of a CPU vendor. -type Vendor int - -const ( - Other Vendor = iota - Intel - AMD - VIA - Transmeta - NSC - KVM // Kernel-based Virtual Machine - MSVM // Microsoft Hyper-V or Windows Virtual PC - VMware - XenHVM -) - -const ( - CMOV = 1 << iota // i686 CMOV - NX // NX (No-Execute) bit - AMD3DNOW // AMD 3DNOW - AMD3DNOWEXT // AMD 3DNowExt - MMX // standard MMX - MMXEXT // SSE integer functions or AMD MMX ext - SSE // SSE functions - SSE2 // P4 SSE functions - SSE3 // Prescott SSE3 functions - SSSE3 // Conroe SSSE3 functions - SSE4 // Penryn SSE4.1 functions - SSE4A // AMD Barcelona microarchitecture SSE4a instructions - SSE42 // Nehalem SSE4.2 functions - AVX // AVX functions - AVX2 // AVX2 functions - FMA3 // Intel FMA 3 - FMA4 // Bulldozer FMA4 functions - XOP // Bulldozer XOP functions - F16C // Half-precision floating-point conversion - BMI1 // Bit Manipulation Instruction Set 1 - BMI2 // Bit Manipulation Instruction Set 2 - TBM // AMD Trailing Bit Manipulation - LZCNT // LZCNT instruction - POPCNT // POPCNT instruction - AESNI // Advanced Encryption Standard New Instructions - CLMUL // Carry-less Multiplication - HTT // Hyperthreading (enabled) - HLE // Hardware Lock Elision - RTM // Restricted Transactional Memory - RDRAND // RDRAND instruction is available - RDSEED // RDSEED instruction is available - ADX // Intel ADX (Multi-Precision Add-Carry Instruction Extensions) - SHA // Intel SHA Extensions - AVX512F // AVX-512 Foundation - AVX512DQ // AVX-512 Doubleword and Quadword Instructions - AVX512IFMA // AVX-512 Integer Fused Multiply-Add Instructions - AVX512PF // AVX-512 Prefetch Instructions - AVX512ER // AVX-512 Exponential and Reciprocal Instructions - AVX512CD // AVX-512 Conflict Detection Instructions - AVX512BW // AVX-512 Byte and Word Instructions - AVX512VL // AVX-512 Vector Length Extensions - AVX512VBMI // AVX-512 Vector Bit Manipulation Instructions - MPX // Intel MPX (Memory Protection Extensions) - ERMS // Enhanced REP MOVSB/STOSB - RDTSCP // RDTSCP Instruction - CX16 // CMPXCHG16B Instruction - SGX // Software Guard Extensions - - // Performance indicators - SSE2SLOW // SSE2 is supported, but usually not faster - SSE3SLOW // SSE3 is supported, but usually not faster - ATOM // Atom processor, some SSSE3 instructions are slower -) - -var flagNames = map[Flags]string{ - CMOV: "CMOV", // i686 CMOV - NX: "NX", // NX (No-Execute) bit - AMD3DNOW: "AMD3DNOW", // AMD 3DNOW - AMD3DNOWEXT: "AMD3DNOWEXT", // AMD 3DNowExt - MMX: "MMX", // Standard MMX - MMXEXT: "MMXEXT", // SSE integer functions or AMD MMX ext - SSE: "SSE", // SSE functions - SSE2: "SSE2", // P4 SSE2 functions - SSE3: "SSE3", // Prescott SSE3 functions - SSSE3: "SSSE3", // Conroe SSSE3 functions - SSE4: "SSE4.1", // Penryn SSE4.1 functions - SSE4A: "SSE4A", // AMD Barcelona microarchitecture SSE4a instructions - SSE42: "SSE4.2", // Nehalem SSE4.2 functions - AVX: "AVX", // AVX functions - AVX2: "AVX2", // AVX functions - FMA3: "FMA3", // Intel FMA 3 - FMA4: "FMA4", // Bulldozer FMA4 functions - XOP: "XOP", // Bulldozer XOP functions - F16C: "F16C", // Half-precision floating-point conversion - BMI1: "BMI1", // Bit Manipulation Instruction Set 1 - BMI2: "BMI2", // Bit Manipulation Instruction Set 2 - TBM: "TBM", // AMD Trailing Bit Manipulation - LZCNT: "LZCNT", // LZCNT instruction - POPCNT: "POPCNT", // POPCNT instruction - AESNI: "AESNI", // Advanced Encryption Standard New Instructions - CLMUL: "CLMUL", // Carry-less Multiplication - HTT: "HTT", // Hyperthreading (enabled) - HLE: "HLE", // Hardware Lock Elision - RTM: "RTM", // Restricted Transactional Memory - RDRAND: "RDRAND", // RDRAND instruction is available - RDSEED: "RDSEED", // RDSEED instruction is available - ADX: "ADX", // Intel ADX (Multi-Precision Add-Carry Instruction Extensions) - SHA: "SHA", // Intel SHA Extensions - AVX512F: "AVX512F", // AVX-512 Foundation - AVX512DQ: "AVX512DQ", // AVX-512 Doubleword and Quadword Instructions - AVX512IFMA: "AVX512IFMA", // AVX-512 Integer Fused Multiply-Add Instructions - AVX512PF: "AVX512PF", // AVX-512 Prefetch Instructions - AVX512ER: "AVX512ER", // AVX-512 Exponential and Reciprocal Instructions - AVX512CD: "AVX512CD", // AVX-512 Conflict Detection Instructions - AVX512BW: "AVX512BW", // AVX-512 Byte and Word Instructions - AVX512VL: "AVX512VL", // AVX-512 Vector Length Extensions - AVX512VBMI: "AVX512VBMI", // AVX-512 Vector Bit Manipulation Instructions - MPX: "MPX", // Intel MPX (Memory Protection Extensions) - ERMS: "ERMS", // Enhanced REP MOVSB/STOSB - RDTSCP: "RDTSCP", // RDTSCP Instruction - CX16: "CX16", // CMPXCHG16B Instruction - SGX: "SGX", // Software Guard Extensions - - // Performance indicators - SSE2SLOW: "SSE2SLOW", // SSE2 supported, but usually not faster - SSE3SLOW: "SSE3SLOW", // SSE3 supported, but usually not faster - ATOM: "ATOM", // Atom processor, some SSSE3 instructions are slower - -} - -// CPUInfo contains information about the detected system CPU. -type CPUInfo struct { - BrandName string // Brand name reported by the CPU - VendorID Vendor // Comparable CPU vendor ID - Features Flags // Features of the CPU - PhysicalCores int // Number of physical processor cores in your CPU. Will be 0 if undetectable. - ThreadsPerCore int // Number of threads per physical core. Will be 1 if undetectable. - LogicalCores int // Number of physical cores times threads that can run on each core through the use of hyperthreading. Will be 0 if undetectable. - Family int // CPU family number - Model int // CPU model number - CacheLine int // Cache line size in bytes. Will be 0 if undetectable. - Cache struct { - L1I int // L1 Instruction Cache (per core or shared). Will be -1 if undetected - L1D int // L1 Data Cache (per core or shared). Will be -1 if undetected - L2 int // L2 Cache (per core or shared). Will be -1 if undetected - L3 int // L3 Instruction Cache (per core or shared). Will be -1 if undetected - } - SGX SGXSupport - maxFunc uint32 - maxExFunc uint32 -} - -var cpuid func(op uint32) (eax, ebx, ecx, edx uint32) -var cpuidex func(op, op2 uint32) (eax, ebx, ecx, edx uint32) -var xgetbv func(index uint32) (eax, edx uint32) -var rdtscpAsm func() (eax, ebx, ecx, edx uint32) - -// CPU contains information about the CPU as detected on startup, -// or when Detect last was called. -// -// Use this as the primary entry point to you data, -// this way queries are -var CPU CPUInfo - -func init() { - initCPU() - Detect() -} - -// Detect will re-detect current CPU info. -// This will replace the content of the exported CPU variable. -// -// Unless you expect the CPU to change while you are running your program -// you should not need to call this function. -// If you call this, you must ensure that no other goroutine is accessing the -// exported CPU variable. -func Detect() { - CPU.maxFunc = maxFunctionID() - CPU.maxExFunc = maxExtendedFunction() - CPU.BrandName = brandName() - CPU.CacheLine = cacheLine() - CPU.Family, CPU.Model = familyModel() - CPU.Features = support() - CPU.SGX = hasSGX(CPU.Features&SGX != 0) - CPU.ThreadsPerCore = threadsPerCore() - CPU.LogicalCores = logicalCores() - CPU.PhysicalCores = physicalCores() - CPU.VendorID = vendorID() - CPU.cacheSize() -} - -// Generated here: http://play.golang.org/p/BxFH2Gdc0G - -// Cmov indicates support of CMOV instructions -func (c CPUInfo) Cmov() bool { - return c.Features&CMOV != 0 -} - -// Amd3dnow indicates support of AMD 3DNOW! instructions -func (c CPUInfo) Amd3dnow() bool { - return c.Features&AMD3DNOW != 0 -} - -// Amd3dnowExt indicates support of AMD 3DNOW! Extended instructions -func (c CPUInfo) Amd3dnowExt() bool { - return c.Features&AMD3DNOWEXT != 0 -} - -// MMX indicates support of MMX instructions -func (c CPUInfo) MMX() bool { - return c.Features&MMX != 0 -} - -// MMXExt indicates support of MMXEXT instructions -// (SSE integer functions or AMD MMX ext) -func (c CPUInfo) MMXExt() bool { - return c.Features&MMXEXT != 0 -} - -// SSE indicates support of SSE instructions -func (c CPUInfo) SSE() bool { - return c.Features&SSE != 0 -} - -// SSE2 indicates support of SSE 2 instructions -func (c CPUInfo) SSE2() bool { - return c.Features&SSE2 != 0 -} - -// SSE3 indicates support of SSE 3 instructions -func (c CPUInfo) SSE3() bool { - return c.Features&SSE3 != 0 -} - -// SSSE3 indicates support of SSSE 3 instructions -func (c CPUInfo) SSSE3() bool { - return c.Features&SSSE3 != 0 -} - -// SSE4 indicates support of SSE 4 (also called SSE 4.1) instructions -func (c CPUInfo) SSE4() bool { - return c.Features&SSE4 != 0 -} - -// SSE42 indicates support of SSE4.2 instructions -func (c CPUInfo) SSE42() bool { - return c.Features&SSE42 != 0 -} - -// AVX indicates support of AVX instructions -// and operating system support of AVX instructions -func (c CPUInfo) AVX() bool { - return c.Features&AVX != 0 -} - -// AVX2 indicates support of AVX2 instructions -func (c CPUInfo) AVX2() bool { - return c.Features&AVX2 != 0 -} - -// FMA3 indicates support of FMA3 instructions -func (c CPUInfo) FMA3() bool { - return c.Features&FMA3 != 0 -} - -// FMA4 indicates support of FMA4 instructions -func (c CPUInfo) FMA4() bool { - return c.Features&FMA4 != 0 -} - -// XOP indicates support of XOP instructions -func (c CPUInfo) XOP() bool { - return c.Features&XOP != 0 -} - -// F16C indicates support of F16C instructions -func (c CPUInfo) F16C() bool { - return c.Features&F16C != 0 -} - -// BMI1 indicates support of BMI1 instructions -func (c CPUInfo) BMI1() bool { - return c.Features&BMI1 != 0 -} - -// BMI2 indicates support of BMI2 instructions -func (c CPUInfo) BMI2() bool { - return c.Features&BMI2 != 0 -} - -// TBM indicates support of TBM instructions -// (AMD Trailing Bit Manipulation) -func (c CPUInfo) TBM() bool { - return c.Features&TBM != 0 -} - -// Lzcnt indicates support of LZCNT instruction -func (c CPUInfo) Lzcnt() bool { - return c.Features&LZCNT != 0 -} - -// Popcnt indicates support of POPCNT instruction -func (c CPUInfo) Popcnt() bool { - return c.Features&POPCNT != 0 -} - -// HTT indicates the processor has Hyperthreading enabled -func (c CPUInfo) HTT() bool { - return c.Features&HTT != 0 -} - -// SSE2Slow indicates that SSE2 may be slow on this processor -func (c CPUInfo) SSE2Slow() bool { - return c.Features&SSE2SLOW != 0 -} - -// SSE3Slow indicates that SSE3 may be slow on this processor -func (c CPUInfo) SSE3Slow() bool { - return c.Features&SSE3SLOW != 0 -} - -// AesNi indicates support of AES-NI instructions -// (Advanced Encryption Standard New Instructions) -func (c CPUInfo) AesNi() bool { - return c.Features&AESNI != 0 -} - -// Clmul indicates support of CLMUL instructions -// (Carry-less Multiplication) -func (c CPUInfo) Clmul() bool { - return c.Features&CLMUL != 0 -} - -// NX indicates support of NX (No-Execute) bit -func (c CPUInfo) NX() bool { - return c.Features&NX != 0 -} - -// SSE4A indicates support of AMD Barcelona microarchitecture SSE4a instructions -func (c CPUInfo) SSE4A() bool { - return c.Features&SSE4A != 0 -} - -// HLE indicates support of Hardware Lock Elision -func (c CPUInfo) HLE() bool { - return c.Features&HLE != 0 -} - -// RTM indicates support of Restricted Transactional Memory -func (c CPUInfo) RTM() bool { - return c.Features&RTM != 0 -} - -// Rdrand indicates support of RDRAND instruction is available -func (c CPUInfo) Rdrand() bool { - return c.Features&RDRAND != 0 -} - -// Rdseed indicates support of RDSEED instruction is available -func (c CPUInfo) Rdseed() bool { - return c.Features&RDSEED != 0 -} - -// ADX indicates support of Intel ADX (Multi-Precision Add-Carry Instruction Extensions) -func (c CPUInfo) ADX() bool { - return c.Features&ADX != 0 -} - -// SHA indicates support of Intel SHA Extensions -func (c CPUInfo) SHA() bool { - return c.Features&SHA != 0 -} - -// AVX512F indicates support of AVX-512 Foundation -func (c CPUInfo) AVX512F() bool { - return c.Features&AVX512F != 0 -} - -// AVX512DQ indicates support of AVX-512 Doubleword and Quadword Instructions -func (c CPUInfo) AVX512DQ() bool { - return c.Features&AVX512DQ != 0 -} - -// AVX512IFMA indicates support of AVX-512 Integer Fused Multiply-Add Instructions -func (c CPUInfo) AVX512IFMA() bool { - return c.Features&AVX512IFMA != 0 -} - -// AVX512PF indicates support of AVX-512 Prefetch Instructions -func (c CPUInfo) AVX512PF() bool { - return c.Features&AVX512PF != 0 -} - -// AVX512ER indicates support of AVX-512 Exponential and Reciprocal Instructions -func (c CPUInfo) AVX512ER() bool { - return c.Features&AVX512ER != 0 -} - -// AVX512CD indicates support of AVX-512 Conflict Detection Instructions -func (c CPUInfo) AVX512CD() bool { - return c.Features&AVX512CD != 0 -} - -// AVX512BW indicates support of AVX-512 Byte and Word Instructions -func (c CPUInfo) AVX512BW() bool { - return c.Features&AVX512BW != 0 -} - -// AVX512VL indicates support of AVX-512 Vector Length Extensions -func (c CPUInfo) AVX512VL() bool { - return c.Features&AVX512VL != 0 -} - -// AVX512VBMI indicates support of AVX-512 Vector Bit Manipulation Instructions -func (c CPUInfo) AVX512VBMI() bool { - return c.Features&AVX512VBMI != 0 -} - -// MPX indicates support of Intel MPX (Memory Protection Extensions) -func (c CPUInfo) MPX() bool { - return c.Features&MPX != 0 -} - -// ERMS indicates support of Enhanced REP MOVSB/STOSB -func (c CPUInfo) ERMS() bool { - return c.Features&ERMS != 0 -} - -// RDTSCP Instruction is available. -func (c CPUInfo) RDTSCP() bool { - return c.Features&RDTSCP != 0 -} - -// CX16 indicates if CMPXCHG16B instruction is available. -func (c CPUInfo) CX16() bool { - return c.Features&CX16 != 0 -} - -// TSX is split into HLE (Hardware Lock Elision) and RTM (Restricted Transactional Memory) detection. -// So TSX simply checks that. -func (c CPUInfo) TSX() bool { - return c.Features&(MPX|RTM) == MPX|RTM -} - -// Atom indicates an Atom processor -func (c CPUInfo) Atom() bool { - return c.Features&ATOM != 0 -} - -// Intel returns true if vendor is recognized as Intel -func (c CPUInfo) Intel() bool { - return c.VendorID == Intel -} - -// AMD returns true if vendor is recognized as AMD -func (c CPUInfo) AMD() bool { - return c.VendorID == AMD -} - -// Transmeta returns true if vendor is recognized as Transmeta -func (c CPUInfo) Transmeta() bool { - return c.VendorID == Transmeta -} - -// NSC returns true if vendor is recognized as National Semiconductor -func (c CPUInfo) NSC() bool { - return c.VendorID == NSC -} - -// VIA returns true if vendor is recognized as VIA -func (c CPUInfo) VIA() bool { - return c.VendorID == VIA -} - -// RTCounter returns the 64-bit time-stamp counter -// Uses the RDTSCP instruction. The value 0 is returned -// if the CPU does not support the instruction. -func (c CPUInfo) RTCounter() uint64 { - if !c.RDTSCP() { - return 0 - } - a, _, _, d := rdtscpAsm() - return uint64(a) | (uint64(d) << 32) -} - -// Ia32TscAux returns the IA32_TSC_AUX part of the RDTSCP. -// This variable is OS dependent, but on Linux contains information -// about the current cpu/core the code is running on. -// If the RDTSCP instruction isn't supported on the CPU, the value 0 is returned. -func (c CPUInfo) Ia32TscAux() uint32 { - if !c.RDTSCP() { - return 0 - } - _, _, ecx, _ := rdtscpAsm() - return ecx -} - -// LogicalCPU will return the Logical CPU the code is currently executing on. -// This is likely to change when the OS re-schedules the running thread -// to another CPU. -// If the current core cannot be detected, -1 will be returned. -func (c CPUInfo) LogicalCPU() int { - if c.maxFunc < 1 { - return -1 - } - _, ebx, _, _ := cpuid(1) - return int(ebx >> 24) -} - -// VM Will return true if the cpu id indicates we are in -// a virtual machine. This is only a hint, and will very likely -// have many false negatives. -func (c CPUInfo) VM() bool { - switch c.VendorID { - case MSVM, KVM, VMware, XenHVM: - return true - } - return false -} - -// Flags contains detected cpu features and caracteristics -type Flags uint64 - -// String returns a string representation of the detected -// CPU features. -func (f Flags) String() string { - return strings.Join(f.Strings(), ",") -} - -// Strings returns and array of the detected features. -func (f Flags) Strings() []string { - s := support() - r := make([]string, 0, 20) - for i := uint(0); i < 64; i++ { - key := Flags(1 << i) - val := flagNames[key] - if s&key != 0 { - r = append(r, val) - } - } - return r -} - -func maxExtendedFunction() uint32 { - eax, _, _, _ := cpuid(0x80000000) - return eax -} - -func maxFunctionID() uint32 { - a, _, _, _ := cpuid(0) - return a -} - -func brandName() string { - if maxExtendedFunction() >= 0x80000004 { - v := make([]uint32, 0, 48) - for i := uint32(0); i < 3; i++ { - a, b, c, d := cpuid(0x80000002 + i) - v = append(v, a, b, c, d) - } - return strings.Trim(string(valAsString(v...)), " ") - } - return "unknown" -} - -func threadsPerCore() int { - mfi := maxFunctionID() - if mfi < 0x4 || vendorID() != Intel { - return 1 - } - - if mfi < 0xb { - _, b, _, d := cpuid(1) - if (d & (1 << 28)) != 0 { - // v will contain logical core count - v := (b >> 16) & 255 - if v > 1 { - a4, _, _, _ := cpuid(4) - // physical cores - v2 := (a4 >> 26) + 1 - if v2 > 0 { - return int(v) / int(v2) - } - } - } - return 1 - } - _, b, _, _ := cpuidex(0xb, 0) - if b&0xffff == 0 { - return 1 - } - return int(b & 0xffff) -} - -func logicalCores() int { - mfi := maxFunctionID() - switch vendorID() { - case Intel: - // Use this on old Intel processors - if mfi < 0xb { - if mfi < 1 { - return 0 - } - // CPUID.1:EBX[23:16] represents the maximum number of addressable IDs (initial APIC ID) - // that can be assigned to logical processors in a physical package. - // The value may not be the same as the number of logical processors that are present in the hardware of a physical package. - _, ebx, _, _ := cpuid(1) - logical := (ebx >> 16) & 0xff - return int(logical) - } - _, b, _, _ := cpuidex(0xb, 1) - return int(b & 0xffff) - case AMD: - _, b, _, _ := cpuid(1) - return int((b >> 16) & 0xff) - default: - return 0 - } -} - -func familyModel() (int, int) { - if maxFunctionID() < 0x1 { - return 0, 0 - } - eax, _, _, _ := cpuid(1) - family := ((eax >> 8) & 0xf) + ((eax >> 20) & 0xff) - model := ((eax >> 4) & 0xf) + ((eax >> 12) & 0xf0) - return int(family), int(model) -} - -func physicalCores() int { - switch vendorID() { - case Intel: - return logicalCores() / threadsPerCore() - case AMD: - if maxExtendedFunction() >= 0x80000008 { - _, _, c, _ := cpuid(0x80000008) - return int(c&0xff) + 1 - } - } - return 0 -} - -// Except from http://en.wikipedia.org/wiki/CPUID#EAX.3D0:_Get_vendor_ID -var vendorMapping = map[string]Vendor{ - "AMDisbetter!": AMD, - "AuthenticAMD": AMD, - "CentaurHauls": VIA, - "GenuineIntel": Intel, - "TransmetaCPU": Transmeta, - "GenuineTMx86": Transmeta, - "Geode by NSC": NSC, - "VIA VIA VIA ": VIA, - "KVMKVMKVMKVM": KVM, - "Microsoft Hv": MSVM, - "VMwareVMware": VMware, - "XenVMMXenVMM": XenHVM, -} - -func vendorID() Vendor { - _, b, c, d := cpuid(0) - v := valAsString(b, d, c) - vend, ok := vendorMapping[string(v)] - if !ok { - return Other - } - return vend -} - -func cacheLine() int { - if maxFunctionID() < 0x1 { - return 0 - } - - _, ebx, _, _ := cpuid(1) - cache := (ebx & 0xff00) >> 5 // cflush size - if cache == 0 && maxExtendedFunction() >= 0x80000006 { - _, _, ecx, _ := cpuid(0x80000006) - cache = ecx & 0xff // cacheline size - } - // TODO: Read from Cache and TLB Information - return int(cache) -} - -func (c *CPUInfo) cacheSize() { - c.Cache.L1D = -1 - c.Cache.L1I = -1 - c.Cache.L2 = -1 - c.Cache.L3 = -1 - vendor := vendorID() - switch vendor { - case Intel: - if maxFunctionID() < 4 { - return - } - for i := uint32(0); ; i++ { - eax, ebx, ecx, _ := cpuidex(4, i) - cacheType := eax & 15 - if cacheType == 0 { - break - } - cacheLevel := (eax >> 5) & 7 - coherency := int(ebx&0xfff) + 1 - partitions := int((ebx>>12)&0x3ff) + 1 - associativity := int((ebx>>22)&0x3ff) + 1 - sets := int(ecx) + 1 - size := associativity * partitions * coherency * sets - switch cacheLevel { - case 1: - if cacheType == 1 { - // 1 = Data Cache - c.Cache.L1D = size - } else if cacheType == 2 { - // 2 = Instruction Cache - c.Cache.L1I = size - } else { - if c.Cache.L1D < 0 { - c.Cache.L1I = size - } - if c.Cache.L1I < 0 { - c.Cache.L1I = size - } - } - case 2: - c.Cache.L2 = size - case 3: - c.Cache.L3 = size - } - } - case AMD: - // Untested. - if maxExtendedFunction() < 0x80000005 { - return - } - _, _, ecx, edx := cpuid(0x80000005) - c.Cache.L1D = int(((ecx >> 24) & 0xFF) * 1024) - c.Cache.L1I = int(((edx >> 24) & 0xFF) * 1024) - - if maxExtendedFunction() < 0x80000006 { - return - } - _, _, ecx, _ = cpuid(0x80000006) - c.Cache.L2 = int(((ecx >> 16) & 0xFFFF) * 1024) - } - - return -} - -type SGXSupport struct { - Available bool - SGX1Supported bool - SGX2Supported bool - MaxEnclaveSizeNot64 int64 - MaxEnclaveSize64 int64 -} - -func hasSGX(available bool) (rval SGXSupport) { - rval.Available = available - - if !available { - return - } - - a, _, _, d := cpuidex(0x12, 0) - rval.SGX1Supported = a&0x01 != 0 - rval.SGX2Supported = a&0x02 != 0 - rval.MaxEnclaveSizeNot64 = 1 << (d & 0xFF) // pow 2 - rval.MaxEnclaveSize64 = 1 << ((d >> 8) & 0xFF) // pow 2 - - return -} - -func support() Flags { - mfi := maxFunctionID() - vend := vendorID() - if mfi < 0x1 { - return 0 - } - rval := uint64(0) - _, _, c, d := cpuid(1) - if (d & (1 << 15)) != 0 { - rval |= CMOV - } - if (d & (1 << 23)) != 0 { - rval |= MMX - } - if (d & (1 << 25)) != 0 { - rval |= MMXEXT - } - if (d & (1 << 25)) != 0 { - rval |= SSE - } - if (d & (1 << 26)) != 0 { - rval |= SSE2 - } - if (c & 1) != 0 { - rval |= SSE3 - } - if (c & 0x00000200) != 0 { - rval |= SSSE3 - } - if (c & 0x00080000) != 0 { - rval |= SSE4 - } - if (c & 0x00100000) != 0 { - rval |= SSE42 - } - if (c & (1 << 25)) != 0 { - rval |= AESNI - } - if (c & (1 << 1)) != 0 { - rval |= CLMUL - } - if c&(1<<23) != 0 { - rval |= POPCNT - } - if c&(1<<30) != 0 { - rval |= RDRAND - } - if c&(1<<29) != 0 { - rval |= F16C - } - if c&(1<<13) != 0 { - rval |= CX16 - } - if vend == Intel && (d&(1<<28)) != 0 && mfi >= 4 { - if threadsPerCore() > 1 { - rval |= HTT - } - } - - // Check XGETBV, OXSAVE and AVX bits - if c&(1<<26) != 0 && c&(1<<27) != 0 && c&(1<<28) != 0 { - // Check for OS support - eax, _ := xgetbv(0) - if (eax & 0x6) == 0x6 { - rval |= AVX - if (c & 0x00001000) != 0 { - rval |= FMA3 - } - } - } - - // Check AVX2, AVX2 requires OS support, but BMI1/2 don't. - if mfi >= 7 { - _, ebx, ecx, _ := cpuidex(7, 0) - if (rval&AVX) != 0 && (ebx&0x00000020) != 0 { - rval |= AVX2 - } - if (ebx & 0x00000008) != 0 { - rval |= BMI1 - if (ebx & 0x00000100) != 0 { - rval |= BMI2 - } - } - if ebx&(1<<2) != 0 { - rval |= SGX - } - if ebx&(1<<4) != 0 { - rval |= HLE - } - if ebx&(1<<9) != 0 { - rval |= ERMS - } - if ebx&(1<<11) != 0 { - rval |= RTM - } - if ebx&(1<<14) != 0 { - rval |= MPX - } - if ebx&(1<<18) != 0 { - rval |= RDSEED - } - if ebx&(1<<19) != 0 { - rval |= ADX - } - if ebx&(1<<29) != 0 { - rval |= SHA - } - - // Only detect AVX-512 features if XGETBV is supported - if c&((1<<26)|(1<<27)) == (1<<26)|(1<<27) { - // Check for OS support - eax, _ := xgetbv(0) - - // Verify that XCR0[7:5] = ‘111b’ (OPMASK state, upper 256-bit of ZMM0-ZMM15 and - // ZMM16-ZMM31 state are enabled by OS) - /// and that XCR0[2:1] = ‘11b’ (XMM state and YMM state are enabled by OS). - if (eax>>5)&7 == 7 && (eax>>1)&3 == 3 { - if ebx&(1<<16) != 0 { - rval |= AVX512F - } - if ebx&(1<<17) != 0 { - rval |= AVX512DQ - } - if ebx&(1<<21) != 0 { - rval |= AVX512IFMA - } - if ebx&(1<<26) != 0 { - rval |= AVX512PF - } - if ebx&(1<<27) != 0 { - rval |= AVX512ER - } - if ebx&(1<<28) != 0 { - rval |= AVX512CD - } - if ebx&(1<<30) != 0 { - rval |= AVX512BW - } - if ebx&(1<<31) != 0 { - rval |= AVX512VL - } - // ecx - if ecx&(1<<1) != 0 { - rval |= AVX512VBMI - } - } - } - } - - if maxExtendedFunction() >= 0x80000001 { - _, _, c, d := cpuid(0x80000001) - if (c & (1 << 5)) != 0 { - rval |= LZCNT - rval |= POPCNT - } - if (d & (1 << 31)) != 0 { - rval |= AMD3DNOW - } - if (d & (1 << 30)) != 0 { - rval |= AMD3DNOWEXT - } - if (d & (1 << 23)) != 0 { - rval |= MMX - } - if (d & (1 << 22)) != 0 { - rval |= MMXEXT - } - if (c & (1 << 6)) != 0 { - rval |= SSE4A - } - if d&(1<<20) != 0 { - rval |= NX - } - if d&(1<<27) != 0 { - rval |= RDTSCP - } - - /* Allow for selectively disabling SSE2 functions on AMD processors - with SSE2 support but not SSE4a. This includes Athlon64, some - Opteron, and some Sempron processors. MMX, SSE, or 3DNow! are faster - than SSE2 often enough to utilize this special-case flag. - AV_CPU_FLAG_SSE2 and AV_CPU_FLAG_SSE2SLOW are both set in this case - so that SSE2 is used unless explicitly disabled by checking - AV_CPU_FLAG_SSE2SLOW. */ - if vendorID() != Intel && - rval&SSE2 != 0 && (c&0x00000040) == 0 { - rval |= SSE2SLOW - } - - /* XOP and FMA4 use the AVX instruction coding scheme, so they can't be - * used unless the OS has AVX support. */ - if (rval & AVX) != 0 { - if (c & 0x00000800) != 0 { - rval |= XOP - } - if (c & 0x00010000) != 0 { - rval |= FMA4 - } - } - - if vendorID() == Intel { - family, model := familyModel() - if family == 6 && (model == 9 || model == 13 || model == 14) { - /* 6/9 (pentium-m "banias"), 6/13 (pentium-m "dothan"), and - * 6/14 (core1 "yonah") theoretically support sse2, but it's - * usually slower than mmx. */ - if (rval & SSE2) != 0 { - rval |= SSE2SLOW - } - if (rval & SSE3) != 0 { - rval |= SSE3SLOW - } - } - /* The Atom processor has SSSE3 support, which is useful in many cases, - * but sometimes the SSSE3 version is slower than the SSE2 equivalent - * on the Atom, but is generally faster on other processors supporting - * SSSE3. This flag allows for selectively disabling certain SSSE3 - * functions on the Atom. */ - if family == 6 && model == 28 { - rval |= ATOM - } - } - } - return Flags(rval) -} - -func valAsString(values ...uint32) []byte { - r := make([]byte, 4*len(values)) - for i, v := range values { - dst := r[i*4:] - dst[0] = byte(v & 0xff) - dst[1] = byte((v >> 8) & 0xff) - dst[2] = byte((v >> 16) & 0xff) - dst[3] = byte((v >> 24) & 0xff) - switch { - case dst[0] == 0: - return r[:i*4] - case dst[1] == 0: - return r[:i*4+1] - case dst[2] == 0: - return r[:i*4+2] - case dst[3] == 0: - return r[:i*4+3] - } - } - return r -} diff --git a/vendor/github.com/klauspost/cpuid/cpuid_386.s b/vendor/github.com/klauspost/cpuid/cpuid_386.s deleted file mode 100644 index 4d731711..00000000 --- a/vendor/github.com/klauspost/cpuid/cpuid_386.s +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. - -// +build 386,!gccgo - -// func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) -TEXT ·asmCpuid(SB), 7, $0 - XORL CX, CX - MOVL op+0(FP), AX - CPUID - MOVL AX, eax+4(FP) - MOVL BX, ebx+8(FP) - MOVL CX, ecx+12(FP) - MOVL DX, edx+16(FP) - RET - -// func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) -TEXT ·asmCpuidex(SB), 7, $0 - MOVL op+0(FP), AX - MOVL op2+4(FP), CX - CPUID - MOVL AX, eax+8(FP) - MOVL BX, ebx+12(FP) - MOVL CX, ecx+16(FP) - MOVL DX, edx+20(FP) - RET - -// func xgetbv(index uint32) (eax, edx uint32) -TEXT ·asmXgetbv(SB), 7, $0 - MOVL index+0(FP), CX - BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV - MOVL AX, eax+4(FP) - MOVL DX, edx+8(FP) - RET - -// func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) -TEXT ·asmRdtscpAsm(SB), 7, $0 - BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP - MOVL AX, eax+0(FP) - MOVL BX, ebx+4(FP) - MOVL CX, ecx+8(FP) - MOVL DX, edx+12(FP) - RET diff --git a/vendor/github.com/klauspost/cpuid/cpuid_amd64.s b/vendor/github.com/klauspost/cpuid/cpuid_amd64.s deleted file mode 100644 index 3c1d60e4..00000000 --- a/vendor/github.com/klauspost/cpuid/cpuid_amd64.s +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. - -//+build amd64,!gccgo - -// func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) -TEXT ·asmCpuid(SB), 7, $0 - XORQ CX, CX - MOVL op+0(FP), AX - CPUID - MOVL AX, eax+8(FP) - MOVL BX, ebx+12(FP) - MOVL CX, ecx+16(FP) - MOVL DX, edx+20(FP) - RET - -// func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) -TEXT ·asmCpuidex(SB), 7, $0 - MOVL op+0(FP), AX - MOVL op2+4(FP), CX - CPUID - MOVL AX, eax+8(FP) - MOVL BX, ebx+12(FP) - MOVL CX, ecx+16(FP) - MOVL DX, edx+20(FP) - RET - -// func asmXgetbv(index uint32) (eax, edx uint32) -TEXT ·asmXgetbv(SB), 7, $0 - MOVL index+0(FP), CX - BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV - MOVL AX, eax+8(FP) - MOVL DX, edx+12(FP) - RET - -// func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) -TEXT ·asmRdtscpAsm(SB), 7, $0 - BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP - MOVL AX, eax+0(FP) - MOVL BX, ebx+4(FP) - MOVL CX, ecx+8(FP) - MOVL DX, edx+12(FP) - RET diff --git a/vendor/github.com/klauspost/cpuid/detect_intel.go b/vendor/github.com/klauspost/cpuid/detect_intel.go deleted file mode 100644 index a5f04dd6..00000000 --- a/vendor/github.com/klauspost/cpuid/detect_intel.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. - -// +build 386,!gccgo amd64,!gccgo - -package cpuid - -func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) -func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) -func asmXgetbv(index uint32) (eax, edx uint32) -func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) - -func initCPU() { - cpuid = asmCpuid - cpuidex = asmCpuidex - xgetbv = asmXgetbv - rdtscpAsm = asmRdtscpAsm -} diff --git a/vendor/github.com/klauspost/cpuid/detect_ref.go b/vendor/github.com/klauspost/cpuid/detect_ref.go deleted file mode 100644 index 909c5d9a..00000000 --- a/vendor/github.com/klauspost/cpuid/detect_ref.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. - -// +build !amd64,!386 gccgo - -package cpuid - -func initCPU() { - cpuid = func(op uint32) (eax, ebx, ecx, edx uint32) { - return 0, 0, 0, 0 - } - - cpuidex = func(op, op2 uint32) (eax, ebx, ecx, edx uint32) { - return 0, 0, 0, 0 - } - - xgetbv = func(index uint32) (eax, edx uint32) { - return 0, 0 - } - - rdtscpAsm = func() (eax, ebx, ecx, edx uint32) { - return 0, 0, 0, 0 - } -} diff --git a/vendor/github.com/klauspost/cpuid/generate.go b/vendor/github.com/klauspost/cpuid/generate.go deleted file mode 100644 index 90e7a98d..00000000 --- a/vendor/github.com/klauspost/cpuid/generate.go +++ /dev/null @@ -1,4 +0,0 @@ -package cpuid - -//go:generate go run private-gen.go -//go:generate gofmt -w ./private diff --git a/vendor/github.com/klauspost/cpuid/private-gen.go b/vendor/github.com/klauspost/cpuid/private-gen.go deleted file mode 100644 index 437333d2..00000000 --- a/vendor/github.com/klauspost/cpuid/private-gen.go +++ /dev/null @@ -1,476 +0,0 @@ -// +build ignore - -package main - -import ( - "bytes" - "fmt" - "go/ast" - "go/parser" - "go/printer" - "go/token" - "io" - "io/ioutil" - "log" - "os" - "reflect" - "strings" - "unicode" - "unicode/utf8" -) - -var inFiles = []string{"cpuid.go", "cpuid_test.go"} -var copyFiles = []string{"cpuid_amd64.s", "cpuid_386.s", "detect_ref.go", "detect_intel.go"} -var fileSet = token.NewFileSet() -var reWrites = []rewrite{ - initRewrite("CPUInfo -> cpuInfo"), - initRewrite("Vendor -> vendor"), - initRewrite("Flags -> flags"), - initRewrite("Detect -> detect"), - initRewrite("CPU -> cpu"), -} -var excludeNames = map[string]bool{"string": true, "join": true, "trim": true, - // cpuid_test.go - "t": true, "println": true, "logf": true, "log": true, "fatalf": true, "fatal": true, -} - -var excludePrefixes = []string{"test", "benchmark"} - -func main() { - Package := "private" - parserMode := parser.ParseComments - exported := make(map[string]rewrite) - for _, file := range inFiles { - in, err := os.Open(file) - if err != nil { - log.Fatalf("opening input", err) - } - - src, err := ioutil.ReadAll(in) - if err != nil { - log.Fatalf("reading input", err) - } - - astfile, err := parser.ParseFile(fileSet, file, src, parserMode) - if err != nil { - log.Fatalf("parsing input", err) - } - - for _, rw := range reWrites { - astfile = rw(astfile) - } - - // Inspect the AST and print all identifiers and literals. - var startDecl token.Pos - var endDecl token.Pos - ast.Inspect(astfile, func(n ast.Node) bool { - var s string - switch x := n.(type) { - case *ast.Ident: - if x.IsExported() { - t := strings.ToLower(x.Name) - for _, pre := range excludePrefixes { - if strings.HasPrefix(t, pre) { - return true - } - } - if excludeNames[t] != true { - //if x.Pos() > startDecl && x.Pos() < endDecl { - exported[x.Name] = initRewrite(x.Name + " -> " + t) - } - } - - case *ast.GenDecl: - if x.Tok == token.CONST && x.Lparen > 0 { - startDecl = x.Lparen - endDecl = x.Rparen - // fmt.Printf("Decl:%s -> %s\n", fileSet.Position(startDecl), fileSet.Position(endDecl)) - } - } - if s != "" { - fmt.Printf("%s:\t%s\n", fileSet.Position(n.Pos()), s) - } - return true - }) - - for _, rw := range exported { - astfile = rw(astfile) - } - - var buf bytes.Buffer - - printer.Fprint(&buf, fileSet, astfile) - - // Remove package documentation and insert information - s := buf.String() - ind := strings.Index(buf.String(), "\npackage cpuid") - s = s[ind:] - s = "// Generated, DO NOT EDIT,\n" + - "// but copy it to your own project and rename the package.\n" + - "// See more at http://github.com/klauspost/cpuid\n" + - s - - outputName := Package + string(os.PathSeparator) + file - - err = ioutil.WriteFile(outputName, []byte(s), 0644) - if err != nil { - log.Fatalf("writing output: %s", err) - } - log.Println("Generated", outputName) - } - - for _, file := range copyFiles { - dst := "" - if strings.HasPrefix(file, "cpuid") { - dst = Package + string(os.PathSeparator) + file - } else { - dst = Package + string(os.PathSeparator) + "cpuid_" + file - } - err := copyFile(file, dst) - if err != nil { - log.Fatalf("copying file: %s", err) - } - log.Println("Copied", dst) - } -} - -// CopyFile copies a file from src to dst. If src and dst files exist, and are -// the same, then return success. Copy the file contents from src to dst. -func copyFile(src, dst string) (err error) { - sfi, err := os.Stat(src) - if err != nil { - return - } - if !sfi.Mode().IsRegular() { - // cannot copy non-regular files (e.g., directories, - // symlinks, devices, etc.) - return fmt.Errorf("CopyFile: non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String()) - } - dfi, err := os.Stat(dst) - if err != nil { - if !os.IsNotExist(err) { - return - } - } else { - if !(dfi.Mode().IsRegular()) { - return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String()) - } - if os.SameFile(sfi, dfi) { - return - } - } - err = copyFileContents(src, dst) - return -} - -// copyFileContents copies the contents of the file named src to the file named -// by dst. The file will be created if it does not already exist. If the -// destination file exists, all it's contents will be replaced by the contents -// of the source file. -func copyFileContents(src, dst string) (err error) { - in, err := os.Open(src) - if err != nil { - return - } - defer in.Close() - out, err := os.Create(dst) - if err != nil { - return - } - defer func() { - cerr := out.Close() - if err == nil { - err = cerr - } - }() - if _, err = io.Copy(out, in); err != nil { - return - } - err = out.Sync() - return -} - -type rewrite func(*ast.File) *ast.File - -// Mostly copied from gofmt -func initRewrite(rewriteRule string) rewrite { - f := strings.Split(rewriteRule, "->") - if len(f) != 2 { - fmt.Fprintf(os.Stderr, "rewrite rule must be of the form 'pattern -> replacement'\n") - os.Exit(2) - } - pattern := parseExpr(f[0], "pattern") - replace := parseExpr(f[1], "replacement") - return func(p *ast.File) *ast.File { return rewriteFile(pattern, replace, p) } -} - -// parseExpr parses s as an expression. -// It might make sense to expand this to allow statement patterns, -// but there are problems with preserving formatting and also -// with what a wildcard for a statement looks like. -func parseExpr(s, what string) ast.Expr { - x, err := parser.ParseExpr(s) - if err != nil { - fmt.Fprintf(os.Stderr, "parsing %s %s at %s\n", what, s, err) - os.Exit(2) - } - return x -} - -// Keep this function for debugging. -/* -func dump(msg string, val reflect.Value) { - fmt.Printf("%s:\n", msg) - ast.Print(fileSet, val.Interface()) - fmt.Println() -} -*/ - -// rewriteFile applies the rewrite rule 'pattern -> replace' to an entire file. -func rewriteFile(pattern, replace ast.Expr, p *ast.File) *ast.File { - cmap := ast.NewCommentMap(fileSet, p, p.Comments) - m := make(map[string]reflect.Value) - pat := reflect.ValueOf(pattern) - repl := reflect.ValueOf(replace) - - var rewriteVal func(val reflect.Value) reflect.Value - rewriteVal = func(val reflect.Value) reflect.Value { - // don't bother if val is invalid to start with - if !val.IsValid() { - return reflect.Value{} - } - for k := range m { - delete(m, k) - } - val = apply(rewriteVal, val) - if match(m, pat, val) { - val = subst(m, repl, reflect.ValueOf(val.Interface().(ast.Node).Pos())) - } - return val - } - - r := apply(rewriteVal, reflect.ValueOf(p)).Interface().(*ast.File) - r.Comments = cmap.Filter(r).Comments() // recreate comments list - return r -} - -// set is a wrapper for x.Set(y); it protects the caller from panics if x cannot be changed to y. -func set(x, y reflect.Value) { - // don't bother if x cannot be set or y is invalid - if !x.CanSet() || !y.IsValid() { - return - } - defer func() { - if x := recover(); x != nil { - if s, ok := x.(string); ok && - (strings.Contains(s, "type mismatch") || strings.Contains(s, "not assignable")) { - // x cannot be set to y - ignore this rewrite - return - } - panic(x) - } - }() - x.Set(y) -} - -// Values/types for special cases. -var ( - objectPtrNil = reflect.ValueOf((*ast.Object)(nil)) - scopePtrNil = reflect.ValueOf((*ast.Scope)(nil)) - - identType = reflect.TypeOf((*ast.Ident)(nil)) - objectPtrType = reflect.TypeOf((*ast.Object)(nil)) - positionType = reflect.TypeOf(token.NoPos) - callExprType = reflect.TypeOf((*ast.CallExpr)(nil)) - scopePtrType = reflect.TypeOf((*ast.Scope)(nil)) -) - -// apply replaces each AST field x in val with f(x), returning val. -// To avoid extra conversions, f operates on the reflect.Value form. -func apply(f func(reflect.Value) reflect.Value, val reflect.Value) reflect.Value { - if !val.IsValid() { - return reflect.Value{} - } - - // *ast.Objects introduce cycles and are likely incorrect after - // rewrite; don't follow them but replace with nil instead - if val.Type() == objectPtrType { - return objectPtrNil - } - - // similarly for scopes: they are likely incorrect after a rewrite; - // replace them with nil - if val.Type() == scopePtrType { - return scopePtrNil - } - - switch v := reflect.Indirect(val); v.Kind() { - case reflect.Slice: - for i := 0; i < v.Len(); i++ { - e := v.Index(i) - set(e, f(e)) - } - case reflect.Struct: - for i := 0; i < v.NumField(); i++ { - e := v.Field(i) - set(e, f(e)) - } - case reflect.Interface: - e := v.Elem() - set(v, f(e)) - } - return val -} - -func isWildcard(s string) bool { - rune, size := utf8.DecodeRuneInString(s) - return size == len(s) && unicode.IsLower(rune) -} - -// match returns true if pattern matches val, -// recording wildcard submatches in m. -// If m == nil, match checks whether pattern == val. -func match(m map[string]reflect.Value, pattern, val reflect.Value) bool { - // Wildcard matches any expression. If it appears multiple - // times in the pattern, it must match the same expression - // each time. - if m != nil && pattern.IsValid() && pattern.Type() == identType { - name := pattern.Interface().(*ast.Ident).Name - if isWildcard(name) && val.IsValid() { - // wildcards only match valid (non-nil) expressions. - if _, ok := val.Interface().(ast.Expr); ok && !val.IsNil() { - if old, ok := m[name]; ok { - return match(nil, old, val) - } - m[name] = val - return true - } - } - } - - // Otherwise, pattern and val must match recursively. - if !pattern.IsValid() || !val.IsValid() { - return !pattern.IsValid() && !val.IsValid() - } - if pattern.Type() != val.Type() { - return false - } - - // Special cases. - switch pattern.Type() { - case identType: - // For identifiers, only the names need to match - // (and none of the other *ast.Object information). - // This is a common case, handle it all here instead - // of recursing down any further via reflection. - p := pattern.Interface().(*ast.Ident) - v := val.Interface().(*ast.Ident) - return p == nil && v == nil || p != nil && v != nil && p.Name == v.Name - case objectPtrType, positionType: - // object pointers and token positions always match - return true - case callExprType: - // For calls, the Ellipsis fields (token.Position) must - // match since that is how f(x) and f(x...) are different. - // Check them here but fall through for the remaining fields. - p := pattern.Interface().(*ast.CallExpr) - v := val.Interface().(*ast.CallExpr) - if p.Ellipsis.IsValid() != v.Ellipsis.IsValid() { - return false - } - } - - p := reflect.Indirect(pattern) - v := reflect.Indirect(val) - if !p.IsValid() || !v.IsValid() { - return !p.IsValid() && !v.IsValid() - } - - switch p.Kind() { - case reflect.Slice: - if p.Len() != v.Len() { - return false - } - for i := 0; i < p.Len(); i++ { - if !match(m, p.Index(i), v.Index(i)) { - return false - } - } - return true - - case reflect.Struct: - for i := 0; i < p.NumField(); i++ { - if !match(m, p.Field(i), v.Field(i)) { - return false - } - } - return true - - case reflect.Interface: - return match(m, p.Elem(), v.Elem()) - } - - // Handle token integers, etc. - return p.Interface() == v.Interface() -} - -// subst returns a copy of pattern with values from m substituted in place -// of wildcards and pos used as the position of tokens from the pattern. -// if m == nil, subst returns a copy of pattern and doesn't change the line -// number information. -func subst(m map[string]reflect.Value, pattern reflect.Value, pos reflect.Value) reflect.Value { - if !pattern.IsValid() { - return reflect.Value{} - } - - // Wildcard gets replaced with map value. - if m != nil && pattern.Type() == identType { - name := pattern.Interface().(*ast.Ident).Name - if isWildcard(name) { - if old, ok := m[name]; ok { - return subst(nil, old, reflect.Value{}) - } - } - } - - if pos.IsValid() && pattern.Type() == positionType { - // use new position only if old position was valid in the first place - if old := pattern.Interface().(token.Pos); !old.IsValid() { - return pattern - } - return pos - } - - // Otherwise copy. - switch p := pattern; p.Kind() { - case reflect.Slice: - v := reflect.MakeSlice(p.Type(), p.Len(), p.Len()) - for i := 0; i < p.Len(); i++ { - v.Index(i).Set(subst(m, p.Index(i), pos)) - } - return v - - case reflect.Struct: - v := reflect.New(p.Type()).Elem() - for i := 0; i < p.NumField(); i++ { - v.Field(i).Set(subst(m, p.Field(i), pos)) - } - return v - - case reflect.Ptr: - v := reflect.New(p.Type()).Elem() - if elem := p.Elem(); elem.IsValid() { - v.Set(subst(m, elem, pos).Addr()) - } - return v - - case reflect.Interface: - v := reflect.New(p.Type()).Elem() - if elem := p.Elem(); elem.IsValid() { - v.Set(subst(m, elem, pos)) - } - return v - } - - return pattern -} diff --git a/vendor/github.com/valyala/bytebufferpool/.travis.yml b/vendor/github.com/valyala/bytebufferpool/.travis.yml deleted file mode 100644 index 6a6ec2eb..00000000 --- a/vendor/github.com/valyala/bytebufferpool/.travis.yml +++ /dev/null @@ -1,15 +0,0 @@ -language: go - -go: - - 1.6 - -script: - # build test for supported platforms - - GOOS=linux go build - - GOOS=darwin go build - - GOOS=freebsd go build - - GOOS=windows go build - - GOARCH=386 go build - - # run tests on a standard platform - - go test -v ./... diff --git a/vendor/github.com/valyala/bytebufferpool/LICENSE b/vendor/github.com/valyala/bytebufferpool/LICENSE deleted file mode 100644 index f7c935c2..00000000 --- a/vendor/github.com/valyala/bytebufferpool/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2016 Aliaksandr Valialkin, VertaMedia - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/vendor/github.com/valyala/bytebufferpool/README.md b/vendor/github.com/valyala/bytebufferpool/README.md deleted file mode 100644 index 061357e8..00000000 --- a/vendor/github.com/valyala/bytebufferpool/README.md +++ /dev/null @@ -1,21 +0,0 @@ -[![Build Status](https://travis-ci.org/valyala/bytebufferpool.svg)](https://travis-ci.org/valyala/bytebufferpool) -[![GoDoc](https://godoc.org/github.com/valyala/bytebufferpool?status.svg)](http://godoc.org/github.com/valyala/bytebufferpool) -[![Go Report](http://goreportcard.com/badge/valyala/bytebufferpool)](http://goreportcard.com/report/valyala/bytebufferpool) - -# bytebufferpool - -An implementation of a pool of byte buffers with anti-memory-waste protection. - -The pool may waste limited amount of memory due to fragmentation. -This amount equals to the maximum total size of the byte buffers -in concurrent use. - -# Benchmark results -Currently bytebufferpool is fastest and most effective buffer pool written in Go. - -You can find results [here](https://omgnull.github.io/go-benchmark/buffer/). - -# bytebufferpool users - -* [fasthttp](https://github.com/valyala/fasthttp) -* [quicktemplate](https://github.com/valyala/quicktemplate) diff --git a/vendor/github.com/valyala/bytebufferpool/bytebuffer.go b/vendor/github.com/valyala/bytebufferpool/bytebuffer.go deleted file mode 100644 index 07a055a2..00000000 --- a/vendor/github.com/valyala/bytebufferpool/bytebuffer.go +++ /dev/null @@ -1,111 +0,0 @@ -package bytebufferpool - -import "io" - -// ByteBuffer provides byte buffer, which can be used for minimizing -// memory allocations. -// -// ByteBuffer may be used with functions appending data to the given []byte -// slice. See example code for details. -// -// Use Get for obtaining an empty byte buffer. -type ByteBuffer struct { - - // B is a byte buffer to use in append-like workloads. - // See example code for details. - B []byte -} - -// Len returns the size of the byte buffer. -func (b *ByteBuffer) Len() int { - return len(b.B) -} - -// ReadFrom implements io.ReaderFrom. -// -// The function appends all the data read from r to b. -func (b *ByteBuffer) ReadFrom(r io.Reader) (int64, error) { - p := b.B - nStart := int64(len(p)) - nMax := int64(cap(p)) - n := nStart - if nMax == 0 { - nMax = 64 - p = make([]byte, nMax) - } else { - p = p[:nMax] - } - for { - if n == nMax { - nMax *= 2 - bNew := make([]byte, nMax) - copy(bNew, p) - p = bNew - } - nn, err := r.Read(p[n:]) - n += int64(nn) - if err != nil { - b.B = p[:n] - n -= nStart - if err == io.EOF { - return n, nil - } - return n, err - } - } -} - -// WriteTo implements io.WriterTo. -func (b *ByteBuffer) WriteTo(w io.Writer) (int64, error) { - n, err := w.Write(b.B) - return int64(n), err -} - -// Bytes returns b.B, i.e. all the bytes accumulated in the buffer. -// -// The purpose of this function is bytes.Buffer compatibility. -func (b *ByteBuffer) Bytes() []byte { - return b.B -} - -// Write implements io.Writer - it appends p to ByteBuffer.B -func (b *ByteBuffer) Write(p []byte) (int, error) { - b.B = append(b.B, p...) - return len(p), nil -} - -// WriteByte appends the byte c to the buffer. -// -// The purpose of this function is bytes.Buffer compatibility. -// -// The function always returns nil. -func (b *ByteBuffer) WriteByte(c byte) error { - b.B = append(b.B, c) - return nil -} - -// WriteString appends s to ByteBuffer.B. -func (b *ByteBuffer) WriteString(s string) (int, error) { - b.B = append(b.B, s...) - return len(s), nil -} - -// Set sets ByteBuffer.B to p. -func (b *ByteBuffer) Set(p []byte) { - b.B = append(b.B[:0], p...) -} - -// SetString sets ByteBuffer.B to s. -func (b *ByteBuffer) SetString(s string) { - b.B = append(b.B[:0], s...) -} - -// String returns string representation of ByteBuffer.B. -func (b *ByteBuffer) String() string { - return string(b.B) -} - -// Reset makes ByteBuffer.B empty. -func (b *ByteBuffer) Reset() { - b.B = b.B[:0] -} diff --git a/vendor/github.com/valyala/bytebufferpool/doc.go b/vendor/github.com/valyala/bytebufferpool/doc.go deleted file mode 100644 index e511b7c5..00000000 --- a/vendor/github.com/valyala/bytebufferpool/doc.go +++ /dev/null @@ -1,7 +0,0 @@ -// Package bytebufferpool implements a pool of byte buffers -// with anti-fragmentation protection. -// -// The pool may waste limited amount of memory due to fragmentation. -// This amount equals to the maximum total size of the byte buffers -// in concurrent use. -package bytebufferpool diff --git a/vendor/github.com/valyala/bytebufferpool/pool.go b/vendor/github.com/valyala/bytebufferpool/pool.go deleted file mode 100644 index 8bb4134d..00000000 --- a/vendor/github.com/valyala/bytebufferpool/pool.go +++ /dev/null @@ -1,151 +0,0 @@ -package bytebufferpool - -import ( - "sort" - "sync" - "sync/atomic" -) - -const ( - minBitSize = 6 // 2**6=64 is a CPU cache line size - steps = 20 - - minSize = 1 << minBitSize - maxSize = 1 << (minBitSize + steps - 1) - - calibrateCallsThreshold = 42000 - maxPercentile = 0.95 -) - -// Pool represents byte buffer pool. -// -// Distinct pools may be used for distinct types of byte buffers. -// Properly determined byte buffer types with their own pools may help reducing -// memory waste. -type Pool struct { - calls [steps]uint64 - calibrating uint64 - - defaultSize uint64 - maxSize uint64 - - pool sync.Pool -} - -var defaultPool Pool - -// Get returns an empty byte buffer from the pool. -// -// Got byte buffer may be returned to the pool via Put call. -// This reduces the number of memory allocations required for byte buffer -// management. -func Get() *ByteBuffer { return defaultPool.Get() } - -// Get returns new byte buffer with zero length. -// -// The byte buffer may be returned to the pool via Put after the use -// in order to minimize GC overhead. -func (p *Pool) Get() *ByteBuffer { - v := p.pool.Get() - if v != nil { - return v.(*ByteBuffer) - } - return &ByteBuffer{ - B: make([]byte, 0, atomic.LoadUint64(&p.defaultSize)), - } -} - -// Put returns byte buffer to the pool. -// -// ByteBuffer.B mustn't be touched after returning it to the pool. -// Otherwise data races will occur. -func Put(b *ByteBuffer) { defaultPool.Put(b) } - -// Put releases byte buffer obtained via Get to the pool. -// -// The buffer mustn't be accessed after returning to the pool. -func (p *Pool) Put(b *ByteBuffer) { - idx := index(len(b.B)) - - if atomic.AddUint64(&p.calls[idx], 1) > calibrateCallsThreshold { - p.calibrate() - } - - maxSize := int(atomic.LoadUint64(&p.maxSize)) - if maxSize == 0 || cap(b.B) <= maxSize { - b.Reset() - p.pool.Put(b) - } -} - -func (p *Pool) calibrate() { - if !atomic.CompareAndSwapUint64(&p.calibrating, 0, 1) { - return - } - - a := make(callSizes, 0, steps) - var callsSum uint64 - for i := uint64(0); i < steps; i++ { - calls := atomic.SwapUint64(&p.calls[i], 0) - callsSum += calls - a = append(a, callSize{ - calls: calls, - size: minSize << i, - }) - } - sort.Sort(a) - - defaultSize := a[0].size - maxSize := defaultSize - - maxSum := uint64(float64(callsSum) * maxPercentile) - callsSum = 0 - for i := 0; i < steps; i++ { - if callsSum > maxSum { - break - } - callsSum += a[i].calls - size := a[i].size - if size > maxSize { - maxSize = size - } - } - - atomic.StoreUint64(&p.defaultSize, defaultSize) - atomic.StoreUint64(&p.maxSize, maxSize) - - atomic.StoreUint64(&p.calibrating, 0) -} - -type callSize struct { - calls uint64 - size uint64 -} - -type callSizes []callSize - -func (ci callSizes) Len() int { - return len(ci) -} - -func (ci callSizes) Less(i, j int) bool { - return ci[i].calls > ci[j].calls -} - -func (ci callSizes) Swap(i, j int) { - ci[i], ci[j] = ci[j], ci[i] -} - -func index(n int) int { - n-- - n >>= minBitSize - idx := 0 - for n > 0 { - n >>= 1 - idx++ - } - if idx >= steps { - idx = steps - 1 - } - return idx -} diff --git a/vendor/github.com/valyala/fasthttp/.gitignore b/vendor/github.com/valyala/fasthttp/.gitignore deleted file mode 100644 index 7b58ce45..00000000 --- a/vendor/github.com/valyala/fasthttp/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -tags -*.pprof -*.fasthttp.gz diff --git a/vendor/github.com/valyala/fasthttp/.travis.yml b/vendor/github.com/valyala/fasthttp/.travis.yml deleted file mode 100644 index fca54a82..00000000 --- a/vendor/github.com/valyala/fasthttp/.travis.yml +++ /dev/null @@ -1,36 +0,0 @@ -language: go - -go: - - tip - - 1.11 - - 1.10.x - - 1.9.x - -os: - - linux - - osx - -matrix: - allow_failures: - - tip - fast_finish: true - -before_install: - - go get -t -v ./... - # - go get -v golang.org/x/tools/cmd/goimports - -script: - # TODO(@kirilldanshin) - # - test -z "$(goimports -d $(find . -type f -name '*.go' -not -path "./vendor/*"))" - # build test for supported platforms - - GOOS=linux go build - - GOOS=darwin go build - - GOOS=freebsd go build - - GOOS=windows go build - - GOARCH=386 go build - - # run tests on a standard platform - - go test -v ./... - - # run tests with the race detector as well - - go test -race -v ./... diff --git a/vendor/github.com/valyala/fasthttp/LICENSE b/vendor/github.com/valyala/fasthttp/LICENSE deleted file mode 100644 index b098914a..00000000 --- a/vendor/github.com/valyala/fasthttp/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015-present Aliaksandr Valialkin, VertaMedia -Copyright (c) 2018-present Kirill Danshin -Copyright (c) 2018-present Erik Dubbelboer -Copyright (c) 2018-present FastHTTP Authors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/vendor/github.com/valyala/fasthttp/README.md b/vendor/github.com/valyala/fasthttp/README.md deleted file mode 100644 index 1a8bff83..00000000 --- a/vendor/github.com/valyala/fasthttp/README.md +++ /dev/null @@ -1,584 +0,0 @@ -[![Build Status](https://travis-ci.org/valyala/fasthttp.svg)](https://travis-ci.org/valyala/fasthttp) -[![GoDoc](https://godoc.org/github.com/valyala/fasthttp?status.svg)](http://godoc.org/github.com/valyala/fasthttp) -[![Go Report](https://goreportcard.com/badge/github.com/valyala/fasthttp)](https://goreportcard.com/report/github.com/valyala/fasthttp) - -# fasthttp -Fast HTTP implementation for Go. - -Currently fasthttp is successfully used by [VertaMedia](https://vertamedia.com/) -in a production serving up to 200K rps from more than 1.5M concurrent keep-alive -connections per physical server. - -[TechEmpower Benchmark round 12 results](https://www.techempower.com/benchmarks/#section=data-r12&hw=peak&test=plaintext) - -[Server Benchmarks](#http-server-performance-comparison-with-nethttp) - -[Client Benchmarks](#http-client-comparison-with-nethttp) - -[Install](#install) - -[Documentation](https://godoc.org/github.com/valyala/fasthttp) - -[Examples from docs](https://godoc.org/github.com/valyala/fasthttp#pkg-examples) - -[Code examples](examples) - -[Awesome fasthttp tools](https://github.com/fasthttp) - -[Switching from net/http to fasthttp](#switching-from-nethttp-to-fasthttp) - -[Fasthttp best practices](#fasthttp-best-practices) - -[Tricks with byte buffers](#tricks-with-byte-buffers) - -[Related projects](#related-projects) - -[FAQ](#faq) - -# HTTP server performance comparison with [net/http](https://golang.org/pkg/net/http/) - -In short, fasthttp server is up to 10 times faster than net/http. -Below are benchmark results. - -*GOMAXPROCS=1* - -net/http server: -``` -$ GOMAXPROCS=1 go test -bench=NetHTTPServerGet -benchmem -benchtime=10s -BenchmarkNetHTTPServerGet1ReqPerConn 1000000 12052 ns/op 2297 B/op 29 allocs/op -BenchmarkNetHTTPServerGet2ReqPerConn 1000000 12278 ns/op 2327 B/op 24 allocs/op -BenchmarkNetHTTPServerGet10ReqPerConn 2000000 8903 ns/op 2112 B/op 19 allocs/op -BenchmarkNetHTTPServerGet10KReqPerConn 2000000 8451 ns/op 2058 B/op 18 allocs/op -BenchmarkNetHTTPServerGet1ReqPerConn10KClients 500000 26733 ns/op 3229 B/op 29 allocs/op -BenchmarkNetHTTPServerGet2ReqPerConn10KClients 1000000 23351 ns/op 3211 B/op 24 allocs/op -BenchmarkNetHTTPServerGet10ReqPerConn10KClients 1000000 13390 ns/op 2483 B/op 19 allocs/op -BenchmarkNetHTTPServerGet100ReqPerConn10KClients 1000000 13484 ns/op 2171 B/op 18 allocs/op -``` - -fasthttp server: -``` -$ GOMAXPROCS=1 go test -bench=kServerGet -benchmem -benchtime=10s -BenchmarkServerGet1ReqPerConn 10000000 1559 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet2ReqPerConn 10000000 1248 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet10ReqPerConn 20000000 797 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet10KReqPerConn 20000000 716 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet1ReqPerConn10KClients 10000000 1974 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet2ReqPerConn10KClients 10000000 1352 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet10ReqPerConn10KClients 20000000 789 ns/op 2 B/op 0 allocs/op -BenchmarkServerGet100ReqPerConn10KClients 20000000 604 ns/op 0 B/op 0 allocs/op -``` - -*GOMAXPROCS=4* - -net/http server: -``` -$ GOMAXPROCS=4 go test -bench=NetHTTPServerGet -benchmem -benchtime=10s -BenchmarkNetHTTPServerGet1ReqPerConn-4 3000000 4529 ns/op 2389 B/op 29 allocs/op -BenchmarkNetHTTPServerGet2ReqPerConn-4 5000000 3896 ns/op 2418 B/op 24 allocs/op -BenchmarkNetHTTPServerGet10ReqPerConn-4 5000000 3145 ns/op 2160 B/op 19 allocs/op -BenchmarkNetHTTPServerGet10KReqPerConn-4 5000000 3054 ns/op 2065 B/op 18 allocs/op -BenchmarkNetHTTPServerGet1ReqPerConn10KClients-4 1000000 10321 ns/op 3710 B/op 30 allocs/op -BenchmarkNetHTTPServerGet2ReqPerConn10KClients-4 2000000 7556 ns/op 3296 B/op 24 allocs/op -BenchmarkNetHTTPServerGet10ReqPerConn10KClients-4 5000000 3905 ns/op 2349 B/op 19 allocs/op -BenchmarkNetHTTPServerGet100ReqPerConn10KClients-4 5000000 3435 ns/op 2130 B/op 18 allocs/op -``` - -fasthttp server: -``` -$ GOMAXPROCS=4 go test -bench=kServerGet -benchmem -benchtime=10s -BenchmarkServerGet1ReqPerConn-4 10000000 1141 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet2ReqPerConn-4 20000000 707 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet10ReqPerConn-4 30000000 341 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet10KReqPerConn-4 50000000 310 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet1ReqPerConn10KClients-4 10000000 1119 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet2ReqPerConn10KClients-4 20000000 644 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet10ReqPerConn10KClients-4 30000000 346 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet100ReqPerConn10KClients-4 50000000 282 ns/op 0 B/op 0 allocs/op -``` - -# HTTP client comparison with net/http - -In short, fasthttp client is up to 10 times faster than net/http. -Below are benchmark results. - -*GOMAXPROCS=1* - -net/http client: -``` -$ GOMAXPROCS=1 go test -bench='HTTPClient(Do|GetEndToEnd)' -benchmem -benchtime=10s -BenchmarkNetHTTPClientDoFastServer 1000000 12567 ns/op 2616 B/op 35 allocs/op -BenchmarkNetHTTPClientGetEndToEnd1TCP 200000 67030 ns/op 5028 B/op 56 allocs/op -BenchmarkNetHTTPClientGetEndToEnd10TCP 300000 51098 ns/op 5031 B/op 56 allocs/op -BenchmarkNetHTTPClientGetEndToEnd100TCP 300000 45096 ns/op 5026 B/op 55 allocs/op -BenchmarkNetHTTPClientGetEndToEnd1Inmemory 500000 24779 ns/op 5035 B/op 57 allocs/op -BenchmarkNetHTTPClientGetEndToEnd10Inmemory 1000000 26425 ns/op 5035 B/op 57 allocs/op -BenchmarkNetHTTPClientGetEndToEnd100Inmemory 500000 28515 ns/op 5045 B/op 57 allocs/op -BenchmarkNetHTTPClientGetEndToEnd1000Inmemory 500000 39511 ns/op 5096 B/op 56 allocs/op -``` - -fasthttp client: -``` -$ GOMAXPROCS=1 go test -bench='kClient(Do|GetEndToEnd)' -benchmem -benchtime=10s -BenchmarkClientDoFastServer 20000000 865 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd1TCP 1000000 18711 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd10TCP 1000000 14664 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd100TCP 1000000 14043 ns/op 1 B/op 0 allocs/op -BenchmarkClientGetEndToEnd1Inmemory 5000000 3965 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd10Inmemory 3000000 4060 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd100Inmemory 5000000 3396 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd1000Inmemory 5000000 3306 ns/op 2 B/op 0 allocs/op -``` - -*GOMAXPROCS=4* - -net/http client: -``` -$ GOMAXPROCS=4 go test -bench='HTTPClient(Do|GetEndToEnd)' -benchmem -benchtime=10s -BenchmarkNetHTTPClientDoFastServer-4 2000000 8774 ns/op 2619 B/op 35 allocs/op -BenchmarkNetHTTPClientGetEndToEnd1TCP-4 500000 22951 ns/op 5047 B/op 56 allocs/op -BenchmarkNetHTTPClientGetEndToEnd10TCP-4 1000000 19182 ns/op 5037 B/op 55 allocs/op -BenchmarkNetHTTPClientGetEndToEnd100TCP-4 1000000 16535 ns/op 5031 B/op 55 allocs/op -BenchmarkNetHTTPClientGetEndToEnd1Inmemory-4 1000000 14495 ns/op 5038 B/op 56 allocs/op -BenchmarkNetHTTPClientGetEndToEnd10Inmemory-4 1000000 10237 ns/op 5034 B/op 56 allocs/op -BenchmarkNetHTTPClientGetEndToEnd100Inmemory-4 1000000 10125 ns/op 5045 B/op 56 allocs/op -BenchmarkNetHTTPClientGetEndToEnd1000Inmemory-4 1000000 11132 ns/op 5136 B/op 56 allocs/op -``` - -fasthttp client: -``` -$ GOMAXPROCS=4 go test -bench='kClient(Do|GetEndToEnd)' -benchmem -benchtime=10s -BenchmarkClientDoFastServer-4 50000000 397 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd1TCP-4 2000000 7388 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd10TCP-4 2000000 6689 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd100TCP-4 3000000 4927 ns/op 1 B/op 0 allocs/op -BenchmarkClientGetEndToEnd1Inmemory-4 10000000 1604 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd10Inmemory-4 10000000 1458 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd100Inmemory-4 10000000 1329 ns/op 0 B/op 0 allocs/op -BenchmarkClientGetEndToEnd1000Inmemory-4 10000000 1316 ns/op 5 B/op 0 allocs/op -``` - - -# Install - -``` -go get -u github.com/valyala/fasthttp -``` - - -# Switching from net/http to fasthttp - -Unfortunately, fasthttp doesn't provide API identical to net/http. -See the [FAQ](#faq) for details. -There is [net/http -> fasthttp handler converter](https://godoc.org/github.com/valyala/fasthttp/fasthttpadaptor), -but it is better to write fasthttp request handlers by hand in order to use -all of the fasthttp advantages (especially high performance :) ). - -Important points: - -* Fasthttp works with [RequestHandler functions](https://godoc.org/github.com/valyala/fasthttp#RequestHandler) -instead of objects implementing [Handler interface](https://golang.org/pkg/net/http/#Handler). -Fortunately, it is easy to pass bound struct methods to fasthttp: - - ```go - type MyHandler struct { - foobar string - } - - // request handler in net/http style, i.e. method bound to MyHandler struct. - func (h *MyHandler) HandleFastHTTP(ctx *fasthttp.RequestCtx) { - // notice that we may access MyHandler properties here - see h.foobar. - fmt.Fprintf(ctx, "Hello, world! Requested path is %q. Foobar is %q", - ctx.Path(), h.foobar) - } - - // request handler in fasthttp style, i.e. just plain function. - func fastHTTPHandler(ctx *fasthttp.RequestCtx) { - fmt.Fprintf(ctx, "Hi there! RequestURI is %q", ctx.RequestURI()) - } - - // pass bound struct method to fasthttp - myHandler := &MyHandler{ - foobar: "foobar", - } - fasthttp.ListenAndServe(":8080", myHandler.HandleFastHTTP) - - // pass plain function to fasthttp - fasthttp.ListenAndServe(":8081", fastHTTPHandler) - ``` - -* The [RequestHandler](https://godoc.org/github.com/valyala/fasthttp#RequestHandler) -accepts only one argument - [RequestCtx](https://godoc.org/github.com/valyala/fasthttp#RequestCtx). -It contains all the functionality required for http request processing -and response writing. Below is an example of a simple request handler conversion -from net/http to fasthttp. - - ```go - // net/http request handler - requestHandler := func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/foo": - fooHandler(w, r) - case "/bar": - barHandler(w, r) - default: - http.Error(w, "Unsupported path", http.StatusNotFound) - } - } - ``` - - ```go - // the corresponding fasthttp request handler - requestHandler := func(ctx *fasthttp.RequestCtx) { - switch string(ctx.Path()) { - case "/foo": - fooHandler(ctx) - case "/bar": - barHandler(ctx) - default: - ctx.Error("Unsupported path", fasthttp.StatusNotFound) - } - } - ``` - -* Fasthttp allows setting response headers and writing response body -in an arbitrary order. There is no 'headers first, then body' restriction -like in net/http. The following code is valid for fasthttp: - - ```go - requestHandler := func(ctx *fasthttp.RequestCtx) { - // set some headers and status code first - ctx.SetContentType("foo/bar") - ctx.SetStatusCode(fasthttp.StatusOK) - - // then write the first part of body - fmt.Fprintf(ctx, "this is the first part of body\n") - - // then set more headers - ctx.Response.Header.Set("Foo-Bar", "baz") - - // then write more body - fmt.Fprintf(ctx, "this is the second part of body\n") - - // then override already written body - ctx.SetBody([]byte("this is completely new body contents")) - - // then update status code - ctx.SetStatusCode(fasthttp.StatusNotFound) - - // basically, anything may be updated many times before - // returning from RequestHandler. - // - // Unlike net/http fasthttp doesn't put response to the wire until - // returning from RequestHandler. - } - ``` - -* Fasthttp doesn't provide [ServeMux](https://golang.org/pkg/net/http/#ServeMux), -but there are more powerful third-party routers and web frameworks -with fasthttp support: - - * [fasthttp-routing](https://github.com/qiangxue/fasthttp-routing) - * [fasthttprouter](https://github.com/buaazp/fasthttprouter) - * [lu](https://github.com/vincentLiuxiang/lu) - * [atreugo](https://github.com/savsgio/atreugo) - - Net/http code with simple ServeMux is trivially converted to fasthttp code: - - ```go - // net/http code - - m := &http.ServeMux{} - m.HandleFunc("/foo", fooHandlerFunc) - m.HandleFunc("/bar", barHandlerFunc) - m.Handle("/baz", bazHandler) - - http.ListenAndServe(":80", m) - ``` - - ```go - // the corresponding fasthttp code - m := func(ctx *fasthttp.RequestCtx) { - switch string(ctx.Path()) { - case "/foo": - fooHandlerFunc(ctx) - case "/bar": - barHandlerFunc(ctx) - case "/baz": - bazHandler.HandlerFunc(ctx) - default: - ctx.Error("not found", fasthttp.StatusNotFound) - } - } - - fasthttp.ListenAndServe(":80", m) - ``` - -* net/http -> fasthttp conversion table: - - * All the pseudocode below assumes w, r and ctx have these types: - ```go - var ( - w http.ResponseWriter - r *http.Request - ctx *fasthttp.RequestCtx - ) - ``` - * r.Body -> [ctx.PostBody()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.PostBody) - * r.URL.Path -> [ctx.Path()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Path) - * r.URL -> [ctx.URI()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.URI) - * r.Method -> [ctx.Method()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Method) - * r.Header -> [ctx.Request.Header](https://godoc.org/github.com/valyala/fasthttp#RequestHeader) - * r.Header.Get() -> [ctx.Request.Header.Peek()](https://godoc.org/github.com/valyala/fasthttp#RequestHeader.Peek) - * r.Host -> [ctx.Host()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Host) - * r.Form -> [ctx.QueryArgs()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.QueryArgs) + - [ctx.PostArgs()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.PostArgs) - * r.PostForm -> [ctx.PostArgs()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.PostArgs) - * r.FormValue() -> [ctx.FormValue()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.FormValue) - * r.FormFile() -> [ctx.FormFile()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.FormFile) - * r.MultipartForm -> [ctx.MultipartForm()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.MultipartForm) - * r.RemoteAddr -> [ctx.RemoteAddr()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.RemoteAddr) - * r.RequestURI -> [ctx.RequestURI()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.RequestURI) - * r.TLS -> [ctx.IsTLS()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.IsTLS) - * r.Cookie() -> [ctx.Request.Header.Cookie()](https://godoc.org/github.com/valyala/fasthttp#RequestHeader.Cookie) - * r.Referer() -> [ctx.Referer()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Referer) - * r.UserAgent() -> [ctx.UserAgent()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.UserAgent) - * w.Header() -> [ctx.Response.Header](https://godoc.org/github.com/valyala/fasthttp#ResponseHeader) - * w.Header().Set() -> [ctx.Response.Header.Set()](https://godoc.org/github.com/valyala/fasthttp#ResponseHeader.Set) - * w.Header().Set("Content-Type") -> [ctx.SetContentType()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.SetContentType) - * w.Header().Set("Set-Cookie") -> [ctx.Response.Header.SetCookie()](https://godoc.org/github.com/valyala/fasthttp#ResponseHeader.SetCookie) - * w.Write() -> [ctx.Write()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Write), - [ctx.SetBody()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.SetBody), - [ctx.SetBodyStream()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.SetBodyStream), - [ctx.SetBodyStreamWriter()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.SetBodyStreamWriter) - * w.WriteHeader() -> [ctx.SetStatusCode()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.SetStatusCode) - * w.(http.Hijacker).Hijack() -> [ctx.Hijack()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Hijack) - * http.Error() -> [ctx.Error()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Error) - * http.FileServer() -> [fasthttp.FSHandler()](https://godoc.org/github.com/valyala/fasthttp#FSHandler), - [fasthttp.FS](https://godoc.org/github.com/valyala/fasthttp#FS) - * http.ServeFile() -> [fasthttp.ServeFile()](https://godoc.org/github.com/valyala/fasthttp#ServeFile) - * http.Redirect() -> [ctx.Redirect()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Redirect) - * http.NotFound() -> [ctx.NotFound()](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.NotFound) - * http.StripPrefix() -> [fasthttp.PathRewriteFunc](https://godoc.org/github.com/valyala/fasthttp#PathRewriteFunc) - -* *VERY IMPORTANT!* Fasthttp disallows holding references -to [RequestCtx](https://godoc.org/github.com/valyala/fasthttp#RequestCtx) or to its' -members after returning from [RequestHandler](https://godoc.org/github.com/valyala/fasthttp#RequestHandler). -Otherwise [data races](http://blog.golang.org/race-detector) are inevitable. -Carefully inspect all the net/http request handlers converted to fasthttp whether -they retain references to RequestCtx or to its' members after returning. -RequestCtx provides the following _band aids_ for this case: - - * Wrap RequestHandler into [TimeoutHandler](https://godoc.org/github.com/valyala/fasthttp#TimeoutHandler). - * Call [TimeoutError](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.TimeoutError) - before returning from RequestHandler if there are references to RequestCtx or to its' members. - See [the example](https://godoc.org/github.com/valyala/fasthttp#example-RequestCtx-TimeoutError) - for more details. - -Use this brilliant tool - [race detector](http://blog.golang.org/race-detector) - -for detecting and eliminating data races in your program. If you detected -data race related to fasthttp in your program, then there is high probability -you forgot calling [TimeoutError](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.TimeoutError) -before returning from [RequestHandler](https://godoc.org/github.com/valyala/fasthttp#RequestHandler). - -* Blind switching from net/http to fasthttp won't give you performance boost. -While fasthttp is optimized for speed, its' performance may be easily saturated -by slow [RequestHandler](https://godoc.org/github.com/valyala/fasthttp#RequestHandler). -So [profile](http://blog.golang.org/profiling-go-programs) and optimize your -code after switching to fasthttp. For instance, use [quicktemplate](https://github.com/valyala/quicktemplate) -instead of [html/template](https://golang.org/pkg/html/template/). - -* See also [fasthttputil](https://godoc.org/github.com/valyala/fasthttp/fasthttputil), -[fasthttpadaptor](https://godoc.org/github.com/valyala/fasthttp/fasthttpadaptor) and -[expvarhandler](https://godoc.org/github.com/valyala/fasthttp/expvarhandler). - - -# Performance optimization tips for multi-core systems - -* Use [reuseport](https://godoc.org/github.com/valyala/fasthttp/reuseport) listener. -* Run a separate server instance per CPU core with GOMAXPROCS=1. -* Pin each server instance to a separate CPU core using [taskset](http://linux.die.net/man/1/taskset). -* Ensure the interrupts of multiqueue network card are evenly distributed between CPU cores. - See [this article](https://blog.cloudflare.com/how-to-achieve-low-latency/) for details. -* Use Go 1.6 as it provides some considerable performance improvements. - - -# Fasthttp best practices - -* Do not allocate objects and `[]byte` buffers - just reuse them as much - as possible. Fasthttp API design encourages this. -* [sync.Pool](https://golang.org/pkg/sync/#Pool) is your best friend. -* [Profile your program](http://blog.golang.org/profiling-go-programs) - in production. - `go tool pprof --alloc_objects your-program mem.pprof` usually gives better - insights for optimization opportunities than `go tool pprof your-program cpu.pprof`. -* Write [tests and benchmarks](https://golang.org/pkg/testing/) for hot paths. -* Avoid conversion between `[]byte` and `string`, since this may result in memory - allocation+copy. Fasthttp API provides functions for both `[]byte` and `string` - - use these functions instead of converting manually between `[]byte` and `string`. - There are some exceptions - see [this wiki page](https://github.com/golang/go/wiki/CompilerOptimizations#string-and-byte) - for more details. -* Verify your tests and production code under - [race detector](https://golang.org/doc/articles/race_detector.html) on a regular basis. -* Prefer [quicktemplate](https://github.com/valyala/quicktemplate) instead of - [html/template](https://golang.org/pkg/html/template/) in your webserver. - - -# Tricks with `[]byte` buffers - -The following tricks are used by fasthttp. Use them in your code too. - -* Standard Go functions accept nil buffers -```go -var ( - // both buffers are uninitialized - dst []byte - src []byte -) -dst = append(dst, src...) // is legal if dst is nil and/or src is nil -copy(dst, src) // is legal if dst is nil and/or src is nil -(string(src) == "") // is true if src is nil -(len(src) == 0) // is true if src is nil -src = src[:0] // works like a charm with nil src - -// this for loop doesn't panic if src is nil -for i, ch := range src { - doSomething(i, ch) -} -``` - -So throw away nil checks for `[]byte` buffers from you code. For example, -```go -srcLen := 0 -if src != nil { - srcLen = len(src) -} -``` - -becomes - -```go -srcLen := len(src) -``` - -* String may be appended to `[]byte` buffer with `append` -```go -dst = append(dst, "foobar"...) -``` - -* `[]byte` buffer may be extended to its' capacity. -```go -buf := make([]byte, 100) -a := buf[:10] // len(a) == 10, cap(a) == 100. -b := a[:100] // is valid, since cap(a) == 100. -``` - -* All fasthttp functions accept nil `[]byte` buffer -```go -statusCode, body, err := fasthttp.Get(nil, "http://google.com/") -uintBuf := fasthttp.AppendUint(nil, 1234) -``` - -# Related projects - - * [fasthttp](https://github.com/fasthttp) - various useful - helpers for projects based on fasthttp. - * [fasthttp-routing](https://github.com/qiangxue/fasthttp-routing) - fast and - powerful routing package for fasthttp servers. - * [fasthttprouter](https://github.com/buaazp/fasthttprouter) - a high - performance fasthttp request router that scales well. - * [gramework](https://github.com/gramework/gramework) - a web framework made by one of fasthttp maintainers - * [lu](https://github.com/vincentLiuxiang/lu) - a high performance - go middleware web framework which is based on fasthttp. - * [websocket](https://github.com/fasthttp/websocket) - Gorilla-based - websocket implementation for fasthttp. - * [fasthttpsession](https://github.com/phachon/fasthttpsession) - a fast and powerful session package for fasthttp servers. - * [atreugo](https://github.com/savsgio/atreugo) - Micro-framework to make simple the use of routing and middlewares. - - -# FAQ - -* *Why creating yet another http package instead of optimizing net/http?* - - Because net/http API limits many optimization opportunities. - For example: - * net/http Request object lifetime isn't limited by request handler execution - time. So the server must create a new request object per each request instead - of reusing existing objects like fasthttp does. - * net/http headers are stored in a `map[string][]string`. So the server - must parse all the headers, convert them from `[]byte` to `string` and put - them into the map before calling user-provided request handler. - This all requires unnecessary memory allocations avoided by fasthttp. - * net/http client API requires creating a new response object per each request. - -* *Why fasthttp API is incompatible with net/http?* - - Because net/http API limits many optimization opportunities. See the answer - above for more details. Also certain net/http API parts are suboptimal - for use: - * Compare [net/http connection hijacking](https://golang.org/pkg/net/http/#Hijacker) - to [fasthttp connection hijacking](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Hijack). - * Compare [net/http Request.Body reading](https://golang.org/pkg/net/http/#Request) - to [fasthttp request body reading](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.PostBody). - -* *Why fasthttp doesn't support HTTP/2.0 and WebSockets?* - - [HTTP/2.0 support](https://github.com/fasthttp/http2) is in progress. [WebSockets](https://github.com/fasthttp/websockets) has been done already. - Third parties also may use [RequestCtx.Hijack](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Hijack) - for implementing these goodies. - -* *Are there known net/http advantages comparing to fasthttp?* - - Yes: - * net/http supports [HTTP/2.0 starting from go1.6](https://http2.golang.org/). - * net/http API is stable, while fasthttp API constantly evolves. - * net/http handles more HTTP corner cases. - * net/http should contain less bugs, since it is used and tested by much - wider audience. - * net/http works on Go older than 1.5. - -* *Why fasthttp API prefers returning `[]byte` instead of `string`?* - - Because `[]byte` to `string` conversion isn't free - it requires memory - allocation and copy. Feel free wrapping returned `[]byte` result into - `string()` if you prefer working with strings instead of byte slices. - But be aware that this has non-zero overhead. - -* *Which GO versions are supported by fasthttp?* - - Go1.5+. Older versions won't be supported, since their standard package - [miss useful functions](https://github.com/valyala/fasthttp/issues/5). - - **NOTE**: Go 1.9.7 is the oldest tested version. We recommend you to update as soon as you can. As of 1.11.3 we will drop 1.9.x support. - -* *Please provide real benchmark data and server information* - - See [this issue](https://github.com/valyala/fasthttp/issues/4). - -* *Are there plans to add request routing to fasthttp?* - - There are no plans to add request routing into fasthttp. - Use third-party routers and web frameworks with fasthttp support: - - * [fasthttp-routing](https://github.com/qiangxue/fasthttp-routing) - * [fasthttprouter](https://github.com/buaazp/fasthttprouter) - * [gramework](https://github.com/gramework/gramework) - * [lu](https://github.com/vincentLiuxiang/lu) - * [atreugo](https://github.com/savsgio/atreugo) - - See also [this issue](https://github.com/valyala/fasthttp/issues/9) for more info. - -* *I detected data race in fasthttp!* - - Cool! [File a bug](https://github.com/valyala/fasthttp/issues/new). But before - doing this check the following in your code: - - * Make sure there are no references to [RequestCtx](https://godoc.org/github.com/valyala/fasthttp#RequestCtx) - or to its' members after returning from [RequestHandler](https://godoc.org/github.com/valyala/fasthttp#RequestHandler). - * Make sure you call [TimeoutError](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.TimeoutError) - before returning from [RequestHandler](https://godoc.org/github.com/valyala/fasthttp#RequestHandler) - if there are references to [RequestCtx](https://godoc.org/github.com/valyala/fasthttp#RequestCtx) - or to its' members, which may be accessed by other goroutines. - -* *I didn't find an answer for my question here* - - Try exploring [these questions](https://github.com/valyala/fasthttp/issues?q=label%3Aquestion). diff --git a/vendor/github.com/valyala/fasthttp/TODO b/vendor/github.com/valyala/fasthttp/TODO deleted file mode 100644 index ce7505f1..00000000 --- a/vendor/github.com/valyala/fasthttp/TODO +++ /dev/null @@ -1,4 +0,0 @@ -- SessionClient with referer and cookies support. -- ProxyHandler similar to FSHandler. -- WebSockets. See https://tools.ietf.org/html/rfc6455 . -- HTTP/2.0. See https://tools.ietf.org/html/rfc7540 . diff --git a/vendor/github.com/valyala/fasthttp/args.go b/vendor/github.com/valyala/fasthttp/args.go deleted file mode 100644 index 4d9b35ee..00000000 --- a/vendor/github.com/valyala/fasthttp/args.go +++ /dev/null @@ -1,522 +0,0 @@ -package fasthttp - -import ( - "bytes" - "errors" - "io" - "sync" - - "github.com/valyala/bytebufferpool" -) - -// AcquireArgs returns an empty Args object from the pool. -// -// The returned Args may be returned to the pool with ReleaseArgs -// when no longer needed. This allows reducing GC load. -func AcquireArgs() *Args { - return argsPool.Get().(*Args) -} - -// ReleaseArgs returns the object acquired via AquireArgs to the pool. -// -// Do not access the released Args object, otherwise data races may occur. -func ReleaseArgs(a *Args) { - a.Reset() - argsPool.Put(a) -} - -var argsPool = &sync.Pool{ - New: func() interface{} { - return &Args{} - }, -} - -// Args represents query arguments. -// -// It is forbidden copying Args instances. Create new instances instead -// and use CopyTo(). -// -// Args instance MUST NOT be used from concurrently running goroutines. -type Args struct { - noCopy noCopy - - args []argsKV - buf []byte -} - -type argsKV struct { - key []byte - value []byte -} - -// Reset clears query args. -func (a *Args) Reset() { - a.args = a.args[:0] -} - -// CopyTo copies all args to dst. -func (a *Args) CopyTo(dst *Args) { - dst.Reset() - dst.args = copyArgs(dst.args, a.args) -} - -// VisitAll calls f for each existing arg. -// -// f must not retain references to key and value after returning. -// Make key and/or value copies if you need storing them after returning. -func (a *Args) VisitAll(f func(key, value []byte)) { - visitArgs(a.args, f) -} - -// Len returns the number of query args. -func (a *Args) Len() int { - return len(a.args) -} - -// Parse parses the given string containing query args. -func (a *Args) Parse(s string) { - a.buf = append(a.buf[:0], s...) - a.ParseBytes(a.buf) -} - -// ParseBytes parses the given b containing query args. -func (a *Args) ParseBytes(b []byte) { - a.Reset() - - var s argsScanner - s.b = b - - var kv *argsKV - a.args, kv = allocArg(a.args) - for s.next(kv) { - if len(kv.key) > 0 || len(kv.value) > 0 { - a.args, kv = allocArg(a.args) - } - } - a.args = releaseArg(a.args) -} - -// String returns string representation of query args. -func (a *Args) String() string { - return string(a.QueryString()) -} - -// QueryString returns query string for the args. -// -// The returned value is valid until the next call to Args methods. -func (a *Args) QueryString() []byte { - a.buf = a.AppendBytes(a.buf[:0]) - return a.buf -} - -// AppendBytes appends query string to dst and returns the extended dst. -func (a *Args) AppendBytes(dst []byte) []byte { - for i, n := 0, len(a.args); i < n; i++ { - kv := &a.args[i] - dst = AppendQuotedArg(dst, kv.key) - if len(kv.value) > 0 { - dst = append(dst, '=') - dst = AppendQuotedArg(dst, kv.value) - } - if i+1 < n { - dst = append(dst, '&') - } - } - return dst -} - -// WriteTo writes query string to w. -// -// WriteTo implements io.WriterTo interface. -func (a *Args) WriteTo(w io.Writer) (int64, error) { - n, err := w.Write(a.QueryString()) - return int64(n), err -} - -// Del deletes argument with the given key from query args. -func (a *Args) Del(key string) { - a.args = delAllArgs(a.args, key) -} - -// DelBytes deletes argument with the given key from query args. -func (a *Args) DelBytes(key []byte) { - a.args = delAllArgs(a.args, b2s(key)) -} - -// Add adds 'key=value' argument. -// -// Multiple values for the same key may be added. -func (a *Args) Add(key, value string) { - a.args = appendArg(a.args, key, value) -} - -// AddBytesK adds 'key=value' argument. -// -// Multiple values for the same key may be added. -func (a *Args) AddBytesK(key []byte, value string) { - a.args = appendArg(a.args, b2s(key), value) -} - -// AddBytesV adds 'key=value' argument. -// -// Multiple values for the same key may be added. -func (a *Args) AddBytesV(key string, value []byte) { - a.args = appendArg(a.args, key, b2s(value)) -} - -// AddBytesKV adds 'key=value' argument. -// -// Multiple values for the same key may be added. -func (a *Args) AddBytesKV(key, value []byte) { - a.args = appendArg(a.args, b2s(key), b2s(value)) -} - -// Set sets 'key=value' argument. -func (a *Args) Set(key, value string) { - a.args = setArg(a.args, key, value) -} - -// SetBytesK sets 'key=value' argument. -func (a *Args) SetBytesK(key []byte, value string) { - a.args = setArg(a.args, b2s(key), value) -} - -// SetBytesV sets 'key=value' argument. -func (a *Args) SetBytesV(key string, value []byte) { - a.args = setArg(a.args, key, b2s(value)) -} - -// SetBytesKV sets 'key=value' argument. -func (a *Args) SetBytesKV(key, value []byte) { - a.args = setArgBytes(a.args, key, value) -} - -// Peek returns query arg value for the given key. -// -// Returned value is valid until the next Args call. -func (a *Args) Peek(key string) []byte { - return peekArgStr(a.args, key) -} - -// PeekBytes returns query arg value for the given key. -// -// Returned value is valid until the next Args call. -func (a *Args) PeekBytes(key []byte) []byte { - return peekArgBytes(a.args, key) -} - -// PeekMulti returns all the arg values for the given key. -func (a *Args) PeekMulti(key string) [][]byte { - var values [][]byte - a.VisitAll(func(k, v []byte) { - if string(k) == key { - values = append(values, v) - } - }) - return values -} - -// PeekMultiBytes returns all the arg values for the given key. -func (a *Args) PeekMultiBytes(key []byte) [][]byte { - return a.PeekMulti(b2s(key)) -} - -// Has returns true if the given key exists in Args. -func (a *Args) Has(key string) bool { - return hasArg(a.args, key) -} - -// HasBytes returns true if the given key exists in Args. -func (a *Args) HasBytes(key []byte) bool { - return hasArg(a.args, b2s(key)) -} - -// ErrNoArgValue is returned when Args value with the given key is missing. -var ErrNoArgValue = errors.New("no Args value for the given key") - -// GetUint returns uint value for the given key. -func (a *Args) GetUint(key string) (int, error) { - value := a.Peek(key) - if len(value) == 0 { - return -1, ErrNoArgValue - } - return ParseUint(value) -} - -// SetUint sets uint value for the given key. -func (a *Args) SetUint(key string, value int) { - bb := bytebufferpool.Get() - bb.B = AppendUint(bb.B[:0], value) - a.SetBytesV(key, bb.B) - bytebufferpool.Put(bb) -} - -// SetUintBytes sets uint value for the given key. -func (a *Args) SetUintBytes(key []byte, value int) { - a.SetUint(b2s(key), value) -} - -// GetUintOrZero returns uint value for the given key. -// -// Zero (0) is returned on error. -func (a *Args) GetUintOrZero(key string) int { - n, err := a.GetUint(key) - if err != nil { - n = 0 - } - return n -} - -// GetUfloat returns ufloat value for the given key. -func (a *Args) GetUfloat(key string) (float64, error) { - value := a.Peek(key) - if len(value) == 0 { - return -1, ErrNoArgValue - } - return ParseUfloat(value) -} - -// GetUfloatOrZero returns ufloat value for the given key. -// -// Zero (0) is returned on error. -func (a *Args) GetUfloatOrZero(key string) float64 { - f, err := a.GetUfloat(key) - if err != nil { - f = 0 - } - return f -} - -// GetBool returns boolean value for the given key. -// -// true is returned for "1", "t", "T", "true", "TRUE", "True", "y", "yes", "Y", "YES", "Yes", -// otherwise false is returned. -func (a *Args) GetBool(key string) bool { - switch b2s(a.Peek(key)) { - // Support the same true cases as strconv.ParseBool - // See: https://github.com/golang/go/blob/4e1b11e2c9bdb0ddea1141eed487be1a626ff5be/src/strconv/atob.go#L12 - // and Y and Yes versions. - case "1", "t", "T", "true", "TRUE", "True", "y", "yes", "Y", "YES", "Yes": - return true - default: - return false - } -} - -func visitArgs(args []argsKV, f func(k, v []byte)) { - for i, n := 0, len(args); i < n; i++ { - kv := &args[i] - f(kv.key, kv.value) - } -} - -func copyArgs(dst, src []argsKV) []argsKV { - if cap(dst) < len(src) { - tmp := make([]argsKV, len(src)) - copy(tmp, dst) - dst = tmp - } - n := len(src) - dst = dst[:n] - for i := 0; i < n; i++ { - dstKV := &dst[i] - srcKV := &src[i] - dstKV.key = append(dstKV.key[:0], srcKV.key...) - dstKV.value = append(dstKV.value[:0], srcKV.value...) - } - return dst -} - -func delAllArgsBytes(args []argsKV, key []byte) []argsKV { - return delAllArgs(args, b2s(key)) -} - -func delAllArgs(args []argsKV, key string) []argsKV { - for i, n := 0, len(args); i < n; i++ { - kv := &args[i] - if key == string(kv.key) { - tmp := *kv - copy(args[i:], args[i+1:]) - n-- - args[n] = tmp - args = args[:n] - } - } - return args -} - -func setArgBytes(h []argsKV, key, value []byte) []argsKV { - return setArg(h, b2s(key), b2s(value)) -} - -func setArg(h []argsKV, key, value string) []argsKV { - n := len(h) - for i := 0; i < n; i++ { - kv := &h[i] - if key == string(kv.key) { - kv.value = append(kv.value[:0], value...) - return h - } - } - return appendArg(h, key, value) -} - -func appendArgBytes(h []argsKV, key, value []byte) []argsKV { - return appendArg(h, b2s(key), b2s(value)) -} - -func appendArg(args []argsKV, key, value string) []argsKV { - var kv *argsKV - args, kv = allocArg(args) - kv.key = append(kv.key[:0], key...) - kv.value = append(kv.value[:0], value...) - return args -} - -func allocArg(h []argsKV) ([]argsKV, *argsKV) { - n := len(h) - if cap(h) > n { - h = h[:n+1] - } else { - h = append(h, argsKV{}) - } - return h, &h[n] -} - -func releaseArg(h []argsKV) []argsKV { - return h[:len(h)-1] -} - -func hasArg(h []argsKV, key string) bool { - for i, n := 0, len(h); i < n; i++ { - kv := &h[i] - if key == string(kv.key) { - return true - } - } - return false -} - -func peekArgBytes(h []argsKV, k []byte) []byte { - for i, n := 0, len(h); i < n; i++ { - kv := &h[i] - if bytes.Equal(kv.key, k) { - return kv.value - } - } - return nil -} - -func peekArgStr(h []argsKV, k string) []byte { - for i, n := 0, len(h); i < n; i++ { - kv := &h[i] - if string(kv.key) == k { - return kv.value - } - } - return nil -} - -type argsScanner struct { - b []byte -} - -func (s *argsScanner) next(kv *argsKV) bool { - if len(s.b) == 0 { - return false - } - - isKey := true - k := 0 - for i, c := range s.b { - switch c { - case '=': - if isKey { - isKey = false - kv.key = decodeArgAppend(kv.key[:0], s.b[:i]) - k = i + 1 - } - case '&': - if isKey { - kv.key = decodeArgAppend(kv.key[:0], s.b[:i]) - kv.value = kv.value[:0] - } else { - kv.value = decodeArgAppend(kv.value[:0], s.b[k:i]) - } - s.b = s.b[i+1:] - return true - } - } - - if isKey { - kv.key = decodeArgAppend(kv.key[:0], s.b) - kv.value = kv.value[:0] - } else { - kv.value = decodeArgAppend(kv.value[:0], s.b[k:]) - } - s.b = s.b[len(s.b):] - return true -} - -func decodeArgAppend(dst, src []byte) []byte { - if bytes.IndexByte(src, '%') < 0 && bytes.IndexByte(src, '+') < 0 { - // fast path: src doesn't contain encoded chars - return append(dst, src...) - } - - // slow path - for i := 0; i < len(src); i++ { - c := src[i] - if c == '%' { - if i+2 >= len(src) { - return append(dst, src[i:]...) - } - x2 := hex2intTable[src[i+2]] - x1 := hex2intTable[src[i+1]] - if x1 == 16 || x2 == 16 { - dst = append(dst, '%') - } else { - dst = append(dst, x1<<4|x2) - i += 2 - } - } else if c == '+' { - dst = append(dst, ' ') - } else { - dst = append(dst, c) - } - } - return dst -} - -// decodeArgAppendNoPlus is almost identical to decodeArgAppend, but it doesn't -// substitute '+' with ' '. -// -// The function is copy-pasted from decodeArgAppend due to the performance -// reasons only. -func decodeArgAppendNoPlus(dst, src []byte) []byte { - if bytes.IndexByte(src, '%') < 0 { - // fast path: src doesn't contain encoded chars - return append(dst, src...) - } - - // slow path - for i := 0; i < len(src); i++ { - c := src[i] - if c == '%' { - if i+2 >= len(src) { - return append(dst, src[i:]...) - } - x2 := hex2intTable[src[i+2]] - x1 := hex2intTable[src[i+1]] - if x1 == 16 || x2 == 16 { - dst = append(dst, '%') - } else { - dst = append(dst, x1<<4|x2) - i += 2 - } - } else { - dst = append(dst, c) - } - } - return dst -} diff --git a/vendor/github.com/valyala/fasthttp/bytesconv.go b/vendor/github.com/valyala/fasthttp/bytesconv.go deleted file mode 100644 index 6292957d..00000000 --- a/vendor/github.com/valyala/fasthttp/bytesconv.go +++ /dev/null @@ -1,456 +0,0 @@ -package fasthttp - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - "math" - "net" - "reflect" - "strings" - "sync" - "time" - "unsafe" -) - -// AppendHTMLEscape appends html-escaped s to dst and returns the extended dst. -func AppendHTMLEscape(dst []byte, s string) []byte { - if strings.IndexByte(s, '<') < 0 && - strings.IndexByte(s, '>') < 0 && - strings.IndexByte(s, '"') < 0 && - strings.IndexByte(s, '\'') < 0 { - - // fast path - nothing to escape - return append(dst, s...) - } - - // slow path - var prev int - var sub string - for i, n := 0, len(s); i < n; i++ { - sub = "" - switch s[i] { - case '<': - sub = "<" - case '>': - sub = ">" - case '"': - sub = """ - case '\'': - sub = "'" - } - if len(sub) > 0 { - dst = append(dst, s[prev:i]...) - dst = append(dst, sub...) - prev = i + 1 - } - } - return append(dst, s[prev:]...) -} - -// AppendHTMLEscapeBytes appends html-escaped s to dst and returns -// the extended dst. -func AppendHTMLEscapeBytes(dst, s []byte) []byte { - return AppendHTMLEscape(dst, b2s(s)) -} - -// AppendIPv4 appends string representation of the given ip v4 to dst -// and returns the extended dst. -func AppendIPv4(dst []byte, ip net.IP) []byte { - ip = ip.To4() - if ip == nil { - return append(dst, "non-v4 ip passed to AppendIPv4"...) - } - - dst = AppendUint(dst, int(ip[0])) - for i := 1; i < 4; i++ { - dst = append(dst, '.') - dst = AppendUint(dst, int(ip[i])) - } - return dst -} - -var errEmptyIPStr = errors.New("empty ip address string") - -// ParseIPv4 parses ip address from ipStr into dst and returns the extended dst. -func ParseIPv4(dst net.IP, ipStr []byte) (net.IP, error) { - if len(ipStr) == 0 { - return dst, errEmptyIPStr - } - if len(dst) < net.IPv4len { - dst = make([]byte, net.IPv4len) - } - copy(dst, net.IPv4zero) - dst = dst.To4() - if dst == nil { - panic("BUG: dst must not be nil") - } - - b := ipStr - for i := 0; i < 3; i++ { - n := bytes.IndexByte(b, '.') - if n < 0 { - return dst, fmt.Errorf("cannot find dot in ipStr %q", ipStr) - } - v, err := ParseUint(b[:n]) - if err != nil { - return dst, fmt.Errorf("cannot parse ipStr %q: %s", ipStr, err) - } - if v > 255 { - return dst, fmt.Errorf("cannot parse ipStr %q: ip part cannot exceed 255: parsed %d", ipStr, v) - } - dst[i] = byte(v) - b = b[n+1:] - } - v, err := ParseUint(b) - if err != nil { - return dst, fmt.Errorf("cannot parse ipStr %q: %s", ipStr, err) - } - if v > 255 { - return dst, fmt.Errorf("cannot parse ipStr %q: ip part cannot exceed 255: parsed %d", ipStr, v) - } - dst[3] = byte(v) - - return dst, nil -} - -// AppendHTTPDate appends HTTP-compliant (RFC1123) representation of date -// to dst and returns the extended dst. -func AppendHTTPDate(dst []byte, date time.Time) []byte { - dst = date.In(time.UTC).AppendFormat(dst, time.RFC1123) - copy(dst[len(dst)-3:], strGMT) - return dst -} - -// ParseHTTPDate parses HTTP-compliant (RFC1123) date. -func ParseHTTPDate(date []byte) (time.Time, error) { - return time.Parse(time.RFC1123, b2s(date)) -} - -// AppendUint appends n to dst and returns the extended dst. -func AppendUint(dst []byte, n int) []byte { - if n < 0 { - panic("BUG: int must be positive") - } - - var b [20]byte - buf := b[:] - i := len(buf) - var q int - for n >= 10 { - i-- - q = n / 10 - buf[i] = '0' + byte(n-q*10) - n = q - } - i-- - buf[i] = '0' + byte(n) - - dst = append(dst, buf[i:]...) - return dst -} - -// ParseUint parses uint from buf. -func ParseUint(buf []byte) (int, error) { - v, n, err := parseUintBuf(buf) - if n != len(buf) { - return -1, errUnexpectedTrailingChar - } - return v, err -} - -var ( - errEmptyInt = errors.New("empty integer") - errUnexpectedFirstChar = errors.New("unexpected first char found. Expecting 0-9") - errUnexpectedTrailingChar = errors.New("unexpected trailing char found. Expecting 0-9") - errTooLongInt = errors.New("too long int") -) - -func parseUintBuf(b []byte) (int, int, error) { - n := len(b) - if n == 0 { - return -1, 0, errEmptyInt - } - v := 0 - for i := 0; i < n; i++ { - c := b[i] - k := c - '0' - if k > 9 { - if i == 0 { - return -1, i, errUnexpectedFirstChar - } - return v, i, nil - } - if i >= maxIntChars { - return -1, i, errTooLongInt - } - v = 10*v + int(k) - } - return v, n, nil -} - -var ( - errEmptyFloat = errors.New("empty float number") - errDuplicateFloatPoint = errors.New("duplicate point found in float number") - errUnexpectedFloatEnd = errors.New("unexpected end of float number") - errInvalidFloatExponent = errors.New("invalid float number exponent") - errUnexpectedFloatChar = errors.New("unexpected char found in float number") -) - -// ParseUfloat parses unsigned float from buf. -func ParseUfloat(buf []byte) (float64, error) { - if len(buf) == 0 { - return -1, errEmptyFloat - } - b := buf - var v uint64 - var offset = 1.0 - var pointFound bool - for i, c := range b { - if c < '0' || c > '9' { - if c == '.' { - if pointFound { - return -1, errDuplicateFloatPoint - } - pointFound = true - continue - } - if c == 'e' || c == 'E' { - if i+1 >= len(b) { - return -1, errUnexpectedFloatEnd - } - b = b[i+1:] - minus := -1 - switch b[0] { - case '+': - b = b[1:] - minus = 1 - case '-': - b = b[1:] - default: - minus = 1 - } - vv, err := ParseUint(b) - if err != nil { - return -1, errInvalidFloatExponent - } - return float64(v) * offset * math.Pow10(minus*int(vv)), nil - } - return -1, errUnexpectedFloatChar - } - v = 10*v + uint64(c-'0') - if pointFound { - offset /= 10 - } - } - return float64(v) * offset, nil -} - -var ( - errEmptyHexNum = errors.New("empty hex number") - errTooLargeHexNum = errors.New("too large hex number") -) - -func readHexInt(r *bufio.Reader) (int, error) { - n := 0 - i := 0 - var k int - for { - c, err := r.ReadByte() - if err != nil { - if err == io.EOF && i > 0 { - return n, nil - } - return -1, err - } - k = int(hex2intTable[c]) - if k == 16 { - if i == 0 { - return -1, errEmptyHexNum - } - r.UnreadByte() - return n, nil - } - if i >= maxHexIntChars { - return -1, errTooLargeHexNum - } - n = (n << 4) | k - i++ - } -} - -var hexIntBufPool sync.Pool - -func writeHexInt(w *bufio.Writer, n int) error { - if n < 0 { - panic("BUG: int must be positive") - } - - v := hexIntBufPool.Get() - if v == nil { - v = make([]byte, maxHexIntChars+1) - } - buf := v.([]byte) - i := len(buf) - 1 - for { - buf[i] = int2hexbyte(n & 0xf) - n >>= 4 - if n == 0 { - break - } - i-- - } - _, err := w.Write(buf[i:]) - hexIntBufPool.Put(v) - return err -} - -func int2hexbyte(n int) byte { - if n < 10 { - return '0' + byte(n) - } - return 'a' + byte(n) - 10 -} - -func hexCharUpper(c byte) byte { - if c < 10 { - return '0' + c - } - return c - 10 + 'A' -} - -var hex2intTable = func() []byte { - b := make([]byte, 256) - for i := 0; i < 256; i++ { - c := byte(16) - if i >= '0' && i <= '9' { - c = byte(i) - '0' - } else if i >= 'a' && i <= 'f' { - c = byte(i) - 'a' + 10 - } else if i >= 'A' && i <= 'F' { - c = byte(i) - 'A' + 10 - } - b[i] = c - } - return b -}() - -const toLower = 'a' - 'A' - -var toLowerTable = func() [256]byte { - var a [256]byte - for i := 0; i < 256; i++ { - c := byte(i) - if c >= 'A' && c <= 'Z' { - c += toLower - } - a[i] = c - } - return a -}() - -var toUpperTable = func() [256]byte { - var a [256]byte - for i := 0; i < 256; i++ { - c := byte(i) - if c >= 'a' && c <= 'z' { - c -= toLower - } - a[i] = c - } - return a -}() - -func lowercaseBytes(b []byte) { - for i := 0; i < len(b); i++ { - p := &b[i] - *p = toLowerTable[*p] - } -} - -// b2s converts byte slice to a string without memory allocation. -// See https://groups.google.com/forum/#!msg/Golang-Nuts/ENgbUzYvCuU/90yGx7GUAgAJ . -// -// Note it may break if string and/or slice header will change -// in the future go versions. -func b2s(b []byte) string { - return *(*string)(unsafe.Pointer(&b)) -} - -// s2b converts string to a byte slice without memory allocation. -// -// Note it may break if string and/or slice header will change -// in the future go versions. -func s2b(s string) []byte { - sh := (*reflect.StringHeader)(unsafe.Pointer(&s)) - bh := reflect.SliceHeader{ - Data: sh.Data, - Len: sh.Len, - Cap: sh.Len, - } - return *(*[]byte)(unsafe.Pointer(&bh)) -} - -// AppendUnquotedArg appends url-decoded src to dst and returns appended dst. -// -// dst may point to src. In this case src will be overwritten. -func AppendUnquotedArg(dst, src []byte) []byte { - return decodeArgAppend(dst, src) -} - -// AppendQuotedArg appends url-encoded src to dst and returns appended dst. -func AppendQuotedArg(dst, src []byte) []byte { - for _, c := range src { - // See http://www.w3.org/TR/html5/forms.html#form-submission-algorithm - if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || - c == '*' || c == '-' || c == '.' || c == '_' { - dst = append(dst, c) - } else { - dst = append(dst, '%', hexCharUpper(c>>4), hexCharUpper(c&15)) - } - } - return dst -} - -func appendQuotedPath(dst, src []byte) []byte { - for _, c := range src { - // From the spec: http://tools.ietf.org/html/rfc3986#section-3.3 - // an path can contain zero or more of pchar that is defined as follows: - // pchar = unreserved / pct-encoded / sub-delims / ":" / "@" - // pct-encoded = "%" HEXDIG HEXDIG - // unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" - // sub-delims = "!" / "$" / "&" / "'" / "(" / ")" - // / "*" / "+" / "," / ";" / "=" - if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || - c == '-' || c == '.' || c == '_' || c == '~' || c == '!' || c == '$' || - c == '&' || c == '\'' || c == '(' || c == ')' || c == '*' || c == '+' || - c == ',' || c == ';' || c == '=' || c == ':' || c == '@' || c == '/' { - dst = append(dst, c) - } else { - dst = append(dst, '%', hexCharUpper(c>>4), hexCharUpper(c&15)) - } - } - return dst -} - -// EqualBytesStr returns true if string(b) == s. -// -// This function has no performance benefits comparing to string(b) == s. -// It is left here for backwards compatibility only. -// -// Deprecated: may be deleted soon. -func EqualBytesStr(b []byte, s string) bool { - return string(b) == s -} - -// AppendBytesStr appends src to dst and returns the extended dst. -// -// This function has no performance benefits comparing to append(dst, src...). -// It is left here for backwards compatibility only. -// -// Deprecated: may be deleted soon. -func AppendBytesStr(dst []byte, src string) []byte { - return append(dst, src...) -} diff --git a/vendor/github.com/valyala/fasthttp/bytesconv_32.go b/vendor/github.com/valyala/fasthttp/bytesconv_32.go deleted file mode 100644 index 14377547..00000000 --- a/vendor/github.com/valyala/fasthttp/bytesconv_32.go +++ /dev/null @@ -1,8 +0,0 @@ -// +build !amd64,!arm64,!ppc64 - -package fasthttp - -const ( - maxIntChars = 9 - maxHexIntChars = 7 -) diff --git a/vendor/github.com/valyala/fasthttp/bytesconv_64.go b/vendor/github.com/valyala/fasthttp/bytesconv_64.go deleted file mode 100644 index 09d07ef1..00000000 --- a/vendor/github.com/valyala/fasthttp/bytesconv_64.go +++ /dev/null @@ -1,8 +0,0 @@ -// +build amd64 arm64 ppc64 - -package fasthttp - -const ( - maxIntChars = 18 - maxHexIntChars = 15 -) diff --git a/vendor/github.com/valyala/fasthttp/client.go b/vendor/github.com/valyala/fasthttp/client.go deleted file mode 100644 index 64be717a..00000000 --- a/vendor/github.com/valyala/fasthttp/client.go +++ /dev/null @@ -1,2188 +0,0 @@ -package fasthttp - -import ( - "bufio" - "bytes" - "crypto/tls" - "errors" - "fmt" - "io" - "net" - "strings" - "sync" - "sync/atomic" - "time" -) - -// Do performs the given http request and fills the given http response. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// Client determines the server to be requested in the following order: -// -// - from RequestURI if it contains full url with scheme and host; -// - from Host header otherwise. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// ErrNoFreeConns is returned if all DefaultMaxConnsPerHost connections -// to the requested host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func Do(req *Request, resp *Response) error { - return defaultClient.Do(req, resp) -} - -// DoTimeout performs the given request and waits for response during -// the given timeout duration. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// Client determines the server to be requested in the following order: -// -// - from RequestURI if it contains full url with scheme and host; -// - from Host header otherwise. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// ErrTimeout is returned if the response wasn't returned during -// the given timeout. -// -// ErrNoFreeConns is returned if all DefaultMaxConnsPerHost connections -// to the requested host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func DoTimeout(req *Request, resp *Response, timeout time.Duration) error { - return defaultClient.DoTimeout(req, resp, timeout) -} - -// DoDeadline performs the given request and waits for response until -// the given deadline. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// Client determines the server to be requested in the following order: -// -// - from RequestURI if it contains full url with scheme and host; -// - from Host header otherwise. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// ErrTimeout is returned if the response wasn't returned until -// the given deadline. -// -// ErrNoFreeConns is returned if all DefaultMaxConnsPerHost connections -// to the requested host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func DoDeadline(req *Request, resp *Response, deadline time.Time) error { - return defaultClient.DoDeadline(req, resp, deadline) -} - -// Get returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -func Get(dst []byte, url string) (statusCode int, body []byte, err error) { - return defaultClient.Get(dst, url) -} - -// GetTimeout returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// ErrTimeout error is returned if url contents couldn't be fetched -// during the given timeout. -func GetTimeout(dst []byte, url string, timeout time.Duration) (statusCode int, body []byte, err error) { - return defaultClient.GetTimeout(dst, url, timeout) -} - -// GetDeadline returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// ErrTimeout error is returned if url contents couldn't be fetched -// until the given deadline. -func GetDeadline(dst []byte, url string, deadline time.Time) (statusCode int, body []byte, err error) { - return defaultClient.GetDeadline(dst, url, deadline) -} - -// Post sends POST request to the given url with the given POST arguments. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// Empty POST body is sent if postArgs is nil. -func Post(dst []byte, url string, postArgs *Args) (statusCode int, body []byte, err error) { - return defaultClient.Post(dst, url, postArgs) -} - -var defaultClient Client - -// Client implements http client. -// -// Copying Client by value is prohibited. Create new instance instead. -// -// It is safe calling Client methods from concurrently running goroutines. -type Client struct { - noCopy noCopy - - // Client name. Used in User-Agent request header. - // - // Default client name is used if not set. - Name string - - // Callback for establishing new connections to hosts. - // - // Default Dial is used if not set. - Dial DialFunc - - // Attempt to connect to both ipv4 and ipv6 addresses if set to true. - // - // This option is used only if default TCP dialer is used, - // i.e. if Dial is blank. - // - // By default client connects only to ipv4 addresses, - // since unfortunately ipv6 remains broken in many networks worldwide :) - DialDualStack bool - - // TLS config for https connections. - // - // Default TLS config is used if not set. - TLSConfig *tls.Config - - // Maximum number of connections per each host which may be established. - // - // DefaultMaxConnsPerHost is used if not set. - MaxConnsPerHost int - - // Idle keep-alive connections are closed after this duration. - // - // By default idle connections are closed - // after DefaultMaxIdleConnDuration. - MaxIdleConnDuration time.Duration - - // Per-connection buffer size for responses' reading. - // This also limits the maximum header size. - // - // Default buffer size is used if 0. - ReadBufferSize int - - // Per-connection buffer size for requests' writing. - // - // Default buffer size is used if 0. - WriteBufferSize int - - // Maximum duration for full response reading (including body). - // - // By default response read timeout is unlimited. - ReadTimeout time.Duration - - // Maximum duration for full request writing (including body). - // - // By default request write timeout is unlimited. - WriteTimeout time.Duration - - // Maximum response body size. - // - // The client returns ErrBodyTooLarge if this limit is greater than 0 - // and response body is greater than the limit. - // - // By default response body size is unlimited. - MaxResponseBodySize int - - // Header names are passed as-is without normalization - // if this option is set. - // - // Disabled header names' normalization may be useful only for proxying - // responses to other clients expecting case-sensitive - // header names. See https://github.com/valyala/fasthttp/issues/57 - // for details. - // - // By default request and response header names are normalized, i.e. - // The first letter and the first letters following dashes - // are uppercased, while all the other letters are lowercased. - // Examples: - // - // * HOST -> Host - // * content-type -> Content-Type - // * cONTENT-lenGTH -> Content-Length - DisableHeaderNamesNormalizing bool - - mLock sync.Mutex - m map[string]*HostClient - ms map[string]*HostClient -} - -// Get returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -func (c *Client) Get(dst []byte, url string) (statusCode int, body []byte, err error) { - return clientGetURL(dst, url, c) -} - -// GetTimeout returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// ErrTimeout error is returned if url contents couldn't be fetched -// during the given timeout. -func (c *Client) GetTimeout(dst []byte, url string, timeout time.Duration) (statusCode int, body []byte, err error) { - return clientGetURLTimeout(dst, url, timeout, c) -} - -// GetDeadline returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// ErrTimeout error is returned if url contents couldn't be fetched -// until the given deadline. -func (c *Client) GetDeadline(dst []byte, url string, deadline time.Time) (statusCode int, body []byte, err error) { - return clientGetURLDeadline(dst, url, deadline, c) -} - -// Post sends POST request to the given url with the given POST arguments. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// Empty POST body is sent if postArgs is nil. -func (c *Client) Post(dst []byte, url string, postArgs *Args) (statusCode int, body []byte, err error) { - return clientPostURL(dst, url, postArgs, c) -} - -// DoTimeout performs the given request and waits for response during -// the given timeout duration. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// Client determines the server to be requested in the following order: -// -// - from RequestURI if it contains full url with scheme and host; -// - from Host header otherwise. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// ErrTimeout is returned if the response wasn't returned during -// the given timeout. -// -// ErrNoFreeConns is returned if all Client.MaxConnsPerHost connections -// to the requested host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func (c *Client) DoTimeout(req *Request, resp *Response, timeout time.Duration) error { - return clientDoTimeout(req, resp, timeout, c) -} - -// DoDeadline performs the given request and waits for response until -// the given deadline. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// Client determines the server to be requested in the following order: -// -// - from RequestURI if it contains full url with scheme and host; -// - from Host header otherwise. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// ErrTimeout is returned if the response wasn't returned until -// the given deadline. -// -// ErrNoFreeConns is returned if all Client.MaxConnsPerHost connections -// to the requested host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func (c *Client) DoDeadline(req *Request, resp *Response, deadline time.Time) error { - return clientDoDeadline(req, resp, deadline, c) -} - -// Do performs the given http request and fills the given http response. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// Client determines the server to be requested in the following order: -// -// - from RequestURI if it contains full url with scheme and host; -// - from Host header otherwise. -// -// Response is ignored if resp is nil. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// ErrNoFreeConns is returned if all Client.MaxConnsPerHost connections -// to the requested host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func (c *Client) Do(req *Request, resp *Response) error { - uri := req.URI() - host := uri.Host() - - isTLS := false - scheme := uri.Scheme() - if bytes.Equal(scheme, strHTTPS) { - isTLS = true - } else if !bytes.Equal(scheme, strHTTP) { - return fmt.Errorf("unsupported protocol %q. http and https are supported", scheme) - } - - startCleaner := false - - c.mLock.Lock() - m := c.m - if isTLS { - m = c.ms - } - if m == nil { - m = make(map[string]*HostClient) - if isTLS { - c.ms = m - } else { - c.m = m - } - } - hc := m[string(host)] - if hc == nil { - hc = &HostClient{ - Addr: addMissingPort(string(host), isTLS), - Name: c.Name, - Dial: c.Dial, - DialDualStack: c.DialDualStack, - IsTLS: isTLS, - TLSConfig: c.TLSConfig, - MaxConns: c.MaxConnsPerHost, - MaxIdleConnDuration: c.MaxIdleConnDuration, - ReadBufferSize: c.ReadBufferSize, - WriteBufferSize: c.WriteBufferSize, - ReadTimeout: c.ReadTimeout, - WriteTimeout: c.WriteTimeout, - MaxResponseBodySize: c.MaxResponseBodySize, - DisableHeaderNamesNormalizing: c.DisableHeaderNamesNormalizing, - } - m[string(host)] = hc - if len(m) == 1 { - startCleaner = true - } - } - c.mLock.Unlock() - - if startCleaner { - go c.mCleaner(m) - } - - return hc.Do(req, resp) -} - -func (c *Client) mCleaner(m map[string]*HostClient) { - mustStop := false - for { - t := time.Now() - c.mLock.Lock() - for k, v := range m { - if t.Sub(v.LastUseTime()) > time.Minute { - delete(m, k) - } - } - if len(m) == 0 { - mustStop = true - } - c.mLock.Unlock() - - if mustStop { - break - } - time.Sleep(10 * time.Second) - } -} - -// DefaultMaxConnsPerHost is the maximum number of concurrent connections -// http client may establish per host by default (i.e. if -// Client.MaxConnsPerHost isn't set). -const DefaultMaxConnsPerHost = 512 - -// DefaultMaxIdleConnDuration is the default duration before idle keep-alive -// connection is closed. -const DefaultMaxIdleConnDuration = 10 * time.Second - -// DefaultMaxIdemponentCallAttempts is the default idempotent calls attempts count. -const DefaultMaxIdemponentCallAttempts = 5 - -// DialFunc must establish connection to addr. -// -// There is no need in establishing TLS (SSL) connection for https. -// The client automatically converts connection to TLS -// if HostClient.IsTLS is set. -// -// TCP address passed to DialFunc always contains host and port. -// Example TCP addr values: -// -// - foobar.com:80 -// - foobar.com:443 -// - foobar.com:8080 -type DialFunc func(addr string) (net.Conn, error) - -// HostClient balances http requests among hosts listed in Addr. -// -// HostClient may be used for balancing load among multiple upstream hosts. -// While multiple addresses passed to HostClient.Addr may be used for balancing -// load among them, it would be better using LBClient instead, since HostClient -// may unevenly balance load among upstream hosts. -// -// It is forbidden copying HostClient instances. Create new instances instead. -// -// It is safe calling HostClient methods from concurrently running goroutines. -type HostClient struct { - noCopy noCopy - - // Comma-separated list of upstream HTTP server host addresses, - // which are passed to Dial in a round-robin manner. - // - // Each address may contain port if default dialer is used. - // For example, - // - // - foobar.com:80 - // - foobar.com:443 - // - foobar.com:8080 - Addr string - - // Client name. Used in User-Agent request header. - Name string - - // Callback for establishing new connection to the host. - // - // Default Dial is used if not set. - Dial DialFunc - - // Attempt to connect to both ipv4 and ipv6 host addresses - // if set to true. - // - // This option is used only if default TCP dialer is used, - // i.e. if Dial is blank. - // - // By default client connects only to ipv4 addresses, - // since unfortunately ipv6 remains broken in many networks worldwide :) - DialDualStack bool - - // Whether to use TLS (aka SSL or HTTPS) for host connections. - IsTLS bool - - // Optional TLS config. - TLSConfig *tls.Config - - // Maximum number of connections which may be established to all hosts - // listed in Addr. - // - // DefaultMaxConnsPerHost is used if not set. - MaxConns int - - // Keep-alive connections are closed after this duration. - // - // By default connection duration is unlimited. - MaxConnDuration time.Duration - - // Idle keep-alive connections are closed after this duration. - // - // By default idle connections are closed - // after DefaultMaxIdleConnDuration. - MaxIdleConnDuration time.Duration - - // Maximum number of attempts for idempotent calls - // - // DefaultMaxIdemponentCallAttempts is used if not set. - MaxIdemponentCallAttempts int - - // Per-connection buffer size for responses' reading. - // This also limits the maximum header size. - // - // Default buffer size is used if 0. - ReadBufferSize int - - // Per-connection buffer size for requests' writing. - // - // Default buffer size is used if 0. - WriteBufferSize int - - // Maximum duration for full response reading (including body). - // - // By default response read timeout is unlimited. - ReadTimeout time.Duration - - // Maximum duration for full request writing (including body). - // - // By default request write timeout is unlimited. - WriteTimeout time.Duration - - // Maximum response body size. - // - // The client returns ErrBodyTooLarge if this limit is greater than 0 - // and response body is greater than the limit. - // - // By default response body size is unlimited. - MaxResponseBodySize int - - // Header names are passed as-is without normalization - // if this option is set. - // - // Disabled header names' normalization may be useful only for proxying - // responses to other clients expecting case-sensitive - // header names. See https://github.com/valyala/fasthttp/issues/57 - // for details. - // - // By default request and response header names are normalized, i.e. - // The first letter and the first letters following dashes - // are uppercased, while all the other letters are lowercased. - // Examples: - // - // * HOST -> Host - // * content-type -> Content-Type - // * cONTENT-lenGTH -> Content-Length - DisableHeaderNamesNormalizing bool - - clientName atomic.Value - lastUseTime uint32 - - connsLock sync.Mutex - connsCount int - conns []*clientConn - - addrsLock sync.Mutex - addrs []string - addrIdx uint32 - - tlsConfigMap map[string]*tls.Config - tlsConfigMapLock sync.Mutex - - readerPool sync.Pool - writerPool sync.Pool - - pendingRequests uint64 - - connsCleanerRun bool -} - -type clientConn struct { - c net.Conn - - createdTime time.Time - lastUseTime time.Time - - lastReadDeadlineTime time.Time - lastWriteDeadlineTime time.Time -} - -var startTimeUnix = time.Now().Unix() - -// LastUseTime returns time the client was last used -func (c *HostClient) LastUseTime() time.Time { - n := atomic.LoadUint32(&c.lastUseTime) - return time.Unix(startTimeUnix+int64(n), 0) -} - -// Get returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -func (c *HostClient) Get(dst []byte, url string) (statusCode int, body []byte, err error) { - return clientGetURL(dst, url, c) -} - -// GetTimeout returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// ErrTimeout error is returned if url contents couldn't be fetched -// during the given timeout. -func (c *HostClient) GetTimeout(dst []byte, url string, timeout time.Duration) (statusCode int, body []byte, err error) { - return clientGetURLTimeout(dst, url, timeout, c) -} - -// GetDeadline returns the status code and body of url. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// ErrTimeout error is returned if url contents couldn't be fetched -// until the given deadline. -func (c *HostClient) GetDeadline(dst []byte, url string, deadline time.Time) (statusCode int, body []byte, err error) { - return clientGetURLDeadline(dst, url, deadline, c) -} - -// Post sends POST request to the given url with the given POST arguments. -// -// The contents of dst will be replaced by the body and returned, if the dst -// is too small a new slice will be allocated. -// -// The function follows redirects. Use Do* for manually handling redirects. -// -// Empty POST body is sent if postArgs is nil. -func (c *HostClient) Post(dst []byte, url string, postArgs *Args) (statusCode int, body []byte, err error) { - return clientPostURL(dst, url, postArgs, c) -} - -type clientDoer interface { - Do(req *Request, resp *Response) error -} - -func clientGetURL(dst []byte, url string, c clientDoer) (statusCode int, body []byte, err error) { - req := AcquireRequest() - - statusCode, body, err = doRequestFollowRedirects(req, dst, url, c) - - ReleaseRequest(req) - return statusCode, body, err -} - -func clientGetURLTimeout(dst []byte, url string, timeout time.Duration, c clientDoer) (statusCode int, body []byte, err error) { - deadline := time.Now().Add(timeout) - return clientGetURLDeadline(dst, url, deadline, c) -} - -type clientURLResponse struct { - statusCode int - body []byte - err error -} - -func clientGetURLDeadline(dst []byte, url string, deadline time.Time, c clientDoer) (statusCode int, body []byte, err error) { - timeout := -time.Since(deadline) - if timeout <= 0 { - return 0, dst, ErrTimeout - } - - var ch chan clientURLResponse - chv := clientURLResponseChPool.Get() - if chv == nil { - chv = make(chan clientURLResponse, 1) - } - ch = chv.(chan clientURLResponse) - - req := AcquireRequest() - - // Note that the request continues execution on ErrTimeout until - // client-specific ReadTimeout exceeds. This helps limiting load - // on slow hosts by MaxConns* concurrent requests. - // - // Without this 'hack' the load on slow host could exceed MaxConns* - // concurrent requests, since timed out requests on client side - // usually continue execution on the host. - go func() { - statusCodeCopy, bodyCopy, errCopy := doRequestFollowRedirects(req, dst, url, c) - ch <- clientURLResponse{ - statusCode: statusCodeCopy, - body: bodyCopy, - err: errCopy, - } - }() - - tc := acquireTimer(timeout) - select { - case resp := <-ch: - ReleaseRequest(req) - clientURLResponseChPool.Put(chv) - statusCode = resp.statusCode - body = resp.body - err = resp.err - case <-tc.C: - body = dst - err = ErrTimeout - } - releaseTimer(tc) - - return statusCode, body, err -} - -var clientURLResponseChPool sync.Pool - -func clientPostURL(dst []byte, url string, postArgs *Args, c clientDoer) (statusCode int, body []byte, err error) { - req := AcquireRequest() - req.Header.SetMethodBytes(strPost) - req.Header.SetContentTypeBytes(strPostArgsContentType) - if postArgs != nil { - postArgs.WriteTo(req.BodyWriter()) - } - - statusCode, body, err = doRequestFollowRedirects(req, dst, url, c) - - ReleaseRequest(req) - return statusCode, body, err -} - -var ( - errMissingLocation = errors.New("missing Location header for http redirect") - errTooManyRedirects = errors.New("too many redirects detected when doing the request") -) - -const maxRedirectsCount = 16 - -func doRequestFollowRedirects(req *Request, dst []byte, url string, c clientDoer) (statusCode int, body []byte, err error) { - resp := AcquireResponse() - bodyBuf := resp.bodyBuffer() - resp.keepBodyBuffer = true - oldBody := bodyBuf.B - bodyBuf.B = dst - - redirectsCount := 0 - for { - req.parsedURI = false - req.Header.host = req.Header.host[:0] - req.SetRequestURI(url) - - if err = c.Do(req, resp); err != nil { - break - } - statusCode = resp.Header.StatusCode() - if statusCode != StatusMovedPermanently && - statusCode != StatusFound && - statusCode != StatusSeeOther && - statusCode != StatusTemporaryRedirect && - statusCode != StatusPermanentRedirect { - break - } - - redirectsCount++ - if redirectsCount > maxRedirectsCount { - err = errTooManyRedirects - break - } - location := resp.Header.peek(strLocation) - if len(location) == 0 { - err = errMissingLocation - break - } - url = getRedirectURL(url, location) - } - - body = bodyBuf.B - bodyBuf.B = oldBody - resp.keepBodyBuffer = false - ReleaseResponse(resp) - - return statusCode, body, err -} - -func getRedirectURL(baseURL string, location []byte) string { - u := AcquireURI() - u.Update(baseURL) - u.UpdateBytes(location) - redirectURL := u.String() - ReleaseURI(u) - return redirectURL -} - -var ( - requestPool sync.Pool - responsePool sync.Pool -) - -// AcquireRequest returns an empty Request instance from request pool. -// -// The returned Request instance may be passed to ReleaseRequest when it is -// no longer needed. This allows Request recycling, reduces GC pressure -// and usually improves performance. -func AcquireRequest() *Request { - v := requestPool.Get() - if v == nil { - return &Request{} - } - return v.(*Request) -} - -// ReleaseRequest returns req acquired via AcquireRequest to request pool. -// -// It is forbidden accessing req and/or its' members after returning -// it to request pool. -func ReleaseRequest(req *Request) { - req.Reset() - requestPool.Put(req) -} - -// AcquireResponse returns an empty Response instance from response pool. -// -// The returned Response instance may be passed to ReleaseResponse when it is -// no longer needed. This allows Response recycling, reduces GC pressure -// and usually improves performance. -func AcquireResponse() *Response { - v := responsePool.Get() - if v == nil { - return &Response{} - } - return v.(*Response) -} - -// ReleaseResponse return resp acquired via AcquireResponse to response pool. -// -// It is forbidden accessing resp and/or its' members after returning -// it to response pool. -func ReleaseResponse(resp *Response) { - resp.Reset() - responsePool.Put(resp) -} - -// DoTimeout performs the given request and waits for response during -// the given timeout duration. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// ErrTimeout is returned if the response wasn't returned during -// the given timeout. -// -// ErrNoFreeConns is returned if all HostClient.MaxConns connections -// to the host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func (c *HostClient) DoTimeout(req *Request, resp *Response, timeout time.Duration) error { - return clientDoTimeout(req, resp, timeout, c) -} - -// DoDeadline performs the given request and waits for response until -// the given deadline. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// ErrTimeout is returned if the response wasn't returned until -// the given deadline. -// -// ErrNoFreeConns is returned if all HostClient.MaxConns connections -// to the host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func (c *HostClient) DoDeadline(req *Request, resp *Response, deadline time.Time) error { - return clientDoDeadline(req, resp, deadline, c) -} - -func clientDoTimeout(req *Request, resp *Response, timeout time.Duration, c clientDoer) error { - deadline := time.Now().Add(timeout) - return clientDoDeadline(req, resp, deadline, c) -} - -func clientDoDeadline(req *Request, resp *Response, deadline time.Time, c clientDoer) error { - timeout := -time.Since(deadline) - if timeout <= 0 { - return ErrTimeout - } - - var ch chan error - chv := errorChPool.Get() - if chv == nil { - chv = make(chan error, 1) - } - ch = chv.(chan error) - - // Make req and resp copies, since on timeout they no longer - // may be accessed. - reqCopy := AcquireRequest() - req.copyToSkipBody(reqCopy) - swapRequestBody(req, reqCopy) - respCopy := AcquireResponse() - - // Note that the request continues execution on ErrTimeout until - // client-specific ReadTimeout exceeds. This helps limiting load - // on slow hosts by MaxConns* concurrent requests. - // - // Without this 'hack' the load on slow host could exceed MaxConns* - // concurrent requests, since timed out requests on client side - // usually continue execution on the host. - go func() { - ch <- c.Do(reqCopy, respCopy) - }() - - tc := acquireTimer(timeout) - var err error - select { - case err = <-ch: - if resp != nil { - respCopy.copyToSkipBody(resp) - swapResponseBody(resp, respCopy) - } - swapRequestBody(reqCopy, req) - ReleaseResponse(respCopy) - ReleaseRequest(reqCopy) - errorChPool.Put(chv) - case <-tc.C: - err = ErrTimeout - } - releaseTimer(tc) - - return err -} - -var errorChPool sync.Pool - -// Do performs the given http request and sets the corresponding response. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// ErrNoFreeConns is returned if all HostClient.MaxConns connections -// to the host are busy. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func (c *HostClient) Do(req *Request, resp *Response) error { - var err error - var retry bool - maxAttempts := c.MaxIdemponentCallAttempts - if maxAttempts <= 0 { - maxAttempts = DefaultMaxIdemponentCallAttempts - } - attempts := 0 - - atomic.AddUint64(&c.pendingRequests, 1) - for { - retry, err = c.do(req, resp) - if err == nil || !retry { - break - } - - if !isIdempotent(req) { - // Retry non-idempotent requests if the server closes - // the connection before sending the response. - // - // This case is possible if the server closes the idle - // keep-alive connection on timeout. - // - // Apache and nginx usually do this. - if err != io.EOF { - break - } - } - attempts++ - if attempts >= maxAttempts { - break - } - } - atomic.AddUint64(&c.pendingRequests, ^uint64(0)) - - if err == io.EOF { - err = ErrConnectionClosed - } - return err -} - -// PendingRequests returns the current number of requests the client -// is executing. -// -// This function may be used for balancing load among multiple HostClient -// instances. -func (c *HostClient) PendingRequests() int { - return int(atomic.LoadUint64(&c.pendingRequests)) -} - -func isIdempotent(req *Request) bool { - return req.Header.IsGet() || req.Header.IsHead() || req.Header.IsPut() -} - -func (c *HostClient) do(req *Request, resp *Response) (bool, error) { - nilResp := false - if resp == nil { - nilResp = true - resp = AcquireResponse() - } - - ok, err := c.doNonNilReqResp(req, resp) - - if nilResp { - ReleaseResponse(resp) - } - - return ok, err -} - -func (c *HostClient) doNonNilReqResp(req *Request, resp *Response) (bool, error) { - if req == nil { - panic("BUG: req cannot be nil") - } - if resp == nil { - panic("BUG: resp cannot be nil") - } - - atomic.StoreUint32(&c.lastUseTime, uint32(time.Now().Unix()-startTimeUnix)) - - // Free up resources occupied by response before sending the request, - // so the GC may reclaim these resources (e.g. response body). - resp.Reset() - - cc, err := c.acquireConn() - if err != nil { - return false, err - } - conn := cc.c - - if c.WriteTimeout > 0 { - // Optimization: update write deadline only if more than 25% - // of the last write deadline exceeded. - // See https://github.com/golang/go/issues/15133 for details. - currentTime := time.Now() - if currentTime.Sub(cc.lastWriteDeadlineTime) > (c.WriteTimeout >> 2) { - if err = conn.SetWriteDeadline(currentTime.Add(c.WriteTimeout)); err != nil { - c.closeConn(cc) - return true, err - } - cc.lastWriteDeadlineTime = currentTime - } - } - - resetConnection := false - if c.MaxConnDuration > 0 && time.Since(cc.createdTime) > c.MaxConnDuration && !req.ConnectionClose() { - req.SetConnectionClose() - resetConnection = true - } - - userAgentOld := req.Header.UserAgent() - if len(userAgentOld) == 0 { - req.Header.userAgent = c.getClientName() - } - bw := c.acquireWriter(conn) - err = req.Write(bw) - - if resetConnection { - req.Header.ResetConnectionClose() - } - - if err == nil { - err = bw.Flush() - } - if err != nil { - c.releaseWriter(bw) - c.closeConn(cc) - return true, err - } - c.releaseWriter(bw) - - if c.ReadTimeout > 0 { - // Optimization: update read deadline only if more than 25% - // of the last read deadline exceeded. - // See https://github.com/golang/go/issues/15133 for details. - currentTime := time.Now() - if currentTime.Sub(cc.lastReadDeadlineTime) > (c.ReadTimeout >> 2) { - if err = conn.SetReadDeadline(currentTime.Add(c.ReadTimeout)); err != nil { - c.closeConn(cc) - return true, err - } - cc.lastReadDeadlineTime = currentTime - } - } - - if !req.Header.IsGet() && req.Header.IsHead() { - resp.SkipBody = true - } - if c.DisableHeaderNamesNormalizing { - resp.Header.DisableNormalizing() - } - - br := c.acquireReader(conn) - if err = resp.ReadLimitBody(br, c.MaxResponseBodySize); err != nil { - c.releaseReader(br) - c.closeConn(cc) - return true, err - } - c.releaseReader(br) - - if resetConnection || req.ConnectionClose() || resp.ConnectionClose() { - c.closeConn(cc) - } else { - c.releaseConn(cc) - } - - return false, err -} - -var ( - // ErrNoFreeConns is returned when no free connections available - // to the given host. - // - // Increase the allowed number of connections per host if you - // see this error. - ErrNoFreeConns = errors.New("no free connections available to host") - - // ErrTimeout is returned from timed out calls. - ErrTimeout = errors.New("timeout") - - // ErrConnectionClosed may be returned from client methods if the server - // closes connection before returning the first response byte. - // - // If you see this error, then either fix the server by returning - // 'Connection: close' response header before closing the connection - // or add 'Connection: close' request header before sending requests - // to broken server. - ErrConnectionClosed = errors.New("the server closed connection before returning the first response byte. " + - "Make sure the server returns 'Connection: close' response header before closing the connection") -) - -func (c *HostClient) acquireConn() (*clientConn, error) { - var cc *clientConn - createConn := false - startCleaner := false - - var n int - c.connsLock.Lock() - n = len(c.conns) - if n == 0 { - maxConns := c.MaxConns - if maxConns <= 0 { - maxConns = DefaultMaxConnsPerHost - } - if c.connsCount < maxConns { - c.connsCount++ - createConn = true - if !c.connsCleanerRun { - startCleaner = true - c.connsCleanerRun = true - } - } - } else { - n-- - cc = c.conns[n] - c.conns[n] = nil - c.conns = c.conns[:n] - } - c.connsLock.Unlock() - - if cc != nil { - return cc, nil - } - if !createConn { - return nil, ErrNoFreeConns - } - - if startCleaner { - go c.connsCleaner() - } - - conn, err := c.dialHostHard() - if err != nil { - c.decConnsCount() - return nil, err - } - cc = acquireClientConn(conn) - - return cc, nil -} - -func (c *HostClient) connsCleaner() { - var ( - scratch []*clientConn - maxIdleConnDuration = c.MaxIdleConnDuration - ) - if maxIdleConnDuration <= 0 { - maxIdleConnDuration = DefaultMaxIdleConnDuration - } - for { - currentTime := time.Now() - - // Determine idle connections to be closed. - c.connsLock.Lock() - conns := c.conns - n := len(conns) - i := 0 - for i < n && currentTime.Sub(conns[i].lastUseTime) > maxIdleConnDuration { - i++ - } - sleepFor := maxIdleConnDuration - if i < n { - // + 1 so we actually sleep past the expiration time and not up to it. - // Otherwise the > check above would still fail. - sleepFor = maxIdleConnDuration - currentTime.Sub(conns[i].lastUseTime) + 1 - } - scratch = append(scratch[:0], conns[:i]...) - if i > 0 { - m := copy(conns, conns[i:]) - for i = m; i < n; i++ { - conns[i] = nil - } - c.conns = conns[:m] - } - c.connsLock.Unlock() - - // Close idle connections. - for i, cc := range scratch { - c.closeConn(cc) - scratch[i] = nil - } - - // Determine whether to stop the connsCleaner. - c.connsLock.Lock() - mustStop := c.connsCount == 0 - if mustStop { - c.connsCleanerRun = false - } - c.connsLock.Unlock() - if mustStop { - break - } - - time.Sleep(sleepFor) - } -} - -func (c *HostClient) closeConn(cc *clientConn) { - c.decConnsCount() - cc.c.Close() - releaseClientConn(cc) -} - -func (c *HostClient) decConnsCount() { - c.connsLock.Lock() - c.connsCount-- - c.connsLock.Unlock() -} - -func acquireClientConn(conn net.Conn) *clientConn { - v := clientConnPool.Get() - if v == nil { - v = &clientConn{} - } - cc := v.(*clientConn) - cc.c = conn - cc.createdTime = time.Now() - return cc -} - -func releaseClientConn(cc *clientConn) { - // Reset all fields. - *cc = clientConn{} - clientConnPool.Put(cc) -} - -var clientConnPool sync.Pool - -func (c *HostClient) releaseConn(cc *clientConn) { - cc.lastUseTime = time.Now() - c.connsLock.Lock() - c.conns = append(c.conns, cc) - c.connsLock.Unlock() -} - -func (c *HostClient) acquireWriter(conn net.Conn) *bufio.Writer { - v := c.writerPool.Get() - if v == nil { - n := c.WriteBufferSize - if n <= 0 { - n = defaultWriteBufferSize - } - return bufio.NewWriterSize(conn, n) - } - bw := v.(*bufio.Writer) - bw.Reset(conn) - return bw -} - -func (c *HostClient) releaseWriter(bw *bufio.Writer) { - c.writerPool.Put(bw) -} - -func (c *HostClient) acquireReader(conn net.Conn) *bufio.Reader { - v := c.readerPool.Get() - if v == nil { - n := c.ReadBufferSize - if n <= 0 { - n = defaultReadBufferSize - } - return bufio.NewReaderSize(conn, n) - } - br := v.(*bufio.Reader) - br.Reset(conn) - return br -} - -func (c *HostClient) releaseReader(br *bufio.Reader) { - c.readerPool.Put(br) -} - -func newClientTLSConfig(c *tls.Config, addr string) *tls.Config { - if c == nil { - c = &tls.Config{} - } else { - // TODO: substitute this with c.Clone() after go1.8 becomes mainstream :) - c = &tls.Config{ - Rand: c.Rand, - Time: c.Time, - Certificates: c.Certificates, - NameToCertificate: c.NameToCertificate, - GetCertificate: c.GetCertificate, - RootCAs: c.RootCAs, - NextProtos: c.NextProtos, - ServerName: c.ServerName, - - // Do not copy ClientAuth, since it is server-related stuff - // Do not copy ClientCAs, since it is server-related stuff - - InsecureSkipVerify: c.InsecureSkipVerify, - CipherSuites: c.CipherSuites, - - // Do not copy PreferServerCipherSuites - this is server stuff - - SessionTicketsDisabled: c.SessionTicketsDisabled, - - // Do not copy SessionTicketKey - this is server stuff - - ClientSessionCache: c.ClientSessionCache, - MinVersion: c.MinVersion, - MaxVersion: c.MaxVersion, - CurvePreferences: c.CurvePreferences, - } - } - - if c.ClientSessionCache == nil { - c.ClientSessionCache = tls.NewLRUClientSessionCache(0) - } - - if len(c.ServerName) == 0 { - serverName := tlsServerName(addr) - if serverName == "*" { - c.InsecureSkipVerify = true - } else { - c.ServerName = serverName - } - } - return c -} - -func tlsServerName(addr string) string { - if !strings.Contains(addr, ":") { - return addr - } - host, _, err := net.SplitHostPort(addr) - if err != nil { - return "*" - } - return host -} - -func (c *HostClient) nextAddr() string { - c.addrsLock.Lock() - if c.addrs == nil { - c.addrs = strings.Split(c.Addr, ",") - } - addr := c.addrs[0] - if len(c.addrs) > 1 { - addr = c.addrs[c.addrIdx%uint32(len(c.addrs))] - c.addrIdx++ - } - c.addrsLock.Unlock() - return addr -} - -func (c *HostClient) dialHostHard() (conn net.Conn, err error) { - // attempt to dial all the available hosts before giving up. - - c.addrsLock.Lock() - n := len(c.addrs) - c.addrsLock.Unlock() - - if n == 0 { - // It looks like c.addrs isn't initialized yet. - n = 1 - } - - timeout := c.ReadTimeout + c.WriteTimeout - if timeout <= 0 { - timeout = DefaultDialTimeout - } - deadline := time.Now().Add(timeout) - for n > 0 { - addr := c.nextAddr() - tlsConfig := c.cachedTLSConfig(addr) - conn, err = dialAddr(addr, c.Dial, c.DialDualStack, c.IsTLS, tlsConfig) - if err == nil { - return conn, nil - } - if time.Since(deadline) >= 0 { - break - } - n-- - } - return nil, err -} - -func (c *HostClient) cachedTLSConfig(addr string) *tls.Config { - if !c.IsTLS { - return nil - } - - c.tlsConfigMapLock.Lock() - if c.tlsConfigMap == nil { - c.tlsConfigMap = make(map[string]*tls.Config) - } - cfg := c.tlsConfigMap[addr] - if cfg == nil { - cfg = newClientTLSConfig(c.TLSConfig, addr) - c.tlsConfigMap[addr] = cfg - } - c.tlsConfigMapLock.Unlock() - - return cfg -} - -func dialAddr(addr string, dial DialFunc, dialDualStack, isTLS bool, tlsConfig *tls.Config) (net.Conn, error) { - if dial == nil { - if dialDualStack { - dial = DialDualStack - } else { - dial = Dial - } - addr = addMissingPort(addr, isTLS) - } - conn, err := dial(addr) - if err != nil { - return nil, err - } - if conn == nil { - panic("BUG: DialFunc returned (nil, nil)") - } - if isTLS { - conn = tls.Client(conn, tlsConfig) - } - return conn, nil -} - -func (c *HostClient) getClientName() []byte { - v := c.clientName.Load() - var clientName []byte - if v == nil { - clientName = []byte(c.Name) - if len(clientName) == 0 { - clientName = defaultUserAgent - } - c.clientName.Store(clientName) - } else { - clientName = v.([]byte) - } - return clientName -} - -func addMissingPort(addr string, isTLS bool) string { - n := strings.Index(addr, ":") - if n >= 0 { - return addr - } - port := 80 - if isTLS { - port = 443 - } - return fmt.Sprintf("%s:%d", addr, port) -} - -// PipelineClient pipelines requests over a limited set of concurrent -// connections to the given Addr. -// -// This client may be used in highly loaded HTTP-based RPC systems for reducing -// context switches and network level overhead. -// See https://en.wikipedia.org/wiki/HTTP_pipelining for details. -// -// It is forbidden copying PipelineClient instances. Create new instances -// instead. -// -// It is safe calling PipelineClient methods from concurrently running -// goroutines. -type PipelineClient struct { - noCopy noCopy - - // Address of the host to connect to. - Addr string - - // The maximum number of concurrent connections to the Addr. - // - // A single connection is used by default. - MaxConns int - - // The maximum number of pending pipelined requests over - // a single connection to Addr. - // - // DefaultMaxPendingRequests is used by default. - MaxPendingRequests int - - // The maximum delay before sending pipelined requests as a batch - // to the server. - // - // By default requests are sent immediately to the server. - MaxBatchDelay time.Duration - - // Callback for connection establishing to the host. - // - // Default Dial is used if not set. - Dial DialFunc - - // Attempt to connect to both ipv4 and ipv6 host addresses - // if set to true. - // - // This option is used only if default TCP dialer is used, - // i.e. if Dial is blank. - // - // By default client connects only to ipv4 addresses, - // since unfortunately ipv6 remains broken in many networks worldwide :) - DialDualStack bool - - // Whether to use TLS (aka SSL or HTTPS) for host connections. - IsTLS bool - - // Optional TLS config. - TLSConfig *tls.Config - - // Idle connection to the host is closed after this duration. - // - // By default idle connection is closed after - // DefaultMaxIdleConnDuration. - MaxIdleConnDuration time.Duration - - // Buffer size for responses' reading. - // This also limits the maximum header size. - // - // Default buffer size is used if 0. - ReadBufferSize int - - // Buffer size for requests' writing. - // - // Default buffer size is used if 0. - WriteBufferSize int - - // Maximum duration for full response reading (including body). - // - // By default response read timeout is unlimited. - ReadTimeout time.Duration - - // Maximum duration for full request writing (including body). - // - // By default request write timeout is unlimited. - WriteTimeout time.Duration - - // Logger for logging client errors. - // - // By default standard logger from log package is used. - Logger Logger - - connClients []*pipelineConnClient - connClientsLock sync.Mutex -} - -type pipelineConnClient struct { - noCopy noCopy - - Addr string - MaxPendingRequests int - MaxBatchDelay time.Duration - Dial DialFunc - DialDualStack bool - IsTLS bool - TLSConfig *tls.Config - MaxIdleConnDuration time.Duration - ReadBufferSize int - WriteBufferSize int - ReadTimeout time.Duration - WriteTimeout time.Duration - Logger Logger - - workPool sync.Pool - - chLock sync.Mutex - chW chan *pipelineWork - chR chan *pipelineWork - - tlsConfigLock sync.Mutex - tlsConfig *tls.Config -} - -type pipelineWork struct { - reqCopy Request - respCopy Response - req *Request - resp *Response - t *time.Timer - deadline time.Time - err error - done chan struct{} -} - -// DoTimeout performs the given request and waits for response during -// the given timeout duration. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// The function doesn't follow redirects. -// -// Response is ignored if resp is nil. -// -// ErrTimeout is returned if the response wasn't returned during -// the given timeout. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func (c *PipelineClient) DoTimeout(req *Request, resp *Response, timeout time.Duration) error { - return c.DoDeadline(req, resp, time.Now().Add(timeout)) -} - -// DoDeadline performs the given request and waits for response until -// the given deadline. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// The function doesn't follow redirects. -// -// Response is ignored if resp is nil. -// -// ErrTimeout is returned if the response wasn't returned until -// the given deadline. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func (c *PipelineClient) DoDeadline(req *Request, resp *Response, deadline time.Time) error { - return c.getConnClient().DoDeadline(req, resp, deadline) -} - -func (c *pipelineConnClient) DoDeadline(req *Request, resp *Response, deadline time.Time) error { - c.init() - - timeout := -time.Since(deadline) - if timeout < 0 { - return ErrTimeout - } - - w := acquirePipelineWork(&c.workPool, timeout) - w.req = &w.reqCopy - w.resp = &w.respCopy - - // Make a copy of the request in order to avoid data races on timeouts - req.copyToSkipBody(&w.reqCopy) - swapRequestBody(req, &w.reqCopy) - - // Put the request to outgoing queue - select { - case c.chW <- w: - // Fast path: len(c.ch) < cap(c.ch) - default: - // Slow path - select { - case c.chW <- w: - case <-w.t.C: - releasePipelineWork(&c.workPool, w) - return ErrTimeout - } - } - - // Wait for the response - var err error - select { - case <-w.done: - if resp != nil { - w.respCopy.copyToSkipBody(resp) - swapResponseBody(resp, &w.respCopy) - } - err = w.err - releasePipelineWork(&c.workPool, w) - case <-w.t.C: - err = ErrTimeout - } - - return err -} - -// Do performs the given http request and sets the corresponding response. -// -// Request must contain at least non-zero RequestURI with full url (including -// scheme and host) or non-zero Host header + RequestURI. -// -// The function doesn't follow redirects. Use Get* for following redirects. -// -// Response is ignored if resp is nil. -// -// It is recommended obtaining req and resp via AcquireRequest -// and AcquireResponse in performance-critical code. -func (c *PipelineClient) Do(req *Request, resp *Response) error { - return c.getConnClient().Do(req, resp) -} - -func (c *pipelineConnClient) Do(req *Request, resp *Response) error { - c.init() - - w := acquirePipelineWork(&c.workPool, 0) - w.req = req - if resp != nil { - w.resp = resp - } else { - w.resp = &w.respCopy - } - - // Put the request to outgoing queue - select { - case c.chW <- w: - default: - // Try substituting the oldest w with the current one. - select { - case wOld := <-c.chW: - wOld.err = ErrPipelineOverflow - wOld.done <- struct{}{} - default: - } - select { - case c.chW <- w: - default: - releasePipelineWork(&c.workPool, w) - return ErrPipelineOverflow - } - } - - // Wait for the response - <-w.done - err := w.err - - releasePipelineWork(&c.workPool, w) - - return err -} - -func (c *PipelineClient) getConnClient() *pipelineConnClient { - c.connClientsLock.Lock() - cc := c.getConnClientUnlocked() - c.connClientsLock.Unlock() - return cc -} - -func (c *PipelineClient) getConnClientUnlocked() *pipelineConnClient { - if len(c.connClients) == 0 { - return c.newConnClient() - } - - // Return the client with the minimum number of pending requests. - minCC := c.connClients[0] - minReqs := minCC.PendingRequests() - if minReqs == 0 { - return minCC - } - for i := 1; i < len(c.connClients); i++ { - cc := c.connClients[i] - reqs := cc.PendingRequests() - if reqs == 0 { - return cc - } - if reqs < minReqs { - minCC = cc - minReqs = reqs - } - } - - maxConns := c.MaxConns - if maxConns <= 0 { - maxConns = 1 - } - if len(c.connClients) < maxConns { - return c.newConnClient() - } - return minCC -} - -func (c *PipelineClient) newConnClient() *pipelineConnClient { - cc := &pipelineConnClient{ - Addr: c.Addr, - MaxPendingRequests: c.MaxPendingRequests, - MaxBatchDelay: c.MaxBatchDelay, - Dial: c.Dial, - DialDualStack: c.DialDualStack, - IsTLS: c.IsTLS, - TLSConfig: c.TLSConfig, - MaxIdleConnDuration: c.MaxIdleConnDuration, - ReadBufferSize: c.ReadBufferSize, - WriteBufferSize: c.WriteBufferSize, - ReadTimeout: c.ReadTimeout, - WriteTimeout: c.WriteTimeout, - Logger: c.Logger, - } - c.connClients = append(c.connClients, cc) - return cc -} - -// ErrPipelineOverflow may be returned from PipelineClient.Do* -// if the requests' queue is overflown. -var ErrPipelineOverflow = errors.New("pipelined requests' queue has been overflown. Increase MaxConns and/or MaxPendingRequests") - -// DefaultMaxPendingRequests is the default value -// for PipelineClient.MaxPendingRequests. -const DefaultMaxPendingRequests = 1024 - -func (c *pipelineConnClient) init() { - c.chLock.Lock() - if c.chR == nil { - maxPendingRequests := c.MaxPendingRequests - if maxPendingRequests <= 0 { - maxPendingRequests = DefaultMaxPendingRequests - } - c.chR = make(chan *pipelineWork, maxPendingRequests) - if c.chW == nil { - c.chW = make(chan *pipelineWork, maxPendingRequests) - } - go func() { - if err := c.worker(); err != nil { - c.logger().Printf("error in PipelineClient(%q): %s", c.Addr, err) - if netErr, ok := err.(net.Error); ok && netErr.Temporary() { - // Throttle client reconnections on temporary errors - time.Sleep(time.Second) - } - } - - c.chLock.Lock() - // Do not reset c.chW to nil, since it may contain - // pending requests, which could be served on the next - // connection to the host. - c.chR = nil - c.chLock.Unlock() - }() - } - c.chLock.Unlock() -} - -func (c *pipelineConnClient) worker() error { - tlsConfig := c.cachedTLSConfig() - conn, err := dialAddr(c.Addr, c.Dial, c.DialDualStack, c.IsTLS, tlsConfig) - if err != nil { - return err - } - - // Start reader and writer - stopW := make(chan struct{}) - doneW := make(chan error) - go func() { - doneW <- c.writer(conn, stopW) - }() - stopR := make(chan struct{}) - doneR := make(chan error) - go func() { - doneR <- c.reader(conn, stopR) - }() - - // Wait until reader and writer are stopped - select { - case err = <-doneW: - conn.Close() - close(stopR) - <-doneR - case err = <-doneR: - conn.Close() - close(stopW) - <-doneW - } - - // Notify pending readers - for len(c.chR) > 0 { - w := <-c.chR - w.err = errPipelineConnStopped - w.done <- struct{}{} - } - - return err -} - -func (c *pipelineConnClient) cachedTLSConfig() *tls.Config { - if !c.IsTLS { - return nil - } - - c.tlsConfigLock.Lock() - cfg := c.tlsConfig - if cfg == nil { - cfg = newClientTLSConfig(c.TLSConfig, c.Addr) - c.tlsConfig = cfg - } - c.tlsConfigLock.Unlock() - - return cfg -} - -func (c *pipelineConnClient) writer(conn net.Conn, stopCh <-chan struct{}) error { - writeBufferSize := c.WriteBufferSize - if writeBufferSize <= 0 { - writeBufferSize = defaultWriteBufferSize - } - bw := bufio.NewWriterSize(conn, writeBufferSize) - defer bw.Flush() - chR := c.chR - chW := c.chW - writeTimeout := c.WriteTimeout - - maxIdleConnDuration := c.MaxIdleConnDuration - if maxIdleConnDuration <= 0 { - maxIdleConnDuration = DefaultMaxIdleConnDuration - } - maxBatchDelay := c.MaxBatchDelay - - var ( - stopTimer = time.NewTimer(time.Hour) - flushTimer = time.NewTimer(time.Hour) - flushTimerCh <-chan time.Time - instantTimerCh = make(chan time.Time) - - w *pipelineWork - err error - - lastWriteDeadlineTime time.Time - ) - close(instantTimerCh) - for { - againChW: - select { - case w = <-chW: - // Fast path: len(chW) > 0 - default: - // Slow path - stopTimer.Reset(maxIdleConnDuration) - select { - case w = <-chW: - case <-stopTimer.C: - return nil - case <-stopCh: - return nil - case <-flushTimerCh: - if err = bw.Flush(); err != nil { - return err - } - flushTimerCh = nil - goto againChW - } - } - - if !w.deadline.IsZero() && time.Since(w.deadline) >= 0 { - w.err = ErrTimeout - w.done <- struct{}{} - continue - } - - if writeTimeout > 0 { - // Optimization: update write deadline only if more than 25% - // of the last write deadline exceeded. - // See https://github.com/golang/go/issues/15133 for details. - currentTime := time.Now() - if currentTime.Sub(lastWriteDeadlineTime) > (writeTimeout >> 2) { - if err = conn.SetWriteDeadline(currentTime.Add(writeTimeout)); err != nil { - w.err = err - w.done <- struct{}{} - return err - } - lastWriteDeadlineTime = currentTime - } - } - if err = w.req.Write(bw); err != nil { - w.err = err - w.done <- struct{}{} - return err - } - if flushTimerCh == nil && (len(chW) == 0 || len(chR) == cap(chR)) { - if maxBatchDelay > 0 { - flushTimer.Reset(maxBatchDelay) - flushTimerCh = flushTimer.C - } else { - flushTimerCh = instantTimerCh - } - } - - againChR: - select { - case chR <- w: - // Fast path: len(chR) < cap(chR) - default: - // Slow path - select { - case chR <- w: - case <-stopCh: - w.err = errPipelineConnStopped - w.done <- struct{}{} - return nil - case <-flushTimerCh: - if err = bw.Flush(); err != nil { - w.err = err - w.done <- struct{}{} - return err - } - flushTimerCh = nil - goto againChR - } - } - } -} - -func (c *pipelineConnClient) reader(conn net.Conn, stopCh <-chan struct{}) error { - readBufferSize := c.ReadBufferSize - if readBufferSize <= 0 { - readBufferSize = defaultReadBufferSize - } - br := bufio.NewReaderSize(conn, readBufferSize) - chR := c.chR - readTimeout := c.ReadTimeout - - var ( - w *pipelineWork - err error - - lastReadDeadlineTime time.Time - ) - for { - select { - case w = <-chR: - // Fast path: len(chR) > 0 - default: - // Slow path - select { - case w = <-chR: - case <-stopCh: - return nil - } - } - - if readTimeout > 0 { - // Optimization: update read deadline only if more than 25% - // of the last read deadline exceeded. - // See https://github.com/golang/go/issues/15133 for details. - currentTime := time.Now() - if currentTime.Sub(lastReadDeadlineTime) > (readTimeout >> 2) { - if err = conn.SetReadDeadline(currentTime.Add(readTimeout)); err != nil { - w.err = err - w.done <- struct{}{} - return err - } - lastReadDeadlineTime = currentTime - } - } - if err = w.resp.Read(br); err != nil { - w.err = err - w.done <- struct{}{} - return err - } - - w.done <- struct{}{} - } -} - -func (c *pipelineConnClient) logger() Logger { - if c.Logger != nil { - return c.Logger - } - return defaultLogger -} - -// PendingRequests returns the current number of pending requests pipelined -// to the server. -// -// This number may exceed MaxPendingRequests*MaxConns by up to two times, since -// each connection to the server may keep up to MaxPendingRequests requests -// in the queue before sending them to the server. -// -// This function may be used for balancing load among multiple PipelineClient -// instances. -func (c *PipelineClient) PendingRequests() int { - c.connClientsLock.Lock() - n := 0 - for _, cc := range c.connClients { - n += cc.PendingRequests() - } - c.connClientsLock.Unlock() - return n -} - -func (c *pipelineConnClient) PendingRequests() int { - c.init() - - c.chLock.Lock() - n := len(c.chR) + len(c.chW) - c.chLock.Unlock() - return n -} - -var errPipelineConnStopped = errors.New("pipeline connection has been stopped") - -func acquirePipelineWork(pool *sync.Pool, timeout time.Duration) *pipelineWork { - v := pool.Get() - if v == nil { - v = &pipelineWork{ - done: make(chan struct{}, 1), - } - } - w := v.(*pipelineWork) - if timeout > 0 { - if w.t == nil { - w.t = time.NewTimer(timeout) - } else { - w.t.Reset(timeout) - } - w.deadline = time.Now().Add(timeout) - } else { - w.deadline = zeroTime - } - return w -} - -func releasePipelineWork(pool *sync.Pool, w *pipelineWork) { - if w.t != nil { - w.t.Stop() - } - w.reqCopy.Reset() - w.respCopy.Reset() - w.req = nil - w.resp = nil - w.err = nil - pool.Put(w) -} diff --git a/vendor/github.com/valyala/fasthttp/coarseTime.go b/vendor/github.com/valyala/fasthttp/coarseTime.go deleted file mode 100644 index 4679df68..00000000 --- a/vendor/github.com/valyala/fasthttp/coarseTime.go +++ /dev/null @@ -1,13 +0,0 @@ -package fasthttp - -import ( - "time" -) - -// CoarseTimeNow returns the current time truncated to the nearest second. -// -// Deprecated: This is slower than calling time.Now() directly. -// This is now time.Now().Truncate(time.Second) shortcut. -func CoarseTimeNow() time.Time { - return time.Now().Truncate(time.Second) -} diff --git a/vendor/github.com/valyala/fasthttp/compress.go b/vendor/github.com/valyala/fasthttp/compress.go deleted file mode 100644 index 73a40d3b..00000000 --- a/vendor/github.com/valyala/fasthttp/compress.go +++ /dev/null @@ -1,438 +0,0 @@ -package fasthttp - -import ( - "bytes" - "fmt" - "io" - "os" - "sync" - - "github.com/klauspost/compress/flate" - "github.com/klauspost/compress/gzip" - "github.com/klauspost/compress/zlib" - "github.com/valyala/bytebufferpool" - "github.com/valyala/fasthttp/stackless" -) - -// Supported compression levels. -const ( - CompressNoCompression = flate.NoCompression - CompressBestSpeed = flate.BestSpeed - CompressBestCompression = flate.BestCompression - CompressDefaultCompression = 6 // flate.DefaultCompression - CompressHuffmanOnly = -2 // flate.HuffmanOnly -) - -func acquireGzipReader(r io.Reader) (*gzip.Reader, error) { - v := gzipReaderPool.Get() - if v == nil { - return gzip.NewReader(r) - } - zr := v.(*gzip.Reader) - if err := zr.Reset(r); err != nil { - return nil, err - } - return zr, nil -} - -func releaseGzipReader(zr *gzip.Reader) { - zr.Close() - gzipReaderPool.Put(zr) -} - -var gzipReaderPool sync.Pool - -func acquireFlateReader(r io.Reader) (io.ReadCloser, error) { - v := flateReaderPool.Get() - if v == nil { - zr, err := zlib.NewReader(r) - if err != nil { - return nil, err - } - return zr, nil - } - zr := v.(io.ReadCloser) - if err := resetFlateReader(zr, r); err != nil { - return nil, err - } - return zr, nil -} - -func releaseFlateReader(zr io.ReadCloser) { - zr.Close() - flateReaderPool.Put(zr) -} - -func resetFlateReader(zr io.ReadCloser, r io.Reader) error { - zrr, ok := zr.(zlib.Resetter) - if !ok { - panic("BUG: zlib.Reader doesn't implement zlib.Resetter???") - } - return zrr.Reset(r, nil) -} - -var flateReaderPool sync.Pool - -func acquireStacklessGzipWriter(w io.Writer, level int) stackless.Writer { - nLevel := normalizeCompressLevel(level) - p := stacklessGzipWriterPoolMap[nLevel] - v := p.Get() - if v == nil { - return stackless.NewWriter(w, func(w io.Writer) stackless.Writer { - return acquireRealGzipWriter(w, level) - }) - } - sw := v.(stackless.Writer) - sw.Reset(w) - return sw -} - -func releaseStacklessGzipWriter(sw stackless.Writer, level int) { - sw.Close() - nLevel := normalizeCompressLevel(level) - p := stacklessGzipWriterPoolMap[nLevel] - p.Put(sw) -} - -func acquireRealGzipWriter(w io.Writer, level int) *gzip.Writer { - nLevel := normalizeCompressLevel(level) - p := realGzipWriterPoolMap[nLevel] - v := p.Get() - if v == nil { - zw, err := gzip.NewWriterLevel(w, level) - if err != nil { - panic(fmt.Sprintf("BUG: unexpected error from gzip.NewWriterLevel(%d): %s", level, err)) - } - return zw - } - zw := v.(*gzip.Writer) - zw.Reset(w) - return zw -} - -func releaseRealGzipWriter(zw *gzip.Writer, level int) { - zw.Close() - nLevel := normalizeCompressLevel(level) - p := realGzipWriterPoolMap[nLevel] - p.Put(zw) -} - -var ( - stacklessGzipWriterPoolMap = newCompressWriterPoolMap() - realGzipWriterPoolMap = newCompressWriterPoolMap() -) - -// AppendGzipBytesLevel appends gzipped src to dst using the given -// compression level and returns the resulting dst. -// -// Supported compression levels are: -// -// * CompressNoCompression -// * CompressBestSpeed -// * CompressBestCompression -// * CompressDefaultCompression -// * CompressHuffmanOnly -func AppendGzipBytesLevel(dst, src []byte, level int) []byte { - w := &byteSliceWriter{dst} - WriteGzipLevel(w, src, level) - return w.b -} - -// WriteGzipLevel writes gzipped p to w using the given compression level -// and returns the number of compressed bytes written to w. -// -// Supported compression levels are: -// -// * CompressNoCompression -// * CompressBestSpeed -// * CompressBestCompression -// * CompressDefaultCompression -// * CompressHuffmanOnly -func WriteGzipLevel(w io.Writer, p []byte, level int) (int, error) { - switch w.(type) { - case *byteSliceWriter, - *bytes.Buffer, - *bytebufferpool.ByteBuffer: - // These writers don't block, so we can just use stacklessWriteGzip - ctx := &compressCtx{ - w: w, - p: p, - level: level, - } - stacklessWriteGzip(ctx) - return len(p), nil - default: - zw := acquireStacklessGzipWriter(w, level) - n, err := zw.Write(p) - releaseStacklessGzipWriter(zw, level) - return n, err - } -} - -var stacklessWriteGzip = stackless.NewFunc(nonblockingWriteGzip) - -func nonblockingWriteGzip(ctxv interface{}) { - ctx := ctxv.(*compressCtx) - zw := acquireRealGzipWriter(ctx.w, ctx.level) - - _, err := zw.Write(ctx.p) - if err != nil { - panic(fmt.Sprintf("BUG: gzip.Writer.Write for len(p)=%d returned unexpected error: %s", len(ctx.p), err)) - } - - releaseRealGzipWriter(zw, ctx.level) -} - -// WriteGzip writes gzipped p to w and returns the number of compressed -// bytes written to w. -func WriteGzip(w io.Writer, p []byte) (int, error) { - return WriteGzipLevel(w, p, CompressDefaultCompression) -} - -// AppendGzipBytes appends gzipped src to dst and returns the resulting dst. -func AppendGzipBytes(dst, src []byte) []byte { - return AppendGzipBytesLevel(dst, src, CompressDefaultCompression) -} - -// WriteGunzip writes ungzipped p to w and returns the number of uncompressed -// bytes written to w. -func WriteGunzip(w io.Writer, p []byte) (int, error) { - r := &byteSliceReader{p} - zr, err := acquireGzipReader(r) - if err != nil { - return 0, err - } - n, err := copyZeroAlloc(w, zr) - releaseGzipReader(zr) - nn := int(n) - if int64(nn) != n { - return 0, fmt.Errorf("too much data gunzipped: %d", n) - } - return nn, err -} - -// AppendGunzipBytes appends gunzipped src to dst and returns the resulting dst. -func AppendGunzipBytes(dst, src []byte) ([]byte, error) { - w := &byteSliceWriter{dst} - _, err := WriteGunzip(w, src) - return w.b, err -} - -// AppendDeflateBytesLevel appends deflated src to dst using the given -// compression level and returns the resulting dst. -// -// Supported compression levels are: -// -// * CompressNoCompression -// * CompressBestSpeed -// * CompressBestCompression -// * CompressDefaultCompression -// * CompressHuffmanOnly -func AppendDeflateBytesLevel(dst, src []byte, level int) []byte { - w := &byteSliceWriter{dst} - WriteDeflateLevel(w, src, level) - return w.b -} - -// WriteDeflateLevel writes deflated p to w using the given compression level -// and returns the number of compressed bytes written to w. -// -// Supported compression levels are: -// -// * CompressNoCompression -// * CompressBestSpeed -// * CompressBestCompression -// * CompressDefaultCompression -// * CompressHuffmanOnly -func WriteDeflateLevel(w io.Writer, p []byte, level int) (int, error) { - switch w.(type) { - case *byteSliceWriter, - *bytes.Buffer, - *bytebufferpool.ByteBuffer: - // These writers don't block, so we can just use stacklessWriteDeflate - ctx := &compressCtx{ - w: w, - p: p, - level: level, - } - stacklessWriteDeflate(ctx) - return len(p), nil - default: - zw := acquireStacklessDeflateWriter(w, level) - n, err := zw.Write(p) - releaseStacklessDeflateWriter(zw, level) - return n, err - } -} - -var stacklessWriteDeflate = stackless.NewFunc(nonblockingWriteDeflate) - -func nonblockingWriteDeflate(ctxv interface{}) { - ctx := ctxv.(*compressCtx) - zw := acquireRealDeflateWriter(ctx.w, ctx.level) - - _, err := zw.Write(ctx.p) - if err != nil { - panic(fmt.Sprintf("BUG: zlib.Writer.Write for len(p)=%d returned unexpected error: %s", len(ctx.p), err)) - } - - releaseRealDeflateWriter(zw, ctx.level) -} - -type compressCtx struct { - w io.Writer - p []byte - level int -} - -// WriteDeflate writes deflated p to w and returns the number of compressed -// bytes written to w. -func WriteDeflate(w io.Writer, p []byte) (int, error) { - return WriteDeflateLevel(w, p, CompressDefaultCompression) -} - -// AppendDeflateBytes appends deflated src to dst and returns the resulting dst. -func AppendDeflateBytes(dst, src []byte) []byte { - return AppendDeflateBytesLevel(dst, src, CompressDefaultCompression) -} - -// WriteInflate writes inflated p to w and returns the number of uncompressed -// bytes written to w. -func WriteInflate(w io.Writer, p []byte) (int, error) { - r := &byteSliceReader{p} - zr, err := acquireFlateReader(r) - if err != nil { - return 0, err - } - n, err := copyZeroAlloc(w, zr) - releaseFlateReader(zr) - nn := int(n) - if int64(nn) != n { - return 0, fmt.Errorf("too much data inflated: %d", n) - } - return nn, err -} - -// AppendInflateBytes appends inflated src to dst and returns the resulting dst. -func AppendInflateBytes(dst, src []byte) ([]byte, error) { - w := &byteSliceWriter{dst} - _, err := WriteInflate(w, src) - return w.b, err -} - -type byteSliceWriter struct { - b []byte -} - -func (w *byteSliceWriter) Write(p []byte) (int, error) { - w.b = append(w.b, p...) - return len(p), nil -} - -type byteSliceReader struct { - b []byte -} - -func (r *byteSliceReader) Read(p []byte) (int, error) { - if len(r.b) == 0 { - return 0, io.EOF - } - n := copy(p, r.b) - r.b = r.b[n:] - return n, nil -} - -func acquireStacklessDeflateWriter(w io.Writer, level int) stackless.Writer { - nLevel := normalizeCompressLevel(level) - p := stacklessDeflateWriterPoolMap[nLevel] - v := p.Get() - if v == nil { - return stackless.NewWriter(w, func(w io.Writer) stackless.Writer { - return acquireRealDeflateWriter(w, level) - }) - } - sw := v.(stackless.Writer) - sw.Reset(w) - return sw -} - -func releaseStacklessDeflateWriter(sw stackless.Writer, level int) { - sw.Close() - nLevel := normalizeCompressLevel(level) - p := stacklessDeflateWriterPoolMap[nLevel] - p.Put(sw) -} - -func acquireRealDeflateWriter(w io.Writer, level int) *zlib.Writer { - nLevel := normalizeCompressLevel(level) - p := realDeflateWriterPoolMap[nLevel] - v := p.Get() - if v == nil { - zw, err := zlib.NewWriterLevel(w, level) - if err != nil { - panic(fmt.Sprintf("BUG: unexpected error from zlib.NewWriterLevel(%d): %s", level, err)) - } - return zw - } - zw := v.(*zlib.Writer) - zw.Reset(w) - return zw -} - -func releaseRealDeflateWriter(zw *zlib.Writer, level int) { - zw.Close() - nLevel := normalizeCompressLevel(level) - p := realDeflateWriterPoolMap[nLevel] - p.Put(zw) -} - -var ( - stacklessDeflateWriterPoolMap = newCompressWriterPoolMap() - realDeflateWriterPoolMap = newCompressWriterPoolMap() -) - -func newCompressWriterPoolMap() []*sync.Pool { - // Initialize pools for all the compression levels defined - // in https://golang.org/pkg/compress/flate/#pkg-constants . - // Compression levels are normalized with normalizeCompressLevel, - // so the fit [0..11]. - var m []*sync.Pool - for i := 0; i < 12; i++ { - m = append(m, &sync.Pool{}) - } - return m -} - -func isFileCompressible(f *os.File, minCompressRatio float64) bool { - // Try compressing the first 4kb of of the file - // and see if it can be compressed by more than - // the given minCompressRatio. - b := bytebufferpool.Get() - zw := acquireStacklessGzipWriter(b, CompressDefaultCompression) - lr := &io.LimitedReader{ - R: f, - N: 4096, - } - _, err := copyZeroAlloc(zw, lr) - releaseStacklessGzipWriter(zw, CompressDefaultCompression) - f.Seek(0, 0) - if err != nil { - return false - } - - n := 4096 - lr.N - zn := len(b.B) - bytebufferpool.Put(b) - return float64(zn) < float64(n)*minCompressRatio -} - -// normalizes compression level into [0..11], so it could be used as an index -// in *PoolMap. -func normalizeCompressLevel(level int) int { - // -2 is the lowest compression level - CompressHuffmanOnly - // 9 is the highest compression level - CompressBestCompression - if level < -2 || level > 9 { - level = CompressDefaultCompression - } - return level + 2 -} diff --git a/vendor/github.com/valyala/fasthttp/cookie.go b/vendor/github.com/valyala/fasthttp/cookie.go deleted file mode 100644 index d5abef2f..00000000 --- a/vendor/github.com/valyala/fasthttp/cookie.go +++ /dev/null @@ -1,474 +0,0 @@ -package fasthttp - -import ( - "bytes" - "errors" - "io" - "sync" - "time" -) - -var zeroTime time.Time - -var ( - // CookieExpireDelete may be set on Cookie.Expire for expiring the given cookie. - CookieExpireDelete = time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) - - // CookieExpireUnlimited indicates that the cookie doesn't expire. - CookieExpireUnlimited = zeroTime -) - -// AcquireCookie returns an empty Cookie object from the pool. -// -// The returned object may be returned back to the pool with ReleaseCookie. -// This allows reducing GC load. -func AcquireCookie() *Cookie { - return cookiePool.Get().(*Cookie) -} - -// ReleaseCookie returns the Cookie object acquired with AcquireCookie back -// to the pool. -// -// Do not access released Cookie object, otherwise data races may occur. -func ReleaseCookie(c *Cookie) { - c.Reset() - cookiePool.Put(c) -} - -var cookiePool = &sync.Pool{ - New: func() interface{} { - return &Cookie{} - }, -} - -// Cookie represents HTTP response cookie. -// -// Do not copy Cookie objects. Create new object and use CopyTo instead. -// -// Cookie instance MUST NOT be used from concurrently running goroutines. -type Cookie struct { - noCopy noCopy - - key []byte - value []byte - expire time.Time - maxAge int - domain []byte - path []byte - - httpOnly bool - secure bool - - bufKV argsKV - buf []byte -} - -// CopyTo copies src cookie to c. -func (c *Cookie) CopyTo(src *Cookie) { - c.Reset() - c.key = append(c.key[:0], src.key...) - c.value = append(c.value[:0], src.value...) - c.expire = src.expire - c.maxAge = src.maxAge - c.domain = append(c.domain[:0], src.domain...) - c.path = append(c.path[:0], src.path...) - c.httpOnly = src.httpOnly - c.secure = src.secure -} - -// HTTPOnly returns true if the cookie is http only. -func (c *Cookie) HTTPOnly() bool { - return c.httpOnly -} - -// SetHTTPOnly sets cookie's httpOnly flag to the given value. -func (c *Cookie) SetHTTPOnly(httpOnly bool) { - c.httpOnly = httpOnly -} - -// Secure returns true if the cookie is secure. -func (c *Cookie) Secure() bool { - return c.secure -} - -// SetSecure sets cookie's secure flag to the given value. -func (c *Cookie) SetSecure(secure bool) { - c.secure = secure -} - -// Path returns cookie path. -func (c *Cookie) Path() []byte { - return c.path -} - -// SetPath sets cookie path. -func (c *Cookie) SetPath(path string) { - c.buf = append(c.buf[:0], path...) - c.path = normalizePath(c.path, c.buf) -} - -// SetPathBytes sets cookie path. -func (c *Cookie) SetPathBytes(path []byte) { - c.buf = append(c.buf[:0], path...) - c.path = normalizePath(c.path, c.buf) -} - -// Domain returns cookie domain. -// -// The returned domain is valid until the next Cookie modification method call. -func (c *Cookie) Domain() []byte { - return c.domain -} - -// SetDomain sets cookie domain. -func (c *Cookie) SetDomain(domain string) { - c.domain = append(c.domain[:0], domain...) -} - -// SetDomainBytes sets cookie domain. -func (c *Cookie) SetDomainBytes(domain []byte) { - c.domain = append(c.domain[:0], domain...) -} - -// MaxAge returns the seconds until the cookie is meant to expire or 0 -// if no max age. -func (c *Cookie) MaxAge() int { - return c.maxAge -} - -// SetMaxAge sets cookie expiration time based on seconds. This takes precedence -// over any absolute expiry set on the cookie -// -// Set max age to 0 to unset -func (c *Cookie) SetMaxAge(seconds int) { - c.maxAge = seconds -} - -// Expire returns cookie expiration time. -// -// CookieExpireUnlimited is returned if cookie doesn't expire -func (c *Cookie) Expire() time.Time { - expire := c.expire - if expire.IsZero() { - expire = CookieExpireUnlimited - } - return expire -} - -// SetExpire sets cookie expiration time. -// -// Set expiration time to CookieExpireDelete for expiring (deleting) -// the cookie on the client. -// -// By default cookie lifetime is limited by browser session. -func (c *Cookie) SetExpire(expire time.Time) { - c.expire = expire -} - -// Value returns cookie value. -// -// The returned value is valid until the next Cookie modification method call. -func (c *Cookie) Value() []byte { - return c.value -} - -// SetValue sets cookie value. -func (c *Cookie) SetValue(value string) { - c.value = append(c.value[:0], value...) -} - -// SetValueBytes sets cookie value. -func (c *Cookie) SetValueBytes(value []byte) { - c.value = append(c.value[:0], value...) -} - -// Key returns cookie name. -// -// The returned value is valid until the next Cookie modification method call. -func (c *Cookie) Key() []byte { - return c.key -} - -// SetKey sets cookie name. -func (c *Cookie) SetKey(key string) { - c.key = append(c.key[:0], key...) -} - -// SetKeyBytes sets cookie name. -func (c *Cookie) SetKeyBytes(key []byte) { - c.key = append(c.key[:0], key...) -} - -// Reset clears the cookie. -func (c *Cookie) Reset() { - c.key = c.key[:0] - c.value = c.value[:0] - c.expire = zeroTime - c.maxAge = 0 - c.domain = c.domain[:0] - c.path = c.path[:0] - c.httpOnly = false - c.secure = false -} - -// AppendBytes appends cookie representation to dst and returns -// the extended dst. -func (c *Cookie) AppendBytes(dst []byte) []byte { - if len(c.key) > 0 { - dst = append(dst, c.key...) - dst = append(dst, '=') - } - dst = append(dst, c.value...) - - if c.maxAge > 0 { - dst = append(dst, ';', ' ') - dst = append(dst, strCookieMaxAge...) - dst = append(dst, '=') - dst = AppendUint(dst, c.maxAge) - } else if !c.expire.IsZero() { - c.bufKV.value = AppendHTTPDate(c.bufKV.value[:0], c.expire) - dst = append(dst, ';', ' ') - dst = append(dst, strCookieExpires...) - dst = append(dst, '=') - dst = append(dst, c.bufKV.value...) - } - if len(c.domain) > 0 { - dst = appendCookiePart(dst, strCookieDomain, c.domain) - } - if len(c.path) > 0 { - dst = appendCookiePart(dst, strCookiePath, c.path) - } - if c.httpOnly { - dst = append(dst, ';', ' ') - dst = append(dst, strCookieHTTPOnly...) - } - if c.secure { - dst = append(dst, ';', ' ') - dst = append(dst, strCookieSecure...) - } - return dst -} - -// Cookie returns cookie representation. -// -// The returned value is valid until the next call to Cookie methods. -func (c *Cookie) Cookie() []byte { - c.buf = c.AppendBytes(c.buf[:0]) - return c.buf -} - -// String returns cookie representation. -func (c *Cookie) String() string { - return string(c.Cookie()) -} - -// WriteTo writes cookie representation to w. -// -// WriteTo implements io.WriterTo interface. -func (c *Cookie) WriteTo(w io.Writer) (int64, error) { - n, err := w.Write(c.Cookie()) - return int64(n), err -} - -var errNoCookies = errors.New("no cookies found") - -// Parse parses Set-Cookie header. -func (c *Cookie) Parse(src string) error { - c.buf = append(c.buf[:0], src...) - return c.ParseBytes(c.buf) -} - -// ParseBytes parses Set-Cookie header. -func (c *Cookie) ParseBytes(src []byte) error { - c.Reset() - - var s cookieScanner - s.b = src - - kv := &c.bufKV - if !s.next(kv) { - return errNoCookies - } - - c.key = append(c.key[:0], kv.key...) - c.value = append(c.value[:0], kv.value...) - - for s.next(kv) { - if len(kv.key) != 0 { - // Case insensitive switch on first char - switch kv.key[0] | 0x20 { - case 'm': - if caseInsensitiveCompare(strCookieMaxAge, kv.key) { - maxAge, err := ParseUint(kv.value) - if err != nil { - return err - } - c.maxAge = maxAge - } - - case 'e': // "expires" - if caseInsensitiveCompare(strCookieExpires, kv.key) { - v := b2s(kv.value) - // Try the same two formats as net/http - // See: https://github.com/golang/go/blob/00379be17e63a5b75b3237819392d2dc3b313a27/src/net/http/cookie.go#L133-L135 - exptime, err := time.ParseInLocation(time.RFC1123, v, time.UTC) - if err != nil { - exptime, err = time.Parse("Mon, 02-Jan-2006 15:04:05 MST", v) - if err != nil { - return err - } - } - c.expire = exptime - } - - case 'd': // "domain" - if caseInsensitiveCompare(strCookieDomain, kv.key) { - c.domain = append(c.domain[:0], kv.value...) - } - - case 'p': // "path" - if caseInsensitiveCompare(strCookiePath, kv.key) { - c.path = append(c.path[:0], kv.value...) - } - } - - } else if len(kv.value) != 0 { - // Case insensitive switch on first char - switch kv.value[0] | 0x20 { - case 'h': // "httponly" - if caseInsensitiveCompare(strCookieHTTPOnly, kv.value) { - c.httpOnly = true - } - - case 's': // "secure" - if caseInsensitiveCompare(strCookieSecure, kv.value) { - c.secure = true - } - } - } // else empty or no match - } - return nil -} - -func appendCookiePart(dst, key, value []byte) []byte { - dst = append(dst, ';', ' ') - dst = append(dst, key...) - dst = append(dst, '=') - return append(dst, value...) -} - -func getCookieKey(dst, src []byte) []byte { - n := bytes.IndexByte(src, '=') - if n >= 0 { - src = src[:n] - } - return decodeCookieArg(dst, src, false) -} - -func appendRequestCookieBytes(dst []byte, cookies []argsKV) []byte { - for i, n := 0, len(cookies); i < n; i++ { - kv := &cookies[i] - if len(kv.key) > 0 { - dst = append(dst, kv.key...) - dst = append(dst, '=') - } - dst = append(dst, kv.value...) - if i+1 < n { - dst = append(dst, ';', ' ') - } - } - return dst -} - -// For Response we can not use the above function as response cookies -// already contain the key= in the value. -func appendResponseCookieBytes(dst []byte, cookies []argsKV) []byte { - for i, n := 0, len(cookies); i < n; i++ { - kv := &cookies[i] - dst = append(dst, kv.value...) - if i+1 < n { - dst = append(dst, ';', ' ') - } - } - return dst -} - -func parseRequestCookies(cookies []argsKV, src []byte) []argsKV { - var s cookieScanner - s.b = src - var kv *argsKV - cookies, kv = allocArg(cookies) - for s.next(kv) { - if len(kv.key) > 0 || len(kv.value) > 0 { - cookies, kv = allocArg(cookies) - } - } - return releaseArg(cookies) -} - -type cookieScanner struct { - b []byte -} - -func (s *cookieScanner) next(kv *argsKV) bool { - b := s.b - if len(b) == 0 { - return false - } - - isKey := true - k := 0 - for i, c := range b { - switch c { - case '=': - if isKey { - isKey = false - kv.key = decodeCookieArg(kv.key, b[:i], false) - k = i + 1 - } - case ';': - if isKey { - kv.key = kv.key[:0] - } - kv.value = decodeCookieArg(kv.value, b[k:i], true) - s.b = b[i+1:] - return true - } - } - - if isKey { - kv.key = kv.key[:0] - } - kv.value = decodeCookieArg(kv.value, b[k:], true) - s.b = b[len(b):] - return true -} - -func decodeCookieArg(dst, src []byte, skipQuotes bool) []byte { - for len(src) > 0 && src[0] == ' ' { - src = src[1:] - } - for len(src) > 0 && src[len(src)-1] == ' ' { - src = src[:len(src)-1] - } - if skipQuotes { - if len(src) > 1 && src[0] == '"' && src[len(src)-1] == '"' { - src = src[1 : len(src)-1] - } - } - return append(dst[:0], src...) -} - -// caseInsensitiveCompare does a case insensitive equality comparison of -// two []byte. Assumes only letters need to be matched. -func caseInsensitiveCompare(a, b []byte) bool { - if len(a) != len(b) { - return false - } - for i := 0; i < len(a); i++ { - if a[i]|0x20 != b[i]|0x20 { - return false - } - } - return true -} diff --git a/vendor/github.com/valyala/fasthttp/doc.go b/vendor/github.com/valyala/fasthttp/doc.go deleted file mode 100644 index efcd4a03..00000000 --- a/vendor/github.com/valyala/fasthttp/doc.go +++ /dev/null @@ -1,37 +0,0 @@ -/* -Package fasthttp provides fast HTTP server and client API. - -Fasthttp provides the following features: - - * Optimized for speed. Easily handles more than 100K qps and more than 1M - concurrent keep-alive connections on modern hardware. - * Optimized for low memory usage. - * Easy 'Connection: Upgrade' support via RequestCtx.Hijack. - * Server provides the following anti-DoS limits: - - * The number of concurrent connections. - * The number of concurrent connections per client IP. - * The number of requests per connection. - * Request read timeout. - * Response write timeout. - * Maximum request header size. - * Maximum request body size. - * Maximum request execution time. - * Maximum keep-alive connection lifetime. - * Early filtering out non-GET requests. - - * A lot of additional useful info is exposed to request handler: - - * Server and client address. - * Per-request logger. - * Unique request id. - * Request start time. - * Connection start time. - * Request sequence number for the current connection. - - * Client supports automatic retry on idempotent requests' failure. - * Fasthttp API is designed with the ability to extend existing client - and server implementations or to write custom client and server - implementations from scratch. -*/ -package fasthttp diff --git a/vendor/github.com/valyala/fasthttp/fasthttputil/doc.go b/vendor/github.com/valyala/fasthttp/fasthttputil/doc.go deleted file mode 100644 index 9cf69e71..00000000 --- a/vendor/github.com/valyala/fasthttp/fasthttputil/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package fasthttputil provides utility functions for fasthttp. -package fasthttputil diff --git a/vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.key b/vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.key deleted file mode 100644 index 7e201fc4..00000000 --- a/vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.key +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIBpQbZ6a5jL1Yh4wdP6yZk4MKjYWArD/QOLENFw8vbELoAoGCCqGSM49 -AwEHoUQDQgAEKQCZWgE2IBhb47ot8MIs1D4KSisHYlZ41IWyeutpjb0fjwwIhimh -pl1Qld1/d2j3Z3vVyfa5yD+ncV7qCFZuSg== ------END EC PRIVATE KEY----- diff --git a/vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.pem b/vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.pem deleted file mode 100644 index ca1a7f2e..00000000 --- a/vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.pem +++ /dev/null @@ -1,10 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBbTCCAROgAwIBAgIQPo718S+K+G7hc1SgTEU4QDAKBggqhkjOPQQDAjASMRAw -DgYDVQQKEwdBY21lIENvMB4XDTE3MDQyMDIxMDExNFoXDTE4MDQyMDIxMDExNFow -EjEQMA4GA1UEChMHQWNtZSBDbzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCkA -mVoBNiAYW+O6LfDCLNQ+CkorB2JWeNSFsnrraY29H48MCIYpoaZdUJXdf3do92d7 -1cn2ucg/p3Fe6ghWbkqjSzBJMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggr -BgEFBQcDATAMBgNVHRMBAf8EAjAAMBQGA1UdEQQNMAuCCWxvY2FsaG9zdDAKBggq -hkjOPQQDAgNIADBFAiEAoLAIQkvSuIcHUqyWroA6yWYw2fznlRH/uO9/hMCxUCEC -IClRYb/5O9eD/Eq/ozPnwNpsQHOeYefEhadJ/P82y0lG ------END CERTIFICATE----- diff --git a/vendor/github.com/valyala/fasthttp/fasthttputil/inmemory_listener.go b/vendor/github.com/valyala/fasthttp/fasthttputil/inmemory_listener.go deleted file mode 100644 index 80aca3fb..00000000 --- a/vendor/github.com/valyala/fasthttp/fasthttputil/inmemory_listener.go +++ /dev/null @@ -1,84 +0,0 @@ -package fasthttputil - -import ( - "fmt" - "net" - "sync" -) - -// InmemoryListener provides in-memory dialer<->net.Listener implementation. -// -// It may be used either for fast in-process client<->server communications -// without network stack overhead or for client<->server tests. -type InmemoryListener struct { - lock sync.Mutex - closed bool - conns chan net.Conn -} - -// NewInmemoryListener returns new in-memory dialer<->net.Listener. -func NewInmemoryListener() *InmemoryListener { - return &InmemoryListener{ - conns: make(chan net.Conn, 1024), - } -} - -// Accept implements net.Listener's Accept. -// -// It is safe calling Accept from concurrently running goroutines. -// -// Accept returns new connection per each Dial call. -func (ln *InmemoryListener) Accept() (net.Conn, error) { - c, ok := <-ln.conns - if !ok { - return nil, fmt.Errorf("InmemoryListener is already closed: use of closed network connection") - } - return c, nil -} - -// Close implements net.Listener's Close. -func (ln *InmemoryListener) Close() error { - var err error - - ln.lock.Lock() - if !ln.closed { - close(ln.conns) - ln.closed = true - } else { - err = fmt.Errorf("InmemoryListener is already closed") - } - ln.lock.Unlock() - return err -} - -// Addr implements net.Listener's Addr. -func (ln *InmemoryListener) Addr() net.Addr { - return &net.UnixAddr{ - Name: "InmemoryListener", - Net: "memory", - } -} - -// Dial creates new client<->server connection, enqueues server side -// of the connection to Accept and returns client side of the connection. -// -// It is safe calling Dial from concurrently running goroutines. -func (ln *InmemoryListener) Dial() (net.Conn, error) { - pc := NewPipeConns() - cConn := pc.Conn1() - sConn := pc.Conn2() - ln.lock.Lock() - if !ln.closed { - ln.conns <- sConn - } else { - sConn.Close() - cConn.Close() - cConn = nil - } - ln.lock.Unlock() - - if cConn == nil { - return nil, fmt.Errorf("InmemoryListener is already closed") - } - return cConn, nil -} diff --git a/vendor/github.com/valyala/fasthttp/fasthttputil/pipeconns.go b/vendor/github.com/valyala/fasthttp/fasthttputil/pipeconns.go deleted file mode 100644 index aa92b6ff..00000000 --- a/vendor/github.com/valyala/fasthttp/fasthttputil/pipeconns.go +++ /dev/null @@ -1,283 +0,0 @@ -package fasthttputil - -import ( - "errors" - "io" - "net" - "sync" - "time" -) - -// NewPipeConns returns new bi-directional connection pipe. -func NewPipeConns() *PipeConns { - ch1 := make(chan *byteBuffer, 4) - ch2 := make(chan *byteBuffer, 4) - - pc := &PipeConns{ - stopCh: make(chan struct{}), - } - pc.c1.rCh = ch1 - pc.c1.wCh = ch2 - pc.c2.rCh = ch2 - pc.c2.wCh = ch1 - pc.c1.pc = pc - pc.c2.pc = pc - return pc -} - -// PipeConns provides bi-directional connection pipe, -// which use in-process memory as a transport. -// -// PipeConns must be created by calling NewPipeConns. -// -// PipeConns has the following additional features comparing to connections -// returned from net.Pipe(): -// -// * It is faster. -// * It buffers Write calls, so there is no need to have concurrent goroutine -// calling Read in order to unblock each Write call. -// * It supports read and write deadlines. -// -type PipeConns struct { - c1 pipeConn - c2 pipeConn - stopCh chan struct{} - stopChLock sync.Mutex -} - -// Conn1 returns the first end of bi-directional pipe. -// -// Data written to Conn1 may be read from Conn2. -// Data written to Conn2 may be read from Conn1. -func (pc *PipeConns) Conn1() net.Conn { - return &pc.c1 -} - -// Conn2 returns the second end of bi-directional pipe. -// -// Data written to Conn2 may be read from Conn1. -// Data written to Conn1 may be read from Conn2. -func (pc *PipeConns) Conn2() net.Conn { - return &pc.c2 -} - -// Close closes pipe connections. -func (pc *PipeConns) Close() error { - pc.stopChLock.Lock() - select { - case <-pc.stopCh: - default: - close(pc.stopCh) - } - pc.stopChLock.Unlock() - - return nil -} - -type pipeConn struct { - b *byteBuffer - bb []byte - - rCh chan *byteBuffer - wCh chan *byteBuffer - pc *PipeConns - - readDeadlineTimer *time.Timer - writeDeadlineTimer *time.Timer - - readDeadlineCh <-chan time.Time - writeDeadlineCh <-chan time.Time -} - -func (c *pipeConn) Write(p []byte) (int, error) { - b := acquireByteBuffer() - b.b = append(b.b[:0], p...) - - select { - case <-c.pc.stopCh: - releaseByteBuffer(b) - return 0, errConnectionClosed - default: - } - - select { - case c.wCh <- b: - default: - select { - case c.wCh <- b: - case <-c.writeDeadlineCh: - c.writeDeadlineCh = closedDeadlineCh - return 0, ErrTimeout - case <-c.pc.stopCh: - releaseByteBuffer(b) - return 0, errConnectionClosed - } - } - - return len(p), nil -} - -func (c *pipeConn) Read(p []byte) (int, error) { - mayBlock := true - nn := 0 - for len(p) > 0 { - n, err := c.read(p, mayBlock) - nn += n - if err != nil { - if !mayBlock && err == errWouldBlock { - err = nil - } - return nn, err - } - p = p[n:] - mayBlock = false - } - - return nn, nil -} - -func (c *pipeConn) read(p []byte, mayBlock bool) (int, error) { - if len(c.bb) == 0 { - if err := c.readNextByteBuffer(mayBlock); err != nil { - return 0, err - } - } - n := copy(p, c.bb) - c.bb = c.bb[n:] - - return n, nil -} - -func (c *pipeConn) readNextByteBuffer(mayBlock bool) error { - releaseByteBuffer(c.b) - c.b = nil - - select { - case c.b = <-c.rCh: - default: - if !mayBlock { - return errWouldBlock - } - select { - case c.b = <-c.rCh: - case <-c.readDeadlineCh: - c.readDeadlineCh = closedDeadlineCh - // rCh may contain data when deadline is reached. - // Read the data before returning ErrTimeout. - select { - case c.b = <-c.rCh: - default: - return ErrTimeout - } - case <-c.pc.stopCh: - // rCh may contain data when stopCh is closed. - // Read the data before returning EOF. - select { - case c.b = <-c.rCh: - default: - return io.EOF - } - } - } - - c.bb = c.b.b - return nil -} - -var ( - errWouldBlock = errors.New("would block") - errConnectionClosed = errors.New("connection closed") - - // ErrTimeout is returned from Read() or Write() on timeout. - ErrTimeout = errors.New("timeout") -) - -func (c *pipeConn) Close() error { - return c.pc.Close() -} - -func (c *pipeConn) LocalAddr() net.Addr { - return pipeAddr(0) -} - -func (c *pipeConn) RemoteAddr() net.Addr { - return pipeAddr(0) -} - -func (c *pipeConn) SetDeadline(deadline time.Time) error { - c.SetReadDeadline(deadline) - c.SetWriteDeadline(deadline) - return nil -} - -func (c *pipeConn) SetReadDeadline(deadline time.Time) error { - if c.readDeadlineTimer == nil { - c.readDeadlineTimer = time.NewTimer(time.Hour) - } - c.readDeadlineCh = updateTimer(c.readDeadlineTimer, deadline) - return nil -} - -func (c *pipeConn) SetWriteDeadline(deadline time.Time) error { - if c.writeDeadlineTimer == nil { - c.writeDeadlineTimer = time.NewTimer(time.Hour) - } - c.writeDeadlineCh = updateTimer(c.writeDeadlineTimer, deadline) - return nil -} - -func updateTimer(t *time.Timer, deadline time.Time) <-chan time.Time { - if !t.Stop() { - select { - case <-t.C: - default: - } - } - if deadline.IsZero() { - return nil - } - d := -time.Since(deadline) - if d <= 0 { - return closedDeadlineCh - } - t.Reset(d) - return t.C -} - -var closedDeadlineCh = func() <-chan time.Time { - ch := make(chan time.Time) - close(ch) - return ch -}() - -type pipeAddr int - -func (pipeAddr) Network() string { - return "pipe" -} - -func (pipeAddr) String() string { - return "pipe" -} - -type byteBuffer struct { - b []byte -} - -func acquireByteBuffer() *byteBuffer { - return byteBufferPool.Get().(*byteBuffer) -} - -func releaseByteBuffer(b *byteBuffer) { - if b != nil { - byteBufferPool.Put(b) - } -} - -var byteBufferPool = &sync.Pool{ - New: func() interface{} { - return &byteBuffer{ - b: make([]byte, 1024), - } - }, -} diff --git a/vendor/github.com/valyala/fasthttp/fasthttputil/rsa.key b/vendor/github.com/valyala/fasthttp/fasthttputil/rsa.key deleted file mode 100644 index 00a79a3b..00000000 --- a/vendor/github.com/valyala/fasthttp/fasthttputil/rsa.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQD4IQusAs8PJdnG -3mURt/AXtgC+ceqLOatJ49JJE1VPTkMAy+oE1f1XvkMrYsHqmDf6GWVzgVXryL4U -wq2/nJSm56ddhN55nI8oSN3dtywUB8/ShelEN73nlN77PeD9tl6NksPwWaKrqxq0 -FlabRPZSQCfmgZbhDV8Sa8mfCkFU0G0lit6kLGceCKMvmW+9Bz7ebsYmVdmVMxmf -IJStFD44lWFTdUc65WISKEdW2ELcUefb0zOLw+0PCbXFGJH5x5ktksW8+BBk2Hkg -GeQRL/qPCccthbScO0VgNj3zJ3ZZL0ObSDAbvNDG85joeNjDNq5DT/BAZ0bOSbEF -sh+f9BAzAgMBAAECggEBAJWv2cq7Jw6MVwSRxYca38xuD6TUNBopgBvjREixURW2 -sNUaLuMb9Omp7fuOaE2N5rcJ+xnjPGIxh/oeN5MQctz9gwn3zf6vY+15h97pUb4D -uGvYPRDaT8YVGS+X9NMZ4ZCmqW2lpWzKnCFoGHcy8yZLbcaxBsRdvKzwOYGoPiFb -K2QuhXZ/1UPmqK9i2DFKtj40X6vBszTNboFxOVpXrPu0FJwLVSDf2hSZ4fMM0DH3 -YqwKcYf5te+hxGKgrqRA3tn0NCWii0in6QIwXMC+kMw1ebg/tZKqyDLMNptAK8J+ -DVw9m5X1seUHS5ehU/g2jrQrtK5WYn7MrFK4lBzlRwECgYEA/d1TeANYECDWRRDk -B0aaRZs87Rwl/J9PsvbsKvtU/bX+OfSOUjOa9iQBqn0LmU8GqusEET/QVUfocVwV -Bggf/5qDLxz100Rj0ags/yE/kNr0Bb31kkkKHFMnCT06YasR7qKllwrAlPJvQv9x -IzBKq+T/Dx08Wep9bCRSFhzRCnsCgYEA+jdeZXTDr/Vz+D2B3nAw1frqYFfGnEVY -wqmoK3VXMDkGuxsloO2rN+SyiUo3JNiQNPDub/t7175GH5pmKtZOlftePANsUjBj -wZ1D0rI5Bxu/71ibIUYIRVmXsTEQkh/ozoh3jXCZ9+bLgYiYx7789IUZZSokFQ3D -FICUT9KJ36kCgYAGoq9Y1rWJjmIrYfqj2guUQC+CfxbbGIrrwZqAsRsSmpwvhZ3m -tiSZxG0quKQB+NfSxdvQW5ulbwC7Xc3K35F+i9pb8+TVBdeaFkw+yu6vaZmxQLrX -fQM/pEjD7A7HmMIaO7QaU5SfEAsqdCTP56Y8AftMuNXn/8IRfo2KuGwaWwKBgFpU -ILzJoVdlad9E/Rw7LjYhZfkv1uBVXIyxyKcfrkEXZSmozDXDdxsvcZCEfVHM6Ipk -K/+7LuMcqp4AFEAEq8wTOdq6daFaHLkpt/FZK6M4TlruhtpFOPkoNc3e45eM83OT -6mziKINJC1CQ6m65sQHpBtjxlKMRG8rL/D6wx9s5AoGBAMRlqNPMwglT3hvDmsAt -9Lf9pdmhERUlHhD8bj8mDaBj2Aqv7f6VRJaYZqP403pKKQexuqcn80mtjkSAPFkN -Cj7BVt/RXm5uoxDTnfi26RF9F6yNDEJ7UU9+peBr99aazF/fTgW/1GcMkQnum8uV -c257YgaWmjK9uB0Y2r2VxS0G ------END PRIVATE KEY----- diff --git a/vendor/github.com/valyala/fasthttp/fasthttputil/rsa.pem b/vendor/github.com/valyala/fasthttp/fasthttputil/rsa.pem deleted file mode 100644 index 93e77cd9..00000000 --- a/vendor/github.com/valyala/fasthttp/fasthttputil/rsa.pem +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICujCCAaKgAwIBAgIJAMbXnKZ/cikUMA0GCSqGSIb3DQEBCwUAMBUxEzARBgNV -BAMTCnVidW50dS5uYW4wHhcNMTUwMjA0MDgwMTM5WhcNMjUwMjAxMDgwMTM5WjAV -MRMwEQYDVQQDEwp1YnVudHUubmFuMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEA+CELrALPDyXZxt5lEbfwF7YAvnHqizmrSePSSRNVT05DAMvqBNX9V75D -K2LB6pg3+hllc4FV68i+FMKtv5yUpuenXYTeeZyPKEjd3bcsFAfP0oXpRDe955Te -+z3g/bZejZLD8Fmiq6satBZWm0T2UkAn5oGW4Q1fEmvJnwpBVNBtJYrepCxnHgij -L5lvvQc+3m7GJlXZlTMZnyCUrRQ+OJVhU3VHOuViEihHVthC3FHn29Mzi8PtDwm1 -xRiR+ceZLZLFvPgQZNh5IBnkES/6jwnHLYW0nDtFYDY98yd2WS9Dm0gwG7zQxvOY -6HjYwzauQ0/wQGdGzkmxBbIfn/QQMwIDAQABow0wCzAJBgNVHRMEAjAAMA0GCSqG -SIb3DQEBCwUAA4IBAQBQjKm/4KN/iTgXbLTL3i7zaxYXFLXsnT1tF+ay4VA8aj98 -L3JwRTciZ3A5iy/W4VSCt3eASwOaPWHKqDBB5RTtL73LoAqsWmO3APOGQAbixcQ2 -45GXi05OKeyiYRi1Nvq7Unv9jUkRDHUYVPZVSAjCpsXzPhFkmZoTRxmx5l0ZF7Li -K91lI5h+eFq0dwZwrmlPambyh1vQUi70VHv8DNToVU29kel7YLbxGbuqETfhrcy6 -X+Mha6RYITkAn5FqsZcKMsc9eYGEF4l3XV+oS7q6xfTxktYJMFTI18J0lQ2Lv/CI -whdMnYGntDQBE/iFCrJEGNsKGc38796GBOb5j+zd ------END CERTIFICATE----- diff --git a/vendor/github.com/valyala/fasthttp/fs.go b/vendor/github.com/valyala/fasthttp/fs.go deleted file mode 100644 index 7a73a795..00000000 --- a/vendor/github.com/valyala/fasthttp/fs.go +++ /dev/null @@ -1,1271 +0,0 @@ -package fasthttp - -import ( - "bytes" - "errors" - "fmt" - "html" - "io" - "io/ioutil" - "mime" - "net/http" - "os" - "path/filepath" - "sort" - "strings" - "sync" - "time" - - "github.com/klauspost/compress/gzip" - "github.com/valyala/bytebufferpool" -) - -// ServeFileBytesUncompressed returns HTTP response containing file contents -// from the given path. -// -// Directory contents is returned if path points to directory. -// -// ServeFileBytes may be used for saving network traffic when serving files -// with good compression ratio. -// -// See also RequestCtx.SendFileBytes. -func ServeFileBytesUncompressed(ctx *RequestCtx, path []byte) { - ServeFileUncompressed(ctx, b2s(path)) -} - -// ServeFileUncompressed returns HTTP response containing file contents -// from the given path. -// -// Directory contents is returned if path points to directory. -// -// ServeFile may be used for saving network traffic when serving files -// with good compression ratio. -// -// See also RequestCtx.SendFile. -func ServeFileUncompressed(ctx *RequestCtx, path string) { - ctx.Request.Header.DelBytes(strAcceptEncoding) - ServeFile(ctx, path) -} - -// ServeFileBytes returns HTTP response containing compressed file contents -// from the given path. -// -// HTTP response may contain uncompressed file contents in the following cases: -// -// * Missing 'Accept-Encoding: gzip' request header. -// * No write access to directory containing the file. -// -// Directory contents is returned if path points to directory. -// -// Use ServeFileBytesUncompressed is you don't need serving compressed -// file contents. -// -// See also RequestCtx.SendFileBytes. -func ServeFileBytes(ctx *RequestCtx, path []byte) { - ServeFile(ctx, b2s(path)) -} - -// ServeFile returns HTTP response containing compressed file contents -// from the given path. -// -// HTTP response may contain uncompressed file contents in the following cases: -// -// * Missing 'Accept-Encoding: gzip' request header. -// * No write access to directory containing the file. -// -// Directory contents is returned if path points to directory. -// -// Use ServeFileUncompressed is you don't need serving compressed file contents. -// -// See also RequestCtx.SendFile. -func ServeFile(ctx *RequestCtx, path string) { - rootFSOnce.Do(func() { - rootFSHandler = rootFS.NewRequestHandler() - }) - if len(path) == 0 || path[0] != '/' { - // extend relative path to absolute path - var err error - if path, err = filepath.Abs(path); err != nil { - ctx.Logger().Printf("cannot resolve path %q to absolute file path: %s", path, err) - ctx.Error("Internal Server Error", StatusInternalServerError) - return - } - } - ctx.Request.SetRequestURI(path) - rootFSHandler(ctx) -} - -var ( - rootFSOnce sync.Once - rootFS = &FS{ - Root: "/", - GenerateIndexPages: true, - Compress: true, - AcceptByteRange: true, - } - rootFSHandler RequestHandler -) - -// PathRewriteFunc must return new request path based on arbitrary ctx -// info such as ctx.Path(). -// -// Path rewriter is used in FS for translating the current request -// to the local filesystem path relative to FS.Root. -// -// The returned path must not contain '/../' substrings due to security reasons, -// since such paths may refer files outside FS.Root. -// -// The returned path may refer to ctx members. For example, ctx.Path(). -type PathRewriteFunc func(ctx *RequestCtx) []byte - -// NewVHostPathRewriter returns path rewriter, which strips slashesCount -// leading slashes from the path and prepends the path with request's host, -// thus simplifying virtual hosting for static files. -// -// Examples: -// -// * host=foobar.com, slashesCount=0, original path="/foo/bar". -// Resulting path: "/foobar.com/foo/bar" -// -// * host=img.aaa.com, slashesCount=1, original path="/images/123/456.jpg" -// Resulting path: "/img.aaa.com/123/456.jpg" -// -func NewVHostPathRewriter(slashesCount int) PathRewriteFunc { - return func(ctx *RequestCtx) []byte { - path := stripLeadingSlashes(ctx.Path(), slashesCount) - host := ctx.Host() - if n := bytes.IndexByte(host, '/'); n >= 0 { - host = nil - } - if len(host) == 0 { - host = strInvalidHost - } - b := bytebufferpool.Get() - b.B = append(b.B, '/') - b.B = append(b.B, host...) - b.B = append(b.B, path...) - ctx.URI().SetPathBytes(b.B) - bytebufferpool.Put(b) - - return ctx.Path() - } -} - -var strInvalidHost = []byte("invalid-host") - -// NewPathSlashesStripper returns path rewriter, which strips slashesCount -// leading slashes from the path. -// -// Examples: -// -// * slashesCount = 0, original path: "/foo/bar", result: "/foo/bar" -// * slashesCount = 1, original path: "/foo/bar", result: "/bar" -// * slashesCount = 2, original path: "/foo/bar", result: "" -// -// The returned path rewriter may be used as FS.PathRewrite . -func NewPathSlashesStripper(slashesCount int) PathRewriteFunc { - return func(ctx *RequestCtx) []byte { - return stripLeadingSlashes(ctx.Path(), slashesCount) - } -} - -// NewPathPrefixStripper returns path rewriter, which removes prefixSize bytes -// from the path prefix. -// -// Examples: -// -// * prefixSize = 0, original path: "/foo/bar", result: "/foo/bar" -// * prefixSize = 3, original path: "/foo/bar", result: "o/bar" -// * prefixSize = 7, original path: "/foo/bar", result: "r" -// -// The returned path rewriter may be used as FS.PathRewrite . -func NewPathPrefixStripper(prefixSize int) PathRewriteFunc { - return func(ctx *RequestCtx) []byte { - path := ctx.Path() - if len(path) >= prefixSize { - path = path[prefixSize:] - } - return path - } -} - -// FS represents settings for request handler serving static files -// from the local filesystem. -// -// It is prohibited copying FS values. Create new values instead. -type FS struct { - noCopy noCopy - - // Path to the root directory to serve files from. - Root string - - // List of index file names to try opening during directory access. - // - // For example: - // - // * index.html - // * index.htm - // * my-super-index.xml - // - // By default the list is empty. - IndexNames []string - - // Index pages for directories without files matching IndexNames - // are automatically generated if set. - // - // Directory index generation may be quite slow for directories - // with many files (more than 1K), so it is discouraged enabling - // index pages' generation for such directories. - // - // By default index pages aren't generated. - GenerateIndexPages bool - - // Transparently compresses responses if set to true. - // - // The server tries minimizing CPU usage by caching compressed files. - // It adds CompressedFileSuffix suffix to the original file name and - // tries saving the resulting compressed file under the new file name. - // So it is advisable to give the server write access to Root - // and to all inner folders in order to minimize CPU usage when serving - // compressed responses. - // - // Transparent compression is disabled by default. - Compress bool - - // Enables byte range requests if set to true. - // - // Byte range requests are disabled by default. - AcceptByteRange bool - - // Path rewriting function. - // - // By default request path is not modified. - PathRewrite PathRewriteFunc - - // PathNotFound fires when file is not found in filesystem - // this functions tries to replace "Cannot open requested path" - // server response giving to the programmer the control of server flow. - // - // By default PathNotFound returns - // "Cannot open requested path" - PathNotFound RequestHandler - - // Expiration duration for inactive file handlers. - // - // FSHandlerCacheDuration is used by default. - CacheDuration time.Duration - - // Suffix to add to the name of cached compressed file. - // - // This value has sense only if Compress is set. - // - // FSCompressedFileSuffix is used by default. - CompressedFileSuffix string - - once sync.Once - h RequestHandler -} - -// FSCompressedFileSuffix is the suffix FS adds to the original file names -// when trying to store compressed file under the new file name. -// See FS.Compress for details. -const FSCompressedFileSuffix = ".fasthttp.gz" - -// FSHandlerCacheDuration is the default expiration duration for inactive -// file handlers opened by FS. -const FSHandlerCacheDuration = 10 * time.Second - -// FSHandler returns request handler serving static files from -// the given root folder. -// -// stripSlashes indicates how many leading slashes must be stripped -// from requested path before searching requested file in the root folder. -// Examples: -// -// * stripSlashes = 0, original path: "/foo/bar", result: "/foo/bar" -// * stripSlashes = 1, original path: "/foo/bar", result: "/bar" -// * stripSlashes = 2, original path: "/foo/bar", result: "" -// -// The returned request handler automatically generates index pages -// for directories without index.html. -// -// The returned handler caches requested file handles -// for FSHandlerCacheDuration. -// Make sure your program has enough 'max open files' limit aka -// 'ulimit -n' if root folder contains many files. -// -// Do not create multiple request handler instances for the same -// (root, stripSlashes) arguments - just reuse a single instance. -// Otherwise goroutine leak will occur. -func FSHandler(root string, stripSlashes int) RequestHandler { - fs := &FS{ - Root: root, - IndexNames: []string{"index.html"}, - GenerateIndexPages: true, - AcceptByteRange: true, - } - if stripSlashes > 0 { - fs.PathRewrite = NewPathSlashesStripper(stripSlashes) - } - return fs.NewRequestHandler() -} - -// NewRequestHandler returns new request handler with the given FS settings. -// -// The returned handler caches requested file handles -// for FS.CacheDuration. -// Make sure your program has enough 'max open files' limit aka -// 'ulimit -n' if FS.Root folder contains many files. -// -// Do not create multiple request handlers from a single FS instance - -// just reuse a single request handler. -func (fs *FS) NewRequestHandler() RequestHandler { - fs.once.Do(fs.initRequestHandler) - return fs.h -} - -func (fs *FS) initRequestHandler() { - root := fs.Root - - // serve files from the current working directory if root is empty - if len(root) == 0 { - root = "." - } - - // strip trailing slashes from the root path - for len(root) > 0 && root[len(root)-1] == '/' { - root = root[:len(root)-1] - } - - cacheDuration := fs.CacheDuration - if cacheDuration <= 0 { - cacheDuration = FSHandlerCacheDuration - } - compressedFileSuffix := fs.CompressedFileSuffix - if len(compressedFileSuffix) == 0 { - compressedFileSuffix = FSCompressedFileSuffix - } - - h := &fsHandler{ - root: root, - indexNames: fs.IndexNames, - pathRewrite: fs.PathRewrite, - generateIndexPages: fs.GenerateIndexPages, - compress: fs.Compress, - pathNotFound: fs.PathNotFound, - acceptByteRange: fs.AcceptByteRange, - cacheDuration: cacheDuration, - compressedFileSuffix: compressedFileSuffix, - cache: make(map[string]*fsFile), - compressedCache: make(map[string]*fsFile), - } - - go func() { - var pendingFiles []*fsFile - for { - time.Sleep(cacheDuration / 2) - pendingFiles = h.cleanCache(pendingFiles) - } - }() - - fs.h = h.handleRequest -} - -type fsHandler struct { - root string - indexNames []string - pathRewrite PathRewriteFunc - pathNotFound RequestHandler - generateIndexPages bool - compress bool - acceptByteRange bool - cacheDuration time.Duration - compressedFileSuffix string - - cache map[string]*fsFile - compressedCache map[string]*fsFile - cacheLock sync.Mutex - - smallFileReaderPool sync.Pool -} - -type fsFile struct { - h *fsHandler - f *os.File - dirIndex []byte - contentType string - contentLength int - compressed bool - - lastModified time.Time - lastModifiedStr []byte - - t time.Time - readersCount int - - bigFiles []*bigFileReader - bigFilesLock sync.Mutex -} - -func (ff *fsFile) NewReader() (io.Reader, error) { - if ff.isBig() { - r, err := ff.bigFileReader() - if err != nil { - ff.decReadersCount() - } - return r, err - } - return ff.smallFileReader(), nil -} - -func (ff *fsFile) smallFileReader() io.Reader { - v := ff.h.smallFileReaderPool.Get() - if v == nil { - v = &fsSmallFileReader{} - } - r := v.(*fsSmallFileReader) - r.ff = ff - r.endPos = ff.contentLength - if r.startPos > 0 { - panic("BUG: fsSmallFileReader with non-nil startPos found in the pool") - } - return r -} - -// files bigger than this size are sent with sendfile -const maxSmallFileSize = 2 * 4096 - -func (ff *fsFile) isBig() bool { - return ff.contentLength > maxSmallFileSize && len(ff.dirIndex) == 0 -} - -func (ff *fsFile) bigFileReader() (io.Reader, error) { - if ff.f == nil { - panic("BUG: ff.f must be non-nil in bigFileReader") - } - - var r io.Reader - - ff.bigFilesLock.Lock() - n := len(ff.bigFiles) - if n > 0 { - r = ff.bigFiles[n-1] - ff.bigFiles = ff.bigFiles[:n-1] - } - ff.bigFilesLock.Unlock() - - if r != nil { - return r, nil - } - - f, err := os.Open(ff.f.Name()) - if err != nil { - return nil, fmt.Errorf("cannot open already opened file: %s", err) - } - return &bigFileReader{ - f: f, - ff: ff, - r: f, - }, nil -} - -func (ff *fsFile) Release() { - if ff.f != nil { - ff.f.Close() - - if ff.isBig() { - ff.bigFilesLock.Lock() - for _, r := range ff.bigFiles { - r.f.Close() - } - ff.bigFilesLock.Unlock() - } - } -} - -func (ff *fsFile) decReadersCount() { - ff.h.cacheLock.Lock() - ff.readersCount-- - if ff.readersCount < 0 { - panic("BUG: negative fsFile.readersCount!") - } - ff.h.cacheLock.Unlock() -} - -// bigFileReader attempts to trigger sendfile -// for sending big files over the wire. -type bigFileReader struct { - f *os.File - ff *fsFile - r io.Reader - lr io.LimitedReader -} - -func (r *bigFileReader) UpdateByteRange(startPos, endPos int) error { - if _, err := r.f.Seek(int64(startPos), 0); err != nil { - return err - } - r.r = &r.lr - r.lr.R = r.f - r.lr.N = int64(endPos - startPos + 1) - return nil -} - -func (r *bigFileReader) Read(p []byte) (int, error) { - return r.r.Read(p) -} - -func (r *bigFileReader) WriteTo(w io.Writer) (int64, error) { - if rf, ok := w.(io.ReaderFrom); ok { - // fast path. Senfile must be triggered - return rf.ReadFrom(r.r) - } - - // slow path - return copyZeroAlloc(w, r.r) -} - -func (r *bigFileReader) Close() error { - r.r = r.f - n, err := r.f.Seek(0, 0) - if err == nil { - if n != 0 { - panic("BUG: File.Seek(0,0) returned (non-zero, nil)") - } - - ff := r.ff - ff.bigFilesLock.Lock() - ff.bigFiles = append(ff.bigFiles, r) - ff.bigFilesLock.Unlock() - } else { - r.f.Close() - } - r.ff.decReadersCount() - return err -} - -type fsSmallFileReader struct { - ff *fsFile - startPos int - endPos int -} - -func (r *fsSmallFileReader) Close() error { - ff := r.ff - ff.decReadersCount() - r.ff = nil - r.startPos = 0 - r.endPos = 0 - ff.h.smallFileReaderPool.Put(r) - return nil -} - -func (r *fsSmallFileReader) UpdateByteRange(startPos, endPos int) error { - r.startPos = startPos - r.endPos = endPos + 1 - return nil -} - -func (r *fsSmallFileReader) Read(p []byte) (int, error) { - tailLen := r.endPos - r.startPos - if tailLen <= 0 { - return 0, io.EOF - } - if len(p) > tailLen { - p = p[:tailLen] - } - - ff := r.ff - if ff.f != nil { - n, err := ff.f.ReadAt(p, int64(r.startPos)) - r.startPos += n - return n, err - } - - n := copy(p, ff.dirIndex[r.startPos:]) - r.startPos += n - return n, nil -} - -func (r *fsSmallFileReader) WriteTo(w io.Writer) (int64, error) { - ff := r.ff - - var n int - var err error - if ff.f == nil { - n, err = w.Write(ff.dirIndex[r.startPos:r.endPos]) - return int64(n), err - } - - if rf, ok := w.(io.ReaderFrom); ok { - return rf.ReadFrom(r) - } - - curPos := r.startPos - bufv := copyBufPool.Get() - buf := bufv.([]byte) - for err == nil { - tailLen := r.endPos - curPos - if tailLen <= 0 { - break - } - if len(buf) > tailLen { - buf = buf[:tailLen] - } - n, err = ff.f.ReadAt(buf, int64(curPos)) - nw, errw := w.Write(buf[:n]) - curPos += nw - if errw == nil && nw != n { - panic("BUG: Write(p) returned (n, nil), where n != len(p)") - } - if err == nil { - err = errw - } - } - copyBufPool.Put(bufv) - - if err == io.EOF { - err = nil - } - return int64(curPos - r.startPos), err -} - -func (h *fsHandler) cleanCache(pendingFiles []*fsFile) []*fsFile { - var filesToRelease []*fsFile - - h.cacheLock.Lock() - - // Close files which couldn't be closed before due to non-zero - // readers count on the previous run. - var remainingFiles []*fsFile - for _, ff := range pendingFiles { - if ff.readersCount > 0 { - remainingFiles = append(remainingFiles, ff) - } else { - filesToRelease = append(filesToRelease, ff) - } - } - pendingFiles = remainingFiles - - pendingFiles, filesToRelease = cleanCacheNolock(h.cache, pendingFiles, filesToRelease, h.cacheDuration) - pendingFiles, filesToRelease = cleanCacheNolock(h.compressedCache, pendingFiles, filesToRelease, h.cacheDuration) - - h.cacheLock.Unlock() - - for _, ff := range filesToRelease { - ff.Release() - } - - return pendingFiles -} - -func cleanCacheNolock(cache map[string]*fsFile, pendingFiles, filesToRelease []*fsFile, cacheDuration time.Duration) ([]*fsFile, []*fsFile) { - t := time.Now() - for k, ff := range cache { - if t.Sub(ff.t) > cacheDuration { - if ff.readersCount > 0 { - // There are pending readers on stale file handle, - // so we cannot close it. Put it into pendingFiles - // so it will be closed later. - pendingFiles = append(pendingFiles, ff) - } else { - filesToRelease = append(filesToRelease, ff) - } - delete(cache, k) - } - } - return pendingFiles, filesToRelease -} - -func (h *fsHandler) handleRequest(ctx *RequestCtx) { - var path []byte - if h.pathRewrite != nil { - path = h.pathRewrite(ctx) - } else { - path = ctx.Path() - } - path = stripTrailingSlashes(path) - - if n := bytes.IndexByte(path, 0); n >= 0 { - ctx.Logger().Printf("cannot serve path with nil byte at position %d: %q", n, path) - ctx.Error("Are you a hacker?", StatusBadRequest) - return - } - if h.pathRewrite != nil { - // There is no need to check for '/../' if path = ctx.Path(), - // since ctx.Path must normalize and sanitize the path. - - if n := bytes.Index(path, strSlashDotDotSlash); n >= 0 { - ctx.Logger().Printf("cannot serve path with '/../' at position %d due to security reasons: %q", n, path) - ctx.Error("Internal Server Error", StatusInternalServerError) - return - } - } - - mustCompress := false - fileCache := h.cache - byteRange := ctx.Request.Header.peek(strRange) - if len(byteRange) == 0 && h.compress && ctx.Request.Header.HasAcceptEncodingBytes(strGzip) { - mustCompress = true - fileCache = h.compressedCache - } - - h.cacheLock.Lock() - ff, ok := fileCache[string(path)] - if ok { - ff.readersCount++ - } - h.cacheLock.Unlock() - - if !ok { - pathStr := string(path) - filePath := h.root + pathStr - var err error - ff, err = h.openFSFile(filePath, mustCompress) - if mustCompress && err == errNoCreatePermission { - ctx.Logger().Printf("insufficient permissions for saving compressed file for %q. Serving uncompressed file. "+ - "Allow write access to the directory with this file in order to improve fasthttp performance", filePath) - mustCompress = false - ff, err = h.openFSFile(filePath, mustCompress) - } - if err == errDirIndexRequired { - ff, err = h.openIndexFile(ctx, filePath, mustCompress) - if err != nil { - ctx.Logger().Printf("cannot open dir index %q: %s", filePath, err) - ctx.Error("Directory index is forbidden", StatusForbidden) - return - } - } else if err != nil { - ctx.Logger().Printf("cannot open file %q: %s", filePath, err) - if h.pathNotFound == nil { - ctx.Error("Cannot open requested path", StatusNotFound) - } else { - ctx.SetStatusCode(StatusNotFound) - h.pathNotFound(ctx) - } - return - } - - h.cacheLock.Lock() - ff1, ok := fileCache[pathStr] - if !ok { - fileCache[pathStr] = ff - ff.readersCount++ - } else { - ff1.readersCount++ - } - h.cacheLock.Unlock() - - if ok { - // The file has been already opened by another - // goroutine, so close the current file and use - // the file opened by another goroutine instead. - ff.Release() - ff = ff1 - } - } - - if !ctx.IfModifiedSince(ff.lastModified) { - ff.decReadersCount() - ctx.NotModified() - return - } - - r, err := ff.NewReader() - if err != nil { - ctx.Logger().Printf("cannot obtain file reader for path=%q: %s", path, err) - ctx.Error("Internal Server Error", StatusInternalServerError) - return - } - - hdr := &ctx.Response.Header - if ff.compressed { - hdr.SetCanonical(strContentEncoding, strGzip) - } - - statusCode := StatusOK - contentLength := ff.contentLength - if h.acceptByteRange { - hdr.SetCanonical(strAcceptRanges, strBytes) - if len(byteRange) > 0 { - startPos, endPos, err := ParseByteRange(byteRange, contentLength) - if err != nil { - r.(io.Closer).Close() - ctx.Logger().Printf("cannot parse byte range %q for path=%q: %s", byteRange, path, err) - ctx.Error("Range Not Satisfiable", StatusRequestedRangeNotSatisfiable) - return - } - - if err = r.(byteRangeUpdater).UpdateByteRange(startPos, endPos); err != nil { - r.(io.Closer).Close() - ctx.Logger().Printf("cannot seek byte range %q for path=%q: %s", byteRange, path, err) - ctx.Error("Internal Server Error", StatusInternalServerError) - return - } - - hdr.SetContentRange(startPos, endPos, contentLength) - contentLength = endPos - startPos + 1 - statusCode = StatusPartialContent - } - } - - hdr.SetCanonical(strLastModified, ff.lastModifiedStr) - if !ctx.IsHead() { - ctx.SetBodyStream(r, contentLength) - } else { - ctx.Response.ResetBody() - ctx.Response.SkipBody = true - ctx.Response.Header.SetContentLength(contentLength) - if rc, ok := r.(io.Closer); ok { - if err := rc.Close(); err != nil { - ctx.Logger().Printf("cannot close file reader: %s", err) - ctx.Error("Internal Server Error", StatusInternalServerError) - return - } - } - } - hdr.noDefaultContentType = true - if len(hdr.ContentType()) == 0 { - ctx.SetContentType(ff.contentType) - } - ctx.SetStatusCode(statusCode) -} - -type byteRangeUpdater interface { - UpdateByteRange(startPos, endPos int) error -} - -// ParseByteRange parses 'Range: bytes=...' header value. -// -// It follows https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35 . -func ParseByteRange(byteRange []byte, contentLength int) (startPos, endPos int, err error) { - b := byteRange - if !bytes.HasPrefix(b, strBytes) { - return 0, 0, fmt.Errorf("unsupported range units: %q. Expecting %q", byteRange, strBytes) - } - - b = b[len(strBytes):] - if len(b) == 0 || b[0] != '=' { - return 0, 0, fmt.Errorf("missing byte range in %q", byteRange) - } - b = b[1:] - - n := bytes.IndexByte(b, '-') - if n < 0 { - return 0, 0, fmt.Errorf("missing the end position of byte range in %q", byteRange) - } - - if n == 0 { - v, err := ParseUint(b[n+1:]) - if err != nil { - return 0, 0, err - } - startPos := contentLength - v - if startPos < 0 { - startPos = 0 - } - return startPos, contentLength - 1, nil - } - - if startPos, err = ParseUint(b[:n]); err != nil { - return 0, 0, err - } - if startPos >= contentLength { - return 0, 0, fmt.Errorf("the start position of byte range cannot exceed %d. byte range %q", contentLength-1, byteRange) - } - - b = b[n+1:] - if len(b) == 0 { - return startPos, contentLength - 1, nil - } - - if endPos, err = ParseUint(b); err != nil { - return 0, 0, err - } - if endPos >= contentLength { - endPos = contentLength - 1 - } - if endPos < startPos { - return 0, 0, fmt.Errorf("the start position of byte range cannot exceed the end position. byte range %q", byteRange) - } - return startPos, endPos, nil -} - -func (h *fsHandler) openIndexFile(ctx *RequestCtx, dirPath string, mustCompress bool) (*fsFile, error) { - for _, indexName := range h.indexNames { - indexFilePath := dirPath + "/" + indexName - ff, err := h.openFSFile(indexFilePath, mustCompress) - if err == nil { - return ff, nil - } - if !os.IsNotExist(err) { - return nil, fmt.Errorf("cannot open file %q: %s", indexFilePath, err) - } - } - - if !h.generateIndexPages { - return nil, fmt.Errorf("cannot access directory without index page. Directory %q", dirPath) - } - - return h.createDirIndex(ctx.URI(), dirPath, mustCompress) -} - -var ( - errDirIndexRequired = errors.New("directory index required") - errNoCreatePermission = errors.New("no 'create file' permissions") -) - -func (h *fsHandler) createDirIndex(base *URI, dirPath string, mustCompress bool) (*fsFile, error) { - w := &bytebufferpool.ByteBuffer{} - - basePathEscaped := html.EscapeString(string(base.Path())) - fmt.Fprintf(w, "%s", basePathEscaped) - fmt.Fprintf(w, "

%s

", basePathEscaped) - fmt.Fprintf(w, "
    ") - - if len(basePathEscaped) > 1 { - var parentURI URI - base.CopyTo(&parentURI) - parentURI.Update(string(base.Path()) + "/..") - parentPathEscaped := html.EscapeString(string(parentURI.Path())) - fmt.Fprintf(w, `
  • ..
    • `, parentPathEscaped) - } - - f, err := os.Open(dirPath) - if err != nil { - return nil, err - } - - fileinfos, err := f.Readdir(0) - f.Close() - if err != nil { - return nil, err - } - - fm := make(map[string]os.FileInfo, len(fileinfos)) - var filenames []string - for _, fi := range fileinfos { - name := fi.Name() - if strings.HasSuffix(name, h.compressedFileSuffix) { - // Do not show compressed files on index page. - continue - } - fm[name] = fi - filenames = append(filenames, name) - } - - var u URI - base.CopyTo(&u) - u.Update(string(u.Path()) + "/") - - sort.Sort(sort.StringSlice(filenames)) - for _, name := range filenames { - u.Update(name) - pathEscaped := html.EscapeString(string(u.Path())) - fi := fm[name] - auxStr := "dir" - className := "dir" - if !fi.IsDir() { - auxStr = fmt.Sprintf("file, %d bytes", fi.Size()) - className = "file" - } - fmt.Fprintf(w, `
  • %s, %s, last modified %s
    • `, - pathEscaped, className, html.EscapeString(name), auxStr, fsModTime(fi.ModTime())) - } - - fmt.Fprintf(w, "
") - - if mustCompress { - var zbuf bytebufferpool.ByteBuffer - zbuf.B = AppendGzipBytesLevel(zbuf.B, w.B, CompressDefaultCompression) - w = &zbuf - } - - dirIndex := w.B - lastModified := time.Now() - ff := &fsFile{ - h: h, - dirIndex: dirIndex, - contentType: "text/html; charset=utf-8", - contentLength: len(dirIndex), - compressed: mustCompress, - lastModified: lastModified, - lastModifiedStr: AppendHTTPDate(nil, lastModified), - - t: lastModified, - } - return ff, nil -} - -const ( - fsMinCompressRatio = 0.8 - fsMaxCompressibleFileSize = 8 * 1024 * 1024 -) - -func (h *fsHandler) compressAndOpenFSFile(filePath string) (*fsFile, error) { - f, err := os.Open(filePath) - if err != nil { - return nil, err - } - - fileInfo, err := f.Stat() - if err != nil { - f.Close() - return nil, fmt.Errorf("cannot obtain info for file %q: %s", filePath, err) - } - - if fileInfo.IsDir() { - f.Close() - return nil, errDirIndexRequired - } - - if strings.HasSuffix(filePath, h.compressedFileSuffix) || - fileInfo.Size() > fsMaxCompressibleFileSize || - !isFileCompressible(f, fsMinCompressRatio) { - return h.newFSFile(f, fileInfo, false) - } - - compressedFilePath := filePath + h.compressedFileSuffix - absPath, err := filepath.Abs(compressedFilePath) - if err != nil { - f.Close() - return nil, fmt.Errorf("cannot determine absolute path for %q: %s", compressedFilePath, err) - } - - flock := getFileLock(absPath) - flock.Lock() - ff, err := h.compressFileNolock(f, fileInfo, filePath, compressedFilePath) - flock.Unlock() - - return ff, err -} - -func (h *fsHandler) compressFileNolock(f *os.File, fileInfo os.FileInfo, filePath, compressedFilePath string) (*fsFile, error) { - // Attempt to open compressed file created by another concurrent - // goroutine. - // It is safe opening such a file, since the file creation - // is guarded by file mutex - see getFileLock call. - if _, err := os.Stat(compressedFilePath); err == nil { - f.Close() - return h.newCompressedFSFile(compressedFilePath) - } - - // Create temporary file, so concurrent goroutines don't use - // it until it is created. - tmpFilePath := compressedFilePath + ".tmp" - zf, err := os.Create(tmpFilePath) - if err != nil { - f.Close() - if !os.IsPermission(err) { - return nil, fmt.Errorf("cannot create temporary file %q: %s", tmpFilePath, err) - } - return nil, errNoCreatePermission - } - - zw := acquireStacklessGzipWriter(zf, CompressDefaultCompression) - _, err = copyZeroAlloc(zw, f) - if err1 := zw.Flush(); err == nil { - err = err1 - } - releaseStacklessGzipWriter(zw, CompressDefaultCompression) - zf.Close() - f.Close() - if err != nil { - return nil, fmt.Errorf("error when compressing file %q to %q: %s", filePath, tmpFilePath, err) - } - if err = os.Chtimes(tmpFilePath, time.Now(), fileInfo.ModTime()); err != nil { - return nil, fmt.Errorf("cannot change modification time to %s for tmp file %q: %s", - fileInfo.ModTime(), tmpFilePath, err) - } - if err = os.Rename(tmpFilePath, compressedFilePath); err != nil { - return nil, fmt.Errorf("cannot move compressed file from %q to %q: %s", tmpFilePath, compressedFilePath, err) - } - return h.newCompressedFSFile(compressedFilePath) -} - -func (h *fsHandler) newCompressedFSFile(filePath string) (*fsFile, error) { - f, err := os.Open(filePath) - if err != nil { - return nil, fmt.Errorf("cannot open compressed file %q: %s", filePath, err) - } - fileInfo, err := f.Stat() - if err != nil { - f.Close() - return nil, fmt.Errorf("cannot obtain info for compressed file %q: %s", filePath, err) - } - return h.newFSFile(f, fileInfo, true) -} - -func (h *fsHandler) openFSFile(filePath string, mustCompress bool) (*fsFile, error) { - filePathOriginal := filePath - if mustCompress { - filePath += h.compressedFileSuffix - } - - f, err := os.Open(filePath) - if err != nil { - if mustCompress && os.IsNotExist(err) { - return h.compressAndOpenFSFile(filePathOriginal) - } - return nil, err - } - - fileInfo, err := f.Stat() - if err != nil { - f.Close() - return nil, fmt.Errorf("cannot obtain info for file %q: %s", filePath, err) - } - - if fileInfo.IsDir() { - f.Close() - if mustCompress { - return nil, fmt.Errorf("directory with unexpected suffix found: %q. Suffix: %q", - filePath, h.compressedFileSuffix) - } - return nil, errDirIndexRequired - } - - if mustCompress { - fileInfoOriginal, err := os.Stat(filePathOriginal) - if err != nil { - f.Close() - return nil, fmt.Errorf("cannot obtain info for original file %q: %s", filePathOriginal, err) - } - - if fileInfoOriginal.ModTime() != fileInfo.ModTime() { - // The compressed file became stale. Re-create it. - f.Close() - os.Remove(filePath) - return h.compressAndOpenFSFile(filePathOriginal) - } - } - - return h.newFSFile(f, fileInfo, mustCompress) -} - -func (h *fsHandler) newFSFile(f *os.File, fileInfo os.FileInfo, compressed bool) (*fsFile, error) { - n := fileInfo.Size() - contentLength := int(n) - if n != int64(contentLength) { - f.Close() - return nil, fmt.Errorf("too big file: %d bytes", n) - } - - // detect content-type - ext := fileExtension(fileInfo.Name(), compressed, h.compressedFileSuffix) - contentType := mime.TypeByExtension(ext) - if len(contentType) == 0 { - data, err := readFileHeader(f, compressed) - if err != nil { - return nil, fmt.Errorf("cannot read header of the file %q: %s", f.Name(), err) - } - contentType = http.DetectContentType(data) - } - - lastModified := fileInfo.ModTime() - ff := &fsFile{ - h: h, - f: f, - contentType: contentType, - contentLength: contentLength, - compressed: compressed, - lastModified: lastModified, - lastModifiedStr: AppendHTTPDate(nil, lastModified), - - t: time.Now(), - } - return ff, nil -} - -func readFileHeader(f *os.File, compressed bool) ([]byte, error) { - r := io.Reader(f) - var zr *gzip.Reader - if compressed { - var err error - if zr, err = acquireGzipReader(f); err != nil { - return nil, err - } - r = zr - } - - lr := &io.LimitedReader{ - R: r, - N: 512, - } - data, err := ioutil.ReadAll(lr) - f.Seek(0, 0) - - if zr != nil { - releaseGzipReader(zr) - } - - return data, err -} - -func stripLeadingSlashes(path []byte, stripSlashes int) []byte { - for stripSlashes > 0 && len(path) > 0 { - if path[0] != '/' { - panic("BUG: path must start with slash") - } - n := bytes.IndexByte(path[1:], '/') - if n < 0 { - path = path[:0] - break - } - path = path[n+1:] - stripSlashes-- - } - return path -} - -func stripTrailingSlashes(path []byte) []byte { - for len(path) > 0 && path[len(path)-1] == '/' { - path = path[:len(path)-1] - } - return path -} - -func fileExtension(path string, compressed bool, compressedFileSuffix string) string { - if compressed && strings.HasSuffix(path, compressedFileSuffix) { - path = path[:len(path)-len(compressedFileSuffix)] - } - n := strings.LastIndexByte(path, '.') - if n < 0 { - return "" - } - return path[n:] -} - -// FileLastModified returns last modified time for the file. -func FileLastModified(path string) (time.Time, error) { - f, err := os.Open(path) - if err != nil { - return zeroTime, err - } - fileInfo, err := f.Stat() - f.Close() - if err != nil { - return zeroTime, err - } - return fsModTime(fileInfo.ModTime()), nil -} - -func fsModTime(t time.Time) time.Time { - return t.In(time.UTC).Truncate(time.Second) -} - -var ( - filesLockMap = make(map[string]*sync.Mutex) - filesLockMapLock sync.Mutex -) - -func getFileLock(absPath string) *sync.Mutex { - filesLockMapLock.Lock() - flock := filesLockMap[absPath] - if flock == nil { - flock = &sync.Mutex{} - filesLockMap[absPath] = flock - } - filesLockMapLock.Unlock() - return flock -} diff --git a/vendor/github.com/valyala/fasthttp/go.mod b/vendor/github.com/valyala/fasthttp/go.mod deleted file mode 100644 index 8434ca1e..00000000 --- a/vendor/github.com/valyala/fasthttp/go.mod +++ /dev/null @@ -1,9 +0,0 @@ -module github.com/valyala/fasthttp - -require ( - github.com/klauspost/compress v1.4.0 - github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e // indirect - github.com/valyala/bytebufferpool v1.0.0 - github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a - golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 -) diff --git a/vendor/github.com/valyala/fasthttp/go.sum b/vendor/github.com/valyala/fasthttp/go.sum deleted file mode 100644 index 93f38fcf..00000000 --- a/vendor/github.com/valyala/fasthttp/go.sum +++ /dev/null @@ -1,10 +0,0 @@ -github.com/klauspost/compress v1.4.0 h1:8nsMz3tWa9SWWPL60G1V6CUsf4lLjWLTNEtibhe8gh8= -github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e h1:+lIPJOWl+jSiJOc70QXJ07+2eg2Jy2EC7Mi11BWujeM= -github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 h1:czFLhve3vsQetD6JOJ8NZZvGQIXlnN3/yXxbT6/awxI= -golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/vendor/github.com/valyala/fasthttp/header.go b/vendor/github.com/valyala/fasthttp/header.go deleted file mode 100644 index 0371327c..00000000 --- a/vendor/github.com/valyala/fasthttp/header.go +++ /dev/null @@ -1,2147 +0,0 @@ -package fasthttp - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - "sync/atomic" - "time" -) - -// ResponseHeader represents HTTP response header. -// -// It is forbidden copying ResponseHeader instances. -// Create new instances instead and use CopyTo. -// -// ResponseHeader instance MUST NOT be used from concurrently running -// goroutines. -type ResponseHeader struct { - noCopy noCopy - - disableNormalizing bool - noHTTP11 bool - connectionClose bool - noDefaultContentType bool - - statusCode int - contentLength int - contentLengthBytes []byte - - contentType []byte - server []byte - - h []argsKV - bufKV argsKV - - cookies []argsKV -} - -// RequestHeader represents HTTP request header. -// -// It is forbidden copying RequestHeader instances. -// Create new instances instead and use CopyTo. -// -// RequestHeader instance MUST NOT be used from concurrently running -// goroutines. -type RequestHeader struct { - noCopy noCopy - - disableNormalizing bool - noHTTP11 bool - connectionClose bool - - // These two fields have been moved close to other bool fields - // for reducing RequestHeader object size. - cookiesCollected bool - rawHeadersParsed bool - - contentLength int - contentLengthBytes []byte - - method []byte - requestURI []byte - host []byte - contentType []byte - userAgent []byte - - h []argsKV - bufKV argsKV - - cookies []argsKV - - rawHeaders []byte -} - -// SetContentRange sets 'Content-Range: bytes startPos-endPos/contentLength' -// header. -func (h *ResponseHeader) SetContentRange(startPos, endPos, contentLength int) { - b := h.bufKV.value[:0] - b = append(b, strBytes...) - b = append(b, ' ') - b = AppendUint(b, startPos) - b = append(b, '-') - b = AppendUint(b, endPos) - b = append(b, '/') - b = AppendUint(b, contentLength) - h.bufKV.value = b - - h.SetCanonical(strContentRange, h.bufKV.value) -} - -// SetByteRange sets 'Range: bytes=startPos-endPos' header. -// -// * If startPos is negative, then 'bytes=-startPos' value is set. -// * If endPos is negative, then 'bytes=startPos-' value is set. -func (h *RequestHeader) SetByteRange(startPos, endPos int) { - h.parseRawHeaders() - - b := h.bufKV.value[:0] - b = append(b, strBytes...) - b = append(b, '=') - if startPos >= 0 { - b = AppendUint(b, startPos) - } else { - endPos = -startPos - } - b = append(b, '-') - if endPos >= 0 { - b = AppendUint(b, endPos) - } - h.bufKV.value = b - - h.SetCanonical(strRange, h.bufKV.value) -} - -// StatusCode returns response status code. -func (h *ResponseHeader) StatusCode() int { - if h.statusCode == 0 { - return StatusOK - } - return h.statusCode -} - -// SetStatusCode sets response status code. -func (h *ResponseHeader) SetStatusCode(statusCode int) { - h.statusCode = statusCode -} - -// SetLastModified sets 'Last-Modified' header to the given value. -func (h *ResponseHeader) SetLastModified(t time.Time) { - h.bufKV.value = AppendHTTPDate(h.bufKV.value[:0], t) - h.SetCanonical(strLastModified, h.bufKV.value) -} - -// ConnectionClose returns true if 'Connection: close' header is set. -func (h *ResponseHeader) ConnectionClose() bool { - return h.connectionClose -} - -// SetConnectionClose sets 'Connection: close' header. -func (h *ResponseHeader) SetConnectionClose() { - h.connectionClose = true -} - -// ResetConnectionClose clears 'Connection: close' header if it exists. -func (h *ResponseHeader) ResetConnectionClose() { - if h.connectionClose { - h.connectionClose = false - h.h = delAllArgsBytes(h.h, strConnection) - } -} - -// ConnectionClose returns true if 'Connection: close' header is set. -func (h *RequestHeader) ConnectionClose() bool { - h.parseRawHeaders() - return h.connectionClose -} - -// SetConnectionClose sets 'Connection: close' header. -func (h *RequestHeader) SetConnectionClose() { - // h.parseRawHeaders() isn't called for performance reasons. - h.connectionClose = true -} - -// ResetConnectionClose clears 'Connection: close' header if it exists. -func (h *RequestHeader) ResetConnectionClose() { - h.parseRawHeaders() - if h.connectionClose { - h.connectionClose = false - h.h = delAllArgsBytes(h.h, strConnection) - } -} - -// ConnectionUpgrade returns true if 'Connection: Upgrade' header is set. -func (h *ResponseHeader) ConnectionUpgrade() bool { - return hasHeaderValue(h.Peek("Connection"), strUpgrade) -} - -// ConnectionUpgrade returns true if 'Connection: Upgrade' header is set. -func (h *RequestHeader) ConnectionUpgrade() bool { - h.parseRawHeaders() - return hasHeaderValue(h.Peek("Connection"), strUpgrade) -} - -// PeekCookie is able to returns cookie by a given key from response. -func (h *ResponseHeader) PeekCookie(key string) []byte { - return peekArgStr(h.cookies, key) -} - -// ContentLength returns Content-Length header value. -// -// It may be negative: -// -1 means Transfer-Encoding: chunked. -// -2 means Transfer-Encoding: identity. -func (h *ResponseHeader) ContentLength() int { - return h.contentLength -} - -// SetContentLength sets Content-Length header value. -// -// Content-Length may be negative: -// -1 means Transfer-Encoding: chunked. -// -2 means Transfer-Encoding: identity. -func (h *ResponseHeader) SetContentLength(contentLength int) { - if h.mustSkipContentLength() { - return - } - h.contentLength = contentLength - if contentLength >= 0 { - h.contentLengthBytes = AppendUint(h.contentLengthBytes[:0], contentLength) - h.h = delAllArgsBytes(h.h, strTransferEncoding) - } else { - h.contentLengthBytes = h.contentLengthBytes[:0] - value := strChunked - if contentLength == -2 { - h.SetConnectionClose() - value = strIdentity - } - h.h = setArgBytes(h.h, strTransferEncoding, value) - } -} - -func (h *ResponseHeader) mustSkipContentLength() bool { - // From http/1.1 specs: - // All 1xx (informational), 204 (no content), and 304 (not modified) responses MUST NOT include a message-body - statusCode := h.StatusCode() - - // Fast path. - if statusCode < 100 || statusCode == StatusOK { - return false - } - - // Slow path. - return statusCode == StatusNotModified || statusCode == StatusNoContent || statusCode < 200 -} - -// ContentLength returns Content-Length header value. -// -// It may be negative: -// -1 means Transfer-Encoding: chunked. -func (h *RequestHeader) ContentLength() int { - if h.ignoreBody() { - return 0 - } - return h.realContentLength() -} - -// realContentLength returns the actual Content-Length set in the request, -// including positive lengths for GET/HEAD requests. -func (h *RequestHeader) realContentLength() int { - h.parseRawHeaders() - return h.contentLength -} - -// SetContentLength sets Content-Length header value. -// -// Negative content-length sets 'Transfer-Encoding: chunked' header. -func (h *RequestHeader) SetContentLength(contentLength int) { - h.parseRawHeaders() - h.contentLength = contentLength - if contentLength >= 0 { - h.contentLengthBytes = AppendUint(h.contentLengthBytes[:0], contentLength) - h.h = delAllArgsBytes(h.h, strTransferEncoding) - } else { - h.contentLengthBytes = h.contentLengthBytes[:0] - h.h = setArgBytes(h.h, strTransferEncoding, strChunked) - } -} - -func (h *ResponseHeader) isCompressibleContentType() bool { - contentType := h.ContentType() - return bytes.HasPrefix(contentType, strTextSlash) || - bytes.HasPrefix(contentType, strApplicationSlash) -} - -// ContentType returns Content-Type header value. -func (h *ResponseHeader) ContentType() []byte { - contentType := h.contentType - if !h.noDefaultContentType && len(h.contentType) == 0 { - contentType = defaultContentType - } - return contentType -} - -// SetContentType sets Content-Type header value. -func (h *ResponseHeader) SetContentType(contentType string) { - h.contentType = append(h.contentType[:0], contentType...) -} - -// SetContentTypeBytes sets Content-Type header value. -func (h *ResponseHeader) SetContentTypeBytes(contentType []byte) { - h.contentType = append(h.contentType[:0], contentType...) -} - -// Server returns Server header value. -func (h *ResponseHeader) Server() []byte { - return h.server -} - -// SetServer sets Server header value. -func (h *ResponseHeader) SetServer(server string) { - h.server = append(h.server[:0], server...) -} - -// SetServerBytes sets Server header value. -func (h *ResponseHeader) SetServerBytes(server []byte) { - h.server = append(h.server[:0], server...) -} - -// ContentType returns Content-Type header value. -func (h *RequestHeader) ContentType() []byte { - h.parseRawHeaders() - return h.contentType -} - -// SetContentType sets Content-Type header value. -func (h *RequestHeader) SetContentType(contentType string) { - h.parseRawHeaders() - h.contentType = append(h.contentType[:0], contentType...) -} - -// SetContentTypeBytes sets Content-Type header value. -func (h *RequestHeader) SetContentTypeBytes(contentType []byte) { - h.parseRawHeaders() - h.contentType = append(h.contentType[:0], contentType...) -} - -// SetMultipartFormBoundary sets the following Content-Type: -// 'multipart/form-data; boundary=...' -// where ... is substituted by the given boundary. -func (h *RequestHeader) SetMultipartFormBoundary(boundary string) { - h.parseRawHeaders() - - b := h.bufKV.value[:0] - b = append(b, strMultipartFormData...) - b = append(b, ';', ' ') - b = append(b, strBoundary...) - b = append(b, '=') - b = append(b, boundary...) - h.bufKV.value = b - - h.SetContentTypeBytes(h.bufKV.value) -} - -// SetMultipartFormBoundaryBytes sets the following Content-Type: -// 'multipart/form-data; boundary=...' -// where ... is substituted by the given boundary. -func (h *RequestHeader) SetMultipartFormBoundaryBytes(boundary []byte) { - h.parseRawHeaders() - - b := h.bufKV.value[:0] - b = append(b, strMultipartFormData...) - b = append(b, ';', ' ') - b = append(b, strBoundary...) - b = append(b, '=') - b = append(b, boundary...) - h.bufKV.value = b - - h.SetContentTypeBytes(h.bufKV.value) -} - -// MultipartFormBoundary returns boundary part -// from 'multipart/form-data; boundary=...' Content-Type. -func (h *RequestHeader) MultipartFormBoundary() []byte { - b := h.ContentType() - if !bytes.HasPrefix(b, strMultipartFormData) { - return nil - } - b = b[len(strMultipartFormData):] - if len(b) == 0 || b[0] != ';' { - return nil - } - - var n int - for len(b) > 0 { - n++ - for len(b) > n && b[n] == ' ' { - n++ - } - b = b[n:] - if !bytes.HasPrefix(b, strBoundary) { - if n = bytes.IndexByte(b, ';'); n < 0 { - return nil - } - continue - } - - b = b[len(strBoundary):] - if len(b) == 0 || b[0] != '=' { - return nil - } - b = b[1:] - if n = bytes.IndexByte(b, ';'); n >= 0 { - b = b[:n] - } - if len(b) > 1 && b[0] == '"' && b[len(b)-1] == '"' { - b = b[1 : len(b)-1] - } - return b - } - return nil -} - -// Host returns Host header value. -func (h *RequestHeader) Host() []byte { - if len(h.host) > 0 { - return h.host - } - if !h.rawHeadersParsed { - // fast path without employing full headers parsing. - host := peekRawHeader(h.rawHeaders, strHost) - if len(host) > 0 { - h.host = append(h.host[:0], host...) - return h.host - } - } - - // slow path. - h.parseRawHeaders() - return h.host -} - -// SetHost sets Host header value. -func (h *RequestHeader) SetHost(host string) { - h.parseRawHeaders() - h.host = append(h.host[:0], host...) -} - -// SetHostBytes sets Host header value. -func (h *RequestHeader) SetHostBytes(host []byte) { - h.parseRawHeaders() - h.host = append(h.host[:0], host...) -} - -// UserAgent returns User-Agent header value. -func (h *RequestHeader) UserAgent() []byte { - h.parseRawHeaders() - return h.userAgent -} - -// SetUserAgent sets User-Agent header value. -func (h *RequestHeader) SetUserAgent(userAgent string) { - h.parseRawHeaders() - h.userAgent = append(h.userAgent[:0], userAgent...) -} - -// SetUserAgentBytes sets User-Agent header value. -func (h *RequestHeader) SetUserAgentBytes(userAgent []byte) { - h.parseRawHeaders() - h.userAgent = append(h.userAgent[:0], userAgent...) -} - -// Referer returns Referer header value. -func (h *RequestHeader) Referer() []byte { - return h.PeekBytes(strReferer) -} - -// SetReferer sets Referer header value. -func (h *RequestHeader) SetReferer(referer string) { - h.SetBytesK(strReferer, referer) -} - -// SetRefererBytes sets Referer header value. -func (h *RequestHeader) SetRefererBytes(referer []byte) { - h.SetCanonical(strReferer, referer) -} - -// Method returns HTTP request method. -func (h *RequestHeader) Method() []byte { - if len(h.method) == 0 { - return strGet - } - return h.method -} - -// SetMethod sets HTTP request method. -func (h *RequestHeader) SetMethod(method string) { - h.method = append(h.method[:0], method...) -} - -// SetMethodBytes sets HTTP request method. -func (h *RequestHeader) SetMethodBytes(method []byte) { - h.method = append(h.method[:0], method...) -} - -// RequestURI returns RequestURI from the first HTTP request line. -func (h *RequestHeader) RequestURI() []byte { - requestURI := h.requestURI - if len(requestURI) == 0 { - requestURI = strSlash - } - return requestURI -} - -// SetRequestURI sets RequestURI for the first HTTP request line. -// RequestURI must be properly encoded. -// Use URI.RequestURI for constructing proper RequestURI if unsure. -func (h *RequestHeader) SetRequestURI(requestURI string) { - h.requestURI = append(h.requestURI[:0], requestURI...) -} - -// SetRequestURIBytes sets RequestURI for the first HTTP request line. -// RequestURI must be properly encoded. -// Use URI.RequestURI for constructing proper RequestURI if unsure. -func (h *RequestHeader) SetRequestURIBytes(requestURI []byte) { - h.requestURI = append(h.requestURI[:0], requestURI...) -} - -// IsGet returns true if request method is GET. -func (h *RequestHeader) IsGet() bool { - return bytes.Equal(h.Method(), strGet) -} - -// IsPost returns true if request method is POST. -func (h *RequestHeader) IsPost() bool { - return bytes.Equal(h.Method(), strPost) -} - -// IsPut returns true if request method is PUT. -func (h *RequestHeader) IsPut() bool { - return bytes.Equal(h.Method(), strPut) -} - -// IsHead returns true if request method is HEAD. -func (h *RequestHeader) IsHead() bool { - return bytes.Equal(h.Method(), strHead) -} - -// IsDelete returns true if request method is DELETE. -func (h *RequestHeader) IsDelete() bool { - return bytes.Equal(h.Method(), strDelete) -} - -// IsConnect returns true if request method is CONNECT. -func (h *RequestHeader) IsConnect() bool { - return bytes.Equal(h.Method(), strConnect) -} - -// IsOptions returns true if request method is OPTIONS. -func (h *RequestHeader) IsOptions() bool { - return bytes.Equal(h.Method(), strOptions) -} - -// IsTrace returns true if request method is TRACE. -func (h *RequestHeader) IsTrace() bool { - return bytes.Equal(h.Method(), strTrace) -} - -// IsPatch returns true if request method is PATCH. -func (h *RequestHeader) IsPatch() bool { - return bytes.Equal(h.Method(), strPatch) -} - -// IsHTTP11 returns true if the request is HTTP/1.1. -func (h *RequestHeader) IsHTTP11() bool { - return !h.noHTTP11 -} - -// IsHTTP11 returns true if the response is HTTP/1.1. -func (h *ResponseHeader) IsHTTP11() bool { - return !h.noHTTP11 -} - -// HasAcceptEncoding returns true if the header contains -// the given Accept-Encoding value. -func (h *RequestHeader) HasAcceptEncoding(acceptEncoding string) bool { - h.bufKV.value = append(h.bufKV.value[:0], acceptEncoding...) - return h.HasAcceptEncodingBytes(h.bufKV.value) -} - -// HasAcceptEncodingBytes returns true if the header contains -// the given Accept-Encoding value. -func (h *RequestHeader) HasAcceptEncodingBytes(acceptEncoding []byte) bool { - ae := h.peek(strAcceptEncoding) - n := bytes.Index(ae, acceptEncoding) - if n < 0 { - return false - } - b := ae[n+len(acceptEncoding):] - if len(b) > 0 && b[0] != ',' { - return false - } - if n == 0 { - return true - } - return ae[n-1] == ' ' -} - -// Len returns the number of headers set, -// i.e. the number of times f is called in VisitAll. -func (h *ResponseHeader) Len() int { - n := 0 - h.VisitAll(func(k, v []byte) { n++ }) - return n -} - -// Len returns the number of headers set, -// i.e. the number of times f is called in VisitAll. -func (h *RequestHeader) Len() int { - n := 0 - h.VisitAll(func(k, v []byte) { n++ }) - return n -} - -// DisableNormalizing disables header names' normalization. -// -// By default all the header names are normalized by uppercasing -// the first letter and all the first letters following dashes, -// while lowercasing all the other letters. -// Examples: -// -// * CONNECTION -> Connection -// * conteNT-tYPE -> Content-Type -// * foo-bar-baz -> Foo-Bar-Baz -// -// Disable header names' normalization only if know what are you doing. -func (h *RequestHeader) DisableNormalizing() { - h.disableNormalizing = true -} - -// DisableNormalizing disables header names' normalization. -// -// By default all the header names are normalized by uppercasing -// the first letter and all the first letters following dashes, -// while lowercasing all the other letters. -// Examples: -// -// * CONNECTION -> Connection -// * conteNT-tYPE -> Content-Type -// * foo-bar-baz -> Foo-Bar-Baz -// -// Disable header names' normalization only if know what are you doing. -func (h *ResponseHeader) DisableNormalizing() { - h.disableNormalizing = true -} - -// Reset clears response header. -func (h *ResponseHeader) Reset() { - h.disableNormalizing = false - h.noDefaultContentType = false - h.resetSkipNormalize() -} - -func (h *ResponseHeader) resetSkipNormalize() { - h.noHTTP11 = false - h.connectionClose = false - - h.statusCode = 0 - h.contentLength = 0 - h.contentLengthBytes = h.contentLengthBytes[:0] - - h.contentType = h.contentType[:0] - h.server = h.server[:0] - - h.h = h.h[:0] - h.cookies = h.cookies[:0] -} - -// Reset clears request header. -func (h *RequestHeader) Reset() { - h.disableNormalizing = false - h.resetSkipNormalize() -} - -func (h *RequestHeader) resetSkipNormalize() { - h.noHTTP11 = false - h.connectionClose = false - - h.contentLength = 0 - h.contentLengthBytes = h.contentLengthBytes[:0] - - h.method = h.method[:0] - h.requestURI = h.requestURI[:0] - h.host = h.host[:0] - h.contentType = h.contentType[:0] - h.userAgent = h.userAgent[:0] - - h.h = h.h[:0] - h.cookies = h.cookies[:0] - h.cookiesCollected = false - - h.rawHeaders = h.rawHeaders[:0] - h.rawHeadersParsed = false -} - -// CopyTo copies all the headers to dst. -func (h *ResponseHeader) CopyTo(dst *ResponseHeader) { - dst.Reset() - - dst.disableNormalizing = h.disableNormalizing - dst.noHTTP11 = h.noHTTP11 - dst.connectionClose = h.connectionClose - dst.noDefaultContentType = h.noDefaultContentType - - dst.statusCode = h.statusCode - dst.contentLength = h.contentLength - dst.contentLengthBytes = append(dst.contentLengthBytes[:0], h.contentLengthBytes...) - dst.contentType = append(dst.contentType[:0], h.contentType...) - dst.server = append(dst.server[:0], h.server...) - dst.h = copyArgs(dst.h, h.h) - dst.cookies = copyArgs(dst.cookies, h.cookies) -} - -// CopyTo copies all the headers to dst. -func (h *RequestHeader) CopyTo(dst *RequestHeader) { - dst.Reset() - - dst.disableNormalizing = h.disableNormalizing - dst.noHTTP11 = h.noHTTP11 - dst.connectionClose = h.connectionClose - - dst.contentLength = h.contentLength - dst.contentLengthBytes = append(dst.contentLengthBytes[:0], h.contentLengthBytes...) - dst.method = append(dst.method[:0], h.method...) - dst.requestURI = append(dst.requestURI[:0], h.requestURI...) - dst.host = append(dst.host[:0], h.host...) - dst.contentType = append(dst.contentType[:0], h.contentType...) - dst.userAgent = append(dst.userAgent[:0], h.userAgent...) - dst.h = copyArgs(dst.h, h.h) - dst.cookies = copyArgs(dst.cookies, h.cookies) - dst.cookiesCollected = h.cookiesCollected - dst.rawHeaders = append(dst.rawHeaders[:0], h.rawHeaders...) - dst.rawHeadersParsed = h.rawHeadersParsed -} - -// VisitAll calls f for each header. -// -// f must not retain references to key and/or value after returning. -// Copy key and/or value contents before returning if you need retaining them. -func (h *ResponseHeader) VisitAll(f func(key, value []byte)) { - if len(h.contentLengthBytes) > 0 { - f(strContentLength, h.contentLengthBytes) - } - contentType := h.ContentType() - if len(contentType) > 0 { - f(strContentType, contentType) - } - server := h.Server() - if len(server) > 0 { - f(strServer, server) - } - if len(h.cookies) > 0 { - visitArgs(h.cookies, func(k, v []byte) { - f(strSetCookie, v) - }) - } - visitArgs(h.h, f) - if h.ConnectionClose() { - f(strConnection, strClose) - } -} - -// VisitAllCookie calls f for each response cookie. -// -// Cookie name is passed in key and the whole Set-Cookie header value -// is passed in value on each f invocation. Value may be parsed -// with Cookie.ParseBytes(). -// -// f must not retain references to key and/or value after returning. -func (h *ResponseHeader) VisitAllCookie(f func(key, value []byte)) { - visitArgs(h.cookies, f) -} - -// VisitAllCookie calls f for each request cookie. -// -// f must not retain references to key and/or value after returning. -func (h *RequestHeader) VisitAllCookie(f func(key, value []byte)) { - h.parseRawHeaders() - h.collectCookies() - visitArgs(h.cookies, f) -} - -// VisitAll calls f for each header. -// -// f must not retain references to key and/or value after returning. -// Copy key and/or value contents before returning if you need retaining them. -func (h *RequestHeader) VisitAll(f func(key, value []byte)) { - h.parseRawHeaders() - host := h.Host() - if len(host) > 0 { - f(strHost, host) - } - if len(h.contentLengthBytes) > 0 { - f(strContentLength, h.contentLengthBytes) - } - contentType := h.ContentType() - if len(contentType) > 0 { - f(strContentType, contentType) - } - userAgent := h.UserAgent() - if len(userAgent) > 0 { - f(strUserAgent, userAgent) - } - - h.collectCookies() - if len(h.cookies) > 0 { - h.bufKV.value = appendRequestCookieBytes(h.bufKV.value[:0], h.cookies) - f(strCookie, h.bufKV.value) - } - visitArgs(h.h, f) - if h.ConnectionClose() { - f(strConnection, strClose) - } -} - -// Del deletes header with the given key. -func (h *ResponseHeader) Del(key string) { - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - h.del(k) -} - -// DelBytes deletes header with the given key. -func (h *ResponseHeader) DelBytes(key []byte) { - h.bufKV.key = append(h.bufKV.key[:0], key...) - normalizeHeaderKey(h.bufKV.key, h.disableNormalizing) - h.del(h.bufKV.key) -} - -func (h *ResponseHeader) del(key []byte) { - switch string(key) { - case "Content-Type": - h.contentType = h.contentType[:0] - case "Server": - h.server = h.server[:0] - case "Set-Cookie": - h.cookies = h.cookies[:0] - case "Content-Length": - h.contentLength = 0 - h.contentLengthBytes = h.contentLengthBytes[:0] - case "Connection": - h.connectionClose = false - } - h.h = delAllArgsBytes(h.h, key) -} - -// Del deletes header with the given key. -func (h *RequestHeader) Del(key string) { - h.parseRawHeaders() - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - h.del(k) -} - -// DelBytes deletes header with the given key. -func (h *RequestHeader) DelBytes(key []byte) { - h.parseRawHeaders() - h.bufKV.key = append(h.bufKV.key[:0], key...) - normalizeHeaderKey(h.bufKV.key, h.disableNormalizing) - h.del(h.bufKV.key) -} - -func (h *RequestHeader) del(key []byte) { - switch string(key) { - case "Host": - h.host = h.host[:0] - case "Content-Type": - h.contentType = h.contentType[:0] - case "User-Agent": - h.userAgent = h.userAgent[:0] - case "Cookie": - h.cookies = h.cookies[:0] - case "Content-Length": - h.contentLength = 0 - h.contentLengthBytes = h.contentLengthBytes[:0] - case "Connection": - h.connectionClose = false - } - h.h = delAllArgsBytes(h.h, key) -} - -// Add adds the given 'key: value' header. -// -// Multiple headers with the same key may be added with this function. -// Use Set for setting a single header for the given key. -func (h *ResponseHeader) Add(key, value string) { - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - h.h = appendArg(h.h, b2s(k), value) -} - -// AddBytesK adds the given 'key: value' header. -// -// Multiple headers with the same key may be added with this function. -// Use SetBytesK for setting a single header for the given key. -func (h *ResponseHeader) AddBytesK(key []byte, value string) { - h.Add(b2s(key), value) -} - -// AddBytesV adds the given 'key: value' header. -// -// Multiple headers with the same key may be added with this function. -// Use SetBytesV for setting a single header for the given key. -func (h *ResponseHeader) AddBytesV(key string, value []byte) { - h.Add(key, b2s(value)) -} - -// AddBytesKV adds the given 'key: value' header. -// -// Multiple headers with the same key may be added with this function. -// Use SetBytesKV for setting a single header for the given key. -func (h *ResponseHeader) AddBytesKV(key, value []byte) { - h.Add(b2s(key), b2s(value)) -} - -// Set sets the given 'key: value' header. -// -// Use Add for setting multiple header values under the same key. -func (h *ResponseHeader) Set(key, value string) { - initHeaderKV(&h.bufKV, key, value, h.disableNormalizing) - h.SetCanonical(h.bufKV.key, h.bufKV.value) -} - -// SetBytesK sets the given 'key: value' header. -// -// Use AddBytesK for setting multiple header values under the same key. -func (h *ResponseHeader) SetBytesK(key []byte, value string) { - h.bufKV.value = append(h.bufKV.value[:0], value...) - h.SetBytesKV(key, h.bufKV.value) -} - -// SetBytesV sets the given 'key: value' header. -// -// Use AddBytesV for setting multiple header values under the same key. -func (h *ResponseHeader) SetBytesV(key string, value []byte) { - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - h.SetCanonical(k, value) -} - -// SetBytesKV sets the given 'key: value' header. -// -// Use AddBytesKV for setting multiple header values under the same key. -func (h *ResponseHeader) SetBytesKV(key, value []byte) { - h.bufKV.key = append(h.bufKV.key[:0], key...) - normalizeHeaderKey(h.bufKV.key, h.disableNormalizing) - h.SetCanonical(h.bufKV.key, value) -} - -// SetCanonical sets the given 'key: value' header assuming that -// key is in canonical form. -func (h *ResponseHeader) SetCanonical(key, value []byte) { - switch string(key) { - case "Content-Type": - h.SetContentTypeBytes(value) - case "Server": - h.SetServerBytes(value) - case "Set-Cookie": - var kv *argsKV - h.cookies, kv = allocArg(h.cookies) - kv.key = getCookieKey(kv.key, value) - kv.value = append(kv.value[:0], value...) - case "Content-Length": - if contentLength, err := parseContentLength(value); err == nil { - h.contentLength = contentLength - h.contentLengthBytes = append(h.contentLengthBytes[:0], value...) - } - case "Connection": - if bytes.Equal(strClose, value) { - h.SetConnectionClose() - } else { - h.ResetConnectionClose() - h.h = setArgBytes(h.h, key, value) - } - case "Transfer-Encoding": - // Transfer-Encoding is managed automatically. - case "Date": - // Date is managed automatically. - default: - h.h = setArgBytes(h.h, key, value) - } -} - -// SetCookie sets the given response cookie. -// -// It is save re-using the cookie after the function returns. -func (h *ResponseHeader) SetCookie(cookie *Cookie) { - h.cookies = setArgBytes(h.cookies, cookie.Key(), cookie.Cookie()) -} - -// SetCookie sets 'key: value' cookies. -func (h *RequestHeader) SetCookie(key, value string) { - h.parseRawHeaders() - h.collectCookies() - h.cookies = setArg(h.cookies, key, value) -} - -// SetCookieBytesK sets 'key: value' cookies. -func (h *RequestHeader) SetCookieBytesK(key []byte, value string) { - h.SetCookie(b2s(key), value) -} - -// SetCookieBytesKV sets 'key: value' cookies. -func (h *RequestHeader) SetCookieBytesKV(key, value []byte) { - h.SetCookie(b2s(key), b2s(value)) -} - -// DelClientCookie instructs the client to remove the given cookie. -// -// Use DelCookie if you want just removing the cookie from response header. -func (h *ResponseHeader) DelClientCookie(key string) { - h.DelCookie(key) - - c := AcquireCookie() - c.SetKey(key) - c.SetExpire(CookieExpireDelete) - h.SetCookie(c) - ReleaseCookie(c) -} - -// DelClientCookieBytes instructs the client to remove the given cookie. -// -// Use DelCookieBytes if you want just removing the cookie from response header. -func (h *ResponseHeader) DelClientCookieBytes(key []byte) { - h.DelClientCookie(b2s(key)) -} - -// DelCookie removes cookie under the given key from response header. -// -// Note that DelCookie doesn't remove the cookie from the client. -// Use DelClientCookie instead. -func (h *ResponseHeader) DelCookie(key string) { - h.cookies = delAllArgs(h.cookies, key) -} - -// DelCookieBytes removes cookie under the given key from response header. -// -// Note that DelCookieBytes doesn't remove the cookie from the client. -// Use DelClientCookieBytes instead. -func (h *ResponseHeader) DelCookieBytes(key []byte) { - h.DelCookie(b2s(key)) -} - -// DelCookie removes cookie under the given key. -func (h *RequestHeader) DelCookie(key string) { - h.parseRawHeaders() - h.collectCookies() - h.cookies = delAllArgs(h.cookies, key) -} - -// DelCookieBytes removes cookie under the given key. -func (h *RequestHeader) DelCookieBytes(key []byte) { - h.DelCookie(b2s(key)) -} - -// DelAllCookies removes all the cookies from response headers. -func (h *ResponseHeader) DelAllCookies() { - h.cookies = h.cookies[:0] -} - -// DelAllCookies removes all the cookies from request headers. -func (h *RequestHeader) DelAllCookies() { - h.parseRawHeaders() - h.collectCookies() - h.cookies = h.cookies[:0] -} - -// Add adds the given 'key: value' header. -// -// Multiple headers with the same key may be added with this function. -// Use Set for setting a single header for the given key. -func (h *RequestHeader) Add(key, value string) { - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - h.h = appendArg(h.h, b2s(k), value) -} - -// AddBytesK adds the given 'key: value' header. -// -// Multiple headers with the same key may be added with this function. -// Use SetBytesK for setting a single header for the given key. -func (h *RequestHeader) AddBytesK(key []byte, value string) { - h.Add(b2s(key), value) -} - -// AddBytesV adds the given 'key: value' header. -// -// Multiple headers with the same key may be added with this function. -// Use SetBytesV for setting a single header for the given key. -func (h *RequestHeader) AddBytesV(key string, value []byte) { - h.Add(key, b2s(value)) -} - -// AddBytesKV adds the given 'key: value' header. -// -// Multiple headers with the same key may be added with this function. -// Use SetBytesKV for setting a single header for the given key. -func (h *RequestHeader) AddBytesKV(key, value []byte) { - h.Add(b2s(key), b2s(value)) -} - -// Set sets the given 'key: value' header. -// -// Use Add for setting multiple header values under the same key. -func (h *RequestHeader) Set(key, value string) { - initHeaderKV(&h.bufKV, key, value, h.disableNormalizing) - h.SetCanonical(h.bufKV.key, h.bufKV.value) -} - -// SetBytesK sets the given 'key: value' header. -// -// Use AddBytesK for setting multiple header values under the same key. -func (h *RequestHeader) SetBytesK(key []byte, value string) { - h.bufKV.value = append(h.bufKV.value[:0], value...) - h.SetBytesKV(key, h.bufKV.value) -} - -// SetBytesV sets the given 'key: value' header. -// -// Use AddBytesV for setting multiple header values under the same key. -func (h *RequestHeader) SetBytesV(key string, value []byte) { - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - h.SetCanonical(k, value) -} - -// SetBytesKV sets the given 'key: value' header. -// -// Use AddBytesKV for setting multiple header values under the same key. -func (h *RequestHeader) SetBytesKV(key, value []byte) { - h.bufKV.key = append(h.bufKV.key[:0], key...) - normalizeHeaderKey(h.bufKV.key, h.disableNormalizing) - h.SetCanonical(h.bufKV.key, value) -} - -// SetCanonical sets the given 'key: value' header assuming that -// key is in canonical form. -func (h *RequestHeader) SetCanonical(key, value []byte) { - h.parseRawHeaders() - switch string(key) { - case "Host": - h.SetHostBytes(value) - case "Content-Type": - h.SetContentTypeBytes(value) - case "User-Agent": - h.SetUserAgentBytes(value) - case "Cookie": - h.collectCookies() - h.cookies = parseRequestCookies(h.cookies, value) - case "Content-Length": - if contentLength, err := parseContentLength(value); err == nil { - h.contentLength = contentLength - h.contentLengthBytes = append(h.contentLengthBytes[:0], value...) - } - case "Connection": - if bytes.Equal(strClose, value) { - h.SetConnectionClose() - } else { - h.ResetConnectionClose() - h.h = setArgBytes(h.h, key, value) - } - case "Transfer-Encoding": - // Transfer-Encoding is managed automatically. - default: - h.h = setArgBytes(h.h, key, value) - } -} - -// Peek returns header value for the given key. -// -// Returned value is valid until the next call to ResponseHeader. -// Do not store references to returned value. Make copies instead. -func (h *ResponseHeader) Peek(key string) []byte { - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - return h.peek(k) -} - -// PeekBytes returns header value for the given key. -// -// Returned value is valid until the next call to ResponseHeader. -// Do not store references to returned value. Make copies instead. -func (h *ResponseHeader) PeekBytes(key []byte) []byte { - h.bufKV.key = append(h.bufKV.key[:0], key...) - normalizeHeaderKey(h.bufKV.key, h.disableNormalizing) - return h.peek(h.bufKV.key) -} - -// Peek returns header value for the given key. -// -// Returned value is valid until the next call to RequestHeader. -// Do not store references to returned value. Make copies instead. -func (h *RequestHeader) Peek(key string) []byte { - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - return h.peek(k) -} - -// PeekBytes returns header value for the given key. -// -// Returned value is valid until the next call to RequestHeader. -// Do not store references to returned value. Make copies instead. -func (h *RequestHeader) PeekBytes(key []byte) []byte { - h.bufKV.key = append(h.bufKV.key[:0], key...) - normalizeHeaderKey(h.bufKV.key, h.disableNormalizing) - return h.peek(h.bufKV.key) -} - -func (h *ResponseHeader) peek(key []byte) []byte { - switch string(key) { - case "Content-Type": - return h.ContentType() - case "Server": - return h.Server() - case "Connection": - if h.ConnectionClose() { - return strClose - } - return peekArgBytes(h.h, key) - case "Content-Length": - return h.contentLengthBytes - case "Set-Cookie": - return appendResponseCookieBytes(nil, h.cookies) - default: - return peekArgBytes(h.h, key) - } -} - -func (h *RequestHeader) peek(key []byte) []byte { - h.parseRawHeaders() - switch string(key) { - case "Host": - return h.Host() - case "Content-Type": - return h.ContentType() - case "User-Agent": - return h.UserAgent() - case "Connection": - if h.ConnectionClose() { - return strClose - } - return peekArgBytes(h.h, key) - case "Content-Length": - return h.contentLengthBytes - case "Cookie": - if h.cookiesCollected { - return appendRequestCookieBytes(nil, h.cookies) - } else { - return peekArgBytes(h.h, key) - } - default: - return peekArgBytes(h.h, key) - } -} - -// Cookie returns cookie for the given key. -func (h *RequestHeader) Cookie(key string) []byte { - h.parseRawHeaders() - h.collectCookies() - return peekArgStr(h.cookies, key) -} - -// CookieBytes returns cookie for the given key. -func (h *RequestHeader) CookieBytes(key []byte) []byte { - h.parseRawHeaders() - h.collectCookies() - return peekArgBytes(h.cookies, key) -} - -// Cookie fills cookie for the given cookie.Key. -// -// Returns false if cookie with the given cookie.Key is missing. -func (h *ResponseHeader) Cookie(cookie *Cookie) bool { - v := peekArgBytes(h.cookies, cookie.Key()) - if v == nil { - return false - } - cookie.ParseBytes(v) - return true -} - -// Read reads response header from r. -// -// io.EOF is returned if r is closed before reading the first header byte. -func (h *ResponseHeader) Read(r *bufio.Reader) error { - n := 1 - for { - err := h.tryRead(r, n) - if err == nil { - return nil - } - if err != errNeedMore { - h.resetSkipNormalize() - return err - } - n = r.Buffered() + 1 - } -} - -func (h *ResponseHeader) tryRead(r *bufio.Reader, n int) error { - h.resetSkipNormalize() - b, err := r.Peek(n) - if len(b) == 0 { - // treat all errors on the first byte read as EOF - if n == 1 || err == io.EOF { - return io.EOF - } - - // This is for go 1.6 bug. See https://github.com/golang/go/issues/14121 . - if err == bufio.ErrBufferFull { - return &ErrSmallBuffer{ - error: fmt.Errorf("error when reading response headers: %s", errSmallBuffer), - } - } - - return fmt.Errorf("error when reading response headers: %s", err) - } - b = mustPeekBuffered(r) - headersLen, errParse := h.parse(b) - if errParse != nil { - return headerError("response", err, errParse, b) - } - mustDiscard(r, headersLen) - return nil -} - -func headerError(typ string, err, errParse error, b []byte) error { - if errParse != errNeedMore { - return headerErrorMsg(typ, errParse, b) - } - if err == nil { - return errNeedMore - } - - // Buggy servers may leave trailing CRLFs after http body. - // Treat this case as EOF. - if isOnlyCRLF(b) { - return io.EOF - } - - if err != bufio.ErrBufferFull { - return headerErrorMsg(typ, err, b) - } - return &ErrSmallBuffer{ - error: headerErrorMsg(typ, errSmallBuffer, b), - } -} - -func headerErrorMsg(typ string, err error, b []byte) error { - return fmt.Errorf("error when reading %s headers: %s. Buffer size=%d, contents: %s", typ, err, len(b), bufferSnippet(b)) -} - -// Read reads request header from r. -// -// io.EOF is returned if r is closed before reading the first header byte. -func (h *RequestHeader) Read(r *bufio.Reader) error { - n := 1 - for { - err := h.tryRead(r, n) - if err == nil { - return nil - } - if err != errNeedMore { - h.resetSkipNormalize() - return err - } - n = r.Buffered() + 1 - } -} - -func (h *RequestHeader) tryRead(r *bufio.Reader, n int) error { - h.resetSkipNormalize() - b, err := r.Peek(n) - if len(b) == 0 { - if err == io.EOF { - return err - } - - if err == nil { - panic("bufio.Reader.Peek() returned nil, nil") - } - - // This is for go 1.6 bug. See https://github.com/golang/go/issues/14121 . - if err == bufio.ErrBufferFull { - return &ErrSmallBuffer{ - error: fmt.Errorf("error when reading request headers: %s", errSmallBuffer), - } - } - - return fmt.Errorf("error when reading request headers: %s", err) - } - b = mustPeekBuffered(r) - headersLen, errParse := h.parse(b) - if errParse != nil { - return headerError("request", err, errParse, b) - } - mustDiscard(r, headersLen) - return nil -} - -func bufferSnippet(b []byte) string { - n := len(b) - start := 200 - end := n - start - if start >= end { - start = n - end = n - } - bStart, bEnd := b[:start], b[end:] - if len(bEnd) == 0 { - return fmt.Sprintf("%q", b) - } - return fmt.Sprintf("%q...%q", bStart, bEnd) -} - -func isOnlyCRLF(b []byte) bool { - for _, ch := range b { - if ch != '\r' && ch != '\n' { - return false - } - } - return true -} - -func init() { - refreshServerDate() - go func() { - for { - time.Sleep(time.Second) - refreshServerDate() - } - }() -} - -var serverDate atomic.Value - -func refreshServerDate() { - b := AppendHTTPDate(nil, time.Now()) - serverDate.Store(b) -} - -// Write writes response header to w. -func (h *ResponseHeader) Write(w *bufio.Writer) error { - _, err := w.Write(h.Header()) - return err -} - -// WriteTo writes response header to w. -// -// WriteTo implements io.WriterTo interface. -func (h *ResponseHeader) WriteTo(w io.Writer) (int64, error) { - n, err := w.Write(h.Header()) - return int64(n), err -} - -// Header returns response header representation. -// -// The returned value is valid until the next call to ResponseHeader methods. -func (h *ResponseHeader) Header() []byte { - h.bufKV.value = h.AppendBytes(h.bufKV.value[:0]) - return h.bufKV.value -} - -// String returns response header representation. -func (h *ResponseHeader) String() string { - return string(h.Header()) -} - -// AppendBytes appends response header representation to dst and returns -// the extended dst. -func (h *ResponseHeader) AppendBytes(dst []byte) []byte { - statusCode := h.StatusCode() - if statusCode < 0 { - statusCode = StatusOK - } - dst = append(dst, statusLine(statusCode)...) - - server := h.Server() - if len(server) != 0 { - dst = appendHeaderLine(dst, strServer, server) - } - dst = appendHeaderLine(dst, strDate, serverDate.Load().([]byte)) - - // Append Content-Type only for non-zero responses - // or if it is explicitly set. - // See https://github.com/valyala/fasthttp/issues/28 . - if h.ContentLength() != 0 || len(h.contentType) > 0 { - dst = appendHeaderLine(dst, strContentType, h.ContentType()) - } - - if len(h.contentLengthBytes) > 0 { - dst = appendHeaderLine(dst, strContentLength, h.contentLengthBytes) - } - - for i, n := 0, len(h.h); i < n; i++ { - kv := &h.h[i] - if !bytes.Equal(kv.key, strDate) { - dst = appendHeaderLine(dst, kv.key, kv.value) - } - } - - n := len(h.cookies) - if n > 0 { - for i := 0; i < n; i++ { - kv := &h.cookies[i] - dst = appendHeaderLine(dst, strSetCookie, kv.value) - } - } - - if h.ConnectionClose() { - dst = appendHeaderLine(dst, strConnection, strClose) - } - - return append(dst, strCRLF...) -} - -// Write writes request header to w. -func (h *RequestHeader) Write(w *bufio.Writer) error { - _, err := w.Write(h.Header()) - return err -} - -// WriteTo writes request header to w. -// -// WriteTo implements io.WriterTo interface. -func (h *RequestHeader) WriteTo(w io.Writer) (int64, error) { - n, err := w.Write(h.Header()) - return int64(n), err -} - -// Header returns request header representation. -// -// The returned representation is valid until the next call to RequestHeader methods. -func (h *RequestHeader) Header() []byte { - h.bufKV.value = h.AppendBytes(h.bufKV.value[:0]) - return h.bufKV.value -} - -// String returns request header representation. -func (h *RequestHeader) String() string { - return string(h.Header()) -} - -// AppendBytes appends request header representation to dst and returns -// the extended dst. -func (h *RequestHeader) AppendBytes(dst []byte) []byte { - // there is no need in h.parseRawHeaders() here - raw headers are specially handled below. - dst = append(dst, h.Method()...) - dst = append(dst, ' ') - dst = append(dst, h.RequestURI()...) - dst = append(dst, ' ') - dst = append(dst, strHTTP11...) - dst = append(dst, strCRLF...) - - if !h.rawHeadersParsed && len(h.rawHeaders) > 0 { - return append(dst, h.rawHeaders...) - } - - userAgent := h.UserAgent() - if len(userAgent) == 0 { - userAgent = defaultUserAgent - } - dst = appendHeaderLine(dst, strUserAgent, userAgent) - - host := h.Host() - if len(host) > 0 { - dst = appendHeaderLine(dst, strHost, host) - } - - contentType := h.ContentType() - if !h.ignoreBody() { - if len(contentType) == 0 { - contentType = strPostArgsContentType - } - dst = appendHeaderLine(dst, strContentType, contentType) - - if len(h.contentLengthBytes) > 0 { - dst = appendHeaderLine(dst, strContentLength, h.contentLengthBytes) - } - } else if len(contentType) > 0 { - dst = appendHeaderLine(dst, strContentType, contentType) - } - - for i, n := 0, len(h.h); i < n; i++ { - kv := &h.h[i] - dst = appendHeaderLine(dst, kv.key, kv.value) - } - - // there is no need in h.collectCookies() here, since if cookies aren't collected yet, - // they all are located in h.h. - n := len(h.cookies) - if n > 0 { - dst = append(dst, strCookie...) - dst = append(dst, strColonSpace...) - dst = appendRequestCookieBytes(dst, h.cookies) - dst = append(dst, strCRLF...) - } - - if h.ConnectionClose() { - dst = appendHeaderLine(dst, strConnection, strClose) - } - - return append(dst, strCRLF...) -} - -func appendHeaderLine(dst, key, value []byte) []byte { - dst = append(dst, key...) - dst = append(dst, strColonSpace...) - dst = append(dst, value...) - return append(dst, strCRLF...) -} - -func (h *ResponseHeader) parse(buf []byte) (int, error) { - m, err := h.parseFirstLine(buf) - if err != nil { - return 0, err - } - n, err := h.parseHeaders(buf[m:]) - if err != nil { - return 0, err - } - return m + n, nil -} - -func (h *RequestHeader) ignoreBody() bool { - return h.IsGet() || h.IsHead() -} - -func (h *RequestHeader) parse(buf []byte) (int, error) { - m, err := h.parseFirstLine(buf) - if err != nil { - return 0, err - } - - var n int - if !h.ignoreBody() || h.noHTTP11 { - n, err = h.parseHeaders(buf[m:]) - if err != nil { - return 0, err - } - h.rawHeadersParsed = true - } else { - var rawHeaders []byte - rawHeaders, n, err = readRawHeaders(h.rawHeaders[:0], buf[m:]) - if err != nil { - return 0, err - } - h.rawHeaders = rawHeaders - } - return m + n, nil -} - -func (h *ResponseHeader) parseFirstLine(buf []byte) (int, error) { - bNext := buf - var b []byte - var err error - for len(b) == 0 { - if b, bNext, err = nextLine(bNext); err != nil { - return 0, err - } - } - - // parse protocol - n := bytes.IndexByte(b, ' ') - if n < 0 { - return 0, fmt.Errorf("cannot find whitespace in the first line of response %q", buf) - } - h.noHTTP11 = !bytes.Equal(b[:n], strHTTP11) - b = b[n+1:] - - // parse status code - h.statusCode, n, err = parseUintBuf(b) - if err != nil { - return 0, fmt.Errorf("cannot parse response status code: %s. Response %q", err, buf) - } - if len(b) > n && b[n] != ' ' { - return 0, fmt.Errorf("unexpected char at the end of status code. Response %q", buf) - } - - return len(buf) - len(bNext), nil -} - -func (h *RequestHeader) parseFirstLine(buf []byte) (int, error) { - bNext := buf - var b []byte - var err error - for len(b) == 0 { - if b, bNext, err = nextLine(bNext); err != nil { - return 0, err - } - } - - // parse method - n := bytes.IndexByte(b, ' ') - if n <= 0 { - return 0, fmt.Errorf("cannot find http request method in %q", buf) - } - h.method = append(h.method[:0], b[:n]...) - b = b[n+1:] - - // parse requestURI - n = bytes.LastIndexByte(b, ' ') - if n < 0 { - h.noHTTP11 = true - n = len(b) - } else if n == 0 { - return 0, fmt.Errorf("requestURI cannot be empty in %q", buf) - } else if !bytes.Equal(b[n+1:], strHTTP11) { - h.noHTTP11 = true - } - h.requestURI = append(h.requestURI[:0], b[:n]...) - - return len(buf) - len(bNext), nil -} - -func peekRawHeader(buf, key []byte) []byte { - n := bytes.Index(buf, key) - if n < 0 { - return nil - } - if n > 0 && buf[n-1] != '\n' { - return nil - } - n += len(key) - if n >= len(buf) { - return nil - } - if buf[n] != ':' { - return nil - } - n++ - if buf[n] != ' ' { - return nil - } - n++ - buf = buf[n:] - n = bytes.IndexByte(buf, '\n') - if n < 0 { - return nil - } - if n > 0 && buf[n-1] == '\r' { - n-- - } - return buf[:n] -} - -func readRawHeaders(dst, buf []byte) ([]byte, int, error) { - n := bytes.IndexByte(buf, '\n') - if n < 0 { - return nil, 0, errNeedMore - } - if (n == 1 && buf[0] == '\r') || n == 0 { - // empty headers - return dst, n + 1, nil - } - - n++ - b := buf - m := n - for { - b = b[m:] - m = bytes.IndexByte(b, '\n') - if m < 0 { - return nil, 0, errNeedMore - } - m++ - n += m - if (m == 2 && b[0] == '\r') || m == 1 { - dst = append(dst, buf[:n]...) - return dst, n, nil - } - } -} - -func (h *ResponseHeader) parseHeaders(buf []byte) (int, error) { - // 'identity' content-length by default - h.contentLength = -2 - - var s headerScanner - s.b = buf - s.disableNormalizing = h.disableNormalizing - var err error - var kv *argsKV - for s.next() { - if len(s.key) > 0 { - switch s.key[0] | 0x20 { - case 'c': - if caseInsensitiveCompare(s.key, strContentType) { - h.contentType = append(h.contentType[:0], s.value...) - continue - } - if caseInsensitiveCompare(s.key, strContentLength) { - if h.contentLength != -1 { - if h.contentLength, err = parseContentLength(s.value); err != nil { - h.contentLength = -2 - } else { - h.contentLengthBytes = append(h.contentLengthBytes[:0], s.value...) - } - } - continue - } - if caseInsensitiveCompare(s.key, strConnection) { - if bytes.Equal(s.value, strClose) { - h.connectionClose = true - } else { - h.connectionClose = false - h.h = appendArgBytes(h.h, s.key, s.value) - } - continue - } - case 's': - if caseInsensitiveCompare(s.key, strServer) { - h.server = append(h.server[:0], s.value...) - continue - } - if caseInsensitiveCompare(s.key, strSetCookie) { - h.cookies, kv = allocArg(h.cookies) - kv.key = getCookieKey(kv.key, s.value) - kv.value = append(kv.value[:0], s.value...) - continue - } - case 't': - if caseInsensitiveCompare(s.key, strTransferEncoding) { - if !bytes.Equal(s.value, strIdentity) { - h.contentLength = -1 - h.h = setArgBytes(h.h, strTransferEncoding, strChunked) - } - continue - } - } - h.h = appendArgBytes(h.h, s.key, s.value) - } - } - if s.err != nil { - h.connectionClose = true - return 0, s.err - } - - if h.contentLength < 0 { - h.contentLengthBytes = h.contentLengthBytes[:0] - } - if h.contentLength == -2 && !h.ConnectionUpgrade() && !h.mustSkipContentLength() { - h.h = setArgBytes(h.h, strTransferEncoding, strIdentity) - h.connectionClose = true - } - if h.noHTTP11 && !h.connectionClose { - // close connection for non-http/1.1 response unless 'Connection: keep-alive' is set. - v := peekArgBytes(h.h, strConnection) - h.connectionClose = !hasHeaderValue(v, strKeepAlive) && !hasHeaderValue(v, strKeepAliveCamelCase) - } - - return len(buf) - len(s.b), nil -} - -func (h *RequestHeader) parseHeaders(buf []byte) (int, error) { - h.contentLength = -2 - - var s headerScanner - s.b = buf - s.disableNormalizing = h.disableNormalizing - var err error - for s.next() { - if len(s.key) > 0 { - switch s.key[0] | 0x20 { - case 'h': - if caseInsensitiveCompare(s.key, strHost) { - h.host = append(h.host[:0], s.value...) - continue - } - case 'u': - if caseInsensitiveCompare(s.key, strUserAgent) { - h.userAgent = append(h.userAgent[:0], s.value...) - continue - } - case 'c': - if caseInsensitiveCompare(s.key, strContentType) { - h.contentType = append(h.contentType[:0], s.value...) - continue - } - if caseInsensitiveCompare(s.key, strContentLength) { - if h.contentLength != -1 { - if h.contentLength, err = parseContentLength(s.value); err != nil { - h.contentLength = -2 - } else { - h.contentLengthBytes = append(h.contentLengthBytes[:0], s.value...) - } - } - continue - } - if caseInsensitiveCompare(s.key, strConnection) { - if bytes.Equal(s.value, strClose) { - h.connectionClose = true - } else { - h.connectionClose = false - h.h = appendArgBytes(h.h, s.key, s.value) - } - continue - } - case 't': - if caseInsensitiveCompare(s.key, strTransferEncoding) { - if !bytes.Equal(s.value, strIdentity) { - h.contentLength = -1 - h.h = setArgBytes(h.h, strTransferEncoding, strChunked) - } - continue - } - } - } - h.h = appendArgBytes(h.h, s.key, s.value) - } - if s.err != nil { - h.connectionClose = true - return 0, s.err - } - - if h.contentLength < 0 { - h.contentLengthBytes = h.contentLengthBytes[:0] - } - if h.noHTTP11 && !h.connectionClose { - // close connection for non-http/1.1 request unless 'Connection: keep-alive' is set. - v := peekArgBytes(h.h, strConnection) - h.connectionClose = !hasHeaderValue(v, strKeepAlive) && !hasHeaderValue(v, strKeepAliveCamelCase) - } - - return len(buf) - len(s.b), nil -} - -func (h *RequestHeader) parseRawHeaders() { - if h.rawHeadersParsed { - return - } - h.rawHeadersParsed = true - if len(h.rawHeaders) == 0 { - return - } - h.parseHeaders(h.rawHeaders) -} - -func (h *RequestHeader) collectCookies() { - if h.cookiesCollected { - return - } - - for i, n := 0, len(h.h); i < n; i++ { - kv := &h.h[i] - if bytes.Equal(kv.key, strCookie) { - h.cookies = parseRequestCookies(h.cookies, kv.value) - tmp := *kv - copy(h.h[i:], h.h[i+1:]) - n-- - i-- - h.h[n] = tmp - h.h = h.h[:n] - } - } - h.cookiesCollected = true -} - -func parseContentLength(b []byte) (int, error) { - v, n, err := parseUintBuf(b) - if err != nil { - return -1, err - } - if n != len(b) { - return -1, fmt.Errorf("non-numeric chars at the end of Content-Length") - } - return v, nil -} - -type headerScanner struct { - b []byte - key []byte - value []byte - err error - - disableNormalizing bool -} - -func (s *headerScanner) next() bool { - bLen := len(s.b) - if bLen >= 2 && s.b[0] == '\r' && s.b[1] == '\n' { - s.b = s.b[2:] - return false - } - if bLen >= 1 && s.b[0] == '\n' { - s.b = s.b[1:] - return false - } - n := bytes.IndexByte(s.b, ':') - if n < 0 { - s.err = errNeedMore - return false - } - s.key = s.b[:n] - normalizeHeaderKey(s.key, s.disableNormalizing) - n++ - for len(s.b) > n && s.b[n] == ' ' { - n++ - } - s.b = s.b[n:] - n = bytes.IndexByte(s.b, '\n') - if n < 0 { - s.err = errNeedMore - return false - } - s.value = s.b[:n] - s.b = s.b[n+1:] - - if n > 0 && s.value[n-1] == '\r' { - n-- - } - for n > 0 && s.value[n-1] == ' ' { - n-- - } - s.value = s.value[:n] - return true -} - -type headerValueScanner struct { - b []byte - value []byte -} - -func (s *headerValueScanner) next() bool { - b := s.b - if len(b) == 0 { - return false - } - n := bytes.IndexByte(b, ',') - if n < 0 { - s.value = stripSpace(b) - s.b = b[len(b):] - return true - } - s.value = stripSpace(b[:n]) - s.b = b[n+1:] - return true -} - -func stripSpace(b []byte) []byte { - for len(b) > 0 && b[0] == ' ' { - b = b[1:] - } - for len(b) > 0 && b[len(b)-1] == ' ' { - b = b[:len(b)-1] - } - return b -} - -func hasHeaderValue(s, value []byte) bool { - var vs headerValueScanner - vs.b = s - for vs.next() { - if bytes.Equal(vs.value, value) { - return true - } - } - return false -} - -func nextLine(b []byte) ([]byte, []byte, error) { - nNext := bytes.IndexByte(b, '\n') - if nNext < 0 { - return nil, nil, errNeedMore - } - n := nNext - if n > 0 && b[n-1] == '\r' { - n-- - } - return b[:n], b[nNext+1:], nil -} - -func initHeaderKV(kv *argsKV, key, value string, disableNormalizing bool) { - kv.key = getHeaderKeyBytes(kv, key, disableNormalizing) - kv.value = append(kv.value[:0], value...) -} - -func getHeaderKeyBytes(kv *argsKV, key string, disableNormalizing bool) []byte { - kv.key = append(kv.key[:0], key...) - normalizeHeaderKey(kv.key, disableNormalizing) - return kv.key -} - -func normalizeHeaderKey(b []byte, disableNormalizing bool) { - if disableNormalizing { - return - } - - n := len(b) - if n == 0 { - return - } - - b[0] = toUpperTable[b[0]] - for i := 1; i < n; i++ { - p := &b[i] - if *p == '-' { - i++ - if i < n { - b[i] = toUpperTable[b[i]] - } - continue - } - *p = toLowerTable[*p] - } -} - -// AppendNormalizedHeaderKey appends normalized header key (name) to dst -// and returns the resulting dst. -// -// Normalized header key starts with uppercase letter. The first letters -// after dashes are also uppercased. All the other letters are lowercased. -// Examples: -// -// * coNTENT-TYPe -> Content-Type -// * HOST -> Host -// * foo-bar-baz -> Foo-Bar-Baz -func AppendNormalizedHeaderKey(dst []byte, key string) []byte { - dst = append(dst, key...) - normalizeHeaderKey(dst[len(dst)-len(key):], false) - return dst -} - -// AppendNormalizedHeaderKeyBytes appends normalized header key (name) to dst -// and returns the resulting dst. -// -// Normalized header key starts with uppercase letter. The first letters -// after dashes are also uppercased. All the other letters are lowercased. -// Examples: -// -// * coNTENT-TYPe -> Content-Type -// * HOST -> Host -// * foo-bar-baz -> Foo-Bar-Baz -func AppendNormalizedHeaderKeyBytes(dst, key []byte) []byte { - return AppendNormalizedHeaderKey(dst, b2s(key)) -} - -var ( - errNeedMore = errors.New("need more data: cannot find trailing lf") - errSmallBuffer = errors.New("small read buffer. Increase ReadBufferSize") -) - -// ErrSmallBuffer is returned when the provided buffer size is too small -// for reading request and/or response headers. -// -// ReadBufferSize value from Server or clients should reduce the number -// of such errors. -type ErrSmallBuffer struct { - error -} - -func mustPeekBuffered(r *bufio.Reader) []byte { - buf, err := r.Peek(r.Buffered()) - if len(buf) == 0 || err != nil { - panic(fmt.Sprintf("bufio.Reader.Peek() returned unexpected data (%q, %v)", buf, err)) - } - return buf -} - -func mustDiscard(r *bufio.Reader, n int) { - if _, err := r.Discard(n); err != nil { - panic(fmt.Sprintf("bufio.Reader.Discard(%d) failed: %s", n, err)) - } -} diff --git a/vendor/github.com/valyala/fasthttp/http.go b/vendor/github.com/valyala/fasthttp/http.go deleted file mode 100644 index f8c38fc5..00000000 --- a/vendor/github.com/valyala/fasthttp/http.go +++ /dev/null @@ -1,1720 +0,0 @@ -package fasthttp - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - "mime/multipart" - "os" - "sync" - - "github.com/valyala/bytebufferpool" -) - -// Request represents HTTP request. -// -// It is forbidden copying Request instances. Create new instances -// and use CopyTo instead. -// -// Request instance MUST NOT be used from concurrently running goroutines. -type Request struct { - noCopy noCopy - - // Request header - // - // Copying Header by value is forbidden. Use pointer to Header instead. - Header RequestHeader - - uri URI - postArgs Args - - bodyStream io.Reader - w requestBodyWriter - body *bytebufferpool.ByteBuffer - - multipartForm *multipart.Form - multipartFormBoundary string - - // Group bool members in order to reduce Request object size. - parsedURI bool - parsedPostArgs bool - - keepBodyBuffer bool - - isTLS bool -} - -// Response represents HTTP response. -// -// It is forbidden copying Response instances. Create new instances -// and use CopyTo instead. -// -// Response instance MUST NOT be used from concurrently running goroutines. -type Response struct { - noCopy noCopy - - // Response header - // - // Copying Header by value is forbidden. Use pointer to Header instead. - Header ResponseHeader - - bodyStream io.Reader - w responseBodyWriter - body *bytebufferpool.ByteBuffer - - // Response.Read() skips reading body if set to true. - // Use it for reading HEAD responses. - // - // Response.Write() skips writing body if set to true. - // Use it for writing HEAD responses. - SkipBody bool - - keepBodyBuffer bool -} - -// SetHost sets host for the request. -func (req *Request) SetHost(host string) { - req.URI().SetHost(host) -} - -// SetHostBytes sets host for the request. -func (req *Request) SetHostBytes(host []byte) { - req.URI().SetHostBytes(host) -} - -// Host returns the host for the given request. -func (req *Request) Host() []byte { - return req.URI().Host() -} - -// SetRequestURI sets RequestURI. -func (req *Request) SetRequestURI(requestURI string) { - req.Header.SetRequestURI(requestURI) - req.parsedURI = false -} - -// SetRequestURIBytes sets RequestURI. -func (req *Request) SetRequestURIBytes(requestURI []byte) { - req.Header.SetRequestURIBytes(requestURI) - req.parsedURI = false -} - -// RequestURI returns request's URI. -func (req *Request) RequestURI() []byte { - if req.parsedURI { - requestURI := req.uri.RequestURI() - req.SetRequestURIBytes(requestURI) - } - return req.Header.RequestURI() -} - -// StatusCode returns response status code. -func (resp *Response) StatusCode() int { - return resp.Header.StatusCode() -} - -// SetStatusCode sets response status code. -func (resp *Response) SetStatusCode(statusCode int) { - resp.Header.SetStatusCode(statusCode) -} - -// ConnectionClose returns true if 'Connection: close' header is set. -func (resp *Response) ConnectionClose() bool { - return resp.Header.ConnectionClose() -} - -// SetConnectionClose sets 'Connection: close' header. -func (resp *Response) SetConnectionClose() { - resp.Header.SetConnectionClose() -} - -// ConnectionClose returns true if 'Connection: close' header is set. -func (req *Request) ConnectionClose() bool { - return req.Header.ConnectionClose() -} - -// SetConnectionClose sets 'Connection: close' header. -func (req *Request) SetConnectionClose() { - req.Header.SetConnectionClose() -} - -// SendFile registers file on the given path to be used as response body -// when Write is called. -// -// Note that SendFile doesn't set Content-Type, so set it yourself -// with Header.SetContentType. -func (resp *Response) SendFile(path string) error { - f, err := os.Open(path) - if err != nil { - return err - } - fileInfo, err := f.Stat() - if err != nil { - f.Close() - return err - } - size64 := fileInfo.Size() - size := int(size64) - if int64(size) != size64 { - size = -1 - } - - resp.Header.SetLastModified(fileInfo.ModTime()) - resp.SetBodyStream(f, size) - return nil -} - -// SetBodyStream sets request body stream and, optionally body size. -// -// If bodySize is >= 0, then the bodyStream must provide exactly bodySize bytes -// before returning io.EOF. -// -// If bodySize < 0, then bodyStream is read until io.EOF. -// -// bodyStream.Close() is called after finishing reading all body data -// if it implements io.Closer. -// -// Note that GET and HEAD requests cannot have body. -// -// See also SetBodyStreamWriter. -func (req *Request) SetBodyStream(bodyStream io.Reader, bodySize int) { - req.ResetBody() - req.bodyStream = bodyStream - req.Header.SetContentLength(bodySize) -} - -// SetBodyStream sets response body stream and, optionally body size. -// -// If bodySize is >= 0, then the bodyStream must provide exactly bodySize bytes -// before returning io.EOF. -// -// If bodySize < 0, then bodyStream is read until io.EOF. -// -// bodyStream.Close() is called after finishing reading all body data -// if it implements io.Closer. -// -// See also SetBodyStreamWriter. -func (resp *Response) SetBodyStream(bodyStream io.Reader, bodySize int) { - resp.ResetBody() - resp.bodyStream = bodyStream - resp.Header.SetContentLength(bodySize) -} - -// IsBodyStream returns true if body is set via SetBodyStream* -func (req *Request) IsBodyStream() bool { - return req.bodyStream != nil -} - -// IsBodyStream returns true if body is set via SetBodyStream* -func (resp *Response) IsBodyStream() bool { - return resp.bodyStream != nil -} - -// SetBodyStreamWriter registers the given sw for populating request body. -// -// This function may be used in the following cases: -// -// * if request body is too big (more than 10MB). -// * if request body is streamed from slow external sources. -// * if request body must be streamed to the server in chunks -// (aka `http client push` or `chunked transfer-encoding`). -// -// Note that GET and HEAD requests cannot have body. -// -/// See also SetBodyStream. -func (req *Request) SetBodyStreamWriter(sw StreamWriter) { - sr := NewStreamReader(sw) - req.SetBodyStream(sr, -1) -} - -// SetBodyStreamWriter registers the given sw for populating response body. -// -// This function may be used in the following cases: -// -// * if response body is too big (more than 10MB). -// * if response body is streamed from slow external sources. -// * if response body must be streamed to the client in chunks -// (aka `http server push` or `chunked transfer-encoding`). -// -// See also SetBodyStream. -func (resp *Response) SetBodyStreamWriter(sw StreamWriter) { - sr := NewStreamReader(sw) - resp.SetBodyStream(sr, -1) -} - -// BodyWriter returns writer for populating response body. -// -// If used inside RequestHandler, the returned writer must not be used -// after returning from RequestHandler. Use RequestCtx.Write -// or SetBodyStreamWriter in this case. -func (resp *Response) BodyWriter() io.Writer { - resp.w.r = resp - return &resp.w -} - -// BodyWriter returns writer for populating request body. -func (req *Request) BodyWriter() io.Writer { - req.w.r = req - return &req.w -} - -type responseBodyWriter struct { - r *Response -} - -func (w *responseBodyWriter) Write(p []byte) (int, error) { - w.r.AppendBody(p) - return len(p), nil -} - -type requestBodyWriter struct { - r *Request -} - -func (w *requestBodyWriter) Write(p []byte) (int, error) { - w.r.AppendBody(p) - return len(p), nil -} - -// Body returns response body. -// -// The returned body is valid until the response modification. -func (resp *Response) Body() []byte { - if resp.bodyStream != nil { - bodyBuf := resp.bodyBuffer() - bodyBuf.Reset() - _, err := copyZeroAlloc(bodyBuf, resp.bodyStream) - resp.closeBodyStream() - if err != nil { - bodyBuf.SetString(err.Error()) - } - } - return resp.bodyBytes() -} - -func (resp *Response) bodyBytes() []byte { - if resp.body == nil { - return nil - } - return resp.body.B -} - -func (req *Request) bodyBytes() []byte { - if req.body == nil { - return nil - } - return req.body.B -} - -func (resp *Response) bodyBuffer() *bytebufferpool.ByteBuffer { - if resp.body == nil { - resp.body = responseBodyPool.Get() - } - return resp.body -} - -func (req *Request) bodyBuffer() *bytebufferpool.ByteBuffer { - if req.body == nil { - req.body = requestBodyPool.Get() - } - return req.body -} - -var ( - responseBodyPool bytebufferpool.Pool - requestBodyPool bytebufferpool.Pool -) - -// BodyGunzip returns un-gzipped body data. -// -// This method may be used if the request header contains -// 'Content-Encoding: gzip' for reading un-gzipped body. -// Use Body for reading gzipped request body. -func (req *Request) BodyGunzip() ([]byte, error) { - return gunzipData(req.Body()) -} - -// BodyGunzip returns un-gzipped body data. -// -// This method may be used if the response header contains -// 'Content-Encoding: gzip' for reading un-gzipped body. -// Use Body for reading gzipped response body. -func (resp *Response) BodyGunzip() ([]byte, error) { - return gunzipData(resp.Body()) -} - -func gunzipData(p []byte) ([]byte, error) { - var bb bytebufferpool.ByteBuffer - _, err := WriteGunzip(&bb, p) - if err != nil { - return nil, err - } - return bb.B, nil -} - -// BodyInflate returns inflated body data. -// -// This method may be used if the response header contains -// 'Content-Encoding: deflate' for reading inflated request body. -// Use Body for reading deflated request body. -func (req *Request) BodyInflate() ([]byte, error) { - return inflateData(req.Body()) -} - -// BodyInflate returns inflated body data. -// -// This method may be used if the response header contains -// 'Content-Encoding: deflate' for reading inflated response body. -// Use Body for reading deflated response body. -func (resp *Response) BodyInflate() ([]byte, error) { - return inflateData(resp.Body()) -} - -func inflateData(p []byte) ([]byte, error) { - var bb bytebufferpool.ByteBuffer - _, err := WriteInflate(&bb, p) - if err != nil { - return nil, err - } - return bb.B, nil -} - -// BodyWriteTo writes request body to w. -func (req *Request) BodyWriteTo(w io.Writer) error { - if req.bodyStream != nil { - _, err := copyZeroAlloc(w, req.bodyStream) - req.closeBodyStream() - return err - } - if req.onlyMultipartForm() { - return WriteMultipartForm(w, req.multipartForm, req.multipartFormBoundary) - } - _, err := w.Write(req.bodyBytes()) - return err -} - -// BodyWriteTo writes response body to w. -func (resp *Response) BodyWriteTo(w io.Writer) error { - if resp.bodyStream != nil { - _, err := copyZeroAlloc(w, resp.bodyStream) - resp.closeBodyStream() - return err - } - _, err := w.Write(resp.bodyBytes()) - return err -} - -// AppendBody appends p to response body. -// -// It is safe re-using p after the function returns. -func (resp *Response) AppendBody(p []byte) { - resp.AppendBodyString(b2s(p)) -} - -// AppendBodyString appends s to response body. -func (resp *Response) AppendBodyString(s string) { - resp.closeBodyStream() - resp.bodyBuffer().WriteString(s) -} - -// SetBody sets response body. -// -// It is safe re-using body argument after the function returns. -func (resp *Response) SetBody(body []byte) { - resp.SetBodyString(b2s(body)) -} - -// SetBodyString sets response body. -func (resp *Response) SetBodyString(body string) { - resp.closeBodyStream() - bodyBuf := resp.bodyBuffer() - bodyBuf.Reset() - bodyBuf.WriteString(body) -} - -// ResetBody resets response body. -func (resp *Response) ResetBody() { - resp.closeBodyStream() - if resp.body != nil { - if resp.keepBodyBuffer { - resp.body.Reset() - } else { - responseBodyPool.Put(resp.body) - resp.body = nil - } - } -} - -// ReleaseBody retires the response body if it is greater than "size" bytes. -// -// This permits GC to reclaim the large buffer. If used, must be before -// ReleaseResponse. -// -// Use this method only if you really understand how it works. -// The majority of workloads don't need this method. -func (resp *Response) ReleaseBody(size int) { - if cap(resp.body.B) > size { - resp.closeBodyStream() - resp.body = nil - } -} - -// ReleaseBody retires the request body if it is greater than "size" bytes. -// -// This permits GC to reclaim the large buffer. If used, must be before -// ReleaseRequest. -// -// Use this method only if you really understand how it works. -// The majority of workloads don't need this method. -func (req *Request) ReleaseBody(size int) { - if cap(req.body.B) > size { - req.closeBodyStream() - req.body = nil - } -} - -// SwapBody swaps response body with the given body and returns -// the previous response body. -// -// It is forbidden to use the body passed to SwapBody after -// the function returns. -func (resp *Response) SwapBody(body []byte) []byte { - bb := resp.bodyBuffer() - - if resp.bodyStream != nil { - bb.Reset() - _, err := copyZeroAlloc(bb, resp.bodyStream) - resp.closeBodyStream() - if err != nil { - bb.Reset() - bb.SetString(err.Error()) - } - } - - oldBody := bb.B - bb.B = body - return oldBody -} - -// SwapBody swaps request body with the given body and returns -// the previous request body. -// -// It is forbidden to use the body passed to SwapBody after -// the function returns. -func (req *Request) SwapBody(body []byte) []byte { - bb := req.bodyBuffer() - - if req.bodyStream != nil { - bb.Reset() - _, err := copyZeroAlloc(bb, req.bodyStream) - req.closeBodyStream() - if err != nil { - bb.Reset() - bb.SetString(err.Error()) - } - } - - oldBody := bb.B - bb.B = body - return oldBody -} - -// Body returns request body. -// -// The returned body is valid until the request modification. -func (req *Request) Body() []byte { - if req.bodyStream != nil { - bodyBuf := req.bodyBuffer() - bodyBuf.Reset() - _, err := copyZeroAlloc(bodyBuf, req.bodyStream) - req.closeBodyStream() - if err != nil { - bodyBuf.SetString(err.Error()) - } - } else if req.onlyMultipartForm() { - body, err := marshalMultipartForm(req.multipartForm, req.multipartFormBoundary) - if err != nil { - return []byte(err.Error()) - } - return body - } - return req.bodyBytes() -} - -// AppendBody appends p to request body. -// -// It is safe re-using p after the function returns. -func (req *Request) AppendBody(p []byte) { - req.AppendBodyString(b2s(p)) -} - -// AppendBodyString appends s to request body. -func (req *Request) AppendBodyString(s string) { - req.RemoveMultipartFormFiles() - req.closeBodyStream() - req.bodyBuffer().WriteString(s) -} - -// SetBody sets request body. -// -// It is safe re-using body argument after the function returns. -func (req *Request) SetBody(body []byte) { - req.SetBodyString(b2s(body)) -} - -// SetBodyString sets request body. -func (req *Request) SetBodyString(body string) { - req.RemoveMultipartFormFiles() - req.closeBodyStream() - req.bodyBuffer().SetString(body) -} - -// ResetBody resets request body. -func (req *Request) ResetBody() { - req.RemoveMultipartFormFiles() - req.closeBodyStream() - if req.body != nil { - if req.keepBodyBuffer { - req.body.Reset() - } else { - requestBodyPool.Put(req.body) - req.body = nil - } - } -} - -// CopyTo copies req contents to dst except of body stream. -func (req *Request) CopyTo(dst *Request) { - req.copyToSkipBody(dst) - if req.body != nil { - dst.bodyBuffer().Set(req.body.B) - } else if dst.body != nil { - dst.body.Reset() - } -} - -func (req *Request) copyToSkipBody(dst *Request) { - dst.Reset() - req.Header.CopyTo(&dst.Header) - - req.uri.CopyTo(&dst.uri) - dst.parsedURI = req.parsedURI - - req.postArgs.CopyTo(&dst.postArgs) - dst.parsedPostArgs = req.parsedPostArgs - dst.isTLS = req.isTLS - - // do not copy multipartForm - it will be automatically - // re-created on the first call to MultipartForm. -} - -// CopyTo copies resp contents to dst except of body stream. -func (resp *Response) CopyTo(dst *Response) { - resp.copyToSkipBody(dst) - if resp.body != nil { - dst.bodyBuffer().Set(resp.body.B) - } else if dst.body != nil { - dst.body.Reset() - } -} - -func (resp *Response) copyToSkipBody(dst *Response) { - dst.Reset() - resp.Header.CopyTo(&dst.Header) - dst.SkipBody = resp.SkipBody -} - -func swapRequestBody(a, b *Request) { - a.body, b.body = b.body, a.body - a.bodyStream, b.bodyStream = b.bodyStream, a.bodyStream -} - -func swapResponseBody(a, b *Response) { - a.body, b.body = b.body, a.body - a.bodyStream, b.bodyStream = b.bodyStream, a.bodyStream -} - -// URI returns request URI -func (req *Request) URI() *URI { - req.parseURI() - return &req.uri -} - -func (req *Request) parseURI() { - if req.parsedURI { - return - } - req.parsedURI = true - - req.uri.parseQuick(req.Header.RequestURI(), &req.Header, req.isTLS) -} - -// PostArgs returns POST arguments. -func (req *Request) PostArgs() *Args { - req.parsePostArgs() - return &req.postArgs -} - -func (req *Request) parsePostArgs() { - if req.parsedPostArgs { - return - } - req.parsedPostArgs = true - - if !bytes.HasPrefix(req.Header.ContentType(), strPostArgsContentType) { - return - } - req.postArgs.ParseBytes(req.bodyBytes()) -} - -// ErrNoMultipartForm means that the request's Content-Type -// isn't 'multipart/form-data'. -var ErrNoMultipartForm = errors.New("request has no multipart/form-data Content-Type") - -// MultipartForm returns requests's multipart form. -// -// Returns ErrNoMultipartForm if request's Content-Type -// isn't 'multipart/form-data'. -// -// RemoveMultipartFormFiles must be called after returned multipart form -// is processed. -func (req *Request) MultipartForm() (*multipart.Form, error) { - if req.multipartForm != nil { - return req.multipartForm, nil - } - - req.multipartFormBoundary = string(req.Header.MultipartFormBoundary()) - if len(req.multipartFormBoundary) == 0 { - return nil, ErrNoMultipartForm - } - - ce := req.Header.peek(strContentEncoding) - body := req.bodyBytes() - if bytes.Equal(ce, strGzip) { - // Do not care about memory usage here. - var err error - if body, err = AppendGunzipBytes(nil, body); err != nil { - return nil, fmt.Errorf("cannot gunzip request body: %s", err) - } - } else if len(ce) > 0 { - return nil, fmt.Errorf("unsupported Content-Encoding: %q", ce) - } - - f, err := readMultipartForm(bytes.NewReader(body), req.multipartFormBoundary, len(body), len(body)) - if err != nil { - return nil, err - } - req.multipartForm = f - return f, nil -} - -func marshalMultipartForm(f *multipart.Form, boundary string) ([]byte, error) { - var buf bytebufferpool.ByteBuffer - if err := WriteMultipartForm(&buf, f, boundary); err != nil { - return nil, err - } - return buf.B, nil -} - -// WriteMultipartForm writes the given multipart form f with the given -// boundary to w. -func WriteMultipartForm(w io.Writer, f *multipart.Form, boundary string) error { - // Do not care about memory allocations here, since multipart - // form processing is slow. - if len(boundary) == 0 { - panic("BUG: form boundary cannot be empty") - } - - mw := multipart.NewWriter(w) - if err := mw.SetBoundary(boundary); err != nil { - return fmt.Errorf("cannot use form boundary %q: %s", boundary, err) - } - - // marshal values - for k, vv := range f.Value { - for _, v := range vv { - if err := mw.WriteField(k, v); err != nil { - return fmt.Errorf("cannot write form field %q value %q: %s", k, v, err) - } - } - } - - // marshal files - for k, fvv := range f.File { - for _, fv := range fvv { - vw, err := mw.CreateFormFile(k, fv.Filename) - if err != nil { - return fmt.Errorf("cannot create form file %q (%q): %s", k, fv.Filename, err) - } - fh, err := fv.Open() - if err != nil { - return fmt.Errorf("cannot open form file %q (%q): %s", k, fv.Filename, err) - } - if _, err = copyZeroAlloc(vw, fh); err != nil { - return fmt.Errorf("error when copying form file %q (%q): %s", k, fv.Filename, err) - } - if err = fh.Close(); err != nil { - return fmt.Errorf("cannot close form file %q (%q): %s", k, fv.Filename, err) - } - } - } - - if err := mw.Close(); err != nil { - return fmt.Errorf("error when closing multipart form writer: %s", err) - } - - return nil -} - -func readMultipartForm(r io.Reader, boundary string, size, maxInMemoryFileSize int) (*multipart.Form, error) { - // Do not care about memory allocations here, since they are tiny - // compared to multipart data (aka multi-MB files) usually sent - // in multipart/form-data requests. - - if size <= 0 { - panic(fmt.Sprintf("BUG: form size must be greater than 0. Given %d", size)) - } - lr := io.LimitReader(r, int64(size)) - mr := multipart.NewReader(lr, boundary) - f, err := mr.ReadForm(int64(maxInMemoryFileSize)) - if err != nil { - return nil, fmt.Errorf("cannot read multipart/form-data body: %s", err) - } - return f, nil -} - -// Reset clears request contents. -func (req *Request) Reset() { - req.Header.Reset() - req.resetSkipHeader() -} - -func (req *Request) resetSkipHeader() { - req.ResetBody() - req.uri.Reset() - req.parsedURI = false - req.postArgs.Reset() - req.parsedPostArgs = false - req.isTLS = false -} - -// RemoveMultipartFormFiles removes multipart/form-data temporary files -// associated with the request. -func (req *Request) RemoveMultipartFormFiles() { - if req.multipartForm != nil { - // Do not check for error, since these files may be deleted or moved - // to new places by user code. - req.multipartForm.RemoveAll() - req.multipartForm = nil - } - req.multipartFormBoundary = "" -} - -// Reset clears response contents. -func (resp *Response) Reset() { - resp.Header.Reset() - resp.resetSkipHeader() - resp.SkipBody = false -} - -func (resp *Response) resetSkipHeader() { - resp.ResetBody() -} - -// Read reads request (including body) from the given r. -// -// RemoveMultipartFormFiles or Reset must be called after -// reading multipart/form-data request in order to delete temporarily -// uploaded files. -// -// If MayContinue returns true, the caller must: -// -// - Either send StatusExpectationFailed response if request headers don't -// satisfy the caller. -// - Or send StatusContinue response before reading request body -// with ContinueReadBody. -// - Or close the connection. -// -// io.EOF is returned if r is closed before reading the first header byte. -func (req *Request) Read(r *bufio.Reader) error { - return req.ReadLimitBody(r, 0) -} - -const defaultMaxInMemoryFileSize = 16 * 1024 * 1024 - -var errGetOnly = errors.New("non-GET request received") - -// ReadLimitBody reads request from the given r, limiting the body size. -// -// If maxBodySize > 0 and the body size exceeds maxBodySize, -// then ErrBodyTooLarge is returned. -// -// RemoveMultipartFormFiles or Reset must be called after -// reading multipart/form-data request in order to delete temporarily -// uploaded files. -// -// If MayContinue returns true, the caller must: -// -// - Either send StatusExpectationFailed response if request headers don't -// satisfy the caller. -// - Or send StatusContinue response before reading request body -// with ContinueReadBody. -// - Or close the connection. -// -// io.EOF is returned if r is closed before reading the first header byte. -func (req *Request) ReadLimitBody(r *bufio.Reader, maxBodySize int) error { - req.resetSkipHeader() - return req.readLimitBody(r, maxBodySize, false) -} - -func (req *Request) readLimitBody(r *bufio.Reader, maxBodySize int, getOnly bool) error { - // Do not reset the request here - the caller must reset it before - // calling this method. - - err := req.Header.Read(r) - if err != nil { - return err - } - if getOnly && !req.Header.IsGet() { - return errGetOnly - } - - if req.MayContinue() { - // 'Expect: 100-continue' header found. Let the caller deciding - // whether to read request body or - // to return StatusExpectationFailed. - return nil - } - - return req.ContinueReadBody(r, maxBodySize) -} - -// MayContinue returns true if the request contains -// 'Expect: 100-continue' header. -// -// The caller must do one of the following actions if MayContinue returns true: -// -// - Either send StatusExpectationFailed response if request headers don't -// satisfy the caller. -// - Or send StatusContinue response before reading request body -// with ContinueReadBody. -// - Or close the connection. -func (req *Request) MayContinue() bool { - return bytes.Equal(req.Header.peek(strExpect), str100Continue) -} - -// ContinueReadBody reads request body if request header contains -// 'Expect: 100-continue'. -// -// The caller must send StatusContinue response before calling this method. -// -// If maxBodySize > 0 and the body size exceeds maxBodySize, -// then ErrBodyTooLarge is returned. -func (req *Request) ContinueReadBody(r *bufio.Reader, maxBodySize int) error { - var err error - contentLength := req.Header.realContentLength() - if contentLength > 0 { - if maxBodySize > 0 && contentLength > maxBodySize { - return ErrBodyTooLarge - } - - // Pre-read multipart form data of known length. - // This way we limit memory usage for large file uploads, since their contents - // is streamed into temporary files if file size exceeds defaultMaxInMemoryFileSize. - req.multipartFormBoundary = string(req.Header.MultipartFormBoundary()) - if len(req.multipartFormBoundary) > 0 && len(req.Header.peek(strContentEncoding)) == 0 { - req.multipartForm, err = readMultipartForm(r, req.multipartFormBoundary, contentLength, defaultMaxInMemoryFileSize) - if err != nil { - req.Reset() - } - return err - } - } - - if contentLength == -2 { - // identity body has no sense for http requests, since - // the end of body is determined by connection close. - // So just ignore request body for requests without - // 'Content-Length' and 'Transfer-Encoding' headers. - req.Header.SetContentLength(0) - return nil - } - - bodyBuf := req.bodyBuffer() - bodyBuf.Reset() - bodyBuf.B, err = readBody(r, contentLength, maxBodySize, bodyBuf.B) - if err != nil { - req.Reset() - return err - } - req.Header.SetContentLength(len(bodyBuf.B)) - return nil -} - -// Read reads response (including body) from the given r. -// -// io.EOF is returned if r is closed before reading the first header byte. -func (resp *Response) Read(r *bufio.Reader) error { - return resp.ReadLimitBody(r, 0) -} - -// ReadLimitBody reads response from the given r, limiting the body size. -// -// If maxBodySize > 0 and the body size exceeds maxBodySize, -// then ErrBodyTooLarge is returned. -// -// io.EOF is returned if r is closed before reading the first header byte. -func (resp *Response) ReadLimitBody(r *bufio.Reader, maxBodySize int) error { - resp.resetSkipHeader() - err := resp.Header.Read(r) - if err != nil { - return err - } - if resp.Header.StatusCode() == StatusContinue { - // Read the next response according to http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html . - if err = resp.Header.Read(r); err != nil { - return err - } - } - - if !resp.mustSkipBody() { - bodyBuf := resp.bodyBuffer() - bodyBuf.Reset() - bodyBuf.B, err = readBody(r, resp.Header.ContentLength(), maxBodySize, bodyBuf.B) - if err != nil { - resp.Reset() - return err - } - resp.Header.SetContentLength(len(bodyBuf.B)) - } - return nil -} - -func (resp *Response) mustSkipBody() bool { - return resp.SkipBody || resp.Header.mustSkipContentLength() -} - -var errRequestHostRequired = errors.New("missing required Host header in request") - -// WriteTo writes request to w. It implements io.WriterTo. -func (req *Request) WriteTo(w io.Writer) (int64, error) { - return writeBufio(req, w) -} - -// WriteTo writes response to w. It implements io.WriterTo. -func (resp *Response) WriteTo(w io.Writer) (int64, error) { - return writeBufio(resp, w) -} - -func writeBufio(hw httpWriter, w io.Writer) (int64, error) { - sw := acquireStatsWriter(w) - bw := acquireBufioWriter(sw) - err1 := hw.Write(bw) - err2 := bw.Flush() - releaseBufioWriter(bw) - n := sw.bytesWritten - releaseStatsWriter(sw) - - err := err1 - if err == nil { - err = err2 - } - return n, err -} - -type statsWriter struct { - w io.Writer - bytesWritten int64 -} - -func (w *statsWriter) Write(p []byte) (int, error) { - n, err := w.w.Write(p) - w.bytesWritten += int64(n) - return n, err -} - -func acquireStatsWriter(w io.Writer) *statsWriter { - v := statsWriterPool.Get() - if v == nil { - return &statsWriter{ - w: w, - } - } - sw := v.(*statsWriter) - sw.w = w - return sw -} - -func releaseStatsWriter(sw *statsWriter) { - sw.w = nil - sw.bytesWritten = 0 - statsWriterPool.Put(sw) -} - -var statsWriterPool sync.Pool - -func acquireBufioWriter(w io.Writer) *bufio.Writer { - v := bufioWriterPool.Get() - if v == nil { - return bufio.NewWriter(w) - } - bw := v.(*bufio.Writer) - bw.Reset(w) - return bw -} - -func releaseBufioWriter(bw *bufio.Writer) { - bufioWriterPool.Put(bw) -} - -var bufioWriterPool sync.Pool - -func (req *Request) onlyMultipartForm() bool { - return req.multipartForm != nil && (req.body == nil || len(req.body.B) == 0) -} - -// Write writes request to w. -// -// Write doesn't flush request to w for performance reasons. -// -// See also WriteTo. -func (req *Request) Write(w *bufio.Writer) error { - if len(req.Header.Host()) == 0 || req.parsedURI { - uri := req.URI() - host := uri.Host() - if len(host) == 0 { - return errRequestHostRequired - } - req.Header.SetHostBytes(host) - req.Header.SetRequestURIBytes(uri.RequestURI()) - } - - if req.bodyStream != nil { - return req.writeBodyStream(w) - } - - body := req.bodyBytes() - var err error - if req.onlyMultipartForm() { - body, err = marshalMultipartForm(req.multipartForm, req.multipartFormBoundary) - if err != nil { - return fmt.Errorf("error when marshaling multipart form: %s", err) - } - req.Header.SetMultipartFormBoundary(req.multipartFormBoundary) - } - - hasBody := !req.Header.ignoreBody() - if hasBody { - if len(body) == 0 { - body = req.postArgs.QueryString() - } - req.Header.SetContentLength(len(body)) - } - if err = req.Header.Write(w); err != nil { - return err - } - if hasBody { - _, err = w.Write(body) - } else if len(body) > 0 { - return fmt.Errorf("non-zero body for non-POST request. body=%q", body) - } - return err -} - -// WriteGzip writes response with gzipped body to w. -// -// The method gzips response body and sets 'Content-Encoding: gzip' -// header before writing response to w. -// -// WriteGzip doesn't flush response to w for performance reasons. -func (resp *Response) WriteGzip(w *bufio.Writer) error { - return resp.WriteGzipLevel(w, CompressDefaultCompression) -} - -// WriteGzipLevel writes response with gzipped body to w. -// -// Level is the desired compression level: -// -// * CompressNoCompression -// * CompressBestSpeed -// * CompressBestCompression -// * CompressDefaultCompression -// * CompressHuffmanOnly -// -// The method gzips response body and sets 'Content-Encoding: gzip' -// header before writing response to w. -// -// WriteGzipLevel doesn't flush response to w for performance reasons. -func (resp *Response) WriteGzipLevel(w *bufio.Writer, level int) error { - if err := resp.gzipBody(level); err != nil { - return err - } - return resp.Write(w) -} - -// WriteDeflate writes response with deflated body to w. -// -// The method deflates response body and sets 'Content-Encoding: deflate' -// header before writing response to w. -// -// WriteDeflate doesn't flush response to w for performance reasons. -func (resp *Response) WriteDeflate(w *bufio.Writer) error { - return resp.WriteDeflateLevel(w, CompressDefaultCompression) -} - -// WriteDeflateLevel writes response with deflated body to w. -// -// Level is the desired compression level: -// -// * CompressNoCompression -// * CompressBestSpeed -// * CompressBestCompression -// * CompressDefaultCompression -// * CompressHuffmanOnly -// -// The method deflates response body and sets 'Content-Encoding: deflate' -// header before writing response to w. -// -// WriteDeflateLevel doesn't flush response to w for performance reasons. -func (resp *Response) WriteDeflateLevel(w *bufio.Writer, level int) error { - if err := resp.deflateBody(level); err != nil { - return err - } - return resp.Write(w) -} - -func (resp *Response) gzipBody(level int) error { - if len(resp.Header.peek(strContentEncoding)) > 0 { - // It looks like the body is already compressed. - // Do not compress it again. - return nil - } - - if !resp.Header.isCompressibleContentType() { - // The content-type cannot be compressed. - return nil - } - - if resp.bodyStream != nil { - // Reset Content-Length to -1, since it is impossible - // to determine body size beforehand of streamed compression. - // For https://github.com/valyala/fasthttp/issues/176 . - resp.Header.SetContentLength(-1) - - // Do not care about memory allocations here, since gzip is slow - // and allocates a lot of memory by itself. - bs := resp.bodyStream - resp.bodyStream = NewStreamReader(func(sw *bufio.Writer) { - zw := acquireStacklessGzipWriter(sw, level) - fw := &flushWriter{ - wf: zw, - bw: sw, - } - copyZeroAlloc(fw, bs) - releaseStacklessGzipWriter(zw, level) - if bsc, ok := bs.(io.Closer); ok { - bsc.Close() - } - }) - } else { - bodyBytes := resp.bodyBytes() - if len(bodyBytes) < minCompressLen { - // There is no sense in spending CPU time on small body compression, - // since there is a very high probability that the compressed - // body size will be bigger than the original body size. - return nil - } - w := responseBodyPool.Get() - w.B = AppendGzipBytesLevel(w.B, bodyBytes, level) - - // Hack: swap resp.body with w. - if resp.body != nil { - responseBodyPool.Put(resp.body) - } - resp.body = w - } - resp.Header.SetCanonical(strContentEncoding, strGzip) - return nil -} - -func (resp *Response) deflateBody(level int) error { - if len(resp.Header.peek(strContentEncoding)) > 0 { - // It looks like the body is already compressed. - // Do not compress it again. - return nil - } - - if !resp.Header.isCompressibleContentType() { - // The content-type cannot be compressed. - return nil - } - - if resp.bodyStream != nil { - // Reset Content-Length to -1, since it is impossible - // to determine body size beforehand of streamed compression. - // For https://github.com/valyala/fasthttp/issues/176 . - resp.Header.SetContentLength(-1) - - // Do not care about memory allocations here, since flate is slow - // and allocates a lot of memory by itself. - bs := resp.bodyStream - resp.bodyStream = NewStreamReader(func(sw *bufio.Writer) { - zw := acquireStacklessDeflateWriter(sw, level) - fw := &flushWriter{ - wf: zw, - bw: sw, - } - copyZeroAlloc(fw, bs) - releaseStacklessDeflateWriter(zw, level) - if bsc, ok := bs.(io.Closer); ok { - bsc.Close() - } - }) - } else { - bodyBytes := resp.bodyBytes() - if len(bodyBytes) < minCompressLen { - // There is no sense in spending CPU time on small body compression, - // since there is a very high probability that the compressed - // body size will be bigger than the original body size. - return nil - } - w := responseBodyPool.Get() - w.B = AppendDeflateBytesLevel(w.B, bodyBytes, level) - - // Hack: swap resp.body with w. - if resp.body != nil { - responseBodyPool.Put(resp.body) - } - resp.body = w - } - resp.Header.SetCanonical(strContentEncoding, strDeflate) - return nil -} - -// Bodies with sizes smaller than minCompressLen aren't compressed at all -const minCompressLen = 200 - -type writeFlusher interface { - io.Writer - Flush() error -} - -type flushWriter struct { - wf writeFlusher - bw *bufio.Writer -} - -func (w *flushWriter) Write(p []byte) (int, error) { - n, err := w.wf.Write(p) - if err != nil { - return 0, err - } - if err = w.wf.Flush(); err != nil { - return 0, err - } - if err = w.bw.Flush(); err != nil { - return 0, err - } - return n, nil -} - -// Write writes response to w. -// -// Write doesn't flush response to w for performance reasons. -// -// See also WriteTo. -func (resp *Response) Write(w *bufio.Writer) error { - sendBody := !resp.mustSkipBody() - - if resp.bodyStream != nil { - return resp.writeBodyStream(w, sendBody) - } - - body := resp.bodyBytes() - bodyLen := len(body) - if sendBody || bodyLen > 0 { - resp.Header.SetContentLength(bodyLen) - } - if err := resp.Header.Write(w); err != nil { - return err - } - if sendBody { - if _, err := w.Write(body); err != nil { - return err - } - } - return nil -} - -func (req *Request) writeBodyStream(w *bufio.Writer) error { - var err error - - contentLength := req.Header.ContentLength() - if contentLength < 0 { - lrSize := limitedReaderSize(req.bodyStream) - if lrSize >= 0 { - contentLength = int(lrSize) - if int64(contentLength) != lrSize { - contentLength = -1 - } - if contentLength >= 0 { - req.Header.SetContentLength(contentLength) - } - } - } - if contentLength >= 0 { - if err = req.Header.Write(w); err == nil { - err = writeBodyFixedSize(w, req.bodyStream, int64(contentLength)) - } - } else { - req.Header.SetContentLength(-1) - if err = req.Header.Write(w); err == nil { - err = writeBodyChunked(w, req.bodyStream) - } - } - err1 := req.closeBodyStream() - if err == nil { - err = err1 - } - return err -} - -func (resp *Response) writeBodyStream(w *bufio.Writer, sendBody bool) error { - var err error - - contentLength := resp.Header.ContentLength() - if contentLength < 0 { - lrSize := limitedReaderSize(resp.bodyStream) - if lrSize >= 0 { - contentLength = int(lrSize) - if int64(contentLength) != lrSize { - contentLength = -1 - } - if contentLength >= 0 { - resp.Header.SetContentLength(contentLength) - } - } - } - if contentLength >= 0 { - if err = resp.Header.Write(w); err == nil && sendBody { - err = writeBodyFixedSize(w, resp.bodyStream, int64(contentLength)) - } - } else { - resp.Header.SetContentLength(-1) - if err = resp.Header.Write(w); err == nil && sendBody { - err = writeBodyChunked(w, resp.bodyStream) - } - } - err1 := resp.closeBodyStream() - if err == nil { - err = err1 - } - return err -} - -func (req *Request) closeBodyStream() error { - if req.bodyStream == nil { - return nil - } - var err error - if bsc, ok := req.bodyStream.(io.Closer); ok { - err = bsc.Close() - } - req.bodyStream = nil - return err -} - -func (resp *Response) closeBodyStream() error { - if resp.bodyStream == nil { - return nil - } - var err error - if bsc, ok := resp.bodyStream.(io.Closer); ok { - err = bsc.Close() - } - resp.bodyStream = nil - return err -} - -// String returns request representation. -// -// Returns error message instead of request representation on error. -// -// Use Write instead of String for performance-critical code. -func (req *Request) String() string { - return getHTTPString(req) -} - -// String returns response representation. -// -// Returns error message instead of response representation on error. -// -// Use Write instead of String for performance-critical code. -func (resp *Response) String() string { - return getHTTPString(resp) -} - -func getHTTPString(hw httpWriter) string { - w := bytebufferpool.Get() - bw := bufio.NewWriter(w) - if err := hw.Write(bw); err != nil { - return err.Error() - } - if err := bw.Flush(); err != nil { - return err.Error() - } - s := string(w.B) - bytebufferpool.Put(w) - return s -} - -type httpWriter interface { - Write(w *bufio.Writer) error -} - -func writeBodyChunked(w *bufio.Writer, r io.Reader) error { - vbuf := copyBufPool.Get() - buf := vbuf.([]byte) - - var err error - var n int - for { - n, err = r.Read(buf) - if n == 0 { - if err == nil { - panic("BUG: io.Reader returned 0, nil") - } - if err == io.EOF { - if err = writeChunk(w, buf[:0]); err != nil { - break - } - err = nil - } - break - } - if err = writeChunk(w, buf[:n]); err != nil { - break - } - } - - copyBufPool.Put(vbuf) - return err -} - -func limitedReaderSize(r io.Reader) int64 { - lr, ok := r.(*io.LimitedReader) - if !ok { - return -1 - } - return lr.N -} - -func writeBodyFixedSize(w *bufio.Writer, r io.Reader, size int64) error { - if size > maxSmallFileSize { - // w buffer must be empty for triggering - // sendfile path in bufio.Writer.ReadFrom. - if err := w.Flush(); err != nil { - return err - } - } - - // Unwrap a single limited reader for triggering sendfile path - // in net.TCPConn.ReadFrom. - lr, ok := r.(*io.LimitedReader) - if ok { - r = lr.R - } - - n, err := copyZeroAlloc(w, r) - - if ok { - lr.N -= n - } - - if n != size && err == nil { - err = fmt.Errorf("copied %d bytes from body stream instead of %d bytes", n, size) - } - return err -} - -func copyZeroAlloc(w io.Writer, r io.Reader) (int64, error) { - vbuf := copyBufPool.Get() - buf := vbuf.([]byte) - n, err := io.CopyBuffer(w, r, buf) - copyBufPool.Put(vbuf) - return n, err -} - -var copyBufPool = sync.Pool{ - New: func() interface{} { - return make([]byte, 4096) - }, -} - -func writeChunk(w *bufio.Writer, b []byte) error { - n := len(b) - writeHexInt(w, n) - w.Write(strCRLF) - w.Write(b) - _, err := w.Write(strCRLF) - err1 := w.Flush() - if err == nil { - err = err1 - } - return err -} - -// ErrBodyTooLarge is returned if either request or response body exceeds -// the given limit. -var ErrBodyTooLarge = errors.New("body size exceeds the given limit") - -func readBody(r *bufio.Reader, contentLength int, maxBodySize int, dst []byte) ([]byte, error) { - dst = dst[:0] - if contentLength >= 0 { - if maxBodySize > 0 && contentLength > maxBodySize { - return dst, ErrBodyTooLarge - } - return appendBodyFixedSize(r, dst, contentLength) - } - if contentLength == -1 { - return readBodyChunked(r, maxBodySize, dst) - } - return readBodyIdentity(r, maxBodySize, dst) -} - -func readBodyIdentity(r *bufio.Reader, maxBodySize int, dst []byte) ([]byte, error) { - dst = dst[:cap(dst)] - if len(dst) == 0 { - dst = make([]byte, 1024) - } - offset := 0 - for { - nn, err := r.Read(dst[offset:]) - if nn <= 0 { - if err != nil { - if err == io.EOF { - return dst[:offset], nil - } - return dst[:offset], err - } - panic(fmt.Sprintf("BUG: bufio.Read() returned (%d, nil)", nn)) - } - offset += nn - if maxBodySize > 0 && offset > maxBodySize { - return dst[:offset], ErrBodyTooLarge - } - if len(dst) == offset { - n := round2(2 * offset) - if maxBodySize > 0 && n > maxBodySize { - n = maxBodySize + 1 - } - b := make([]byte, n) - copy(b, dst) - dst = b - } - } -} - -func appendBodyFixedSize(r *bufio.Reader, dst []byte, n int) ([]byte, error) { - if n == 0 { - return dst, nil - } - - offset := len(dst) - dstLen := offset + n - if cap(dst) < dstLen { - b := make([]byte, round2(dstLen)) - copy(b, dst) - dst = b - } - dst = dst[:dstLen] - - for { - nn, err := r.Read(dst[offset:]) - if nn <= 0 { - if err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - return dst[:offset], err - } - panic(fmt.Sprintf("BUG: bufio.Read() returned (%d, nil)", nn)) - } - offset += nn - if offset == dstLen { - return dst, nil - } - } -} - -func readBodyChunked(r *bufio.Reader, maxBodySize int, dst []byte) ([]byte, error) { - if len(dst) > 0 { - panic("BUG: expected zero-length buffer") - } - - strCRLFLen := len(strCRLF) - for { - chunkSize, err := parseChunkSize(r) - if err != nil { - return dst, err - } - if maxBodySize > 0 && len(dst)+chunkSize > maxBodySize { - return dst, ErrBodyTooLarge - } - dst, err = appendBodyFixedSize(r, dst, chunkSize+strCRLFLen) - if err != nil { - return dst, err - } - if !bytes.Equal(dst[len(dst)-strCRLFLen:], strCRLF) { - return dst, fmt.Errorf("cannot find crlf at the end of chunk") - } - dst = dst[:len(dst)-strCRLFLen] - if chunkSize == 0 { - return dst, nil - } - } -} - -func parseChunkSize(r *bufio.Reader) (int, error) { - n, err := readHexInt(r) - if err != nil { - return -1, err - } - for { - c, err := r.ReadByte() - if err != nil { - return -1, fmt.Errorf("cannot read '\r' char at the end of chunk size: %s", err) - } - // Skip any trailing whitespace after chunk size. - if c == ' ' { - continue - } - if c != '\r' { - return -1, fmt.Errorf("unexpected char %q at the end of chunk size. Expected %q", c, '\r') - } - break - } - c, err := r.ReadByte() - if err != nil { - return -1, fmt.Errorf("cannot read '\n' char at the end of chunk size: %s", err) - } - if c != '\n' { - return -1, fmt.Errorf("unexpected char %q at the end of chunk size. Expected %q", c, '\n') - } - return n, nil -} - -func round2(n int) int { - if n <= 0 { - return 0 - } - n-- - x := uint(0) - for n > 0 { - n >>= 1 - x++ - } - return 1 << x -} diff --git a/vendor/github.com/valyala/fasthttp/lbclient.go b/vendor/github.com/valyala/fasthttp/lbclient.go deleted file mode 100644 index 12418b6b..00000000 --- a/vendor/github.com/valyala/fasthttp/lbclient.go +++ /dev/null @@ -1,183 +0,0 @@ -package fasthttp - -import ( - "sync" - "sync/atomic" - "time" -) - -// BalancingClient is the interface for clients, which may be passed -// to LBClient.Clients. -type BalancingClient interface { - DoDeadline(req *Request, resp *Response, deadline time.Time) error - PendingRequests() int -} - -// LBClient balances requests among available LBClient.Clients. -// -// It has the following features: -// -// - Balances load among available clients using 'least loaded' + 'round robin' -// hybrid technique. -// - Dynamically decreases load on unhealthy clients. -// -// It is forbidden copying LBClient instances. Create new instances instead. -// -// It is safe calling LBClient methods from concurrently running goroutines. -type LBClient struct { - noCopy noCopy - - // Clients must contain non-zero clients list. - // Incoming requests are balanced among these clients. - Clients []BalancingClient - - // HealthCheck is a callback called after each request. - // - // The request, response and the error returned by the client - // is passed to HealthCheck, so the callback may determine whether - // the client is healthy. - // - // Load on the current client is decreased if HealthCheck returns false. - // - // By default HealthCheck returns false if err != nil. - HealthCheck func(req *Request, resp *Response, err error) bool - - // Timeout is the request timeout used when calling LBClient.Do. - // - // DefaultLBClientTimeout is used by default. - Timeout time.Duration - - cs []*lbClient - - // nextIdx is for spreading requests among equally loaded clients - // in a round-robin fashion. - nextIdx uint32 - - once sync.Once -} - -// DefaultLBClientTimeout is the default request timeout used by LBClient -// when calling LBClient.Do. -// -// The timeout may be overridden via LBClient.Timeout. -const DefaultLBClientTimeout = time.Second - -// DoDeadline calls DoDeadline on the least loaded client -func (cc *LBClient) DoDeadline(req *Request, resp *Response, deadline time.Time) error { - return cc.get().DoDeadline(req, resp, deadline) -} - -// DoTimeout calculates deadline and calls DoDeadline on the least loaded client -func (cc *LBClient) DoTimeout(req *Request, resp *Response, timeout time.Duration) error { - deadline := time.Now().Add(timeout) - return cc.get().DoDeadline(req, resp, deadline) -} - -// Do calls calculates deadline using LBClient.Timeout and calls DoDeadline -// on the least loaded client. -func (cc *LBClient) Do(req *Request, resp *Response) error { - timeout := cc.Timeout - if timeout <= 0 { - timeout = DefaultLBClientTimeout - } - return cc.DoTimeout(req, resp, timeout) -} - -func (cc *LBClient) init() { - if len(cc.Clients) == 0 { - panic("BUG: LBClient.Clients cannot be empty") - } - for _, c := range cc.Clients { - cc.cs = append(cc.cs, &lbClient{ - c: c, - healthCheck: cc.HealthCheck, - }) - } - - // Randomize nextIdx in order to prevent initial servers' - // hammering from a cluster of identical LBClients. - cc.nextIdx = uint32(time.Now().UnixNano()) -} - -func (cc *LBClient) get() *lbClient { - cc.once.Do(cc.init) - - cs := cc.cs - idx := atomic.AddUint32(&cc.nextIdx, 1) - idx %= uint32(len(cs)) - - minC := cs[idx] - minN := minC.PendingRequests() - if minN == 0 { - return minC - } - for _, c := range cs[idx+1:] { - n := c.PendingRequests() - if n == 0 { - return c - } - if n < minN { - minC = c - minN = n - } - } - for _, c := range cs[:idx] { - n := c.PendingRequests() - if n == 0 { - return c - } - if n < minN { - minC = c - minN = n - } - } - return minC -} - -type lbClient struct { - c BalancingClient - healthCheck func(req *Request, resp *Response, err error) bool - penalty uint32 -} - -func (c *lbClient) DoDeadline(req *Request, resp *Response, deadline time.Time) error { - err := c.c.DoDeadline(req, resp, deadline) - if !c.isHealthy(req, resp, err) && c.incPenalty() { - // Penalize the client returning error, so the next requests - // are routed to another clients. - time.AfterFunc(penaltyDuration, c.decPenalty) - } - return err -} - -func (c *lbClient) PendingRequests() int { - n := c.c.PendingRequests() - m := atomic.LoadUint32(&c.penalty) - return n + int(m) -} - -func (c *lbClient) isHealthy(req *Request, resp *Response, err error) bool { - if c.healthCheck == nil { - return err == nil - } - return c.healthCheck(req, resp, err) -} - -func (c *lbClient) incPenalty() bool { - m := atomic.AddUint32(&c.penalty, 1) - if m > maxPenalty { - c.decPenalty() - return false - } - return true -} - -func (c *lbClient) decPenalty() { - atomic.AddUint32(&c.penalty, ^uint32(0)) -} - -const ( - maxPenalty = 300 - - penaltyDuration = 3 * time.Second -) diff --git a/vendor/github.com/valyala/fasthttp/nocopy.go b/vendor/github.com/valyala/fasthttp/nocopy.go deleted file mode 100644 index 32af52e4..00000000 --- a/vendor/github.com/valyala/fasthttp/nocopy.go +++ /dev/null @@ -1,9 +0,0 @@ -package fasthttp - -// Embed this type into a struct, which mustn't be copied, -// so `go vet` gives a warning if this struct is copied. -// -// See https://github.com/golang/go/issues/8005#issuecomment-190753527 for details. -type noCopy struct{} - -func (*noCopy) Lock() {} diff --git a/vendor/github.com/valyala/fasthttp/peripconn.go b/vendor/github.com/valyala/fasthttp/peripconn.go deleted file mode 100644 index afd2a927..00000000 --- a/vendor/github.com/valyala/fasthttp/peripconn.go +++ /dev/null @@ -1,100 +0,0 @@ -package fasthttp - -import ( - "fmt" - "net" - "sync" -) - -type perIPConnCounter struct { - pool sync.Pool - lock sync.Mutex - m map[uint32]int -} - -func (cc *perIPConnCounter) Register(ip uint32) int { - cc.lock.Lock() - if cc.m == nil { - cc.m = make(map[uint32]int) - } - n := cc.m[ip] + 1 - cc.m[ip] = n - cc.lock.Unlock() - return n -} - -func (cc *perIPConnCounter) Unregister(ip uint32) { - cc.lock.Lock() - if cc.m == nil { - cc.lock.Unlock() - panic("BUG: perIPConnCounter.Register() wasn't called") - } - n := cc.m[ip] - 1 - if n < 0 { - cc.lock.Unlock() - panic(fmt.Sprintf("BUG: negative per-ip counter=%d for ip=%d", n, ip)) - } - cc.m[ip] = n - cc.lock.Unlock() -} - -type perIPConn struct { - net.Conn - - ip uint32 - perIPConnCounter *perIPConnCounter -} - -func acquirePerIPConn(conn net.Conn, ip uint32, counter *perIPConnCounter) *perIPConn { - v := counter.pool.Get() - if v == nil { - v = &perIPConn{ - perIPConnCounter: counter, - } - } - c := v.(*perIPConn) - c.Conn = conn - c.ip = ip - return c -} - -func releasePerIPConn(c *perIPConn) { - c.Conn = nil - c.perIPConnCounter.pool.Put(c) -} - -func (c *perIPConn) Close() error { - err := c.Conn.Close() - c.perIPConnCounter.Unregister(c.ip) - releasePerIPConn(c) - return err -} - -func getUint32IP(c net.Conn) uint32 { - return ip2uint32(getConnIP4(c)) -} - -func getConnIP4(c net.Conn) net.IP { - addr := c.RemoteAddr() - ipAddr, ok := addr.(*net.TCPAddr) - if !ok { - return net.IPv4zero - } - return ipAddr.IP.To4() -} - -func ip2uint32(ip net.IP) uint32 { - if len(ip) != 4 { - return 0 - } - return uint32(ip[0])<<24 | uint32(ip[1])<<16 | uint32(ip[2])<<8 | uint32(ip[3]) -} - -func uint322ip(ip uint32) net.IP { - b := make([]byte, 4) - b[0] = byte(ip >> 24) - b[1] = byte(ip >> 16) - b[2] = byte(ip >> 8) - b[3] = byte(ip) - return b -} diff --git a/vendor/github.com/valyala/fasthttp/reuseport/LICENSE b/vendor/github.com/valyala/fasthttp/reuseport/LICENSE deleted file mode 100644 index 5f25159a..00000000 --- a/vendor/github.com/valyala/fasthttp/reuseport/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Max Riveiro - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/valyala/fasthttp/server.go b/vendor/github.com/valyala/fasthttp/server.go deleted file mode 100644 index d0ef421e..00000000 --- a/vendor/github.com/valyala/fasthttp/server.go +++ /dev/null @@ -1,2361 +0,0 @@ -package fasthttp - -import ( - "bufio" - "crypto/tls" - "errors" - "fmt" - "io" - "log" - "mime/multipart" - "net" - "os" - "strings" - "sync" - "sync/atomic" - "time" -) - -var errNoCertOrKeyProvided = errors.New("Cert or key has not provided") - -var ( - // ErrAlreadyServing is returned when calling Serve on a Server - // that is already serving connections. - ErrAlreadyServing = errors.New("Server is already serving connections") -) - -// ServeConn serves HTTP requests from the given connection -// using the given handler. -// -// ServeConn returns nil if all requests from the c are successfully served. -// It returns non-nil error otherwise. -// -// Connection c must immediately propagate all the data passed to Write() -// to the client. Otherwise requests' processing may hang. -// -// ServeConn closes c before returning. -func ServeConn(c net.Conn, handler RequestHandler) error { - v := serverPool.Get() - if v == nil { - v = &Server{} - } - s := v.(*Server) - s.Handler = handler - err := s.ServeConn(c) - s.Handler = nil - serverPool.Put(v) - return err -} - -var serverPool sync.Pool - -// Serve serves incoming connections from the given listener -// using the given handler. -// -// Serve blocks until the given listener returns permanent error. -func Serve(ln net.Listener, handler RequestHandler) error { - s := &Server{ - Handler: handler, - } - return s.Serve(ln) -} - -// ServeTLS serves HTTPS requests from the given net.Listener -// using the given handler. -// -// certFile and keyFile are paths to TLS certificate and key files. -func ServeTLS(ln net.Listener, certFile, keyFile string, handler RequestHandler) error { - s := &Server{ - Handler: handler, - } - return s.ServeTLS(ln, certFile, keyFile) -} - -// ServeTLSEmbed serves HTTPS requests from the given net.Listener -// using the given handler. -// -// certData and keyData must contain valid TLS certificate and key data. -func ServeTLSEmbed(ln net.Listener, certData, keyData []byte, handler RequestHandler) error { - s := &Server{ - Handler: handler, - } - return s.ServeTLSEmbed(ln, certData, keyData) -} - -// ListenAndServe serves HTTP requests from the given TCP addr -// using the given handler. -func ListenAndServe(addr string, handler RequestHandler) error { - s := &Server{ - Handler: handler, - } - return s.ListenAndServe(addr) -} - -// ListenAndServeUNIX serves HTTP requests from the given UNIX addr -// using the given handler. -// -// The function deletes existing file at addr before starting serving. -// -// The server sets the given file mode for the UNIX addr. -func ListenAndServeUNIX(addr string, mode os.FileMode, handler RequestHandler) error { - s := &Server{ - Handler: handler, - } - return s.ListenAndServeUNIX(addr, mode) -} - -// ListenAndServeTLS serves HTTPS requests from the given TCP addr -// using the given handler. -// -// certFile and keyFile are paths to TLS certificate and key files. -func ListenAndServeTLS(addr, certFile, keyFile string, handler RequestHandler) error { - s := &Server{ - Handler: handler, - } - return s.ListenAndServeTLS(addr, certFile, keyFile) -} - -// ListenAndServeTLSEmbed serves HTTPS requests from the given TCP addr -// using the given handler. -// -// certData and keyData must contain valid TLS certificate and key data. -func ListenAndServeTLSEmbed(addr string, certData, keyData []byte, handler RequestHandler) error { - s := &Server{ - Handler: handler, - } - return s.ListenAndServeTLSEmbed(addr, certData, keyData) -} - -// RequestHandler must process incoming requests. -// -// RequestHandler must call ctx.TimeoutError() before returning -// if it keeps references to ctx and/or its' members after the return. -// Consider wrapping RequestHandler into TimeoutHandler if response time -// must be limited. -type RequestHandler func(ctx *RequestCtx) - -// ServeHandler must process tls.Config.NextProto negotiated requests. -type ServeHandler func(c net.Conn) error - -// Server implements HTTP server. -// -// Default Server settings should satisfy the majority of Server users. -// Adjust Server settings only if you really understand the consequences. -// -// It is forbidden copying Server instances. Create new Server instances -// instead. -// -// It is safe to call Server methods from concurrently running goroutines. -type Server struct { - noCopy noCopy - - // Handler for processing incoming requests. - Handler RequestHandler - - // Server name for sending in response headers. - // - // Default server name is used if left blank. - Name string - - // The maximum number of concurrent connections the server may serve. - // - // DefaultConcurrency is used if not set. - Concurrency int - - // Whether to disable keep-alive connections. - // - // The server will close all the incoming connections after sending - // the first response to client if this option is set to true. - // - // By default keep-alive connections are enabled. - DisableKeepalive bool - - // Per-connection buffer size for requests' reading. - // This also limits the maximum header size. - // - // Increase this buffer if your clients send multi-KB RequestURIs - // and/or multi-KB headers (for example, BIG cookies). - // - // Default buffer size is used if not set. - ReadBufferSize int - - // Per-connection buffer size for responses' writing. - // - // Default buffer size is used if not set. - WriteBufferSize int - - // Maximum duration for reading the full request (including body). - // - // This also limits the maximum duration for idle keep-alive - // connections. - // - // By default request read timeout is unlimited. - ReadTimeout time.Duration - - // Maximum duration for writing the full response (including body). - // - // By default response write timeout is unlimited. - WriteTimeout time.Duration - - // Maximum number of concurrent client connections allowed per IP. - // - // By default unlimited number of concurrent connections - // may be established to the server from a single IP address. - MaxConnsPerIP int - - // Maximum number of requests served per connection. - // - // The server closes connection after the last request. - // 'Connection: close' header is added to the last response. - // - // By default unlimited number of requests may be served per connection. - MaxRequestsPerConn int - - // Maximum keep-alive connection lifetime. - // - // The server closes keep-alive connection after its' lifetime - // expiration. - // - // See also ReadTimeout for limiting the duration of idle keep-alive - // connections. - // - // By default keep-alive connection lifetime is unlimited. - MaxKeepaliveDuration time.Duration - - // Whether to enable tcp keep-alive connections. - // - // Whether the operating system should send tcp keep-alive messages on the tcp connection. - // - // By default tcp keep-alive connections are disabled. - TCPKeepalive bool - - // Period between tcp keep-alive messages. - // - // TCP keep-alive period is determined by operation system by default. - TCPKeepalivePeriod time.Duration - - // Maximum request body size. - // - // The server rejects requests with bodies exceeding this limit. - // - // Request body size is limited by DefaultMaxRequestBodySize by default. - MaxRequestBodySize int - - // Aggressively reduces memory usage at the cost of higher CPU usage - // if set to true. - // - // Try enabling this option only if the server consumes too much memory - // serving mostly idle keep-alive connections. This may reduce memory - // usage by more than 50%. - // - // Aggressive memory usage reduction is disabled by default. - ReduceMemoryUsage bool - - // Rejects all non-GET requests if set to true. - // - // This option is useful as anti-DoS protection for servers - // accepting only GET requests. The request size is limited - // by ReadBufferSize if GetOnly is set. - // - // Server accepts all the requests by default. - GetOnly bool - - // Logs all errors, including the most frequent - // 'connection reset by peer', 'broken pipe' and 'connection timeout' - // errors. Such errors are common in production serving real-world - // clients. - // - // By default the most frequent errors such as - // 'connection reset by peer', 'broken pipe' and 'connection timeout' - // are suppressed in order to limit output log traffic. - LogAllErrors bool - - // Header names are passed as-is without normalization - // if this option is set. - // - // Disabled header names' normalization may be useful only for proxying - // incoming requests to other servers expecting case-sensitive - // header names. See https://github.com/valyala/fasthttp/issues/57 - // for details. - // - // By default request and response header names are normalized, i.e. - // The first letter and the first letters following dashes - // are uppercased, while all the other letters are lowercased. - // Examples: - // - // * HOST -> Host - // * content-type -> Content-Type - // * cONTENT-lenGTH -> Content-Length - DisableHeaderNamesNormalizing bool - - // NoDefaultServerHeader, when set to true, causes the default Server header - // to be excluded from the Response. - // - // The default Server header value is the value of the Name field or an - // internal default value in its absence. With this option set to true, - // the only time a Server header will be sent is if a non-zero length - // value is explicitly provided during a request. - NoDefaultServerHeader bool - - // NoDefaultContentType, when set to true, causes the default Content-Type - // header to be excluded from the Response. - // - // The default Content-Type header value is the internal default value. When - // set to true, the Content-Type will not be present. - NoDefaultContentType bool - - // ConnState specifies an optional callback function that is - // called when a client connection changes state. See the - // ConnState type and associated constants for details. - ConnState func(net.Conn, ConnState) - - // Logger, which is used by RequestCtx.Logger(). - // - // By default standard logger from log package is used. - Logger Logger - - tlsConfig *tls.Config - nextProtos map[string]ServeHandler - - concurrency uint32 - concurrencyCh chan struct{} - perIPConnCounter perIPConnCounter - serverName atomic.Value - - ctxPool sync.Pool - readerPool sync.Pool - writerPool sync.Pool - hijackConnPool sync.Pool - bytePool sync.Pool - - // We need to know our listener so we can close it in Shutdown(). - ln net.Listener - - mu sync.Mutex - open int32 - stop int32 -} - -// TimeoutHandler creates RequestHandler, which returns StatusRequestTimeout -// error with the given msg to the client if h didn't return during -// the given duration. -// -// The returned handler may return StatusTooManyRequests error with the given -// msg to the client if there are more than Server.Concurrency concurrent -// handlers h are running at the moment. -func TimeoutHandler(h RequestHandler, timeout time.Duration, msg string) RequestHandler { - if timeout <= 0 { - return h - } - - return func(ctx *RequestCtx) { - concurrencyCh := ctx.s.concurrencyCh - select { - case concurrencyCh <- struct{}{}: - default: - ctx.Error(msg, StatusTooManyRequests) - return - } - - ch := ctx.timeoutCh - if ch == nil { - ch = make(chan struct{}, 1) - ctx.timeoutCh = ch - } - go func() { - h(ctx) - ch <- struct{}{} - <-concurrencyCh - }() - ctx.timeoutTimer = initTimer(ctx.timeoutTimer, timeout) - select { - case <-ch: - case <-ctx.timeoutTimer.C: - ctx.TimeoutError(msg) - } - stopTimer(ctx.timeoutTimer) - } -} - -// CompressHandler returns RequestHandler that transparently compresses -// response body generated by h if the request contains 'gzip' or 'deflate' -// 'Accept-Encoding' header. -func CompressHandler(h RequestHandler) RequestHandler { - return CompressHandlerLevel(h, CompressDefaultCompression) -} - -// CompressHandlerLevel returns RequestHandler that transparently compresses -// response body generated by h if the request contains 'gzip' or 'deflate' -// 'Accept-Encoding' header. -// -// Level is the desired compression level: -// -// * CompressNoCompression -// * CompressBestSpeed -// * CompressBestCompression -// * CompressDefaultCompression -// * CompressHuffmanOnly -func CompressHandlerLevel(h RequestHandler, level int) RequestHandler { - return func(ctx *RequestCtx) { - h(ctx) - if ctx.Request.Header.HasAcceptEncodingBytes(strGzip) { - ctx.Response.gzipBody(level) - } else if ctx.Request.Header.HasAcceptEncodingBytes(strDeflate) { - ctx.Response.deflateBody(level) - } - } -} - -// RequestCtx contains incoming request and manages outgoing response. -// -// It is forbidden copying RequestCtx instances. -// -// RequestHandler should avoid holding references to incoming RequestCtx and/or -// its' members after the return. -// If holding RequestCtx references after the return is unavoidable -// (for instance, ctx is passed to a separate goroutine and ctx lifetime cannot -// be controlled), then the RequestHandler MUST call ctx.TimeoutError() -// before return. -// -// It is unsafe modifying/reading RequestCtx instance from concurrently -// running goroutines. The only exception is TimeoutError*, which may be called -// while other goroutines accessing RequestCtx. -type RequestCtx struct { - noCopy noCopy - - // Incoming request. - // - // Copying Request by value is forbidden. Use pointer to Request instead. - Request Request - - // Outgoing response. - // - // Copying Response by value is forbidden. Use pointer to Response instead. - Response Response - - userValues userData - - lastReadDuration time.Duration - - connID uint64 - connRequestNum uint64 - connTime time.Time - - time time.Time - - logger ctxLogger - s *Server - c net.Conn - fbr firstByteReader - - timeoutResponse *Response - timeoutCh chan struct{} - timeoutTimer *time.Timer - - hijackHandler HijackHandler -} - -// HijackHandler must process the hijacked connection c. -// -// The connection c is automatically closed after returning from HijackHandler. -// -// The connection c must not be used after returning from the handler. -type HijackHandler func(c net.Conn) - -// Hijack registers the given handler for connection hijacking. -// -// The handler is called after returning from RequestHandler -// and sending http response. The current connection is passed -// to the handler. The connection is automatically closed after -// returning from the handler. -// -// The server skips calling the handler in the following cases: -// -// * 'Connection: close' header exists in either request or response. -// * Unexpected error during response writing to the connection. -// -// The server stops processing requests from hijacked connections. -// Server limits such as Concurrency, ReadTimeout, WriteTimeout, etc. -// aren't applied to hijacked connections. -// -// The handler must not retain references to ctx members. -// -// Arbitrary 'Connection: Upgrade' protocols may be implemented -// with HijackHandler. For instance, -// -// * WebSocket ( https://en.wikipedia.org/wiki/WebSocket ) -// * HTTP/2.0 ( https://en.wikipedia.org/wiki/HTTP/2 ) -// -func (ctx *RequestCtx) Hijack(handler HijackHandler) { - ctx.hijackHandler = handler -} - -// Hijacked returns true after Hijack is called. -func (ctx *RequestCtx) Hijacked() bool { - return ctx.hijackHandler != nil -} - -// SetUserValue stores the given value (arbitrary object) -// under the given key in ctx. -// -// The value stored in ctx may be obtained by UserValue*. -// -// This functionality may be useful for passing arbitrary values between -// functions involved in request processing. -// -// All the values are removed from ctx after returning from the top -// RequestHandler. Additionally, Close method is called on each value -// implementing io.Closer before removing the value from ctx. -func (ctx *RequestCtx) SetUserValue(key string, value interface{}) { - ctx.userValues.Set(key, value) -} - -// SetUserValueBytes stores the given value (arbitrary object) -// under the given key in ctx. -// -// The value stored in ctx may be obtained by UserValue*. -// -// This functionality may be useful for passing arbitrary values between -// functions involved in request processing. -// -// All the values stored in ctx are deleted after returning from RequestHandler. -func (ctx *RequestCtx) SetUserValueBytes(key []byte, value interface{}) { - ctx.userValues.SetBytes(key, value) -} - -// UserValue returns the value stored via SetUserValue* under the given key. -func (ctx *RequestCtx) UserValue(key string) interface{} { - return ctx.userValues.Get(key) -} - -// UserValueBytes returns the value stored via SetUserValue* -// under the given key. -func (ctx *RequestCtx) UserValueBytes(key []byte) interface{} { - return ctx.userValues.GetBytes(key) -} - -// VisitUserValues calls visitor for each existing userValue. -// -// visitor must not retain references to key and value after returning. -// Make key and/or value copies if you need storing them after returning. -func (ctx *RequestCtx) VisitUserValues(visitor func([]byte, interface{})) { - for i, n := 0, len(ctx.userValues); i < n; i++ { - kv := &ctx.userValues[i] - visitor(kv.key, kv.value) - } -} - -type connTLSer interface { - ConnectionState() tls.ConnectionState -} - -// IsTLS returns true if the underlying connection is tls.Conn. -// -// tls.Conn is an encrypted connection (aka SSL, HTTPS). -func (ctx *RequestCtx) IsTLS() bool { - // cast to (connTLSer) instead of (*tls.Conn), since it catches - // cases with overridden tls.Conn such as: - // - // type customConn struct { - // *tls.Conn - // - // // other custom fields here - // } - _, ok := ctx.c.(connTLSer) - return ok -} - -// TLSConnectionState returns TLS connection state. -// -// The function returns nil if the underlying connection isn't tls.Conn. -// -// The returned state may be used for verifying TLS version, client certificates, -// etc. -func (ctx *RequestCtx) TLSConnectionState() *tls.ConnectionState { - tlsConn, ok := ctx.c.(connTLSer) - if !ok { - return nil - } - state := tlsConn.ConnectionState() - return &state -} - -type firstByteReader struct { - c net.Conn - ch byte - byteRead bool -} - -func (r *firstByteReader) Read(b []byte) (int, error) { - if len(b) == 0 { - return 0, nil - } - nn := 0 - if !r.byteRead { - b[0] = r.ch - b = b[1:] - r.byteRead = true - nn = 1 - } - n, err := r.c.Read(b) - return n + nn, err -} - -// Logger is used for logging formatted messages. -type Logger interface { - // Printf must have the same semantics as log.Printf. - Printf(format string, args ...interface{}) -} - -var ctxLoggerLock sync.Mutex - -type ctxLogger struct { - ctx *RequestCtx - logger Logger -} - -func (cl *ctxLogger) Printf(format string, args ...interface{}) { - ctxLoggerLock.Lock() - msg := fmt.Sprintf(format, args...) - ctx := cl.ctx - cl.logger.Printf("%.3f %s - %s", time.Since(ctx.Time()).Seconds(), ctx.String(), msg) - ctxLoggerLock.Unlock() -} - -var zeroTCPAddr = &net.TCPAddr{ - IP: net.IPv4zero, -} - -// String returns unique string representation of the ctx. -// -// The returned value may be useful for logging. -func (ctx *RequestCtx) String() string { - return fmt.Sprintf("#%016X - %s<->%s - %s %s", ctx.ID(), ctx.LocalAddr(), ctx.RemoteAddr(), ctx.Request.Header.Method(), ctx.URI().FullURI()) -} - -// ID returns unique ID of the request. -func (ctx *RequestCtx) ID() uint64 { - return (ctx.connID << 32) | ctx.connRequestNum -} - -// ConnID returns unique connection ID. -// -// This ID may be used to match distinct requests to the same incoming -// connection. -func (ctx *RequestCtx) ConnID() uint64 { - return ctx.connID -} - -// Time returns RequestHandler call time. -func (ctx *RequestCtx) Time() time.Time { - return ctx.time -} - -// ConnTime returns the time the server started serving the connection -// the current request came from. -func (ctx *RequestCtx) ConnTime() time.Time { - return ctx.connTime -} - -// ConnRequestNum returns request sequence number -// for the current connection. -// -// Sequence starts with 1. -func (ctx *RequestCtx) ConnRequestNum() uint64 { - return ctx.connRequestNum -} - -// SetConnectionClose sets 'Connection: close' response header and closes -// connection after the RequestHandler returns. -func (ctx *RequestCtx) SetConnectionClose() { - ctx.Response.SetConnectionClose() -} - -// SetStatusCode sets response status code. -func (ctx *RequestCtx) SetStatusCode(statusCode int) { - ctx.Response.SetStatusCode(statusCode) -} - -// SetContentType sets response Content-Type. -func (ctx *RequestCtx) SetContentType(contentType string) { - ctx.Response.Header.SetContentType(contentType) -} - -// SetContentTypeBytes sets response Content-Type. -// -// It is safe modifying contentType buffer after function return. -func (ctx *RequestCtx) SetContentTypeBytes(contentType []byte) { - ctx.Response.Header.SetContentTypeBytes(contentType) -} - -// RequestURI returns RequestURI. -// -// This uri is valid until returning from RequestHandler. -func (ctx *RequestCtx) RequestURI() []byte { - return ctx.Request.Header.RequestURI() -} - -// URI returns requested uri. -// -// The uri is valid until returning from RequestHandler. -func (ctx *RequestCtx) URI() *URI { - return ctx.Request.URI() -} - -// Referer returns request referer. -// -// The referer is valid until returning from RequestHandler. -func (ctx *RequestCtx) Referer() []byte { - return ctx.Request.Header.Referer() -} - -// UserAgent returns User-Agent header value from the request. -func (ctx *RequestCtx) UserAgent() []byte { - return ctx.Request.Header.UserAgent() -} - -// Path returns requested path. -// -// The path is valid until returning from RequestHandler. -func (ctx *RequestCtx) Path() []byte { - return ctx.URI().Path() -} - -// Host returns requested host. -// -// The host is valid until returning from RequestHandler. -func (ctx *RequestCtx) Host() []byte { - return ctx.URI().Host() -} - -// QueryArgs returns query arguments from RequestURI. -// -// It doesn't return POST'ed arguments - use PostArgs() for this. -// -// Returned arguments are valid until returning from RequestHandler. -// -// See also PostArgs, FormValue and FormFile. -func (ctx *RequestCtx) QueryArgs() *Args { - return ctx.URI().QueryArgs() -} - -// PostArgs returns POST arguments. -// -// It doesn't return query arguments from RequestURI - use QueryArgs for this. -// -// Returned arguments are valid until returning from RequestHandler. -// -// See also QueryArgs, FormValue and FormFile. -func (ctx *RequestCtx) PostArgs() *Args { - return ctx.Request.PostArgs() -} - -// MultipartForm returns requests's multipart form. -// -// Returns ErrNoMultipartForm if request's content-type -// isn't 'multipart/form-data'. -// -// All uploaded temporary files are automatically deleted after -// returning from RequestHandler. Either move or copy uploaded files -// into new place if you want retaining them. -// -// Use SaveMultipartFile function for permanently saving uploaded file. -// -// The returned form is valid until returning from RequestHandler. -// -// See also FormFile and FormValue. -func (ctx *RequestCtx) MultipartForm() (*multipart.Form, error) { - return ctx.Request.MultipartForm() -} - -// FormFile returns uploaded file associated with the given multipart form key. -// -// The file is automatically deleted after returning from RequestHandler, -// so either move or copy uploaded file into new place if you want retaining it. -// -// Use SaveMultipartFile function for permanently saving uploaded file. -// -// The returned file header is valid until returning from RequestHandler. -func (ctx *RequestCtx) FormFile(key string) (*multipart.FileHeader, error) { - mf, err := ctx.MultipartForm() - if err != nil { - return nil, err - } - if mf.File == nil { - return nil, err - } - fhh := mf.File[key] - if fhh == nil { - return nil, ErrMissingFile - } - return fhh[0], nil -} - -// ErrMissingFile may be returned from FormFile when the is no uploaded file -// associated with the given multipart form key. -var ErrMissingFile = errors.New("there is no uploaded file associated with the given key") - -// SaveMultipartFile saves multipart file fh under the given filename path. -func SaveMultipartFile(fh *multipart.FileHeader, path string) error { - f, err := fh.Open() - if err != nil { - return err - } - - if ff, ok := f.(*os.File); ok { - // Windows can't rename files that are opened. - if err := f.Close(); err != nil { - return err - } - - // If renaming fails we try the normal copying method. - // Renaming could fail if the files are on different devices. - if os.Rename(ff.Name(), path) == nil { - return nil - } - - // Reopen f for the code below. - f, err = fh.Open() - if err != nil { - return err - } - } - - defer f.Close() - - ff, err := os.Create(path) - if err != nil { - return err - } - defer ff.Close() - _, err = copyZeroAlloc(ff, f) - return err -} - -// FormValue returns form value associated with the given key. -// -// The value is searched in the following places: -// -// * Query string. -// * POST or PUT body. -// -// There are more fine-grained methods for obtaining form values: -// -// * QueryArgs for obtaining values from query string. -// * PostArgs for obtaining values from POST or PUT body. -// * MultipartForm for obtaining values from multipart form. -// * FormFile for obtaining uploaded files. -// -// The returned value is valid until returning from RequestHandler. -func (ctx *RequestCtx) FormValue(key string) []byte { - v := ctx.QueryArgs().Peek(key) - if len(v) > 0 { - return v - } - v = ctx.PostArgs().Peek(key) - if len(v) > 0 { - return v - } - mf, err := ctx.MultipartForm() - if err == nil && mf.Value != nil { - vv := mf.Value[key] - if len(vv) > 0 { - return []byte(vv[0]) - } - } - return nil -} - -// IsGet returns true if request method is GET. -func (ctx *RequestCtx) IsGet() bool { - return ctx.Request.Header.IsGet() -} - -// IsPost returns true if request method is POST. -func (ctx *RequestCtx) IsPost() bool { - return ctx.Request.Header.IsPost() -} - -// IsPut returns true if request method is PUT. -func (ctx *RequestCtx) IsPut() bool { - return ctx.Request.Header.IsPut() -} - -// IsDelete returns true if request method is DELETE. -func (ctx *RequestCtx) IsDelete() bool { - return ctx.Request.Header.IsDelete() -} - -// IsConnect returns true if request method is CONNECT. -func (ctx *RequestCtx) IsConnect() bool { - return ctx.Request.Header.IsConnect() -} - -// IsOptions returns true if request method is OPTIONS. -func (ctx *RequestCtx) IsOptions() bool { - return ctx.Request.Header.IsOptions() -} - -// IsTrace returns true if request method is TRACE. -func (ctx *RequestCtx) IsTrace() bool { - return ctx.Request.Header.IsTrace() -} - -// IsPatch returns true if request method is PATCH. -func (ctx *RequestCtx) IsPatch() bool { - return ctx.Request.Header.IsPatch() -} - -// Method return request method. -// -// Returned value is valid until returning from RequestHandler. -func (ctx *RequestCtx) Method() []byte { - return ctx.Request.Header.Method() -} - -// IsHead returns true if request method is HEAD. -func (ctx *RequestCtx) IsHead() bool { - return ctx.Request.Header.IsHead() -} - -// RemoteAddr returns client address for the given request. -// -// Always returns non-nil result. -func (ctx *RequestCtx) RemoteAddr() net.Addr { - if ctx.c == nil { - return zeroTCPAddr - } - addr := ctx.c.RemoteAddr() - if addr == nil { - return zeroTCPAddr - } - return addr -} - -// LocalAddr returns server address for the given request. -// -// Always returns non-nil result. -func (ctx *RequestCtx) LocalAddr() net.Addr { - if ctx.c == nil { - return zeroTCPAddr - } - addr := ctx.c.LocalAddr() - if addr == nil { - return zeroTCPAddr - } - return addr -} - -// RemoteIP returns the client ip the request came from. -// -// Always returns non-nil result. -func (ctx *RequestCtx) RemoteIP() net.IP { - return addrToIP(ctx.RemoteAddr()) -} - -// LocalIP returns the server ip the request came to. -// -// Always returns non-nil result. -func (ctx *RequestCtx) LocalIP() net.IP { - return addrToIP(ctx.LocalAddr()) -} - -func addrToIP(addr net.Addr) net.IP { - x, ok := addr.(*net.TCPAddr) - if !ok { - return net.IPv4zero - } - return x.IP -} - -// Error sets response status code to the given value and sets response body -// to the given message. -func (ctx *RequestCtx) Error(msg string, statusCode int) { - ctx.Response.Reset() - ctx.SetStatusCode(statusCode) - ctx.SetContentTypeBytes(defaultContentType) - ctx.SetBodyString(msg) -} - -// Success sets response Content-Type and body to the given values. -func (ctx *RequestCtx) Success(contentType string, body []byte) { - ctx.SetContentType(contentType) - ctx.SetBody(body) -} - -// SuccessString sets response Content-Type and body to the given values. -func (ctx *RequestCtx) SuccessString(contentType, body string) { - ctx.SetContentType(contentType) - ctx.SetBodyString(body) -} - -// Redirect sets 'Location: uri' response header and sets the given statusCode. -// -// statusCode must have one of the following values: -// -// * StatusMovedPermanently (301) -// * StatusFound (302) -// * StatusSeeOther (303) -// * StatusTemporaryRedirect (307) -// -// All other statusCode values are replaced by StatusFound (302). -// -// The redirect uri may be either absolute or relative to the current -// request uri. Fasthttp will always send an absolute uri back to the client. -// To send a relative uri you can use the following code: -// -// strLocation = []byte("Location") // Put this with your top level var () declarations. -// ctx.Response.Header.SetCanonical(strLocation, "/relative?uri") -// ctx.Response.SetStatusCode(fasthttp.StatusMovedPermanently) -// -func (ctx *RequestCtx) Redirect(uri string, statusCode int) { - u := AcquireURI() - ctx.URI().CopyTo(u) - u.Update(uri) - ctx.redirect(u.FullURI(), statusCode) - ReleaseURI(u) -} - -// RedirectBytes sets 'Location: uri' response header and sets -// the given statusCode. -// -// statusCode must have one of the following values: -// -// * StatusMovedPermanently (301) -// * StatusFound (302) -// * StatusSeeOther (303) -// * StatusTemporaryRedirect (307) -// -// All other statusCode values are replaced by StatusFound (302). -// -// The redirect uri may be either absolute or relative to the current -// request uri. Fasthttp will always send an absolute uri back to the client. -// To send a relative uri you can use the following code: -// -// strLocation = []byte("Location") // Put this with your top level var () declarations. -// ctx.Response.Header.SetCanonical(strLocation, "/relative?uri") -// ctx.Response.SetStatusCode(fasthttp.StatusMovedPermanently) -// -func (ctx *RequestCtx) RedirectBytes(uri []byte, statusCode int) { - s := b2s(uri) - ctx.Redirect(s, statusCode) -} - -func (ctx *RequestCtx) redirect(uri []byte, statusCode int) { - ctx.Response.Header.SetCanonical(strLocation, uri) - statusCode = getRedirectStatusCode(statusCode) - ctx.Response.SetStatusCode(statusCode) -} - -func getRedirectStatusCode(statusCode int) int { - if statusCode == StatusMovedPermanently || statusCode == StatusFound || - statusCode == StatusSeeOther || statusCode == StatusTemporaryRedirect { - return statusCode - } - return StatusFound -} - -// SetBody sets response body to the given value. -// -// It is safe re-using body argument after the function returns. -func (ctx *RequestCtx) SetBody(body []byte) { - ctx.Response.SetBody(body) -} - -// SetBodyString sets response body to the given value. -func (ctx *RequestCtx) SetBodyString(body string) { - ctx.Response.SetBodyString(body) -} - -// ResetBody resets response body contents. -func (ctx *RequestCtx) ResetBody() { - ctx.Response.ResetBody() -} - -// SendFile sends local file contents from the given path as response body. -// -// This is a shortcut to ServeFile(ctx, path). -// -// SendFile logs all the errors via ctx.Logger. -// -// See also ServeFile, FSHandler and FS. -func (ctx *RequestCtx) SendFile(path string) { - ServeFile(ctx, path) -} - -// SendFileBytes sends local file contents from the given path as response body. -// -// This is a shortcut to ServeFileBytes(ctx, path). -// -// SendFileBytes logs all the errors via ctx.Logger. -// -// See also ServeFileBytes, FSHandler and FS. -func (ctx *RequestCtx) SendFileBytes(path []byte) { - ServeFileBytes(ctx, path) -} - -// IfModifiedSince returns true if lastModified exceeds 'If-Modified-Since' -// value from the request header. -// -// The function returns true also 'If-Modified-Since' request header is missing. -func (ctx *RequestCtx) IfModifiedSince(lastModified time.Time) bool { - ifModStr := ctx.Request.Header.peek(strIfModifiedSince) - if len(ifModStr) == 0 { - return true - } - ifMod, err := ParseHTTPDate(ifModStr) - if err != nil { - return true - } - lastModified = lastModified.Truncate(time.Second) - return ifMod.Before(lastModified) -} - -// NotModified resets response and sets '304 Not Modified' response status code. -func (ctx *RequestCtx) NotModified() { - ctx.Response.Reset() - ctx.SetStatusCode(StatusNotModified) -} - -// NotFound resets response and sets '404 Not Found' response status code. -func (ctx *RequestCtx) NotFound() { - ctx.Response.Reset() - ctx.SetStatusCode(StatusNotFound) - ctx.SetBodyString("404 Page not found") -} - -// Write writes p into response body. -func (ctx *RequestCtx) Write(p []byte) (int, error) { - ctx.Response.AppendBody(p) - return len(p), nil -} - -// WriteString appends s to response body. -func (ctx *RequestCtx) WriteString(s string) (int, error) { - ctx.Response.AppendBodyString(s) - return len(s), nil -} - -// PostBody returns POST request body. -// -// The returned value is valid until RequestHandler return. -func (ctx *RequestCtx) PostBody() []byte { - return ctx.Request.Body() -} - -// SetBodyStream sets response body stream and, optionally body size. -// -// bodyStream.Close() is called after finishing reading all body data -// if it implements io.Closer. -// -// If bodySize is >= 0, then bodySize bytes must be provided by bodyStream -// before returning io.EOF. -// -// If bodySize < 0, then bodyStream is read until io.EOF. -// -// See also SetBodyStreamWriter. -func (ctx *RequestCtx) SetBodyStream(bodyStream io.Reader, bodySize int) { - ctx.Response.SetBodyStream(bodyStream, bodySize) -} - -// SetBodyStreamWriter registers the given stream writer for populating -// response body. -// -// Access to RequestCtx and/or its' members is forbidden from sw. -// -// This function may be used in the following cases: -// -// * if response body is too big (more than 10MB). -// * if response body is streamed from slow external sources. -// * if response body must be streamed to the client in chunks. -// (aka `http server push`). -func (ctx *RequestCtx) SetBodyStreamWriter(sw StreamWriter) { - ctx.Response.SetBodyStreamWriter(sw) -} - -// IsBodyStream returns true if response body is set via SetBodyStream*. -func (ctx *RequestCtx) IsBodyStream() bool { - return ctx.Response.IsBodyStream() -} - -// Logger returns logger, which may be used for logging arbitrary -// request-specific messages inside RequestHandler. -// -// Each message logged via returned logger contains request-specific information -// such as request id, request duration, local address, remote address, -// request method and request url. -// -// It is safe re-using returned logger for logging multiple messages -// for the current request. -// -// The returned logger is valid until returning from RequestHandler. -func (ctx *RequestCtx) Logger() Logger { - if ctx.logger.ctx == nil { - ctx.logger.ctx = ctx - } - if ctx.logger.logger == nil { - ctx.logger.logger = ctx.s.logger() - } - return &ctx.logger -} - -// TimeoutError sets response status code to StatusRequestTimeout and sets -// body to the given msg. -// -// All response modifications after TimeoutError call are ignored. -// -// TimeoutError MUST be called before returning from RequestHandler if there are -// references to ctx and/or its members in other goroutines remain. -// -// Usage of this function is discouraged. Prefer eliminating ctx references -// from pending goroutines instead of using this function. -func (ctx *RequestCtx) TimeoutError(msg string) { - ctx.TimeoutErrorWithCode(msg, StatusRequestTimeout) -} - -// TimeoutErrorWithCode sets response body to msg and response status -// code to statusCode. -// -// All response modifications after TimeoutErrorWithCode call are ignored. -// -// TimeoutErrorWithCode MUST be called before returning from RequestHandler -// if there are references to ctx and/or its members in other goroutines remain. -// -// Usage of this function is discouraged. Prefer eliminating ctx references -// from pending goroutines instead of using this function. -func (ctx *RequestCtx) TimeoutErrorWithCode(msg string, statusCode int) { - var resp Response - resp.SetStatusCode(statusCode) - resp.SetBodyString(msg) - ctx.TimeoutErrorWithResponse(&resp) -} - -// TimeoutErrorWithResponse marks the ctx as timed out and sends the given -// response to the client. -// -// All ctx modifications after TimeoutErrorWithResponse call are ignored. -// -// TimeoutErrorWithResponse MUST be called before returning from RequestHandler -// if there are references to ctx and/or its members in other goroutines remain. -// -// Usage of this function is discouraged. Prefer eliminating ctx references -// from pending goroutines instead of using this function. -func (ctx *RequestCtx) TimeoutErrorWithResponse(resp *Response) { - respCopy := &Response{} - resp.CopyTo(respCopy) - ctx.timeoutResponse = respCopy -} - -// NextProto adds nph to be processed when key is negotiated when TLS conection is established. -// -// This function can only be called before the server is started. -func (s *Server) NextProto(key string, nph ServeHandler) { - if s.nextProtos == nil { - s.nextProtos = make(map[string]ServeHandler) - } - s.configTLS() - s.tlsConfig.NextProtos = append(s.tlsConfig.NextProtos, key) - s.nextProtos[key] = nph -} - -func (s *Server) hasNextProto(c net.Conn) (proto string) { - tlsConn, ok := c.(connTLSer) - if ok { - proto = tlsConn.ConnectionState().NegotiatedProtocol - } - return -} - -// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted -// connections. It's used by ListenAndServe, ListenAndServeTLS and -// ListenAndServeTLSEmbed so dead TCP connections (e.g. closing laptop mid-download) -// eventually go away. -type tcpKeepaliveListener struct { - *net.TCPListener - keepalivePeriod time.Duration -} - -func (ln tcpKeepaliveListener) Accept() (net.Conn, error) { - tc, err := ln.AcceptTCP() - if err != nil { - return nil, err - } - tc.SetKeepAlive(true) - if ln.keepalivePeriod > 0 { - tc.SetKeepAlivePeriod(ln.keepalivePeriod) - } - return tc, nil -} - -// ListenAndServe serves HTTP requests from the given TCP4 addr. -// -// Pass custom listener to Serve if you need listening on non-TCP4 media -// such as IPv6. -// -// Accepted connections are configured to enable TCP keep-alives. -func (s *Server) ListenAndServe(addr string) error { - ln, err := net.Listen("tcp4", addr) - if err != nil { - return err - } - if s.TCPKeepalive { - if tcpln, ok := ln.(*net.TCPListener); ok { - return s.Serve(tcpKeepaliveListener{ - TCPListener: tcpln, - keepalivePeriod: s.TCPKeepalivePeriod, - }) - } - } - return s.Serve(ln) -} - -// ListenAndServeUNIX serves HTTP requests from the given UNIX addr. -// -// The function deletes existing file at addr before starting serving. -// -// The server sets the given file mode for the UNIX addr. -func (s *Server) ListenAndServeUNIX(addr string, mode os.FileMode) error { - if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("unexpected error when trying to remove unix socket file %q: %s", addr, err) - } - ln, err := net.Listen("unix", addr) - if err != nil { - return err - } - if err = os.Chmod(addr, mode); err != nil { - return fmt.Errorf("cannot chmod %#o for %q: %s", mode, addr, err) - } - return s.Serve(ln) -} - -// ListenAndServeTLS serves HTTPS requests from the given TCP4 addr. -// -// certFile and keyFile are paths to TLS certificate and key files. -// -// Pass custom listener to Serve if you need listening on non-TCP4 media -// such as IPv6. -// -// If the certFile or keyFile has not been provided to the server structure, -// the function will use the previously added TLS configuration. -// -// Accepted connections are configured to enable TCP keep-alives. -func (s *Server) ListenAndServeTLS(addr, certFile, keyFile string) error { - ln, err := net.Listen("tcp4", addr) - if err != nil { - return err - } - if s.TCPKeepalive { - if tcpln, ok := ln.(*net.TCPListener); ok { - return s.ServeTLS(tcpKeepaliveListener{ - TCPListener: tcpln, - keepalivePeriod: s.TCPKeepalivePeriod, - }, certFile, keyFile) - } - } - return s.ServeTLS(ln, certFile, keyFile) -} - -// ListenAndServeTLSEmbed serves HTTPS requests from the given TCP4 addr. -// -// certData and keyData must contain valid TLS certificate and key data. -// -// Pass custom listener to Serve if you need listening on arbitrary media -// such as IPv6. -// -// If the certFile or keyFile has not been provided the server structure, -// the function will use previously added TLS configuration. -// -// Accepted connections are configured to enable TCP keep-alives. -func (s *Server) ListenAndServeTLSEmbed(addr string, certData, keyData []byte) error { - ln, err := net.Listen("tcp4", addr) - if err != nil { - return err - } - if s.TCPKeepalive { - if tcpln, ok := ln.(*net.TCPListener); ok { - return s.ServeTLSEmbed(tcpKeepaliveListener{ - TCPListener: tcpln, - keepalivePeriod: s.TCPKeepalivePeriod, - }, certData, keyData) - } - } - return s.ServeTLSEmbed(ln, certData, keyData) -} - -// ServeTLS serves HTTPS requests from the given listener. -// -// certFile and keyFile are paths to TLS certificate and key files. -// -// If the certFile or keyFile has not been provided the server structure, -// the function will use previously added TLS configuration. -func (s *Server) ServeTLS(ln net.Listener, certFile, keyFile string) error { - err := s.AppendCert(certFile, keyFile) - if err != nil && err != errNoCertOrKeyProvided { - return err - } - if s.tlsConfig == nil { - return errNoCertOrKeyProvided - } - s.tlsConfig.BuildNameToCertificate() - - return s.Serve( - tls.NewListener(ln, s.tlsConfig), - ) -} - -// ServeTLSEmbed serves HTTPS requests from the given listener. -// -// certData and keyData must contain valid TLS certificate and key data. -// -// If the certFile or keyFile has not been provided the server structure, -// the function will use previously added TLS configuration. -func (s *Server) ServeTLSEmbed(ln net.Listener, certData, keyData []byte) error { - err := s.AppendCertEmbed(certData, keyData) - if err != nil && err != errNoCertOrKeyProvided { - return err - } - if s.tlsConfig == nil { - return errNoCertOrKeyProvided - } - s.tlsConfig.BuildNameToCertificate() - - return s.Serve( - tls.NewListener(ln, s.tlsConfig), - ) -} - -// AppendCert appends certificate and keyfile to TLS Configuration. -// -// This function allows programmer to handle multiple domains -// in one server structure. See examples/multidomain -func (s *Server) AppendCert(certFile, keyFile string) error { - if len(certFile) == 0 && len(keyFile) == 0 { - return errNoCertOrKeyProvided - } - - cert, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - return fmt.Errorf("cannot load TLS key pair from certFile=%q and keyFile=%q: %s", certFile, keyFile, err) - } - - s.configTLS() - - s.tlsConfig.Certificates = append(s.tlsConfig.Certificates, cert) - return nil -} - -// AppendCertEmbed does the same as AppendCert but using in-memory data. -func (s *Server) AppendCertEmbed(certData, keyData []byte) error { - if len(certData) == 0 && len(keyData) == 0 { - return errNoCertOrKeyProvided - } - - cert, err := tls.X509KeyPair(certData, keyData) - if err != nil { - return fmt.Errorf("cannot load TLS key pair from the provided certData(%d) and keyData(%d): %s", - len(certData), len(keyData), err) - } - - s.configTLS() - - s.tlsConfig.Certificates = append(s.tlsConfig.Certificates, cert) - return nil -} - -func (s *Server) configTLS() { - if s.tlsConfig == nil { - s.tlsConfig = &tls.Config{ - PreferServerCipherSuites: true, - } - } -} - -// DefaultConcurrency is the maximum number of concurrent connections -// the Server may serve by default (i.e. if Server.Concurrency isn't set). -const DefaultConcurrency = 256 * 1024 - -// Serve serves incoming connections from the given listener. -// -// Serve blocks until the given listener returns permanent error. -func (s *Server) Serve(ln net.Listener) error { - var lastOverflowErrorTime time.Time - var lastPerIPErrorTime time.Time - var c net.Conn - var err error - - s.mu.Lock() - { - if s.ln != nil { - s.mu.Unlock() - return ErrAlreadyServing - } - - s.ln = ln - } - s.mu.Unlock() - - maxWorkersCount := s.getConcurrency() - s.concurrencyCh = make(chan struct{}, maxWorkersCount) - wp := &workerPool{ - WorkerFunc: s.serveConn, - MaxWorkersCount: maxWorkersCount, - LogAllErrors: s.LogAllErrors, - Logger: s.logger(), - connState: s.setState, - } - wp.Start() - - for { - if c, err = acceptConn(s, ln, &lastPerIPErrorTime); err != nil { - wp.Stop() - if err == io.EOF { - return nil - } - return err - } - s.setState(c, StateNew) - atomic.AddInt32(&s.open, 1) - if !wp.Serve(c) { - atomic.AddInt32(&s.open, -1) - s.writeFastError(c, StatusServiceUnavailable, - "The connection cannot be served because Server.Concurrency limit exceeded") - c.Close() - s.setState(c, StateClosed) - if time.Since(lastOverflowErrorTime) > time.Minute { - s.logger().Printf("The incoming connection cannot be served, because %d concurrent connections are served. "+ - "Try increasing Server.Concurrency", maxWorkersCount) - lastOverflowErrorTime = time.Now() - } - - // The current server reached concurrency limit, - // so give other concurrently running servers a chance - // accepting incoming connections on the same address. - // - // There is a hope other servers didn't reach their - // concurrency limits yet :) - time.Sleep(100 * time.Millisecond) - } - c = nil - } -} - -// Shutdown gracefully shuts down the server without interrupting any active connections. -// Shutdown works by first closing all open listeners and then waiting indefinitely for all connections to return to idle and then shut down. -// -// When Shutdown is called, Serve, ListenAndServe, and ListenAndServeTLS immediately return nil. -// Make sure the program doesn't exit and waits instead for Shutdown to return. -// -// Shutdown does not close keepalive connections so its recommended to set ReadTimeout to something else than 0. -func (s *Server) Shutdown() error { - s.mu.Lock() - defer s.mu.Unlock() - - atomic.StoreInt32(&s.stop, 1) - defer atomic.StoreInt32(&s.stop, 0) - - if s.ln == nil { - return nil - } - - if err := s.ln.Close(); err != nil { - return err - } - - // Closing the listener will make Serve() call Stop on the worker pool. - // Setting .stop to 1 will make serveConn() break out of its loop. - // Now we just have to wait until all workers are done. - for { - if open := atomic.LoadInt32(&s.open); open == 0 { - break - } - // This is not an optimal solution but using a sync.WaitGroup - // here causes data races as it's hard to prevent Add() to be called - // while Wait() is waiting. - time.Sleep(time.Millisecond * 100) - } - - s.ln = nil - return nil -} - -func acceptConn(s *Server, ln net.Listener, lastPerIPErrorTime *time.Time) (net.Conn, error) { - for { - c, err := ln.Accept() - if err != nil { - if c != nil { - panic("BUG: net.Listener returned non-nil conn and non-nil error") - } - if netErr, ok := err.(net.Error); ok && netErr.Temporary() { - s.logger().Printf("Temporary error when accepting new connections: %s", netErr) - time.Sleep(time.Second) - continue - } - if err != io.EOF && !strings.Contains(err.Error(), "use of closed network connection") { - s.logger().Printf("Permanent error when accepting new connections: %s", err) - return nil, err - } - return nil, io.EOF - } - if c == nil { - panic("BUG: net.Listener returned (nil, nil)") - } - if s.MaxConnsPerIP > 0 { - pic := wrapPerIPConn(s, c) - if pic == nil { - if time.Since(*lastPerIPErrorTime) > time.Minute { - s.logger().Printf("The number of connections from %s exceeds MaxConnsPerIP=%d", - getConnIP4(c), s.MaxConnsPerIP) - *lastPerIPErrorTime = time.Now() - } - continue - } - c = pic - } - return c, nil - } -} - -func wrapPerIPConn(s *Server, c net.Conn) net.Conn { - ip := getUint32IP(c) - if ip == 0 { - return c - } - n := s.perIPConnCounter.Register(ip) - if n > s.MaxConnsPerIP { - s.perIPConnCounter.Unregister(ip) - s.writeFastError(c, StatusTooManyRequests, "The number of connections from your ip exceeds MaxConnsPerIP") - c.Close() - return nil - } - return acquirePerIPConn(c, ip, &s.perIPConnCounter) -} - -var defaultLogger = Logger(log.New(os.Stderr, "", log.LstdFlags)) - -func (s *Server) logger() Logger { - if s.Logger != nil { - return s.Logger - } - return defaultLogger -} - -var ( - // ErrPerIPConnLimit may be returned from ServeConn if the number of connections - // per ip exceeds Server.MaxConnsPerIP. - ErrPerIPConnLimit = errors.New("too many connections per ip") - - // ErrConcurrencyLimit may be returned from ServeConn if the number - // of concurrently served connections exceeds Server.Concurrency. - ErrConcurrencyLimit = errors.New("cannot serve the connection because Server.Concurrency concurrent connections are served") - - // ErrKeepaliveTimeout is returned from ServeConn - // if the connection lifetime exceeds MaxKeepaliveDuration. - ErrKeepaliveTimeout = errors.New("exceeded MaxKeepaliveDuration") -) - -// ServeConn serves HTTP requests from the given connection. -// -// ServeConn returns nil if all requests from the c are successfully served. -// It returns non-nil error otherwise. -// -// Connection c must immediately propagate all the data passed to Write() -// to the client. Otherwise requests' processing may hang. -// -// ServeConn closes c before returning. -func (s *Server) ServeConn(c net.Conn) error { - if s.MaxConnsPerIP > 0 { - pic := wrapPerIPConn(s, c) - if pic == nil { - return ErrPerIPConnLimit - } - c = pic - } - - n := atomic.AddUint32(&s.concurrency, 1) - if n > uint32(s.getConcurrency()) { - atomic.AddUint32(&s.concurrency, ^uint32(0)) - s.writeFastError(c, StatusServiceUnavailable, "The connection cannot be served because Server.Concurrency limit exceeded") - c.Close() - return ErrConcurrencyLimit - } - - atomic.AddInt32(&s.open, 1) - - err := s.serveConn(c) - - atomic.AddUint32(&s.concurrency, ^uint32(0)) - - if err != errHijacked { - err1 := c.Close() - s.setState(c, StateClosed) - if err == nil { - err = err1 - } - } else { - err = nil - s.setState(c, StateHijacked) - } - return err -} - -var errHijacked = errors.New("connection has been hijacked") - -func (s *Server) getConcurrency() int { - n := s.Concurrency - if n <= 0 { - n = DefaultConcurrency - } - return n -} - -var globalConnID uint64 - -func nextConnID() uint64 { - return atomic.AddUint64(&globalConnID, 1) -} - -// DefaultMaxRequestBodySize is the maximum request body size the server -// reads by default. -// -// See Server.MaxRequestBodySize for details. -const DefaultMaxRequestBodySize = 4 * 1024 * 1024 - -func (s *Server) serveConn(c net.Conn) error { - defer atomic.AddInt32(&s.open, -1) - - if proto := s.hasNextProto(c); proto != "" { - handler, ok := s.nextProtos[proto] - if ok { - return handler(c) - } - } - - var serverName []byte - if !s.NoDefaultServerHeader { - serverName = s.getServerName() - } - connRequestNum := uint64(0) - connID := nextConnID() - currentTime := time.Now() - connTime := currentTime - maxRequestBodySize := s.MaxRequestBodySize - if maxRequestBodySize <= 0 { - maxRequestBodySize = DefaultMaxRequestBodySize - } - - ctx := s.acquireCtx(c) - ctx.connTime = connTime - isTLS := ctx.IsTLS() - var ( - br *bufio.Reader - bw *bufio.Writer - - err error - timeoutResponse *Response - hijackHandler HijackHandler - - lastReadDeadlineTime time.Time - lastWriteDeadlineTime time.Time - - connectionClose bool - isHTTP11 bool - ) - for { - if atomic.LoadInt32(&s.stop) == 1 { - err = nil - break - } - - connRequestNum++ - ctx.time = currentTime - - if s.ReadTimeout > 0 || s.MaxKeepaliveDuration > 0 { - lastReadDeadlineTime = s.updateReadDeadline(c, ctx, lastReadDeadlineTime) - if lastReadDeadlineTime.IsZero() { - err = ErrKeepaliveTimeout - break - } - } - - if !(s.ReduceMemoryUsage || ctx.lastReadDuration > time.Second) || br != nil { - if br == nil { - br = acquireReader(ctx) - } - } else { - br, err = acquireByteReader(&ctx) - } - ctx.Request.isTLS = isTLS - ctx.Response.Header.noDefaultContentType = s.NoDefaultContentType - - if err == nil { - if s.DisableHeaderNamesNormalizing { - ctx.Request.Header.DisableNormalizing() - ctx.Response.Header.DisableNormalizing() - } - // reading Headers and Body - err = ctx.Request.readLimitBody(br, maxRequestBodySize, s.GetOnly) - if br.Buffered() > 0 { - // If we read any bytes off the wire, we're active. - s.setState(c, StateActive) - } - if br.Buffered() == 0 || err != nil { - releaseReader(s, br) - br = nil - } - } - - currentTime = time.Now() - ctx.lastReadDuration = currentTime.Sub(ctx.time) - - if err != nil { - if err == io.EOF { - err = nil - } else { - bw = writeErrorResponse(bw, ctx, serverName, err) - } - break - } - - // 'Expect: 100-continue' request handling. - // See http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html for details. - if !ctx.Request.Header.ignoreBody() && ctx.Request.MayContinue() { - // Send 'HTTP/1.1 100 Continue' response. - if bw == nil { - bw = acquireWriter(ctx) - } - bw.Write(strResponseContinue) - err = bw.Flush() - releaseWriter(s, bw) - bw = nil - if err != nil { - break - } - - // Read request body. - if br == nil { - br = acquireReader(ctx) - } - err = ctx.Request.ContinueReadBody(br, maxRequestBodySize) - if br.Buffered() == 0 || err != nil { - releaseReader(s, br) - br = nil - } - if err != nil { - bw = writeErrorResponse(bw, ctx, serverName, err) - break - } - } - - connectionClose = s.DisableKeepalive || ctx.Request.Header.ConnectionClose() - isHTTP11 = ctx.Request.Header.IsHTTP11() - - if serverName != nil { - ctx.Response.Header.SetServerBytes(serverName) - } - ctx.connID = connID - ctx.connRequestNum = connRequestNum - ctx.time = currentTime - s.Handler(ctx) - - timeoutResponse = ctx.timeoutResponse - if timeoutResponse != nil { - ctx = s.acquireCtx(c) - timeoutResponse.CopyTo(&ctx.Response) - if br != nil { - // Close connection, since br may be attached to the old ctx via ctx.fbr. - ctx.SetConnectionClose() - } - } - - if !ctx.IsGet() && ctx.IsHead() { - ctx.Response.SkipBody = true - } - ctx.Request.Reset() - - hijackHandler = ctx.hijackHandler - ctx.hijackHandler = nil - - ctx.userValues.Reset() - - if s.MaxRequestsPerConn > 0 && connRequestNum >= uint64(s.MaxRequestsPerConn) { - ctx.SetConnectionClose() - } - - if s.WriteTimeout > 0 || s.MaxKeepaliveDuration > 0 { - lastWriteDeadlineTime = s.updateWriteDeadline(c, ctx, lastWriteDeadlineTime) - } - - connectionClose = connectionClose || ctx.Response.ConnectionClose() - if connectionClose { - ctx.Response.Header.SetCanonical(strConnection, strClose) - } else if !isHTTP11 { - // Set 'Connection: keep-alive' response header for non-HTTP/1.1 request. - // There is no need in setting this header for http/1.1, since in http/1.1 - // connections are keep-alive by default. - ctx.Response.Header.SetCanonical(strConnection, strKeepAlive) - } - - if serverName != nil && len(ctx.Response.Header.Server()) == 0 { - ctx.Response.Header.SetServerBytes(serverName) - } - - if bw == nil { - bw = acquireWriter(ctx) - } - if err = writeResponse(ctx, bw); err != nil { - break - } - - if br == nil || connectionClose { - err = bw.Flush() - releaseWriter(s, bw) - bw = nil - if err != nil { - break - } - if connectionClose { - break - } - } - - if hijackHandler != nil { - var hjr io.Reader - hjr = c - if br != nil { - hjr = br - br = nil - - // br may point to ctx.fbr, so do not return ctx into pool. - ctx = s.acquireCtx(c) - } - if bw != nil { - err = bw.Flush() - releaseWriter(s, bw) - bw = nil - if err != nil { - break - } - } - c.SetReadDeadline(zeroTime) - c.SetWriteDeadline(zeroTime) - go hijackConnHandler(hjr, c, s, hijackHandler) - hijackHandler = nil - err = errHijacked - break - } - - currentTime = time.Now() - s.setState(c, StateIdle) - } - - if br != nil { - releaseReader(s, br) - } - if bw != nil { - releaseWriter(s, bw) - } - s.releaseCtx(ctx) - return err -} - -func (s *Server) setState(nc net.Conn, state ConnState) { - if hook := s.ConnState; hook != nil { - hook(nc, state) - } -} - -func (s *Server) updateReadDeadline(c net.Conn, ctx *RequestCtx, lastDeadlineTime time.Time) time.Time { - readTimeout := s.ReadTimeout - currentTime := ctx.time - if s.MaxKeepaliveDuration > 0 { - connTimeout := s.MaxKeepaliveDuration - currentTime.Sub(ctx.connTime) - if connTimeout <= 0 { - return zeroTime - } - if connTimeout < readTimeout { - readTimeout = connTimeout - } - } - - // Optimization: update read deadline only if more than 25% - // of the last read deadline exceeded. - // See https://github.com/golang/go/issues/15133 for details. - if currentTime.Sub(lastDeadlineTime) > (readTimeout >> 2) { - if err := c.SetReadDeadline(currentTime.Add(readTimeout)); err != nil { - panic(fmt.Sprintf("BUG: error in SetReadDeadline(%s): %s", readTimeout, err)) - } - lastDeadlineTime = currentTime - } - return lastDeadlineTime -} - -func (s *Server) updateWriteDeadline(c net.Conn, ctx *RequestCtx, lastDeadlineTime time.Time) time.Time { - writeTimeout := s.WriteTimeout - if s.MaxKeepaliveDuration > 0 { - connTimeout := s.MaxKeepaliveDuration - time.Since(ctx.connTime) - if connTimeout <= 0 { - // MaxKeepAliveDuration exceeded, but let's try sending response anyway - // in 100ms with 'Connection: close' header. - ctx.SetConnectionClose() - connTimeout = 100 * time.Millisecond - } - if connTimeout < writeTimeout { - writeTimeout = connTimeout - } - } - - // Optimization: update write deadline only if more than 25% - // of the last write deadline exceeded. - // See https://github.com/golang/go/issues/15133 for details. - currentTime := time.Now() - if currentTime.Sub(lastDeadlineTime) > (writeTimeout >> 2) { - if err := c.SetWriteDeadline(currentTime.Add(writeTimeout)); err != nil { - panic(fmt.Sprintf("BUG: error in SetWriteDeadline(%s): %s", writeTimeout, err)) - } - lastDeadlineTime = currentTime - } - return lastDeadlineTime -} - -func hijackConnHandler(r io.Reader, c net.Conn, s *Server, h HijackHandler) { - hjc := s.acquireHijackConn(r, c) - h(hjc) - - if br, ok := r.(*bufio.Reader); ok { - releaseReader(s, br) - } - c.Close() - s.releaseHijackConn(hjc) -} - -func (s *Server) acquireHijackConn(r io.Reader, c net.Conn) *hijackConn { - v := s.hijackConnPool.Get() - if v == nil { - hjc := &hijackConn{ - Conn: c, - r: r, - } - return hjc - } - hjc := v.(*hijackConn) - hjc.Conn = c - hjc.r = r - return hjc -} - -func (s *Server) releaseHijackConn(hjc *hijackConn) { - hjc.Conn = nil - hjc.r = nil - s.hijackConnPool.Put(hjc) -} - -type hijackConn struct { - net.Conn - r io.Reader -} - -func (c hijackConn) Read(p []byte) (int, error) { - return c.r.Read(p) -} - -func (c hijackConn) Close() error { - // hijacked conn is closed in hijackConnHandler. - return nil -} - -// LastTimeoutErrorResponse returns the last timeout response set -// via TimeoutError* call. -// -// This function is intended for custom server implementations. -func (ctx *RequestCtx) LastTimeoutErrorResponse() *Response { - return ctx.timeoutResponse -} - -func writeResponse(ctx *RequestCtx, w *bufio.Writer) error { - if ctx.timeoutResponse != nil { - panic("BUG: cannot write timed out response") - } - err := ctx.Response.Write(w) - ctx.Response.Reset() - return err -} - -const ( - defaultReadBufferSize = 4096 - defaultWriteBufferSize = 4096 -) - -func acquireByteReader(ctxP **RequestCtx) (*bufio.Reader, error) { - ctx := *ctxP - s := ctx.s - c := ctx.c - t := ctx.time - s.releaseCtx(ctx) - - // Make GC happy, so it could garbage collect ctx - // while we waiting for the next request. - ctx = nil - *ctxP = nil - - v := s.bytePool.Get() - if v == nil { - v = make([]byte, 1) - } - b := v.([]byte) - n, err := c.Read(b) - ch := b[0] - s.bytePool.Put(v) - ctx = s.acquireCtx(c) - ctx.time = t - *ctxP = ctx - if err != nil { - // Treat all errors as EOF on unsuccessful read - // of the first request byte. - return nil, io.EOF - } - if n != 1 { - panic("BUG: Reader must return at least one byte") - } - - ctx.fbr.c = c - ctx.fbr.ch = ch - ctx.fbr.byteRead = false - r := acquireReader(ctx) - r.Reset(&ctx.fbr) - return r, nil -} - -func acquireReader(ctx *RequestCtx) *bufio.Reader { - v := ctx.s.readerPool.Get() - if v == nil { - n := ctx.s.ReadBufferSize - if n <= 0 { - n = defaultReadBufferSize - } - return bufio.NewReaderSize(ctx.c, n) - } - r := v.(*bufio.Reader) - r.Reset(ctx.c) - return r -} - -func releaseReader(s *Server, r *bufio.Reader) { - s.readerPool.Put(r) -} - -func acquireWriter(ctx *RequestCtx) *bufio.Writer { - v := ctx.s.writerPool.Get() - if v == nil { - n := ctx.s.WriteBufferSize - if n <= 0 { - n = defaultWriteBufferSize - } - return bufio.NewWriterSize(ctx.c, n) - } - w := v.(*bufio.Writer) - w.Reset(ctx.c) - return w -} - -func releaseWriter(s *Server, w *bufio.Writer) { - s.writerPool.Put(w) -} - -func (s *Server) acquireCtx(c net.Conn) (ctx *RequestCtx) { - v := s.ctxPool.Get() - if v == nil { - ctx = &RequestCtx{ - s: s, - } - keepBodyBuffer := !s.ReduceMemoryUsage - ctx.Request.keepBodyBuffer = keepBodyBuffer - ctx.Response.keepBodyBuffer = keepBodyBuffer - } else { - ctx = v.(*RequestCtx) - } - ctx.c = c - return -} - -// Init2 prepares ctx for passing to RequestHandler. -// -// conn is used only for determining local and remote addresses. -// -// This function is intended for custom Server implementations. -// See https://github.com/valyala/httpteleport for details. -func (ctx *RequestCtx) Init2(conn net.Conn, logger Logger, reduceMemoryUsage bool) { - ctx.c = conn - ctx.logger.logger = logger - ctx.connID = nextConnID() - ctx.s = fakeServer - ctx.connRequestNum = 0 - ctx.connTime = time.Now() - ctx.time = ctx.connTime - - keepBodyBuffer := !reduceMemoryUsage - ctx.Request.keepBodyBuffer = keepBodyBuffer - ctx.Response.keepBodyBuffer = keepBodyBuffer -} - -// Init prepares ctx for passing to RequestHandler. -// -// remoteAddr and logger are optional. They are used by RequestCtx.Logger(). -// -// This function is intended for custom Server implementations. -func (ctx *RequestCtx) Init(req *Request, remoteAddr net.Addr, logger Logger) { - if remoteAddr == nil { - remoteAddr = zeroTCPAddr - } - c := &fakeAddrer{ - laddr: zeroTCPAddr, - raddr: remoteAddr, - } - if logger == nil { - logger = defaultLogger - } - ctx.Init2(c, logger, true) - req.CopyTo(&ctx.Request) -} - -var fakeServer = &Server{ - // Initialize concurrencyCh for TimeoutHandler - concurrencyCh: make(chan struct{}, DefaultConcurrency), -} - -type fakeAddrer struct { - net.Conn - laddr net.Addr - raddr net.Addr -} - -func (fa *fakeAddrer) RemoteAddr() net.Addr { - return fa.raddr -} - -func (fa *fakeAddrer) LocalAddr() net.Addr { - return fa.laddr -} - -func (fa *fakeAddrer) Read(p []byte) (int, error) { - panic("BUG: unexpected Read call") -} - -func (fa *fakeAddrer) Write(p []byte) (int, error) { - panic("BUG: unexpected Write call") -} - -func (fa *fakeAddrer) Close() error { - panic("BUG: unexpected Close call") -} - -func (s *Server) releaseCtx(ctx *RequestCtx) { - if ctx.timeoutResponse != nil { - panic("BUG: cannot release timed out RequestCtx") - } - ctx.c = nil - ctx.fbr.c = nil - s.ctxPool.Put(ctx) -} - -func (s *Server) getServerName() []byte { - v := s.serverName.Load() - var serverName []byte - if v == nil { - serverName = []byte(s.Name) - if len(serverName) == 0 { - serverName = defaultServerName - } - s.serverName.Store(serverName) - } else { - serverName = v.([]byte) - } - return serverName -} - -func (s *Server) writeFastError(w io.Writer, statusCode int, msg string) { - w.Write(statusLine(statusCode)) - - server := "" - if !s.NoDefaultServerHeader { - server = fmt.Sprintf("Server: %s\r\n", s.getServerName()) - } - - fmt.Fprintf(w, "Connection: close\r\n"+ - server+ - "Date: %s\r\n"+ - "Content-Type: text/plain\r\n"+ - "Content-Length: %d\r\n"+ - "\r\n"+ - "%s", - serverDate.Load(), len(msg), msg) -} - -func writeErrorResponse(bw *bufio.Writer, ctx *RequestCtx, serverName []byte, err error) *bufio.Writer { - if _, ok := err.(*ErrSmallBuffer); ok { - ctx.Error("Too big request header", StatusRequestHeaderFieldsTooLarge) - } else { - ctx.Error("Error when parsing request", StatusBadRequest) - } - if serverName != nil { - ctx.Response.Header.SetServerBytes(serverName) - } - ctx.SetConnectionClose() - if bw == nil { - bw = acquireWriter(ctx) - } - writeResponse(ctx, bw) - bw.Flush() - return bw -} - -// A ConnState represents the state of a client connection to a server. -// It's used by the optional Server.ConnState hook. -type ConnState int - -const ( - // StateNew represents a new connection that is expected to - // send a request immediately. Connections begin at this - // state and then transition to either StateActive or - // StateClosed. - StateNew ConnState = iota - - // StateActive represents a connection that has read 1 or more - // bytes of a request. The Server.ConnState hook for - // StateActive fires before the request has entered a handler - // and doesn't fire again until the request has been - // handled. After the request is handled, the state - // transitions to StateClosed, StateHijacked, or StateIdle. - // For HTTP/2, StateActive fires on the transition from zero - // to one active request, and only transitions away once all - // active requests are complete. That means that ConnState - // cannot be used to do per-request work; ConnState only notes - // the overall state of the connection. - StateActive - - // StateIdle represents a connection that has finished - // handling a request and is in the keep-alive state, waiting - // for a new request. Connections transition from StateIdle - // to either StateActive or StateClosed. - StateIdle - - // StateHijacked represents a hijacked connection. - // This is a terminal state. It does not transition to StateClosed. - StateHijacked - - // StateClosed represents a closed connection. - // This is a terminal state. Hijacked connections do not - // transition to StateClosed. - StateClosed -) - -var stateName = map[ConnState]string{ - StateNew: "new", - StateActive: "active", - StateIdle: "idle", - StateHijacked: "hijacked", - StateClosed: "closed", -} - -func (c ConnState) String() string { - return stateName[c] -} diff --git a/vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.key b/vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.key deleted file mode 100644 index 00a79a3b..00000000 --- a/vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQD4IQusAs8PJdnG -3mURt/AXtgC+ceqLOatJ49JJE1VPTkMAy+oE1f1XvkMrYsHqmDf6GWVzgVXryL4U -wq2/nJSm56ddhN55nI8oSN3dtywUB8/ShelEN73nlN77PeD9tl6NksPwWaKrqxq0 -FlabRPZSQCfmgZbhDV8Sa8mfCkFU0G0lit6kLGceCKMvmW+9Bz7ebsYmVdmVMxmf -IJStFD44lWFTdUc65WISKEdW2ELcUefb0zOLw+0PCbXFGJH5x5ktksW8+BBk2Hkg -GeQRL/qPCccthbScO0VgNj3zJ3ZZL0ObSDAbvNDG85joeNjDNq5DT/BAZ0bOSbEF -sh+f9BAzAgMBAAECggEBAJWv2cq7Jw6MVwSRxYca38xuD6TUNBopgBvjREixURW2 -sNUaLuMb9Omp7fuOaE2N5rcJ+xnjPGIxh/oeN5MQctz9gwn3zf6vY+15h97pUb4D -uGvYPRDaT8YVGS+X9NMZ4ZCmqW2lpWzKnCFoGHcy8yZLbcaxBsRdvKzwOYGoPiFb -K2QuhXZ/1UPmqK9i2DFKtj40X6vBszTNboFxOVpXrPu0FJwLVSDf2hSZ4fMM0DH3 -YqwKcYf5te+hxGKgrqRA3tn0NCWii0in6QIwXMC+kMw1ebg/tZKqyDLMNptAK8J+ -DVw9m5X1seUHS5ehU/g2jrQrtK5WYn7MrFK4lBzlRwECgYEA/d1TeANYECDWRRDk -B0aaRZs87Rwl/J9PsvbsKvtU/bX+OfSOUjOa9iQBqn0LmU8GqusEET/QVUfocVwV -Bggf/5qDLxz100Rj0ags/yE/kNr0Bb31kkkKHFMnCT06YasR7qKllwrAlPJvQv9x -IzBKq+T/Dx08Wep9bCRSFhzRCnsCgYEA+jdeZXTDr/Vz+D2B3nAw1frqYFfGnEVY -wqmoK3VXMDkGuxsloO2rN+SyiUo3JNiQNPDub/t7175GH5pmKtZOlftePANsUjBj -wZ1D0rI5Bxu/71ibIUYIRVmXsTEQkh/ozoh3jXCZ9+bLgYiYx7789IUZZSokFQ3D -FICUT9KJ36kCgYAGoq9Y1rWJjmIrYfqj2guUQC+CfxbbGIrrwZqAsRsSmpwvhZ3m -tiSZxG0quKQB+NfSxdvQW5ulbwC7Xc3K35F+i9pb8+TVBdeaFkw+yu6vaZmxQLrX -fQM/pEjD7A7HmMIaO7QaU5SfEAsqdCTP56Y8AftMuNXn/8IRfo2KuGwaWwKBgFpU -ILzJoVdlad9E/Rw7LjYhZfkv1uBVXIyxyKcfrkEXZSmozDXDdxsvcZCEfVHM6Ipk -K/+7LuMcqp4AFEAEq8wTOdq6daFaHLkpt/FZK6M4TlruhtpFOPkoNc3e45eM83OT -6mziKINJC1CQ6m65sQHpBtjxlKMRG8rL/D6wx9s5AoGBAMRlqNPMwglT3hvDmsAt -9Lf9pdmhERUlHhD8bj8mDaBj2Aqv7f6VRJaYZqP403pKKQexuqcn80mtjkSAPFkN -Cj7BVt/RXm5uoxDTnfi26RF9F6yNDEJ7UU9+peBr99aazF/fTgW/1GcMkQnum8uV -c257YgaWmjK9uB0Y2r2VxS0G ------END PRIVATE KEY----- diff --git a/vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.pem b/vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.pem deleted file mode 100644 index 93e77cd9..00000000 --- a/vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.pem +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICujCCAaKgAwIBAgIJAMbXnKZ/cikUMA0GCSqGSIb3DQEBCwUAMBUxEzARBgNV -BAMTCnVidW50dS5uYW4wHhcNMTUwMjA0MDgwMTM5WhcNMjUwMjAxMDgwMTM5WjAV -MRMwEQYDVQQDEwp1YnVudHUubmFuMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEA+CELrALPDyXZxt5lEbfwF7YAvnHqizmrSePSSRNVT05DAMvqBNX9V75D -K2LB6pg3+hllc4FV68i+FMKtv5yUpuenXYTeeZyPKEjd3bcsFAfP0oXpRDe955Te -+z3g/bZejZLD8Fmiq6satBZWm0T2UkAn5oGW4Q1fEmvJnwpBVNBtJYrepCxnHgij -L5lvvQc+3m7GJlXZlTMZnyCUrRQ+OJVhU3VHOuViEihHVthC3FHn29Mzi8PtDwm1 -xRiR+ceZLZLFvPgQZNh5IBnkES/6jwnHLYW0nDtFYDY98yd2WS9Dm0gwG7zQxvOY -6HjYwzauQ0/wQGdGzkmxBbIfn/QQMwIDAQABow0wCzAJBgNVHRMEAjAAMA0GCSqG -SIb3DQEBCwUAA4IBAQBQjKm/4KN/iTgXbLTL3i7zaxYXFLXsnT1tF+ay4VA8aj98 -L3JwRTciZ3A5iy/W4VSCt3eASwOaPWHKqDBB5RTtL73LoAqsWmO3APOGQAbixcQ2 -45GXi05OKeyiYRi1Nvq7Unv9jUkRDHUYVPZVSAjCpsXzPhFkmZoTRxmx5l0ZF7Li -K91lI5h+eFq0dwZwrmlPambyh1vQUi70VHv8DNToVU29kel7YLbxGbuqETfhrcy6 -X+Mha6RYITkAn5FqsZcKMsc9eYGEF4l3XV+oS7q6xfTxktYJMFTI18J0lQ2Lv/CI -whdMnYGntDQBE/iFCrJEGNsKGc38796GBOb5j+zd ------END CERTIFICATE----- diff --git a/vendor/github.com/valyala/fasthttp/stackless/doc.go b/vendor/github.com/valyala/fasthttp/stackless/doc.go deleted file mode 100644 index 8c0cc497..00000000 --- a/vendor/github.com/valyala/fasthttp/stackless/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package stackless provides functionality that may save stack space -// for high number of concurrently running goroutines. -package stackless diff --git a/vendor/github.com/valyala/fasthttp/stackless/func.go b/vendor/github.com/valyala/fasthttp/stackless/func.go deleted file mode 100644 index 9a49bcc2..00000000 --- a/vendor/github.com/valyala/fasthttp/stackless/func.go +++ /dev/null @@ -1,79 +0,0 @@ -package stackless - -import ( - "runtime" - "sync" -) - -// NewFunc returns stackless wrapper for the function f. -// -// Unlike f, the returned stackless wrapper doesn't use stack space -// on the goroutine that calls it. -// The wrapper may save a lot of stack space if the following conditions -// are met: -// -// - f doesn't contain blocking calls on network, I/O or channels; -// - f uses a lot of stack space; -// - the wrapper is called from high number of concurrent goroutines. -// -// The stackless wrapper returns false if the call cannot be processed -// at the moment due to high load. -func NewFunc(f func(ctx interface{})) func(ctx interface{}) bool { - if f == nil { - panic("BUG: f cannot be nil") - } - - funcWorkCh := make(chan *funcWork, runtime.GOMAXPROCS(-1)*2048) - onceInit := func() { - n := runtime.GOMAXPROCS(-1) - for i := 0; i < n; i++ { - go funcWorker(funcWorkCh, f) - } - } - var once sync.Once - - return func(ctx interface{}) bool { - once.Do(onceInit) - fw := getFuncWork() - fw.ctx = ctx - - select { - case funcWorkCh <- fw: - default: - putFuncWork(fw) - return false - } - <-fw.done - putFuncWork(fw) - return true - } -} - -func funcWorker(funcWorkCh <-chan *funcWork, f func(ctx interface{})) { - for fw := range funcWorkCh { - f(fw.ctx) - fw.done <- struct{}{} - } -} - -func getFuncWork() *funcWork { - v := funcWorkPool.Get() - if v == nil { - v = &funcWork{ - done: make(chan struct{}, 1), - } - } - return v.(*funcWork) -} - -func putFuncWork(fw *funcWork) { - fw.ctx = nil - funcWorkPool.Put(fw) -} - -var funcWorkPool sync.Pool - -type funcWork struct { - ctx interface{} - done chan struct{} -} diff --git a/vendor/github.com/valyala/fasthttp/stackless/writer.go b/vendor/github.com/valyala/fasthttp/stackless/writer.go deleted file mode 100644 index c2053f9a..00000000 --- a/vendor/github.com/valyala/fasthttp/stackless/writer.go +++ /dev/null @@ -1,139 +0,0 @@ -package stackless - -import ( - "errors" - "fmt" - "io" - - "github.com/valyala/bytebufferpool" -) - -// Writer is an interface stackless writer must conform to. -// -// The interface contains common subset for Writers from compress/* packages. -type Writer interface { - Write(p []byte) (int, error) - Flush() error - Close() error - Reset(w io.Writer) -} - -// NewWriterFunc must return new writer that will be wrapped into -// stackless writer. -type NewWriterFunc func(w io.Writer) Writer - -// NewWriter creates a stackless writer around a writer returned -// from newWriter. -// -// The returned writer writes data to dstW. -// -// Writers that use a lot of stack space may be wrapped into stackless writer, -// thus saving stack space for high number of concurrently running goroutines. -func NewWriter(dstW io.Writer, newWriter NewWriterFunc) Writer { - w := &writer{ - dstW: dstW, - } - w.zw = newWriter(&w.xw) - return w -} - -type writer struct { - dstW io.Writer - zw Writer - xw xWriter - - err error - n int - - p []byte - op op -} - -type op int - -const ( - opWrite op = iota - opFlush - opClose - opReset -) - -func (w *writer) Write(p []byte) (int, error) { - w.p = p - err := w.do(opWrite) - w.p = nil - return w.n, err -} - -func (w *writer) Flush() error { - return w.do(opFlush) -} - -func (w *writer) Close() error { - return w.do(opClose) -} - -func (w *writer) Reset(dstW io.Writer) { - w.xw.Reset() - w.do(opReset) - w.dstW = dstW -} - -func (w *writer) do(op op) error { - w.op = op - if !stacklessWriterFunc(w) { - return errHighLoad - } - err := w.err - if err != nil { - return err - } - if w.xw.bb != nil && len(w.xw.bb.B) > 0 { - _, err = w.dstW.Write(w.xw.bb.B) - } - w.xw.Reset() - - return err -} - -var errHighLoad = errors.New("cannot compress data due to high load") - -var stacklessWriterFunc = NewFunc(writerFunc) - -func writerFunc(ctx interface{}) { - w := ctx.(*writer) - switch w.op { - case opWrite: - w.n, w.err = w.zw.Write(w.p) - case opFlush: - w.err = w.zw.Flush() - case opClose: - w.err = w.zw.Close() - case opReset: - w.zw.Reset(&w.xw) - w.err = nil - default: - panic(fmt.Sprintf("BUG: unexpected op: %d", w.op)) - } -} - -type xWriter struct { - bb *bytebufferpool.ByteBuffer -} - -func (w *xWriter) Write(p []byte) (int, error) { - if w.bb == nil { - w.bb = bufferPool.Get() - } - w.bb.Write(p) - return len(p), nil -} - -func (w *xWriter) Reset() { - if w.bb != nil { - bufferPool.Put(w.bb) - w.bb = nil - } -} - -var bufferPool bytebufferpool.Pool diff --git a/vendor/github.com/valyala/fasthttp/status.go b/vendor/github.com/valyala/fasthttp/status.go deleted file mode 100644 index 6687efb4..00000000 --- a/vendor/github.com/valyala/fasthttp/status.go +++ /dev/null @@ -1,176 +0,0 @@ -package fasthttp - -import ( - "fmt" - "sync/atomic" -) - -// HTTP status codes were stolen from net/http. -const ( - StatusContinue = 100 // RFC 7231, 6.2.1 - StatusSwitchingProtocols = 101 // RFC 7231, 6.2.2 - StatusProcessing = 102 // RFC 2518, 10.1 - - StatusOK = 200 // RFC 7231, 6.3.1 - StatusCreated = 201 // RFC 7231, 6.3.2 - StatusAccepted = 202 // RFC 7231, 6.3.3 - StatusNonAuthoritativeInfo = 203 // RFC 7231, 6.3.4 - StatusNoContent = 204 // RFC 7231, 6.3.5 - StatusResetContent = 205 // RFC 7231, 6.3.6 - StatusPartialContent = 206 // RFC 7233, 4.1 - StatusMultiStatus = 207 // RFC 4918, 11.1 - StatusAlreadyReported = 208 // RFC 5842, 7.1 - StatusIMUsed = 226 // RFC 3229, 10.4.1 - - StatusMultipleChoices = 300 // RFC 7231, 6.4.1 - StatusMovedPermanently = 301 // RFC 7231, 6.4.2 - StatusFound = 302 // RFC 7231, 6.4.3 - StatusSeeOther = 303 // RFC 7231, 6.4.4 - StatusNotModified = 304 // RFC 7232, 4.1 - StatusUseProxy = 305 // RFC 7231, 6.4.5 - _ = 306 // RFC 7231, 6.4.6 (Unused) - StatusTemporaryRedirect = 307 // RFC 7231, 6.4.7 - StatusPermanentRedirect = 308 // RFC 7538, 3 - - StatusBadRequest = 400 // RFC 7231, 6.5.1 - StatusUnauthorized = 401 // RFC 7235, 3.1 - StatusPaymentRequired = 402 // RFC 7231, 6.5.2 - StatusForbidden = 403 // RFC 7231, 6.5.3 - StatusNotFound = 404 // RFC 7231, 6.5.4 - StatusMethodNotAllowed = 405 // RFC 7231, 6.5.5 - StatusNotAcceptable = 406 // RFC 7231, 6.5.6 - StatusProxyAuthRequired = 407 // RFC 7235, 3.2 - StatusRequestTimeout = 408 // RFC 7231, 6.5.7 - StatusConflict = 409 // RFC 7231, 6.5.8 - StatusGone = 410 // RFC 7231, 6.5.9 - StatusLengthRequired = 411 // RFC 7231, 6.5.10 - StatusPreconditionFailed = 412 // RFC 7232, 4.2 - StatusRequestEntityTooLarge = 413 // RFC 7231, 6.5.11 - StatusRequestURITooLong = 414 // RFC 7231, 6.5.12 - StatusUnsupportedMediaType = 415 // RFC 7231, 6.5.13 - StatusRequestedRangeNotSatisfiable = 416 // RFC 7233, 4.4 - StatusExpectationFailed = 417 // RFC 7231, 6.5.14 - StatusTeapot = 418 // RFC 7168, 2.3.3 - StatusUnprocessableEntity = 422 // RFC 4918, 11.2 - StatusLocked = 423 // RFC 4918, 11.3 - StatusFailedDependency = 424 // RFC 4918, 11.4 - StatusUpgradeRequired = 426 // RFC 7231, 6.5.15 - StatusPreconditionRequired = 428 // RFC 6585, 3 - StatusTooManyRequests = 429 // RFC 6585, 4 - StatusRequestHeaderFieldsTooLarge = 431 // RFC 6585, 5 - StatusUnavailableForLegalReasons = 451 // RFC 7725, 3 - - StatusInternalServerError = 500 // RFC 7231, 6.6.1 - StatusNotImplemented = 501 // RFC 7231, 6.6.2 - StatusBadGateway = 502 // RFC 7231, 6.6.3 - StatusServiceUnavailable = 503 // RFC 7231, 6.6.4 - StatusGatewayTimeout = 504 // RFC 7231, 6.6.5 - StatusHTTPVersionNotSupported = 505 // RFC 7231, 6.6.6 - StatusVariantAlsoNegotiates = 506 // RFC 2295, 8.1 - StatusInsufficientStorage = 507 // RFC 4918, 11.5 - StatusLoopDetected = 508 // RFC 5842, 7.2 - StatusNotExtended = 510 // RFC 2774, 7 - StatusNetworkAuthenticationRequired = 511 // RFC 6585, 6 -) - -var ( - statusLines atomic.Value - - statusMessages = map[int]string{ - StatusContinue: "Continue", - StatusSwitchingProtocols: "Switching Protocols", - StatusProcessing: "Processing", - - StatusOK: "OK", - StatusCreated: "Created", - StatusAccepted: "Accepted", - StatusNonAuthoritativeInfo: "Non-Authoritative Information", - StatusNoContent: "No Content", - StatusResetContent: "Reset Content", - StatusPartialContent: "Partial Content", - StatusMultiStatus: "Multi-Status", - StatusAlreadyReported: "Already Reported", - StatusIMUsed: "IM Used", - - StatusMultipleChoices: "Multiple Choices", - StatusMovedPermanently: "Moved Permanently", - StatusFound: "Found", - StatusSeeOther: "See Other", - StatusNotModified: "Not Modified", - StatusUseProxy: "Use Proxy", - StatusTemporaryRedirect: "Temporary Redirect", - StatusPermanentRedirect: "Permanent Redirect", - - StatusBadRequest: "Bad Request", - StatusUnauthorized: "Unauthorized", - StatusPaymentRequired: "Payment Required", - StatusForbidden: "Forbidden", - StatusNotFound: "Not Found", - StatusMethodNotAllowed: "Method Not Allowed", - StatusNotAcceptable: "Not Acceptable", - StatusProxyAuthRequired: "Proxy Authentication Required", - StatusRequestTimeout: "Request Timeout", - StatusConflict: "Conflict", - StatusGone: "Gone", - StatusLengthRequired: "Length Required", - StatusPreconditionFailed: "Precondition Failed", - StatusRequestEntityTooLarge: "Request Entity Too Large", - StatusRequestURITooLong: "Request URI Too Long", - StatusUnsupportedMediaType: "Unsupported Media Type", - StatusRequestedRangeNotSatisfiable: "Requested Range Not Satisfiable", - StatusExpectationFailed: "Expectation Failed", - StatusTeapot: "I'm a teapot", - StatusUnprocessableEntity: "Unprocessable Entity", - StatusLocked: "Locked", - StatusFailedDependency: "Failed Dependency", - StatusUpgradeRequired: "Upgrade Required", - StatusPreconditionRequired: "Precondition Required", - StatusTooManyRequests: "Too Many Requests", - StatusRequestHeaderFieldsTooLarge: "Request Header Fields Too Large", - StatusUnavailableForLegalReasons: "Unavailable For Legal Reasons", - - StatusInternalServerError: "Internal Server Error", - StatusNotImplemented: "Not Implemented", - StatusBadGateway: "Bad Gateway", - StatusServiceUnavailable: "Service Unavailable", - StatusGatewayTimeout: "Gateway Timeout", - StatusHTTPVersionNotSupported: "HTTP Version Not Supported", - StatusVariantAlsoNegotiates: "Variant Also Negotiates", - StatusInsufficientStorage: "Insufficient Storage", - StatusLoopDetected: "Loop Detected", - StatusNotExtended: "Not Extended", - StatusNetworkAuthenticationRequired: "Network Authentication Required", - } -) - -// StatusMessage returns HTTP status message for the given status code. -func StatusMessage(statusCode int) string { - s := statusMessages[statusCode] - if s == "" { - s = "Unknown Status Code" - } - return s -} - -func init() { - statusLines.Store(make(map[int][]byte)) -} - -func statusLine(statusCode int) []byte { - m := statusLines.Load().(map[int][]byte) - h := m[statusCode] - if h != nil { - return h - } - - statusText := StatusMessage(statusCode) - - h = []byte(fmt.Sprintf("HTTP/1.1 %d %s\r\n", statusCode, statusText)) - newM := make(map[int][]byte, len(m)+1) - for k, v := range m { - newM[k] = v - } - newM[statusCode] = h - statusLines.Store(newM) - return h -} diff --git a/vendor/github.com/valyala/fasthttp/stream.go b/vendor/github.com/valyala/fasthttp/stream.go deleted file mode 100644 index aa23b1af..00000000 --- a/vendor/github.com/valyala/fasthttp/stream.go +++ /dev/null @@ -1,54 +0,0 @@ -package fasthttp - -import ( - "bufio" - "io" - "sync" - - "github.com/valyala/fasthttp/fasthttputil" -) - -// StreamWriter must write data to w. -// -// Usually StreamWriter writes data to w in a loop (aka 'data streaming'). -// -// StreamWriter must return immediately if w returns error. -// -// Since the written data is buffered, do not forget calling w.Flush -// when the data must be propagated to reader. -type StreamWriter func(w *bufio.Writer) - -// NewStreamReader returns a reader, which replays all the data generated by sw. -// -// The returned reader may be passed to Response.SetBodyStream. -// -// Close must be called on the returned reader after all the required data -// has been read. Otherwise goroutine leak may occur. -// -// See also Response.SetBodyStreamWriter. -func NewStreamReader(sw StreamWriter) io.ReadCloser { - pc := fasthttputil.NewPipeConns() - pw := pc.Conn1() - pr := pc.Conn2() - - var bw *bufio.Writer - v := streamWriterBufPool.Get() - if v == nil { - bw = bufio.NewWriter(pw) - } else { - bw = v.(*bufio.Writer) - bw.Reset(pw) - } - - go func() { - sw(bw) - bw.Flush() - pw.Close() - - streamWriterBufPool.Put(bw) - }() - - return pr -} - -var streamWriterBufPool sync.Pool diff --git a/vendor/github.com/valyala/fasthttp/strings.go b/vendor/github.com/valyala/fasthttp/strings.go deleted file mode 100644 index afd70b40..00000000 --- a/vendor/github.com/valyala/fasthttp/strings.go +++ /dev/null @@ -1,78 +0,0 @@ -package fasthttp - -var ( - defaultServerName = []byte("fasthttp") - defaultUserAgent = []byte("fasthttp") - defaultContentType = []byte("text/plain; charset=utf-8") -) - -var ( - strSlash = []byte("/") - strSlashSlash = []byte("//") - strSlashDotDot = []byte("/..") - strSlashDotSlash = []byte("/./") - strSlashDotDotSlash = []byte("/../") - strCRLF = []byte("\r\n") - strHTTP = []byte("http") - strHTTPS = []byte("https") - strHTTP11 = []byte("HTTP/1.1") - strColonSlashSlash = []byte("://") - strColonSpace = []byte(": ") - strGMT = []byte("GMT") - - strResponseContinue = []byte("HTTP/1.1 100 Continue\r\n\r\n") - - strGet = []byte("GET") - strHead = []byte("HEAD") - strPost = []byte("POST") - strPut = []byte("PUT") - strDelete = []byte("DELETE") - strConnect = []byte("CONNECT") - strOptions = []byte("OPTIONS") - strTrace = []byte("TRACE") - strPatch = []byte("PATCH") - - strExpect = []byte("Expect") - strConnection = []byte("Connection") - strContentLength = []byte("Content-Length") - strContentType = []byte("Content-Type") - strDate = []byte("Date") - strHost = []byte("Host") - strReferer = []byte("Referer") - strServer = []byte("Server") - strTransferEncoding = []byte("Transfer-Encoding") - strContentEncoding = []byte("Content-Encoding") - strAcceptEncoding = []byte("Accept-Encoding") - strUserAgent = []byte("User-Agent") - strCookie = []byte("Cookie") - strSetCookie = []byte("Set-Cookie") - strLocation = []byte("Location") - strIfModifiedSince = []byte("If-Modified-Since") - strLastModified = []byte("Last-Modified") - strAcceptRanges = []byte("Accept-Ranges") - strRange = []byte("Range") - strContentRange = []byte("Content-Range") - - strCookieExpires = []byte("expires") - strCookieDomain = []byte("domain") - strCookiePath = []byte("path") - strCookieHTTPOnly = []byte("HttpOnly") - strCookieSecure = []byte("secure") - strCookieMaxAge = []byte("max-age") - - strClose = []byte("close") - strGzip = []byte("gzip") - strDeflate = []byte("deflate") - strKeepAlive = []byte("keep-alive") - strKeepAliveCamelCase = []byte("Keep-Alive") - strUpgrade = []byte("Upgrade") - strChunked = []byte("chunked") - strIdentity = []byte("identity") - str100Continue = []byte("100-continue") - strPostArgsContentType = []byte("application/x-www-form-urlencoded") - strMultipartFormData = []byte("multipart/form-data") - strBoundary = []byte("boundary") - strBytes = []byte("bytes") - strTextSlash = []byte("text/") - strApplicationSlash = []byte("application/") -) diff --git a/vendor/github.com/valyala/fasthttp/tcpdialer.go b/vendor/github.com/valyala/fasthttp/tcpdialer.go deleted file mode 100644 index e31fd758..00000000 --- a/vendor/github.com/valyala/fasthttp/tcpdialer.go +++ /dev/null @@ -1,369 +0,0 @@ -package fasthttp - -import ( - "errors" - "net" - "strconv" - "sync" - "sync/atomic" - "time" -) - -// Dial dials the given TCP addr using tcp4. -// -// This function has the following additional features comparing to net.Dial: -// -// * It reduces load on DNS resolver by caching resolved TCP addressed -// for DefaultDNSCacheDuration. -// * It dials all the resolved TCP addresses in round-robin manner until -// connection is established. This may be useful if certain addresses -// are temporarily unreachable. -// * It returns ErrDialTimeout if connection cannot be established during -// DefaultDialTimeout seconds. Use DialTimeout for customizing dial timeout. -// -// This dialer is intended for custom code wrapping before passing -// to Client.Dial or HostClient.Dial. -// -// For instance, per-host counters and/or limits may be implemented -// by such wrappers. -// -// The addr passed to the function must contain port. Example addr values: -// -// * foobar.baz:443 -// * foo.bar:80 -// * aaa.com:8080 -func Dial(addr string) (net.Conn, error) { - return getDialer(DefaultDialTimeout, false)(addr) -} - -// DialTimeout dials the given TCP addr using tcp4 using the given timeout. -// -// This function has the following additional features comparing to net.Dial: -// -// * It reduces load on DNS resolver by caching resolved TCP addressed -// for DefaultDNSCacheDuration. -// * It dials all the resolved TCP addresses in round-robin manner until -// connection is established. This may be useful if certain addresses -// are temporarily unreachable. -// -// This dialer is intended for custom code wrapping before passing -// to Client.Dial or HostClient.Dial. -// -// For instance, per-host counters and/or limits may be implemented -// by such wrappers. -// -// The addr passed to the function must contain port. Example addr values: -// -// * foobar.baz:443 -// * foo.bar:80 -// * aaa.com:8080 -func DialTimeout(addr string, timeout time.Duration) (net.Conn, error) { - return getDialer(timeout, false)(addr) -} - -// DialDualStack dials the given TCP addr using both tcp4 and tcp6. -// -// This function has the following additional features comparing to net.Dial: -// -// * It reduces load on DNS resolver by caching resolved TCP addressed -// for DefaultDNSCacheDuration. -// * It dials all the resolved TCP addresses in round-robin manner until -// connection is established. This may be useful if certain addresses -// are temporarily unreachable. -// * It returns ErrDialTimeout if connection cannot be established during -// DefaultDialTimeout seconds. Use DialDualStackTimeout for custom dial -// timeout. -// -// This dialer is intended for custom code wrapping before passing -// to Client.Dial or HostClient.Dial. -// -// For instance, per-host counters and/or limits may be implemented -// by such wrappers. -// -// The addr passed to the function must contain port. Example addr values: -// -// * foobar.baz:443 -// * foo.bar:80 -// * aaa.com:8080 -func DialDualStack(addr string) (net.Conn, error) { - return getDialer(DefaultDialTimeout, true)(addr) -} - -// DialDualStackTimeout dials the given TCP addr using both tcp4 and tcp6 -// using the given timeout. -// -// This function has the following additional features comparing to net.Dial: -// -// * It reduces load on DNS resolver by caching resolved TCP addressed -// for DefaultDNSCacheDuration. -// * It dials all the resolved TCP addresses in round-robin manner until -// connection is established. This may be useful if certain addresses -// are temporarily unreachable. -// -// This dialer is intended for custom code wrapping before passing -// to Client.Dial or HostClient.Dial. -// -// For instance, per-host counters and/or limits may be implemented -// by such wrappers. -// -// The addr passed to the function must contain port. Example addr values: -// -// * foobar.baz:443 -// * foo.bar:80 -// * aaa.com:8080 -func DialDualStackTimeout(addr string, timeout time.Duration) (net.Conn, error) { - return getDialer(timeout, true)(addr) -} - -func getDialer(timeout time.Duration, dualStack bool) DialFunc { - if timeout <= 0 { - timeout = DefaultDialTimeout - } - timeoutRounded := int(timeout.Seconds()*10 + 9) - - m := dialMap - if dualStack { - m = dialDualStackMap - } - - dialMapLock.Lock() - d := m[timeoutRounded] - if d == nil { - dialer := dialerStd - if dualStack { - dialer = dialerDualStack - } - d = dialer.NewDial(timeout) - m[timeoutRounded] = d - } - dialMapLock.Unlock() - return d -} - -var ( - dialerStd = &tcpDialer{} - dialerDualStack = &tcpDialer{DualStack: true} - - dialMap = make(map[int]DialFunc) - dialDualStackMap = make(map[int]DialFunc) - dialMapLock sync.Mutex -) - -type tcpDialer struct { - DualStack bool - - tcpAddrsLock sync.Mutex - tcpAddrsMap map[string]*tcpAddrEntry - - concurrencyCh chan struct{} - - once sync.Once -} - -const maxDialConcurrency = 1000 - -func (d *tcpDialer) NewDial(timeout time.Duration) DialFunc { - d.once.Do(func() { - d.concurrencyCh = make(chan struct{}, maxDialConcurrency) - d.tcpAddrsMap = make(map[string]*tcpAddrEntry) - go d.tcpAddrsClean() - }) - - return func(addr string) (net.Conn, error) { - addrs, idx, err := d.getTCPAddrs(addr) - if err != nil { - return nil, err - } - network := "tcp4" - if d.DualStack { - network = "tcp" - } - - var conn net.Conn - n := uint32(len(addrs)) - deadline := time.Now().Add(timeout) - for n > 0 { - conn, err = tryDial(network, &addrs[idx%n], deadline, d.concurrencyCh) - if err == nil { - return conn, nil - } - if err == ErrDialTimeout { - return nil, err - } - idx++ - n-- - } - return nil, err - } -} - -func tryDial(network string, addr *net.TCPAddr, deadline time.Time, concurrencyCh chan struct{}) (net.Conn, error) { - timeout := -time.Since(deadline) - if timeout <= 0 { - return nil, ErrDialTimeout - } - - select { - case concurrencyCh <- struct{}{}: - default: - tc := acquireTimer(timeout) - isTimeout := false - select { - case concurrencyCh <- struct{}{}: - case <-tc.C: - isTimeout = true - } - releaseTimer(tc) - if isTimeout { - return nil, ErrDialTimeout - } - } - - timeout = -time.Since(deadline) - if timeout <= 0 { - <-concurrencyCh - return nil, ErrDialTimeout - } - - chv := dialResultChanPool.Get() - if chv == nil { - chv = make(chan dialResult, 1) - } - ch := chv.(chan dialResult) - go func() { - var dr dialResult - dr.conn, dr.err = net.DialTCP(network, nil, addr) - ch <- dr - <-concurrencyCh - }() - - var ( - conn net.Conn - err error - ) - - tc := acquireTimer(timeout) - select { - case dr := <-ch: - conn = dr.conn - err = dr.err - dialResultChanPool.Put(ch) - case <-tc.C: - err = ErrDialTimeout - } - releaseTimer(tc) - - return conn, err -} - -var dialResultChanPool sync.Pool - -type dialResult struct { - conn net.Conn - err error -} - -// ErrDialTimeout is returned when TCP dialing is timed out. -var ErrDialTimeout = errors.New("dialing to the given TCP address timed out") - -// DefaultDialTimeout is timeout used by Dial and DialDualStack -// for establishing TCP connections. -const DefaultDialTimeout = 3 * time.Second - -type tcpAddrEntry struct { - addrs []net.TCPAddr - addrsIdx uint32 - - resolveTime time.Time - pending bool -} - -// DefaultDNSCacheDuration is the duration for caching resolved TCP addresses -// by Dial* functions. -const DefaultDNSCacheDuration = time.Minute - -func (d *tcpDialer) tcpAddrsClean() { - expireDuration := 2 * DefaultDNSCacheDuration - for { - time.Sleep(time.Second) - t := time.Now() - - d.tcpAddrsLock.Lock() - for k, e := range d.tcpAddrsMap { - if t.Sub(e.resolveTime) > expireDuration { - delete(d.tcpAddrsMap, k) - } - } - d.tcpAddrsLock.Unlock() - } -} - -func (d *tcpDialer) getTCPAddrs(addr string) ([]net.TCPAddr, uint32, error) { - d.tcpAddrsLock.Lock() - e := d.tcpAddrsMap[addr] - if e != nil && !e.pending && time.Since(e.resolveTime) > DefaultDNSCacheDuration { - e.pending = true - e = nil - } - d.tcpAddrsLock.Unlock() - - if e == nil { - addrs, err := resolveTCPAddrs(addr, d.DualStack) - if err != nil { - d.tcpAddrsLock.Lock() - e = d.tcpAddrsMap[addr] - if e != nil && e.pending { - e.pending = false - } - d.tcpAddrsLock.Unlock() - return nil, 0, err - } - - e = &tcpAddrEntry{ - addrs: addrs, - resolveTime: time.Now(), - } - - d.tcpAddrsLock.Lock() - d.tcpAddrsMap[addr] = e - d.tcpAddrsLock.Unlock() - } - - idx := atomic.AddUint32(&e.addrsIdx, 1) - return e.addrs, idx, nil -} - -func resolveTCPAddrs(addr string, dualStack bool) ([]net.TCPAddr, error) { - host, portS, err := net.SplitHostPort(addr) - if err != nil { - return nil, err - } - port, err := strconv.Atoi(portS) - if err != nil { - return nil, err - } - - ips, err := net.LookupIP(host) - if err != nil { - return nil, err - } - - n := len(ips) - addrs := make([]net.TCPAddr, 0, n) - for i := 0; i < n; i++ { - ip := ips[i] - if !dualStack && ip.To4() == nil { - continue - } - addrs = append(addrs, net.TCPAddr{ - IP: ip, - Port: port, - }) - } - if len(addrs) == 0 { - return nil, errNoDNSEntries - } - return addrs, nil -} - -var errNoDNSEntries = errors.New("couldn't find DNS entries for the given domain. Try using DialDualStack") diff --git a/vendor/github.com/valyala/fasthttp/timer.go b/vendor/github.com/valyala/fasthttp/timer.go deleted file mode 100644 index bb12acb7..00000000 --- a/vendor/github.com/valyala/fasthttp/timer.go +++ /dev/null @@ -1,44 +0,0 @@ -package fasthttp - -import ( - "sync" - "time" -) - -func initTimer(t *time.Timer, timeout time.Duration) *time.Timer { - if t == nil { - return time.NewTimer(timeout) - } - if t.Reset(timeout) { - panic("BUG: active timer trapped into initTimer()") - } - return t -} - -func stopTimer(t *time.Timer) { - if !t.Stop() { - // Collect possibly added time from the channel - // if timer has been stopped and nobody collected its' value. - select { - case <-t.C: - default: - } - } -} - -func acquireTimer(timeout time.Duration) *time.Timer { - v := timerPool.Get() - if v == nil { - return time.NewTimer(timeout) - } - t := v.(*time.Timer) - initTimer(t, timeout) - return t -} - -func releaseTimer(t *time.Timer) { - stopTimer(t) - timerPool.Put(t) -} - -var timerPool sync.Pool diff --git a/vendor/github.com/valyala/fasthttp/uri.go b/vendor/github.com/valyala/fasthttp/uri.go deleted file mode 100644 index d536f593..00000000 --- a/vendor/github.com/valyala/fasthttp/uri.go +++ /dev/null @@ -1,525 +0,0 @@ -package fasthttp - -import ( - "bytes" - "io" - "sync" -) - -// AcquireURI returns an empty URI instance from the pool. -// -// Release the URI with ReleaseURI after the URI is no longer needed. -// This allows reducing GC load. -func AcquireURI() *URI { - return uriPool.Get().(*URI) -} - -// ReleaseURI releases the URI acquired via AcquireURI. -// -// The released URI mustn't be used after releasing it, otherwise data races -// may occur. -func ReleaseURI(u *URI) { - u.Reset() - uriPool.Put(u) -} - -var uriPool = &sync.Pool{ - New: func() interface{} { - return &URI{} - }, -} - -// URI represents URI :) . -// -// It is forbidden copying URI instances. Create new instance and use CopyTo -// instead. -// -// URI instance MUST NOT be used from concurrently running goroutines. -type URI struct { - noCopy noCopy - - pathOriginal []byte - scheme []byte - path []byte - queryString []byte - hash []byte - host []byte - - queryArgs Args - parsedQueryArgs bool - - fullURI []byte - requestURI []byte - - h *RequestHeader -} - -// CopyTo copies uri contents to dst. -func (u *URI) CopyTo(dst *URI) { - dst.Reset() - dst.pathOriginal = append(dst.pathOriginal[:0], u.pathOriginal...) - dst.scheme = append(dst.scheme[:0], u.scheme...) - dst.path = append(dst.path[:0], u.path...) - dst.queryString = append(dst.queryString[:0], u.queryString...) - dst.hash = append(dst.hash[:0], u.hash...) - dst.host = append(dst.host[:0], u.host...) - - u.queryArgs.CopyTo(&dst.queryArgs) - dst.parsedQueryArgs = u.parsedQueryArgs - - // fullURI and requestURI shouldn't be copied, since they are created - // from scratch on each FullURI() and RequestURI() call. - dst.h = u.h -} - -// Hash returns URI hash, i.e. qwe of http://aaa.com/foo/bar?baz=123#qwe . -// -// The returned value is valid until the next URI method call. -func (u *URI) Hash() []byte { - return u.hash -} - -// SetHash sets URI hash. -func (u *URI) SetHash(hash string) { - u.hash = append(u.hash[:0], hash...) -} - -// SetHashBytes sets URI hash. -func (u *URI) SetHashBytes(hash []byte) { - u.hash = append(u.hash[:0], hash...) -} - -// QueryString returns URI query string, -// i.e. baz=123 of http://aaa.com/foo/bar?baz=123#qwe . -// -// The returned value is valid until the next URI method call. -func (u *URI) QueryString() []byte { - return u.queryString -} - -// SetQueryString sets URI query string. -func (u *URI) SetQueryString(queryString string) { - u.queryString = append(u.queryString[:0], queryString...) - u.parsedQueryArgs = false -} - -// SetQueryStringBytes sets URI query string. -func (u *URI) SetQueryStringBytes(queryString []byte) { - u.queryString = append(u.queryString[:0], queryString...) - u.parsedQueryArgs = false -} - -// Path returns URI path, i.e. /foo/bar of http://aaa.com/foo/bar?baz=123#qwe . -// -// The returned path is always urldecoded and normalized, -// i.e. '//f%20obar/baz/../zzz' becomes '/f obar/zzz'. -// -// The returned value is valid until the next URI method call. -func (u *URI) Path() []byte { - path := u.path - if len(path) == 0 { - path = strSlash - } - return path -} - -// SetPath sets URI path. -func (u *URI) SetPath(path string) { - u.pathOriginal = append(u.pathOriginal[:0], path...) - u.path = normalizePath(u.path, u.pathOriginal) -} - -// SetPathBytes sets URI path. -func (u *URI) SetPathBytes(path []byte) { - u.pathOriginal = append(u.pathOriginal[:0], path...) - u.path = normalizePath(u.path, u.pathOriginal) -} - -// PathOriginal returns the original path from requestURI passed to URI.Parse(). -// -// The returned value is valid until the next URI method call. -func (u *URI) PathOriginal() []byte { - return u.pathOriginal -} - -// Scheme returns URI scheme, i.e. http of http://aaa.com/foo/bar?baz=123#qwe . -// -// Returned scheme is always lowercased. -// -// The returned value is valid until the next URI method call. -func (u *URI) Scheme() []byte { - scheme := u.scheme - if len(scheme) == 0 { - scheme = strHTTP - } - return scheme -} - -// SetScheme sets URI scheme, i.e. http, https, ftp, etc. -func (u *URI) SetScheme(scheme string) { - u.scheme = append(u.scheme[:0], scheme...) - lowercaseBytes(u.scheme) -} - -// SetSchemeBytes sets URI scheme, i.e. http, https, ftp, etc. -func (u *URI) SetSchemeBytes(scheme []byte) { - u.scheme = append(u.scheme[:0], scheme...) - lowercaseBytes(u.scheme) -} - -// Reset clears uri. -func (u *URI) Reset() { - u.pathOriginal = u.pathOriginal[:0] - u.scheme = u.scheme[:0] - u.path = u.path[:0] - u.queryString = u.queryString[:0] - u.hash = u.hash[:0] - - u.host = u.host[:0] - u.queryArgs.Reset() - u.parsedQueryArgs = false - - // There is no need in u.fullURI = u.fullURI[:0], since full uri - // is calculated on each call to FullURI(). - - // There is no need in u.requestURI = u.requestURI[:0], since requestURI - // is calculated on each call to RequestURI(). - - u.h = nil -} - -// Host returns host part, i.e. aaa.com of http://aaa.com/foo/bar?baz=123#qwe . -// -// Host is always lowercased. -func (u *URI) Host() []byte { - if len(u.host) == 0 && u.h != nil { - u.host = append(u.host[:0], u.h.Host()...) - lowercaseBytes(u.host) - u.h = nil - } - return u.host -} - -// SetHost sets host for the uri. -func (u *URI) SetHost(host string) { - u.host = append(u.host[:0], host...) - lowercaseBytes(u.host) -} - -// SetHostBytes sets host for the uri. -func (u *URI) SetHostBytes(host []byte) { - u.host = append(u.host[:0], host...) - lowercaseBytes(u.host) -} - -// Parse initializes URI from the given host and uri. -// -// host may be nil. In this case uri must contain fully qualified uri, -// i.e. with scheme and host. http is assumed if scheme is omitted. -// -// uri may contain e.g. RequestURI without scheme and host if host is non-empty. -func (u *URI) Parse(host, uri []byte) { - u.parse(host, uri, nil) -} - -func (u *URI) parseQuick(uri []byte, h *RequestHeader, isTLS bool) { - u.parse(nil, uri, h) - if isTLS { - u.scheme = append(u.scheme[:0], strHTTPS...) - } -} - -func (u *URI) parse(host, uri []byte, h *RequestHeader) { - u.Reset() - u.h = h - - scheme, host, uri := splitHostURI(host, uri) - u.scheme = append(u.scheme, scheme...) - lowercaseBytes(u.scheme) - u.host = append(u.host, host...) - lowercaseBytes(u.host) - - b := uri - queryIndex := bytes.IndexByte(b, '?') - fragmentIndex := bytes.IndexByte(b, '#') - // Ignore query in fragment part - if fragmentIndex >= 0 && queryIndex > fragmentIndex { - queryIndex = -1 - } - - if queryIndex < 0 && fragmentIndex < 0 { - u.pathOriginal = append(u.pathOriginal, b...) - u.path = normalizePath(u.path, u.pathOriginal) - return - } - - if queryIndex >= 0 { - // Path is everything up to the start of the query - u.pathOriginal = append(u.pathOriginal, b[:queryIndex]...) - u.path = normalizePath(u.path, u.pathOriginal) - - if fragmentIndex < 0 { - u.queryString = append(u.queryString, b[queryIndex+1:]...) - } else { - u.queryString = append(u.queryString, b[queryIndex+1:fragmentIndex]...) - u.hash = append(u.hash, b[fragmentIndex+1:]...) - } - return - } - - // fragmentIndex >= 0 && queryIndex < 0 - // Path is up to the start of fragment - u.pathOriginal = append(u.pathOriginal, b[:fragmentIndex]...) - u.path = normalizePath(u.path, u.pathOriginal) - u.hash = append(u.hash, b[fragmentIndex+1:]...) -} - -func normalizePath(dst, src []byte) []byte { - dst = dst[:0] - dst = addLeadingSlash(dst, src) - dst = decodeArgAppendNoPlus(dst, src) - - // remove duplicate slashes - b := dst - bSize := len(b) - for { - n := bytes.Index(b, strSlashSlash) - if n < 0 { - break - } - b = b[n:] - copy(b, b[1:]) - b = b[:len(b)-1] - bSize-- - } - dst = dst[:bSize] - - // remove /./ parts - b = dst - for { - n := bytes.Index(b, strSlashDotSlash) - if n < 0 { - break - } - nn := n + len(strSlashDotSlash) - 1 - copy(b[n:], b[nn:]) - b = b[:len(b)-nn+n] - } - - // remove /foo/../ parts - for { - n := bytes.Index(b, strSlashDotDotSlash) - if n < 0 { - break - } - nn := bytes.LastIndexByte(b[:n], '/') - if nn < 0 { - nn = 0 - } - n += len(strSlashDotDotSlash) - 1 - copy(b[nn:], b[n:]) - b = b[:len(b)-n+nn] - } - - // remove trailing /foo/.. - n := bytes.LastIndex(b, strSlashDotDot) - if n >= 0 && n+len(strSlashDotDot) == len(b) { - nn := bytes.LastIndexByte(b[:n], '/') - if nn < 0 { - return strSlash - } - b = b[:nn+1] - } - - return b -} - -// RequestURI returns RequestURI - i.e. URI without Scheme and Host. -func (u *URI) RequestURI() []byte { - dst := appendQuotedPath(u.requestURI[:0], u.Path()) - if u.queryArgs.Len() > 0 { - dst = append(dst, '?') - dst = u.queryArgs.AppendBytes(dst) - } else if len(u.queryString) > 0 { - dst = append(dst, '?') - dst = append(dst, u.queryString...) - } - if len(u.hash) > 0 { - dst = append(dst, '#') - dst = append(dst, u.hash...) - } - u.requestURI = dst - return u.requestURI -} - -// LastPathSegment returns the last part of uri path after '/'. -// -// Examples: -// -// * For /foo/bar/baz.html path returns baz.html. -// * For /foo/bar/ returns empty byte slice. -// * For /foobar.js returns foobar.js. -func (u *URI) LastPathSegment() []byte { - path := u.Path() - n := bytes.LastIndexByte(path, '/') - if n < 0 { - return path - } - return path[n+1:] -} - -// Update updates uri. -// -// The following newURI types are accepted: -// -// * Absolute, i.e. http://foobar.com/aaa/bb?cc . In this case the original -// uri is replaced by newURI. -// * Absolute without scheme, i.e. //foobar.com/aaa/bb?cc. In this case -// the original scheme is preserved. -// * Missing host, i.e. /aaa/bb?cc . In this case only RequestURI part -// of the original uri is replaced. -// * Relative path, i.e. xx?yy=abc . In this case the original RequestURI -// is updated according to the new relative path. -func (u *URI) Update(newURI string) { - u.UpdateBytes(s2b(newURI)) -} - -// UpdateBytes updates uri. -// -// The following newURI types are accepted: -// -// * Absolute, i.e. http://foobar.com/aaa/bb?cc . In this case the original -// uri is replaced by newURI. -// * Absolute without scheme, i.e. //foobar.com/aaa/bb?cc. In this case -// the original scheme is preserved. -// * Missing host, i.e. /aaa/bb?cc . In this case only RequestURI part -// of the original uri is replaced. -// * Relative path, i.e. xx?yy=abc . In this case the original RequestURI -// is updated according to the new relative path. -func (u *URI) UpdateBytes(newURI []byte) { - u.requestURI = u.updateBytes(newURI, u.requestURI) -} - -func (u *URI) updateBytes(newURI, buf []byte) []byte { - if len(newURI) == 0 { - return buf - } - - n := bytes.Index(newURI, strSlashSlash) - if n >= 0 { - // absolute uri - var b [32]byte - schemeOriginal := b[:0] - if len(u.scheme) > 0 { - schemeOriginal = append([]byte(nil), u.scheme...) - } - u.Parse(nil, newURI) - if len(schemeOriginal) > 0 && len(u.scheme) == 0 { - u.scheme = append(u.scheme[:0], schemeOriginal...) - } - return buf - } - - if newURI[0] == '/' { - // uri without host - buf = u.appendSchemeHost(buf[:0]) - buf = append(buf, newURI...) - u.Parse(nil, buf) - return buf - } - - // relative path - switch newURI[0] { - case '?': - // query string only update - u.SetQueryStringBytes(newURI[1:]) - return append(buf[:0], u.FullURI()...) - case '#': - // update only hash - u.SetHashBytes(newURI[1:]) - return append(buf[:0], u.FullURI()...) - default: - // update the last path part after the slash - path := u.Path() - n = bytes.LastIndexByte(path, '/') - if n < 0 { - panic("BUG: path must contain at least one slash") - } - buf = u.appendSchemeHost(buf[:0]) - buf = appendQuotedPath(buf, path[:n+1]) - buf = append(buf, newURI...) - u.Parse(nil, buf) - return buf - } -} - -// FullURI returns full uri in the form {Scheme}://{Host}{RequestURI}#{Hash}. -func (u *URI) FullURI() []byte { - u.fullURI = u.AppendBytes(u.fullURI[:0]) - return u.fullURI -} - -// AppendBytes appends full uri to dst and returns the extended dst. -func (u *URI) AppendBytes(dst []byte) []byte { - dst = u.appendSchemeHost(dst) - return append(dst, u.RequestURI()...) -} - -func (u *URI) appendSchemeHost(dst []byte) []byte { - dst = append(dst, u.Scheme()...) - dst = append(dst, strColonSlashSlash...) - return append(dst, u.Host()...) -} - -// WriteTo writes full uri to w. -// -// WriteTo implements io.WriterTo interface. -func (u *URI) WriteTo(w io.Writer) (int64, error) { - n, err := w.Write(u.FullURI()) - return int64(n), err -} - -// String returns full uri. -func (u *URI) String() string { - return string(u.FullURI()) -} - -func splitHostURI(host, uri []byte) ([]byte, []byte, []byte) { - n := bytes.Index(uri, strSlashSlash) - if n < 0 { - return strHTTP, host, uri - } - scheme := uri[:n] - if bytes.IndexByte(scheme, '/') >= 0 { - return strHTTP, host, uri - } - if len(scheme) > 0 && scheme[len(scheme)-1] == ':' { - scheme = scheme[:len(scheme)-1] - } - n += len(strSlashSlash) - uri = uri[n:] - n = bytes.IndexByte(uri, '/') - if n < 0 { - // A hack for bogus urls like foobar.com?a=b without - // slash after host. - if n = bytes.IndexByte(uri, '?'); n >= 0 { - return scheme, uri[:n], uri[n:] - } - return scheme, uri, strSlash - } - return scheme, uri[:n], uri[n:] -} - -// QueryArgs returns query args. -func (u *URI) QueryArgs() *Args { - u.parseQueryArgs() - return &u.queryArgs -} - -func (u *URI) parseQueryArgs() { - if u.parsedQueryArgs { - return - } - u.queryArgs.ParseBytes(u.queryString) - u.parsedQueryArgs = true -} diff --git a/vendor/github.com/valyala/fasthttp/uri_unix.go b/vendor/github.com/valyala/fasthttp/uri_unix.go deleted file mode 100644 index 1e307332..00000000 --- a/vendor/github.com/valyala/fasthttp/uri_unix.go +++ /dev/null @@ -1,12 +0,0 @@ -// +build !windows - -package fasthttp - -func addLeadingSlash(dst, src []byte) []byte { - // add leading slash for unix paths - if len(src) == 0 || src[0] != '/' { - dst = append(dst, '/') - } - - return dst -} diff --git a/vendor/github.com/valyala/fasthttp/uri_windows.go b/vendor/github.com/valyala/fasthttp/uri_windows.go deleted file mode 100644 index 95917a6b..00000000 --- a/vendor/github.com/valyala/fasthttp/uri_windows.go +++ /dev/null @@ -1,12 +0,0 @@ -// +build windows - -package fasthttp - -func addLeadingSlash(dst, src []byte) []byte { - // zero length and "C:/" case - if len(src) == 0 || (len(src) > 2 && src[1] != ':') { - dst = append(dst, '/') - } - - return dst -} diff --git a/vendor/github.com/valyala/fasthttp/userdata.go b/vendor/github.com/valyala/fasthttp/userdata.go deleted file mode 100644 index bd3e28aa..00000000 --- a/vendor/github.com/valyala/fasthttp/userdata.go +++ /dev/null @@ -1,71 +0,0 @@ -package fasthttp - -import ( - "io" -) - -type userDataKV struct { - key []byte - value interface{} -} - -type userData []userDataKV - -func (d *userData) Set(key string, value interface{}) { - args := *d - n := len(args) - for i := 0; i < n; i++ { - kv := &args[i] - if string(kv.key) == key { - kv.value = value - return - } - } - - c := cap(args) - if c > n { - args = args[:n+1] - kv := &args[n] - kv.key = append(kv.key[:0], key...) - kv.value = value - *d = args - return - } - - kv := userDataKV{} - kv.key = append(kv.key[:0], key...) - kv.value = value - *d = append(args, kv) -} - -func (d *userData) SetBytes(key []byte, value interface{}) { - d.Set(b2s(key), value) -} - -func (d *userData) Get(key string) interface{} { - args := *d - n := len(args) - for i := 0; i < n; i++ { - kv := &args[i] - if string(kv.key) == key { - return kv.value - } - } - return nil -} - -func (d *userData) GetBytes(key []byte) interface{} { - return d.Get(b2s(key)) -} - -func (d *userData) Reset() { - args := *d - n := len(args) - for i := 0; i < n; i++ { - v := args[i].value - if vc, ok := v.(io.Closer); ok { - vc.Close() - } - } - *d = (*d)[:0] -} diff --git a/vendor/github.com/valyala/fasthttp/workerpool.go b/vendor/github.com/valyala/fasthttp/workerpool.go deleted file mode 100644 index bfd297c3..00000000 --- a/vendor/github.com/valyala/fasthttp/workerpool.go +++ /dev/null @@ -1,237 +0,0 @@ -package fasthttp - -import ( - "net" - "runtime" - "strings" - "sync" - "time" -) - -// workerPool serves incoming connections via a pool of workers -// in FILO order, i.e. the most recently stopped worker will serve the next -// incoming connection. -// -// Such a scheme keeps CPU caches hot (in theory). -type workerPool struct { - // Function for serving server connections. - // It must leave c unclosed. - WorkerFunc ServeHandler - - MaxWorkersCount int - - LogAllErrors bool - - MaxIdleWorkerDuration time.Duration - - Logger Logger - - lock sync.Mutex - workersCount int - mustStop bool - - ready []*workerChan - - stopCh chan struct{} - - workerChanPool sync.Pool - - connState func(net.Conn, ConnState) -} - -type workerChan struct { - lastUseTime time.Time - ch chan net.Conn -} - -func (wp *workerPool) Start() { - if wp.stopCh != nil { - panic("BUG: workerPool already started") - } - wp.stopCh = make(chan struct{}) - stopCh := wp.stopCh - go func() { - var scratch []*workerChan - for { - wp.clean(&scratch) - select { - case <-stopCh: - return - default: - time.Sleep(wp.getMaxIdleWorkerDuration()) - } - } - }() -} - -func (wp *workerPool) Stop() { - if wp.stopCh == nil { - panic("BUG: workerPool wasn't started") - } - close(wp.stopCh) - wp.stopCh = nil - - // Stop all the workers waiting for incoming connections. - // Do not wait for busy workers - they will stop after - // serving the connection and noticing wp.mustStop = true. - wp.lock.Lock() - ready := wp.ready - for i, ch := range ready { - ch.ch <- nil - ready[i] = nil - } - wp.ready = ready[:0] - wp.mustStop = true - wp.lock.Unlock() -} - -func (wp *workerPool) getMaxIdleWorkerDuration() time.Duration { - if wp.MaxIdleWorkerDuration <= 0 { - return 10 * time.Second - } - return wp.MaxIdleWorkerDuration -} - -func (wp *workerPool) clean(scratch *[]*workerChan) { - maxIdleWorkerDuration := wp.getMaxIdleWorkerDuration() - - // Clean least recently used workers if they didn't serve connections - // for more than maxIdleWorkerDuration. - currentTime := time.Now() - - wp.lock.Lock() - ready := wp.ready - n := len(ready) - i := 0 - for i < n && currentTime.Sub(ready[i].lastUseTime) > maxIdleWorkerDuration { - i++ - } - *scratch = append((*scratch)[:0], ready[:i]...) - if i > 0 { - m := copy(ready, ready[i:]) - for i = m; i < n; i++ { - ready[i] = nil - } - wp.ready = ready[:m] - } - wp.lock.Unlock() - - // Notify obsolete workers to stop. - // This notification must be outside the wp.lock, since ch.ch - // may be blocking and may consume a lot of time if many workers - // are located on non-local CPUs. - tmp := *scratch - for i, ch := range tmp { - ch.ch <- nil - tmp[i] = nil - } -} - -func (wp *workerPool) Serve(c net.Conn) bool { - ch := wp.getCh() - if ch == nil { - return false - } - ch.ch <- c - return true -} - -var workerChanCap = func() int { - // Use blocking workerChan if GOMAXPROCS=1. - // This immediately switches Serve to WorkerFunc, which results - // in higher performance (under go1.5 at least). - if runtime.GOMAXPROCS(0) == 1 { - return 0 - } - - // Use non-blocking workerChan if GOMAXPROCS>1, - // since otherwise the Serve caller (Acceptor) may lag accepting - // new connections if WorkerFunc is CPU-bound. - return 1 -}() - -func (wp *workerPool) getCh() *workerChan { - var ch *workerChan - createWorker := false - - wp.lock.Lock() - ready := wp.ready - n := len(ready) - 1 - if n < 0 { - if wp.workersCount < wp.MaxWorkersCount { - createWorker = true - wp.workersCount++ - } - } else { - ch = ready[n] - ready[n] = nil - wp.ready = ready[:n] - } - wp.lock.Unlock() - - if ch == nil { - if !createWorker { - return nil - } - vch := wp.workerChanPool.Get() - if vch == nil { - vch = &workerChan{ - ch: make(chan net.Conn, workerChanCap), - } - } - ch = vch.(*workerChan) - go func() { - wp.workerFunc(ch) - wp.workerChanPool.Put(vch) - }() - } - return ch -} - -func (wp *workerPool) release(ch *workerChan) bool { - ch.lastUseTime = time.Now() - wp.lock.Lock() - if wp.mustStop { - wp.lock.Unlock() - return false - } - wp.ready = append(wp.ready, ch) - wp.lock.Unlock() - return true -} - -func (wp *workerPool) workerFunc(ch *workerChan) { - var c net.Conn - - var err error - for c = range ch.ch { - if c == nil { - break - } - - if err = wp.WorkerFunc(c); err != nil && err != errHijacked { - errStr := err.Error() - if wp.LogAllErrors || !(strings.Contains(errStr, "broken pipe") || - strings.Contains(errStr, "reset by peer") || - strings.Contains(errStr, "request headers: small read buffer") || - strings.Contains(errStr, "i/o timeout")) { - wp.Logger.Printf("error when serving connection %q<->%q: %s", c.LocalAddr(), c.RemoteAddr(), err) - } - } - if err == errHijacked { - wp.connState(c, StateHijacked) - } else { - c.Close() - wp.connState(c, StateClosed) - } - c = nil - - if !wp.release(ch) { - break - } - } - - wp.lock.Lock() - wp.workersCount-- - wp.lock.Unlock() -}