From 095f4d71023a63ca16d401892cf01ae3a6c33aa0 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 16 May 2021 18:34:51 -0500 Subject: [PATCH] Multi match (#74) * discovery support for multiple matches * switch proxy matcher usage, add random selection * fix multi-match logic * pass match picker func * simplify rand picker * update health params and docs * fix early termination on discovery multi-match * add grouping of sorted matches in sorted result * add mention of live check to readme --- README.md | 9 ++++++ app/discovery/discovery.go | 42 ++++++++++++++++++++----- app/discovery/discovery_test.go | 52 ++++++++++++++++--------------- app/main.go | 10 +++--- app/proxy/proxy.go | 31 +++++++++++++++++-- app/proxy/proxy_test.go | 54 +++++++++++++++++++++++++++++++++ 6 files changed, 159 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 6b4d30a..01e74e8 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Reproxy is a simple edge HTTP(s) server / reverse proxy supporting various provi - Single binary distribution - Docker container distribution - Built-in static assets server +- Live health check and fail-over - Management server with routes info and prometheus metrics --- @@ -192,6 +193,9 @@ reproxy provides 2 endpoints for this purpose: - `/ping` responds with `pong` and indicates what reproxy up and running - `/health` returns `200 OK` status if all destination servers responded to their ping request with `200` or `417 Expectation Failed` if any of servers responded with non-200 code. It also returns json body with details about passed/failed services. +In addition to the controllers above, reproxy supports optional live health checks. In this case (in enabled), each destination checked for ping response periodically and excluded from the destination routes if failed. It is possible to return multiple identical destinations from the same or various providers, and the passed picked. If numerous matches were discovered and passed - the final one picked randomly. +To turn live health check on, user should set `--health-check.enabled` (or env `HEALTH_CHECK_ENABLED=true`). To customize checking interval `--health-check.interval=` can be used. + ## Management API Optional, can be turned on with `--mgmt.enabled`. Exposes 2 endpoints on `mgmt.listen` (address:port): @@ -296,6 +300,11 @@ error: --error.enabled enable html errors reporting [$ERROR_ENABLED] --error.template= error message template file [$ERROR_TEMPLATE] +health-check: + --health-check.enabled enable automatic health-check [$HEALTH_CHECK_ENABLED] + --health-check.interval= automatic health-check interval (default: 300s) [$HEALTH_CHECK_INTERVAL] + + Help Options: -h, --help Show this help message ``` diff --git a/app/discovery/discovery.go b/app/discovery/discovery.go index 91204ef..e3bfbfa 100644 --- a/app/discovery/discovery.go +++ b/app/discovery/discovery.go @@ -42,6 +42,18 @@ type URLMapper struct { dead bool } +// Matches returns result of url mapping. May have multiple routes. Lack of any routes means no match was wound +type Matches struct { + MatchType MatchType + Routes []MatchedRoute +} + +// MatchedRoute contains a single match used to produce multi-matched Matches +type MatchedRoute struct { + Destination string + Alive bool +} + // Provider defines sources of mappers type Provider interface { Events(ctx context.Context) (res <-chan ProviderID) @@ -125,30 +137,41 @@ func (s *Service) Run(ctx context.Context) error { } } -// Match url to all mappers -func (s *Service) Match(srv, src string) (string, MatchType, bool) { +// Match url to all mappers. Returns Matches with potentially multiple destinations for MTProxy. +// For MTStatic always a single match because fail-over doesn't supported for assets +func (s *Service) Match(srv, src string) (res Matches) { s.lock.RLock() defer s.lock.RUnlock() + lastSrcMatch := "" for _, srvName := range []string{srv, "*", ""} { for _, m := range s.mappers[srvName] { + // if the first match found and the next src match is not identical we can stop as src match regexes presorted + if len(res.Routes) > 0 && m.SrcMatch.String() != lastSrcMatch { + return res + } + switch m.MatchType { case MTProxy: dest := m.SrcMatch.ReplaceAllString(src, m.Dst) - if src != dest { - return dest, m.MatchType, m.IsAlive() + if src != dest { // regex matched + lastSrcMatch = m.SrcMatch.String() + res.MatchType = MTProxy + res.Routes = append(res.Routes, MatchedRoute{dest, m.IsAlive()}) } case MTStatic: if src == m.AssetsWebRoot || strings.HasPrefix(src, m.AssetsWebRoot+"/") { - return m.AssetsWebRoot + ":" + m.AssetsLocation, MTStatic, true + res.MatchType = MTStatic + res.Routes = append(res.Routes, MatchedRoute{m.AssetsWebRoot + ":" + m.AssetsLocation, true}) + return res } } } } - return src, MTProxy, false + return res } // ScheduleHealthCheck starts background loop with health-check @@ -206,7 +229,12 @@ func (s *Service) Mappers() (mappers []URLMapper) { mappers = append(mappers, m...) } sort.Slice(mappers, func(i, j int) bool { - return len(mappers[i].SrcMatch.String()) > len(mappers[j].SrcMatch.String()) + // sort by len first, to make longer matches first + if len(mappers[i].SrcMatch.String()) != len(mappers[j].SrcMatch.String()) { + return len(mappers[i].SrcMatch.String()) > len(mappers[j].SrcMatch.String()) + } + // if len identical sort by SrcMatch string to keep same SrcMatch grouped together + return mappers[i].SrcMatch.String() < mappers[j].SrcMatch.String() }) return mappers } diff --git a/app/discovery/discovery_test.go b/app/discovery/discovery_test.go index fab6de3..66a3463 100644 --- a/app/discovery/discovery_test.go +++ b/app/discovery/discovery_test.go @@ -88,8 +88,13 @@ func TestService_Match(t *testing.T) { Dst: "http://127.0.0.2:8080/blah2/$1/abc", ProviderID: PIFile}, {Server: "m.example.com", SrcMatch: *regexp.MustCompile("^/api/svc4/(.*)"), Dst: "http://127.0.0.4:8080/blah2/$1/abc", MatchType: MTProxy, dead: true}, + {Server: "m.example.com", SrcMatch: *regexp.MustCompile("^/api/svc5/(.*)"), Dst: "http://127.0.0.5:8080/blah2/$1/abc", MatchType: MTProxy, dead: false}, + {Server: "m.example.com", SrcMatch: *regexp.MustCompile("^/api/svc5/(.*)"), + Dst: "http://127.0.0.5:8080/blah2/$1/abc/2", MatchType: MTProxy, dead: false}, + {Server: "m.example.com", SrcMatch: *regexp.MustCompile("^/api/svc5/(.*)"), + Dst: "http://127.0.0.5:8080/blah2/$1/abc/3", MatchType: MTProxy, dead: true}, }, nil }, } @@ -115,40 +120,39 @@ func TestService_Match(t *testing.T) { err := svc.Run(ctx) require.Error(t, err) assert.Equal(t, context.DeadlineExceeded, err) - assert.Equal(t, 8, len(svc.Mappers())) + assert.Equal(t, 10, len(svc.Mappers())) tbl := []struct { server, src string - dest string - mt MatchType - ok bool + res Matches }{ + {"example.com", "/api/svc3/xyz/something", Matches{MTProxy, []MatchedRoute{{"http://127.0.0.3:8080/blah3/xyz/something", true}}}}, + {"example.com", "/api/svc3/xyz", Matches{MTProxy, []MatchedRoute{{"http://127.0.0.3:8080/blah3/xyz", true}}}}, + {"abc.example.com", "/api/svc1/1234", Matches{MTProxy, []MatchedRoute{{"http://127.0.0.1:8080/blah1/1234", true}}}}, + {"zzz.example.com", "/aaa/api/svc1/1234", Matches{MTProxy, nil}}, + {"m.example.com", "/api/svc2/1234", Matches{MTProxy, []MatchedRoute{{"http://127.0.0.2:8080/blah2/1234/abc", true}}}}, + {"m1.example.com", "/api/svc2/1234", Matches{MTProxy, nil}}, + {"m.example.com", "/api/svc4/id12345", Matches{MTProxy, []MatchedRoute{{"http://127.0.0.4:8080/blah2/id12345/abc", false}}}}, - {"example.com", "/api/svc3/xyz/something", "http://127.0.0.3:8080/blah3/xyz/something", MTProxy, true}, - {"example.com", "/api/svc3/xyz", "http://127.0.0.3:8080/blah3/xyz", MTProxy, true}, - {"abc.example.com", "/api/svc1/1234", "http://127.0.0.1:8080/blah1/1234", MTProxy, true}, - {"zzz.example.com", "/aaa/api/svc1/1234", "/aaa/api/svc1/1234", MTProxy, false}, - {"m.example.com", "/api/svc2/1234", "http://127.0.0.2:8080/blah2/1234/abc", MTProxy, true}, - {"m1.example.com", "/api/svc2/1234", "/api/svc2/1234", MTProxy, false}, - {"m.example.com", "/api/svc4/id12345", "http://127.0.0.4:8080/blah2/id12345/abc", MTProxy, false}, - {"m.example.com", "/api/svc5/num123456", "http://127.0.0.5:8080/blah2/num123456/abc", MTProxy, true}, - {"m1.example.com", "/web/index.html", "/web:/var/web/", MTStatic, true}, - {"m1.example.com", "/web/", "/web:/var/web/", MTStatic, true}, - {"m1.example.com", "/www/something", "/www:/var/web/", MTStatic, true}, - {"m1.example.com", "/www/", "/www:/var/web/", MTStatic, true}, - {"m1.example.com", "/www", "/www:/var/web/", MTStatic, true}, - {"xyx.example.com", "/path/something", "/path:/var/web/path/", MTStatic, true}, + {"m.example.com", "/api/svc5/num123456", Matches{MTProxy, []MatchedRoute{ + {"http://127.0.0.5:8080/blah2/num123456/abc", true}, + {"http://127.0.0.5:8080/blah2/num123456/abc/2", true}, + {"http://127.0.0.5:8080/blah2/num123456/abc/3", false}, + }}}, + + {"m1.example.com", "/web/index.html", Matches{MTStatic, []MatchedRoute{{"/web:/var/web/", true}}}}, + {"m1.example.com", "/web/", Matches{MTStatic, []MatchedRoute{{"/web:/var/web/", true}}}}, + {"m1.example.com", "/www/something", Matches{MTStatic, []MatchedRoute{{"/www:/var/web/", true}}}}, + {"m1.example.com", "/www/", Matches{MTStatic, []MatchedRoute{{"/www:/var/web/", true}}}}, + {"m1.example.com", "/www", Matches{MTStatic, []MatchedRoute{{"/www:/var/web/", true}}}}, + {"xyx.example.com", "/path/something", Matches{MTStatic, []MatchedRoute{{"/path:/var/web/path/", true}}}}, } for i, tt := range tbl { tt := tt t.Run(strconv.Itoa(i), func(t *testing.T) { - res, mt, ok := svc.Match(tt.server, tt.src) - assert.Equal(t, tt.ok, ok) - assert.Equal(t, tt.dest, res) - if ok { - assert.Equal(t, tt.mt, mt) - } + res := svc.Match(tt.server, tt.src) + assert.Equal(t, tt.res, res) }) } } diff --git a/app/main.go b/app/main.go index d3c68db..13a9d89 100644 --- a/app/main.go +++ b/app/main.go @@ -105,13 +105,13 @@ var opts struct { Template string `long:"template" env:"TEMPLATE" description:"error message template file"` } `group:"error" namespace:"error" env-namespace:"ERROR"` + HealthCheck struct { + Enabled bool `long:"enabled" env:"ENABLED" description:"enable automatic health-check"` + Interval time.Duration `long:"interval" env:"INTERVAL" default:"300s" description:"automatic health-check interval"` + } `group:"health-check" namespace:"health-check" env-namespace:"HEALTH_CHECK"` + Signature bool `long:"signature" env:"SIGNATURE" description:"enable reproxy signature headers"` Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"` - - HealthCheck struct { - Enabled bool `long:"health-check" env:"HEALTH_CHECK" description:"enable automatic health-check"` - Interval time.Duration `long:"health-check-interval" env:"HEALTH_CHECK_INTERVAL" default:"300s" description:"automatic health-check interval"` - } } var revision = "unknown" diff --git a/app/proxy/proxy.go b/app/proxy/proxy.go index bec4e4e..850296b 100644 --- a/app/proxy/proxy.go +++ b/app/proxy/proxy.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "math/rand" "net" "net/http" "net/http/httputil" @@ -44,7 +45,7 @@ type Http struct { // nolint golint // Matcher source info (server and route) to the destination url // If no match found return ok=false type Matcher interface { - Match(srv, src string) (string, discovery.MatchType, bool) + Match(srv, src string) (res discovery.Matches) Servers() (servers []string) Mappers() (mappers []discovery.URLMapper) CheckHealth() (pingResult map[string]error) @@ -111,6 +112,8 @@ func (h *Http) Run(ctx context.Context) error { h.gzipHandler(), ) + rand.Seed(time.Now().UnixNano()) + if len(h.SSLConfig.FQDNs) == 0 && h.SSLConfig.SSLMode == SSLAuto { // discovery async and may happen not right away. Try to get servers for some time for i := 0; i < 100; i++ { @@ -201,7 +204,8 @@ func (h *Http) proxyHandler() http.HandlerFunc { if server == "" { server = strings.Split(r.Host, ":")[0] } - u, mt, ok := h.Match(server, r.URL.Path) + matches := h.Match(server, r.URL.Path) // get all matches for the server:path pair + u, ok := h.getMatch(matches, rand.Intn) if !ok { // no route match if h.isAssetRequest(r) { assetsHandler.ServeHTTP(w, r) @@ -212,7 +216,7 @@ func (h *Http) proxyHandler() http.HandlerFunc { return } - switch mt { + switch matches.MatchType { case discovery.MTProxy: uu, err := url.Parse(u) if err != nil { @@ -239,6 +243,27 @@ func (h *Http) proxyHandler() http.HandlerFunc { } } +func (h *Http) getMatch(mm discovery.Matches, picker func(len int) int) (u string, ok bool) { + if len(mm.Routes) == 0 { + return "", false + } + + var urls []string + for _, m := range mm.Routes { + if m.Alive { + urls = append(urls, m.Destination) + } + } + switch len(urls) { + case 0: + return "", false + case 1: + return urls[0], true + default: + return urls[picker(len(urls))], true + } +} + func (h *Http) assetsHandler() http.HandlerFunc { if h.AssetsLocation == "" || h.AssetsWebRoot == "" { return func(writer http.ResponseWriter, request *http.Request) {} diff --git a/app/proxy/proxy_test.go b/app/proxy/proxy_test.go index fcaa915..4f9b50d 100644 --- a/app/proxy/proxy_test.go +++ b/app/proxy/proxy_test.go @@ -363,3 +363,57 @@ func TestHttp_isAssetRequest(t *testing.T) { } } + +func TestHttp_getMatch(t *testing.T) { + + tbl := []struct { + matches discovery.Matches + res string + ok bool + }{ + { + discovery.Matches{MatchType: discovery.MTProxy, Routes: []discovery.MatchedRoute{}}, "", false, + }, + { + discovery.Matches{MatchType: discovery.MTProxy, Routes: []discovery.MatchedRoute{ + {Destination: "dest1", Alive: false}, + {Destination: "dest2", Alive: true}, + {Destination: "dest3", Alive: false}, + }}, + "dest2", true, + }, + { + discovery.Matches{MatchType: discovery.MTProxy, Routes: []discovery.MatchedRoute{ + {Destination: "dest1", Alive: false}, + {Destination: "dest2", Alive: true}, + {Destination: "dest3", Alive: true}, + }}, + "dest2", true, + }, + { + discovery.Matches{MatchType: discovery.MTProxy, Routes: []discovery.MatchedRoute{ + {Destination: "dest1", Alive: true}, + {Destination: "dest2", Alive: true}, + {Destination: "dest3", Alive: true}, + }}, + "dest1", true, + }, + { + discovery.Matches{MatchType: discovery.MTProxy, Routes: []discovery.MatchedRoute{ + {Destination: "dest1", Alive: false}, + {Destination: "dest2", Alive: false}, + {Destination: "dest3", Alive: false}, + }}, + "", false, + }, + } + + h := Http{} + for i, tt := range tbl { + t.Run(strconv.Itoa(i), func(t *testing.T) { + res, ok := h.getMatch(tt.matches, func(len int) int { return 0 }) + require.Equal(t, tt.ok, ok) + assert.Equal(t, tt.res, res) + }) + } +}