Multi match (#74)

* discovery support for multiple matches

* switch proxy matcher usage, add random selection

* fix multi-match logic

* pass match picker func

* simplify rand picker

* update health params and docs

* fix early termination on discovery multi-match

* add grouping of sorted matches in sorted result

* add mention of live check to readme

README.md

@@ -17,6 +17,7 @@ Reproxy is a simple edge HTTP(s) server / reverse proxy supporting various provi
 - Single binary distribution
 - Docker container distribution
 - Built-in static assets server
+- Live health check and fail-over  
 - Management server with routes info and prometheus metrics
 
 ---
@@ -192,6 +193,9 @@ reproxy provides 2 endpoints for this purpose:
 - `/ping` responds with `pong` and indicates what reproxy up and running
 - `/health` returns `200 OK` status if all destination servers responded to their ping request with `200` or `417 Expectation Failed` if any of servers responded with non-200 code. It also returns json body with details about passed/failed services.
 
+In addition to the controllers above, reproxy supports optional live health checks. In this case (in enabled), each destination checked for ping response periodically and excluded from the destination routes if failed. It is possible to return multiple identical destinations from the same or various providers, and the passed picked. If numerous matches were discovered and passed - the final one picked randomly.
+To turn live health check on, user should set `--health-check.enabled` (or env `HEALTH_CHECK_ENABLED=true`). To customize checking interval `--health-check.interval=` can be used.
+
 ## Management API
 
 Optional, can be turned on with `--mgmt.enabled`. Exposes 2 endpoints on `mgmt.listen` (address:port):
@@ -296,6 +300,11 @@ error:
       --error.enabled               enable html errors reporting [$ERROR_ENABLED]
       --error.template=             error message template file [$ERROR_TEMPLATE]
 
+health-check:
+      --health-check.enabled        enable automatic health-check [$HEALTH_CHECK_ENABLED]
+      --health-check.interval=      automatic health-check interval (default: 300s) [$HEALTH_CHECK_INTERVAL]
+
+
 Help Options:
   -h, --help                        Show this help message
 ```

app/discovery/discovery.go

@@ -42,6 +42,18 @@ type URLMapper struct {
 	dead bool
 }
 
+// Matches returns result of url mapping. May have multiple routes. Lack of any routes means no match was wound
+type Matches struct {
+	MatchType MatchType
+	Routes    []MatchedRoute
+}
+
+// MatchedRoute contains a single match used to produce multi-matched Matches
+type MatchedRoute struct {
+	Destination string
+	Alive       bool
+}
+
 // Provider defines sources of mappers
 type Provider interface {
 	Events(ctx context.Context) (res <-chan ProviderID)
@@ -125,30 +137,41 @@ func (s *Service) Run(ctx context.Context) error {
 	}
 }
 
-// Match url to all mappers
+// Match url to all mappers. Returns Matches with potentially multiple destinations for MTProxy.
-func (s *Service) Match(srv, src string) (string, MatchType, bool) {
+// For MTStatic always a single match because fail-over doesn't supported for assets
+func (s *Service) Match(srv, src string) (res Matches) {
 
 	s.lock.RLock()
 	defer s.lock.RUnlock()
 
+	lastSrcMatch := ""
 	for _, srvName := range []string{srv, "*", ""} {
 		for _, m := range s.mappers[srvName] {
 
+			// if the first match found and the next src match is not identical we can stop as src match regexes presorted
+			if len(res.Routes) > 0 && m.SrcMatch.String() != lastSrcMatch {
+				return res
+			}
+
 			switch m.MatchType {
 			case MTProxy:
 				dest := m.SrcMatch.ReplaceAllString(src, m.Dst)
-				if src != dest {
+				if src != dest { // regex matched
-					return dest, m.MatchType, m.IsAlive()
+					lastSrcMatch = m.SrcMatch.String()
+					res.MatchType = MTProxy
+					res.Routes = append(res.Routes, MatchedRoute{dest, m.IsAlive()})
 				}
 			case MTStatic:
 				if src == m.AssetsWebRoot || strings.HasPrefix(src, m.AssetsWebRoot+"/") {
-					return m.AssetsWebRoot + ":" + m.AssetsLocation, MTStatic, true
+					res.MatchType = MTStatic
+					res.Routes = append(res.Routes, MatchedRoute{m.AssetsWebRoot + ":" + m.AssetsLocation, true})
+					return res
 				}
 			}
 		}
 	}
 
-	return src, MTProxy, false
+	return res
 }
 
 // ScheduleHealthCheck starts background loop with health-check
@@ -206,7 +229,12 @@ func (s *Service) Mappers() (mappers []URLMapper) {
 		mappers = append(mappers, m...)
 	}
 	sort.Slice(mappers, func(i, j int) bool {
-		return len(mappers[i].SrcMatch.String()) > len(mappers[j].SrcMatch.String())
+		// sort by len first, to make longer matches first
+		if len(mappers[i].SrcMatch.String()) != len(mappers[j].SrcMatch.String()) {
+			return len(mappers[i].SrcMatch.String()) > len(mappers[j].SrcMatch.String())
+		}
+		// if len identical sort by SrcMatch string to keep same SrcMatch grouped together
+		return mappers[i].SrcMatch.String() < mappers[j].SrcMatch.String()
 	})
 	return mappers
 }

app/discovery/discovery_test.go

@@ -88,8 +88,13 @@ func TestService_Match(t *testing.T) {
 					Dst: "http://127.0.0.2:8080/blah2/$1/abc", ProviderID: PIFile},
 				{Server: "m.example.com", SrcMatch: *regexp.MustCompile("^/api/svc4/(.*)"),
 					Dst: "http://127.0.0.4:8080/blah2/$1/abc", MatchType: MTProxy, dead: true},
+
 				{Server: "m.example.com", SrcMatch: *regexp.MustCompile("^/api/svc5/(.*)"),
 					Dst: "http://127.0.0.5:8080/blah2/$1/abc", MatchType: MTProxy, dead: false},
+				{Server: "m.example.com", SrcMatch: *regexp.MustCompile("^/api/svc5/(.*)"),
+					Dst: "http://127.0.0.5:8080/blah2/$1/abc/2", MatchType: MTProxy, dead: false},
+				{Server: "m.example.com", SrcMatch: *regexp.MustCompile("^/api/svc5/(.*)"),
+					Dst: "http://127.0.0.5:8080/blah2/$1/abc/3", MatchType: MTProxy, dead: true},
 			}, nil
 		},
 	}
@@ -115,40 +120,39 @@ func TestService_Match(t *testing.T) {
 	err := svc.Run(ctx)
 	require.Error(t, err)
 	assert.Equal(t, context.DeadlineExceeded, err)
-	assert.Equal(t, 8, len(svc.Mappers()))
+	assert.Equal(t, 10, len(svc.Mappers()))
 
 	tbl := []struct {
 		server, src string
-		dest        string
+		res         Matches
-		mt          MatchType
-		ok          bool
 	}{
+		{"example.com", "/api/svc3/xyz/something", Matches{MTProxy, []MatchedRoute{{"http://127.0.0.3:8080/blah3/xyz/something", true}}}},
+		{"example.com", "/api/svc3/xyz", Matches{MTProxy, []MatchedRoute{{"http://127.0.0.3:8080/blah3/xyz", true}}}},
+		{"abc.example.com", "/api/svc1/1234", Matches{MTProxy, []MatchedRoute{{"http://127.0.0.1:8080/blah1/1234", true}}}},
+		{"zzz.example.com", "/aaa/api/svc1/1234", Matches{MTProxy, nil}},
+		{"m.example.com", "/api/svc2/1234", Matches{MTProxy, []MatchedRoute{{"http://127.0.0.2:8080/blah2/1234/abc", true}}}},
+		{"m1.example.com", "/api/svc2/1234", Matches{MTProxy, nil}},
+		{"m.example.com", "/api/svc4/id12345", Matches{MTProxy, []MatchedRoute{{"http://127.0.0.4:8080/blah2/id12345/abc", false}}}},
 
-		{"example.com", "/api/svc3/xyz/something", "http://127.0.0.3:8080/blah3/xyz/something", MTProxy, true},
+		{"m.example.com", "/api/svc5/num123456", Matches{MTProxy, []MatchedRoute{
-		{"example.com", "/api/svc3/xyz", "http://127.0.0.3:8080/blah3/xyz", MTProxy, true},
+			{"http://127.0.0.5:8080/blah2/num123456/abc", true},
-		{"abc.example.com", "/api/svc1/1234", "http://127.0.0.1:8080/blah1/1234", MTProxy, true},
+			{"http://127.0.0.5:8080/blah2/num123456/abc/2", true},
-		{"zzz.example.com", "/aaa/api/svc1/1234", "/aaa/api/svc1/1234", MTProxy, false},
+			{"http://127.0.0.5:8080/blah2/num123456/abc/3", false},
-		{"m.example.com", "/api/svc2/1234", "http://127.0.0.2:8080/blah2/1234/abc", MTProxy, true},
+		}}},
-		{"m1.example.com", "/api/svc2/1234", "/api/svc2/1234", MTProxy, false},
+
-		{"m.example.com", "/api/svc4/id12345", "http://127.0.0.4:8080/blah2/id12345/abc", MTProxy, false},
+		{"m1.example.com", "/web/index.html", Matches{MTStatic, []MatchedRoute{{"/web:/var/web/", true}}}},
-		{"m.example.com", "/api/svc5/num123456", "http://127.0.0.5:8080/blah2/num123456/abc", MTProxy, true},
+		{"m1.example.com", "/web/", Matches{MTStatic, []MatchedRoute{{"/web:/var/web/", true}}}},
-		{"m1.example.com", "/web/index.html", "/web:/var/web/", MTStatic, true},
+		{"m1.example.com", "/www/something", Matches{MTStatic, []MatchedRoute{{"/www:/var/web/", true}}}},
-		{"m1.example.com", "/web/", "/web:/var/web/", MTStatic, true},
+		{"m1.example.com", "/www/", Matches{MTStatic, []MatchedRoute{{"/www:/var/web/", true}}}},
-		{"m1.example.com", "/www/something", "/www:/var/web/", MTStatic, true},
+		{"m1.example.com", "/www", Matches{MTStatic, []MatchedRoute{{"/www:/var/web/", true}}}},
-		{"m1.example.com", "/www/", "/www:/var/web/", MTStatic, true},
+		{"xyx.example.com", "/path/something", Matches{MTStatic, []MatchedRoute{{"/path:/var/web/path/", true}}}},
-		{"m1.example.com", "/www", "/www:/var/web/", MTStatic, true},
-		{"xyx.example.com", "/path/something", "/path:/var/web/path/", MTStatic, true},
 	}
 
 	for i, tt := range tbl {
 		tt := tt
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
-			res, mt, ok := svc.Match(tt.server, tt.src)
+			res := svc.Match(tt.server, tt.src)
-			assert.Equal(t, tt.ok, ok)
+			assert.Equal(t, tt.res, res)
-			assert.Equal(t, tt.dest, res)
-			if ok {
-				assert.Equal(t, tt.mt, mt)
-			}
 		})
 	}
 }

app/main.go

@@ -105,13 +105,13 @@ var opts struct {
 		Template string `long:"template" env:"TEMPLATE" description:"error message template file"`
 	} `group:"error" namespace:"error" env-namespace:"ERROR"`
 
+	HealthCheck struct {
+		Enabled  bool          `long:"enabled" env:"ENABLED" description:"enable automatic health-check"`
+		Interval time.Duration `long:"interval" env:"INTERVAL" default:"300s" description:"automatic health-check interval"`
+	} `group:"health-check" namespace:"health-check" env-namespace:"HEALTH_CHECK"`
+
 	Signature bool `long:"signature" env:"SIGNATURE" description:"enable reproxy signature headers"`
 	Dbg       bool `long:"dbg" env:"DEBUG" description:"debug mode"`
-
-	HealthCheck struct {
-		Enabled  bool          `long:"health-check" env:"HEALTH_CHECK" description:"enable automatic health-check"`
-		Interval time.Duration `long:"health-check-interval" env:"HEALTH_CHECK_INTERVAL" default:"300s" description:"automatic health-check interval"`
-	}
 }
 
 var revision = "unknown"

app/proxy/proxy.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"math/rand"
 	"net"
 	"net/http"
 	"net/http/httputil"
@@ -44,7 +45,7 @@ type Http struct { // nolint golint
 // Matcher source info (server and route) to the destination url
 // If no match found return ok=false
 type Matcher interface {
-	Match(srv, src string) (string, discovery.MatchType, bool)
+	Match(srv, src string) (res discovery.Matches)
 	Servers() (servers []string)
 	Mappers() (mappers []discovery.URLMapper)
 	CheckHealth() (pingResult map[string]error)
@@ -111,6 +112,8 @@ func (h *Http) Run(ctx context.Context) error {
 		h.gzipHandler(),
 	)
 
+	rand.Seed(time.Now().UnixNano())
+
 	if len(h.SSLConfig.FQDNs) == 0 && h.SSLConfig.SSLMode == SSLAuto {
 		// discovery async and may happen not right away. Try to get servers for some time
 		for i := 0; i < 100; i++ {
@@ -201,7 +204,8 @@ func (h *Http) proxyHandler() http.HandlerFunc {
 		if server == "" {
 			server = strings.Split(r.Host, ":")[0]
 		}
-		u, mt, ok := h.Match(server, r.URL.Path)
+		matches := h.Match(server, r.URL.Path) // get all matches for the server:path pair
+		u, ok := h.getMatch(matches, rand.Intn)
 		if !ok { // no route match
 			if h.isAssetRequest(r) {
 				assetsHandler.ServeHTTP(w, r)
@@ -212,7 +216,7 @@ func (h *Http) proxyHandler() http.HandlerFunc {
 			return
 		}
 
-		switch mt {
+		switch matches.MatchType {
 		case discovery.MTProxy:
 			uu, err := url.Parse(u)
 			if err != nil {
@@ -239,6 +243,27 @@ func (h *Http) proxyHandler() http.HandlerFunc {
 	}
 }
 
+func (h *Http) getMatch(mm discovery.Matches, picker func(len int) int) (u string, ok bool) {
+	if len(mm.Routes) == 0 {
+		return "", false
+	}
+
+	var urls []string
+	for _, m := range mm.Routes {
+		if m.Alive {
+			urls = append(urls, m.Destination)
+		}
+	}
+	switch len(urls) {
+	case 0:
+		return "", false
+	case 1:
+		return urls[0], true
+	default:
+		return urls[picker(len(urls))], true
+	}
+}
+
 func (h *Http) assetsHandler() http.HandlerFunc {
 	if h.AssetsLocation == "" || h.AssetsWebRoot == "" {
 		return func(writer http.ResponseWriter, request *http.Request) {}

app/proxy/proxy_test.go

@@ -363,3 +363,57 @@ func TestHttp_isAssetRequest(t *testing.T) {
 	}
 
 }
+
+func TestHttp_getMatch(t *testing.T) {
+
+	tbl := []struct {
+		matches discovery.Matches
+		res     string
+		ok      bool
+	}{
+		{
+			discovery.Matches{MatchType: discovery.MTProxy, Routes: []discovery.MatchedRoute{}}, "", false,
+		},
+		{
+			discovery.Matches{MatchType: discovery.MTProxy, Routes: []discovery.MatchedRoute{
+				{Destination: "dest1", Alive: false},
+				{Destination: "dest2", Alive: true},
+				{Destination: "dest3", Alive: false},
+			}},
+			"dest2", true,
+		},
+		{
+			discovery.Matches{MatchType: discovery.MTProxy, Routes: []discovery.MatchedRoute{
+				{Destination: "dest1", Alive: false},
+				{Destination: "dest2", Alive: true},
+				{Destination: "dest3", Alive: true},
+			}},
+			"dest2", true,
+		},
+		{
+			discovery.Matches{MatchType: discovery.MTProxy, Routes: []discovery.MatchedRoute{
+				{Destination: "dest1", Alive: true},
+				{Destination: "dest2", Alive: true},
+				{Destination: "dest3", Alive: true},
+			}},
+			"dest1", true,
+		},
+		{
+			discovery.Matches{MatchType: discovery.MTProxy, Routes: []discovery.MatchedRoute{
+				{Destination: "dest1", Alive: false},
+				{Destination: "dest2", Alive: false},
+				{Destination: "dest3", Alive: false},
+			}},
+			"", false,
+		},
+	}
+
+	h := Http{}
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			res, ok := h.getMatch(tt.matches, func(len int) int { return 0 })
+			require.Equal(t, tt.ok, ok)
+			assert.Equal(t, tt.res, res)
+		})
+	}
+}