Add Prometheus metrics endpoint

Add the Prometheus http.Handler to serve metrics at MetricsPath ("/metrics"
by default). This allows Prometheus to scrape metrics from OAuth2 Proxy.

Add a new middleware NewRequestMetrics and attach it to the preAuth
chain. This will collect metrics on all requests made to OAuth2 Proxy

Collapse some calls to Prinf() and os.Exit(1) to Fatalf as they are
equivalent. main() has a strict 50 lines limit so brevity in these
calls appreciated

pkg/apis/options/options.go

@@ -22,6 +22,7 @@ type Options struct {
 	ProxyPrefix        string   `flag:"proxy-prefix" cfg:"proxy_prefix"`
 	PingPath           string   `flag:"ping-path" cfg:"ping_path"`
 	PingUserAgent      string   `flag:"ping-user-agent" cfg:"ping_user_agent"`
+	MetricsAddress     string   `flag:"metrics-address" cfg:"metrics_address"`
 	HTTPAddress        string   `flag:"http-address" cfg:"http_address"`
 	HTTPSAddress       string   `flag:"https-address" cfg:"https_address"`
 	ReverseProxy       bool     `flag:"reverse-proxy" cfg:"reverse_proxy"`
@@ -132,14 +133,14 @@ func (o *Options) SetRealClientIPParser(s ipapi.RealClientIPParser) { o.realClie
 // NewOptions constructs a new Options with defaulted values
 func NewOptions() *Options {
 	return &Options{
-		ProxyPrefix:        "/oauth2",
-		ProviderType:       "google",
-		PingPath:           "/ping",
-		HTTPAddress:        "127.0.0.1:4180",
-		HTTPSAddress:       ":443",
-		RealClientIPHeader: "X-Real-IP",
-		ForceHTTPS:         false,
-
+		ProxyPrefix:                      "/oauth2",
+		ProviderType:                     "google",
+		MetricsAddress:                   "",
+		PingPath:                         "/ping",
+		HTTPAddress:                      "127.0.0.1:4180",
+		HTTPSAddress:                     ":443",
+		RealClientIPHeader:               "X-Real-IP",
+		ForceHTTPS:                       false,
 		Cookie:                           cookieDefaults(),
 		Session:                          sessionOptionsDefaults(),
 		Templates:                        templatesDefaults(),
@@ -201,6 +202,7 @@ func NewFlagSet() *pflag.FlagSet {
 	flagSet.String("proxy-prefix", "/oauth2", "the url root path that this proxy should be nested under (e.g. /<oauth2>/sign_in)")
 	flagSet.String("ping-path", "/ping", "the ping endpoint that can be used for basic health checks")
 	flagSet.String("ping-user-agent", "", "special User-Agent that will be used for basic health checks")
+	flagSet.String("metrics-address", "", "the address /metrics will be served on (e.g. \":9100\")")
 	flagSet.String("session-store-type", "cookie", "the session storage provider to use")
 	flagSet.Bool("session-cookie-minimal", false, "strip OAuth tokens from cookie session stores if they aren't needed (cookie session store only)")
 	flagSet.String("redis-connection-url", "", "URL of redis server for redis session storage (eg: redis://HOST[:PORT])")

pkg/middleware/metrics.go

@@ -0,0 +1,122 @@
+package middleware
+
+import (
+	"net/http"
+
+	"github.com/justinas/alice"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+// DefaultMetricsHandler is the default http.Handler for serving metrics from
+// the default prometheus.Registry
+var DefaultMetricsHandler = NewMetricsHandlerWithDefaultRegistry()
+
+// NewMetricsHandlerWithDefaultRegistry creates a new http.Handler for serving
+// metrics from the default prometheus.Registry.
+func NewMetricsHandlerWithDefaultRegistry() http.Handler {
+	return NewMetricsHandler(prometheus.DefaultRegisterer, prometheus.DefaultGatherer)
+}
+
+// NewMetricsHandler creates a new http.Handler for serving metrics from the
+// provided prometheus.Registerer and prometheus.Gatherer
+func NewMetricsHandler(registerer prometheus.Registerer, gatherer prometheus.Gatherer) http.Handler {
+	return promhttp.InstrumentMetricHandler(
+		registerer, promhttp.HandlerFor(gatherer, promhttp.HandlerOpts{}),
+	)
+}
+
+// NewRequestMetricsWithDefaultRegistry returns a middleware that will record
+// metrics for HTTP requests to the default prometheus.Registry
+func NewRequestMetricsWithDefaultRegistry() alice.Constructor {
+	return NewRequestMetrics(prometheus.DefaultRegisterer)
+}
+
+// NewRequestMetrics returns a middleware that will record metrics for HTTP
+// requests to the provided prometheus.Registerer
+func NewRequestMetrics(registerer prometheus.Registerer) alice.Constructor {
+	return func(next http.Handler) http.Handler {
+		// Counter for all requests
+		// This is bucketed based on the response code we set
+		counterHandler := func(next http.Handler) http.Handler {
+			return promhttp.InstrumentHandlerCounter(registerRequestsCounter(registerer), next)
+		}
+
+		// Gauge to all requests currently being handled
+		inFlightHandler := func(next http.Handler) http.Handler {
+			return promhttp.InstrumentHandlerInFlight(registerInflightRequestsGauge(registerer), next)
+		}
+
+		// The latency of all requests bucketed by HTTP method
+		durationHandler := func(next http.Handler) http.Handler {
+			return promhttp.InstrumentHandlerDuration(registerRequestsLatencyHistogram(registerer), next)
+		}
+
+		return alice.New(counterHandler, inFlightHandler, durationHandler).Then(next)
+	}
+}
+
+// registerRequestsCounter registers the 'oauth2_proxy_requests_total' metric
+// This keeps a tally of all received requests bucket by their HTTP response
+// status code
+func registerRequestsCounter(registerer prometheus.Registerer) *prometheus.CounterVec {
+	counter := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "oauth2_proxy_requests_total",
+			Help: "Total number of requests by HTTP status code.",
+		},
+		[]string{"code"},
+	)
+
+	if err := registerer.Register(counter); err != nil {
+		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+			counter = are.ExistingCollector.(*prometheus.CounterVec)
+		} else {
+			panic(err)
+		}
+	}
+
+	return counter
+}
+
+// registerInflightRequestsGauge registers 'oauth2_proxy_requests_in_flight'
+// This only keeps the count of currently in progress HTTP requests
+func registerInflightRequestsGauge(registerer prometheus.Registerer) prometheus.Gauge {
+	gauge := prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "oauth2_proxy_requests_in_flight",
+		Help: "Current number of requests being served.",
+	})
+
+	if err := registerer.Register(gauge); err != nil {
+		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+			gauge = are.ExistingCollector.(prometheus.Gauge)
+		} else {
+			panic(err)
+		}
+	}
+
+	return gauge
+}
+
+// registerRequestsLatencyHistogram registers 'oauth2_proxy_response_duration_seconds'
+// This keeps tally of the requests bucketed by the time taken to process the request
+func registerRequestsLatencyHistogram(registerer prometheus.Registerer) *prometheus.HistogramVec {
+	histogram := prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "oauth2_proxy_response_duration_seconds",
+			Help:    "A histogram of request latencies.",
+			Buckets: prometheus.DefBuckets,
+		},
+		[]string{"method"},
+	)
+
+	if err := registerer.Register(histogram); err != nil {
+		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+			histogram = are.ExistingCollector.(*prometheus.HistogramVec)
+		} else {
+			panic(err)
+		}
+	}
+
+	return histogram
+}

pkg/middleware/metrics_test.go

@@ -0,0 +1,67 @@
+package middleware
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"os"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/extensions/table"
+	. "github.com/onsi/gomega"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+var _ = Describe("Instrumentation suite", func() {
+	type requestTableInput struct {
+		registry        *prometheus.Registry
+		requestString   string
+		expectedHandler http.Handler
+		expectedMetrics []string
+		expectedStatus  int
+		// Prometheus output is large so is stored in testdata
+		expectedResultsFile string
+	}
+
+	DescribeTable("when serving a request",
+		func(in *requestTableInput) {
+			req := httptest.NewRequest("", in.requestString, nil)
+
+			rw := httptest.NewRecorder()
+
+			handler := NewRequestMetrics(in.registry)(in.expectedHandler)
+			handler.ServeHTTP(rw, req)
+
+			Expect(rw.Code).To(Equal(in.expectedStatus))
+
+			expectedPrometheusText, err := os.Open(in.expectedResultsFile)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = testutil.GatherAndCompare(in.registry, expectedPrometheusText, in.expectedMetrics...)
+			Expect(err).NotTo(HaveOccurred())
+		},
+		Entry("successfully", func() *requestTableInput {
+			in := &requestTableInput{
+				registry:      prometheus.NewRegistry(),
+				requestString: "http://example.com/metrics",
+				expectedMetrics: []string{
+					"oauth2_proxy_requests_total",
+				},
+				expectedStatus:      200,
+				expectedResultsFile: "testdata/metrics/successfulrequest.txt",
+			}
+			in.expectedHandler = NewMetricsHandler(in.registry, in.registry)
+
+			return in
+		}()),
+		Entry("with not found", &requestTableInput{
+			registry:            prometheus.NewRegistry(),
+			requestString:       "http://example.com/",
+			expectedHandler:     http.NotFoundHandler(),
+			expectedMetrics:     []string{"oauth2_proxy_requests_total"},
+			expectedStatus:      404,
+			expectedResultsFile: "testdata/metrics/notfoundrequest.txt",
+		}),
+	)
+})

pkg/middleware/testdata/metrics/notfoundrequest.txt

@@ -0,0 +1,3 @@
+# HELP oauth2_proxy_requests_total Total number of requests by HTTP status code.
+# TYPE oauth2_proxy_requests_total counter
+oauth2_proxy_requests_total{code="404"} 1

pkg/middleware/testdata/metrics/successfulrequest.txt

@@ -0,0 +1,3 @@
+# HELP oauth2_proxy_requests_total Total number of requests by HTTP status code.
+# TYPE oauth2_proxy_requests_total counter
+oauth2_proxy_requests_total{code="200"} 1