Add metrics for certificates in the verified chains (#48)

2025-07-12 23:50:14 +02:00 · 2020-09-11 18:20:41 +01:00
parent ddedd5f1b5
commit 17aa4e2d2d
6 changed files with 483 additions and 68 deletions
--- a/README.md
+++ b/README.md
@ -34,6 +34,7 @@ meaningful visualisations and consoles.
      - [&lt;https_probe&gt;](#https_probe)
      - [&lt;tcp_probe&gt;](#tcp_probe)
  - [Example Queries](#example-queries)
+  - [Peer Cerificates vs Verified Chain Certificates](#peer-cerificates-vs-verified-chain-certificates)
  - [Proxying](#proxying)
  - [Grafana](#grafana)

@ -88,12 +89,14 @@ Flags:
 ## Metrics

 | Metric                       | Meaning                                                                                                 | Labels                                                        |
-| ----------------------- | ----------------------------------------------------------------------------------- | --------------------------------------------------- |
-| ssl_cert_not_after      | The date after which the certificate expires. Expressed as a Unix Epoch Time.       | serial_no, issuer_cn, cn, dnsnames, ips, emails, ou |
-| ssl_cert_not_before     | The date before which the certificate is not valid. Expressed as a Unix Epoch Time. | serial_no, issuer_cn, cn, dnsnames, ips, emails, ou |
+| ---------------------------- | ------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- |
+| ssl_cert_not_after           | The date after which a peer certificate expires. Expressed as a Unix Epoch Time.                        | serial_no, issuer_cn, cn, dnsnames, ips, emails, ou           |
+| ssl_cert_not_before          | The date before which a peer certificate is not valid. Expressed as a Unix Epoch Time.                  | serial_no, issuer_cn, cn, dnsnames, ips, emails, ou           |
 | ssl_prober                   | The prober used by the exporter to connect to the target. Boolean.                                      | prober                                                        |
 | ssl_tls_connect_success      | Was the TLS connection successful? Boolean.                                                             |                                                               |
 | ssl_tls_version_info         | The TLS version used. Always 1.                                                                         | version                                                       |
+| ssl_verified_cert_not_after  | The date after which a certificate in the verified chain expires. Expressed as a Unix Epoch Time.       | chain_no, serial_no, issuer_cn, cn, dnsnames, ips, emails, ou |
+| ssl_verified_cert_not_before | The date before which a certificate in the verified chain is not valid. Expressed as a Unix Epoch Time. | chain_no, serial_no, issuer_cn, cn, dnsnames, ips, emails, ou |

 ## Configuration

@ -212,10 +215,17 @@ Wildcard certificates that are expiring:
 ssl_cert_not_after{cn=~"\*.*"} - time() < 86400 * 7
 ```

-Number of certificates in the chain:
+Certificates that expire within 7 days in the verified chain that expires
+latest:

 ```
-count(ssl_cert_not_after) by (instance, serial_no, issuer_cn)
+ssl_verified_cert_not_after{chain_no="0"} - time() < 86400 * 7
+```
+
+Number of certificates presented by the server:
+
+```
+count(ssl_cert_not_after) by (instance)
 ```

 Identify instances that have failed to create a valid SSL connection:
@ -224,6 +234,40 @@ Identify instances that have failed to create a valid SSL connection:
 ssl_tls_connect_success == 0
 ```

+## Peer Cerificates vs Verified Chain Certificates
+
+Metrics are exported for the `NotAfter` and `NotBefore` fields for peer
+certificates as well as for the verified chain that is
+constructed by the client.
+
+The former only includes the certificates that are served explicitly by the
+target, while the latter can contain multiple chains of trust that are
+constructed from root certificates held by the client to the target's server
+certificate.
+
+This has important implications when monitoring certificate expiry.
+
+For instance, it may be the case that `ssl_cert_not_after` reports that the root
+certificate served by the target is expiring soon even though clients can form
+another, much longer lived, chain of trust using another valid root certificate
+held locally. In this case, you may want to use `ssl_verified_cert_not_after` to
+alert on expiry instead, as this will contain the chain that the client actually
+constructs:
+
+```
+ssl_verified_cert_not_after{chain_no="0"} - time() < 86400 * 7
+```
+
+Each chain is numbered by the exporter in reverse order of expiry, so that
+`chain_no="0"` is the chain that will expire the latest. Therefore the query
+above will only alert when the chain of trust between the exporter and the
+target is truly nearing expiry.
+
+It's very important to note that a query of this kind only represents the chain
+of trust between the exporter and the target. Genuine clients may hold different
+root certs than the exporter and therefore have different verified chains of
+trust.
+
 ## Proxying

 The `https` prober supports the use of proxy servers discovered by the
--- a/ssl_exporter.go
+++ b/ssl_exporter.go
@ -5,6 +5,7 @@ import (
 	"crypto/x509"
 	"fmt"
 	"net/http"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@ -48,6 +49,16 @@ var (
 		"NotAfter expressed as a Unix Epoch Time",
 		[]string{"serial_no", "issuer_cn", "cn", "dnsnames", "ips", "emails", "ou"}, nil,
 	)
+	verifiedNotBefore = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "", "verified_cert_not_before"),
+		"NotBefore expressed as a Unix Epoch Time for a certificate in the list of verified chains",
+		[]string{"chain_no", "serial_no", "issuer_cn", "cn", "dnsnames", "ips", "emails", "ou"}, nil,
+	)
+	verifiedNotAfter = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "", "verfied_cert_not_after"),
+		"NotAfter expressed as a Unix Epoch Time for a certificate in the list of verified chains",
+		[]string{"chain_no", "serial_no", "issuer_cn", "cn", "dnsnames", "ips", "emails", "ou"}, nil,
+	)
 )

 // Exporter is the exporter type...
@ -65,6 +76,8 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
 	ch <- proberType
 	ch <- notAfter
 	ch <- notBefore
+	ch <- verifiedNotAfter
+	ch <- verifiedNotBefore
 }

 // Collect metrics
@ -97,6 +110,8 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
 		return
 	}

+	// If there are peer certificates in the connection state then consider
+	// the tls connection a success
 	ch <- prometheus.MustNewConstMetric(
 		tlsConnectSuccess, prometheus.GaugeValue, 1,
 	)
@ -104,41 +119,101 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
 	// Remove duplicate certificates from the response
 	peerCertificates = uniq(peerCertificates)

-	// Loop through returned certificates and create metrics
+	// Loop through peer certificates and create metrics
 	for _, cert := range peerCertificates {
-		var DNSNamesLabel, emailsLabel, ipsLabel, OULabel string
-
-		if len(cert.DNSNames) > 0 {
-			DNSNamesLabel = "," + strings.Join(cert.DNSNames, ",") + ","
-		}
-
-		if len(cert.EmailAddresses) > 0 {
-			emailsLabel = "," + strings.Join(cert.EmailAddresses, ",") + ","
-		}
-
-		if len(cert.IPAddresses) > 0 {
-			ipsLabel = ","
-			for _, ip := range cert.IPAddresses {
-				ipsLabel = ipsLabel + ip.String() + ","
-			}
-		}
-
-		if len(cert.Subject.OrganizationalUnit) > 0 {
-			OULabel = "," + strings.Join(cert.Subject.OrganizationalUnit, ",") + ","
-		}
-
 		if !cert.NotAfter.IsZero() {
 			ch <- prometheus.MustNewConstMetric(
-				notAfter, prometheus.GaugeValue, float64(cert.NotAfter.UnixNano()/1e9), cert.SerialNumber.String(), cert.Issuer.CommonName, cert.Subject.CommonName, DNSNamesLabel, ipsLabel, emailsLabel, OULabel,
+				notAfter,
+				prometheus.GaugeValue,
+				float64(cert.NotAfter.UnixNano()/1e9),
+				cert.SerialNumber.String(),
+				cert.Issuer.CommonName,
+				cert.Subject.CommonName,
+				getDNSNames(cert),
+				getIPAddresses(cert),
+				getEmailAddresses(cert),
+				getOrganizationalUnits(cert),
 			)
 		}

 		if !cert.NotBefore.IsZero() {
 			ch <- prometheus.MustNewConstMetric(
-				notBefore, prometheus.GaugeValue, float64(cert.NotBefore.UnixNano()/1e9), cert.SerialNumber.String(), cert.Issuer.CommonName, cert.Subject.CommonName, DNSNamesLabel, ipsLabel, emailsLabel, OULabel,
+				notBefore,
+				prometheus.GaugeValue,
+				float64(cert.NotBefore.UnixNano()/1e9),
+				cert.SerialNumber.String(),
+				cert.Issuer.CommonName,
+				cert.Subject.CommonName,
+				getDNSNames(cert),
+				getIPAddresses(cert),
+				getEmailAddresses(cert),
+				getOrganizationalUnits(cert),
 			)
 		}
 	}
+
+	// Retrieve the list of verified chains from the connection state
+	verifiedChains := state.VerifiedChains
+
+	// Sort the verified chains from the chain that is valid for longest to the chain
+	// that expires the soonest
+	sort.Slice(verifiedChains, func(i, j int) bool {
+		iExpiry := time.Time{}
+		for _, cert := range verifiedChains[i] {
+			if (iExpiry.IsZero() || cert.NotAfter.Before(iExpiry)) && !cert.NotAfter.IsZero() {
+				iExpiry = cert.NotAfter
+			}
+		}
+		jExpiry := time.Time{}
+		for _, cert := range verifiedChains[j] {
+			if (jExpiry.IsZero() || cert.NotAfter.Before(jExpiry)) && !cert.NotAfter.IsZero() {
+				jExpiry = cert.NotAfter
+			}
+		}
+
+		return iExpiry.After(jExpiry)
+	})
+
+	// Loop through the verified chains creating metrics. Label the metrics
+	// with the index of the chain.
+	for i, chain := range verifiedChains {
+		chain = uniq(chain)
+		for _, cert := range chain {
+			chainNo := strconv.Itoa(i)
+
+			if !cert.NotAfter.IsZero() {
+				ch <- prometheus.MustNewConstMetric(
+					verifiedNotAfter,
+					prometheus.GaugeValue,
+					float64(cert.NotAfter.UnixNano()/1e9),
+					chainNo,
+					cert.SerialNumber.String(),
+					cert.Issuer.CommonName,
+					cert.Subject.CommonName,
+					getDNSNames(cert),
+					getIPAddresses(cert),
+					getEmailAddresses(cert),
+					getOrganizationalUnits(cert),
+				)
+			}
+
+			if !cert.NotBefore.IsZero() {
+				ch <- prometheus.MustNewConstMetric(
+					verifiedNotBefore,
+					prometheus.GaugeValue,
+					float64(cert.NotBefore.UnixNano()/1e9),
+					chainNo,
+					cert.SerialNumber.String(),
+					cert.Issuer.CommonName,
+					cert.Subject.CommonName,
+					getDNSNames(cert),
+					getIPAddresses(cert),
+					getEmailAddresses(cert),
+					getOrganizationalUnits(cert),
+				)
+			}
+		}
+	}
 }

 func probeHandler(w http.ResponseWriter, r *http.Request, conf *config.Config) {
@ -234,6 +309,42 @@ func getTLSVersion(state *tls.ConnectionState) string {
 	}
 }

+func getDNSNames(cert *x509.Certificate) string {
+	if len(cert.DNSNames) > 0 {
+		return "," + strings.Join(cert.DNSNames, ",") + ","
+	}
+
+	return ""
+}
+
+func getEmailAddresses(cert *x509.Certificate) string {
+	if len(cert.EmailAddresses) > 0 {
+		return "," + strings.Join(cert.EmailAddresses, ",") + ","
+	}
+
+	return ""
+}
+
+func getIPAddresses(cert *x509.Certificate) string {
+	if len(cert.IPAddresses) > 0 {
+		ips := ","
+		for _, ip := range cert.IPAddresses {
+			ips = ips + ip.String() + ","
+		}
+		return ips
+	}
+
+	return ""
+}
+
+func getOrganizationalUnits(cert *x509.Certificate) string {
+	if len(cert.Subject.OrganizationalUnit) > 0 {
+		return "," + strings.Join(cert.Subject.OrganizationalUnit, ",") + ","
+	}
+
+	return ""
+}
+
 func init() {
 	prometheus.MustRegister(version.NewCollector(namespace + "_exporter"))
 }
--- a/ssl_exporter_test.go
+++ b/ssl_exporter_test.go
@ -1,10 +1,14 @@
 package main

 import (
+	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
+	"math/big"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@ -56,7 +60,7 @@ func TestProbeHandlerHTTPS(t *testing.T) {
 		t.Errorf("expected `ssl_prober{prober=\"https\"} 1`")
 	}

-	// Check notAfter and notBefore metrics
+	// Check notAfter and notBefore metrics for the peer certificate
 	if err := checkDates(certPEM, rr.Body.String()); err != nil {
 		t.Errorf(err.Error())
 	}
@ -68,6 +72,89 @@ func TestProbeHandlerHTTPS(t *testing.T) {
 	}
 }

+// TestProbeHandlerHTTPSVerifiedChains checks that metrics are generated
+// correctly for the verified chains
+func TestProbeHandlerHTTPSVerifiedChains(t *testing.T) {
+	rootPrivateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	rootCertExpiry := time.Now().AddDate(0, 0, 5)
+	rootCertTmpl := test.GenerateCertificateTemplate(rootCertExpiry)
+	rootCertTmpl.IsCA = true
+	rootCertTmpl.SerialNumber = big.NewInt(1)
+	rootCert, rootCertPem := test.GenerateSelfSignedCertificateWithPrivateKey(rootCertTmpl, rootPrivateKey)
+
+	olderRootCertExpiry := time.Now().AddDate(0, 0, 3)
+	olderRootCertTmpl := test.GenerateCertificateTemplate(olderRootCertExpiry)
+	olderRootCertTmpl.IsCA = true
+	olderRootCertTmpl.SerialNumber = big.NewInt(2)
+	olderRootCert, olderRootCertPem := test.GenerateSelfSignedCertificateWithPrivateKey(olderRootCertTmpl, rootPrivateKey)
+
+	oldestRootCertExpiry := time.Now().AddDate(0, 0, 1)
+	oldestRootCertTmpl := test.GenerateCertificateTemplate(oldestRootCertExpiry)
+	oldestRootCertTmpl.IsCA = true
+	oldestRootCertTmpl.SerialNumber = big.NewInt(3)
+	oldestRootCert, oldestRootCertPem := test.GenerateSelfSignedCertificateWithPrivateKey(oldestRootCertTmpl, rootPrivateKey)
+
+	serverCertExpiry := time.Now().AddDate(0, 0, 4)
+	serverCertTmpl := test.GenerateCertificateTemplate(serverCertExpiry)
+	serverCertTmpl.SerialNumber = big.NewInt(4)
+	serverCert, serverCertPem, serverKey := test.GenerateSignedCertificate(serverCertTmpl, olderRootCert, rootPrivateKey)
+
+	verifiedChains := [][]*x509.Certificate{
+		[]*x509.Certificate{
+			serverCert,
+			rootCert,
+		},
+		[]*x509.Certificate{
+			serverCert,
+			olderRootCert,
+		},
+		[]*x509.Certificate{
+			serverCert,
+			oldestRootCert,
+		},
+	}
+
+	caCertPem := bytes.Join([][]byte{oldestRootCertPem, olderRootCertPem, rootCertPem}, []byte(""))
+
+	server, caFile, teardown, err := test.SetupHTTPSServerWithCertAndKey(
+		caCertPem,
+		serverCertPem,
+		serverKey,
+	)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+	defer teardown()
+
+	server.StartTLS()
+	defer server.Close()
+
+	conf := &config.Config{
+		Modules: map[string]config.Module{
+			"https": config.Module{
+				Prober: "https",
+				TLSConfig: pconfig.TLSConfig{
+					CAFile: caFile,
+				},
+			},
+		},
+	}
+
+	rr, err := probe(server.URL, "https", conf)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	// Check verifiedNotAfter and verifiedNotBefore metrics
+	if err := checkVerifiedChainDates(verifiedChains, rr.Body.String()); err != nil {
+		t.Errorf(err.Error())
+	}
+}
+
 func TestProbeHandlerHTTPSNoServer(t *testing.T) {
 	rr, err := probe("localhost:6666", "https", config.DefaultConfig)
 	if err != nil {
@ -232,6 +319,89 @@ func TestProbeHandlerTCP(t *testing.T) {
 	}
 }

+// TestProbeHandlerTCPVerifiedChains checks that metrics are generated
+// correctly for the verified chains
+func TestProbeHandlerTCPVerifiedChains(t *testing.T) {
+	rootPrivateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	rootCertExpiry := time.Now().AddDate(0, 0, 5)
+	rootCertTmpl := test.GenerateCertificateTemplate(rootCertExpiry)
+	rootCertTmpl.IsCA = true
+	rootCertTmpl.SerialNumber = big.NewInt(1)
+	rootCert, rootCertPem := test.GenerateSelfSignedCertificateWithPrivateKey(rootCertTmpl, rootPrivateKey)
+
+	olderRootCertExpiry := time.Now().AddDate(0, 0, 3)
+	olderRootCertTmpl := test.GenerateCertificateTemplate(olderRootCertExpiry)
+	olderRootCertTmpl.IsCA = true
+	olderRootCertTmpl.SerialNumber = big.NewInt(2)
+	olderRootCert, olderRootCertPem := test.GenerateSelfSignedCertificateWithPrivateKey(olderRootCertTmpl, rootPrivateKey)
+
+	oldestRootCertExpiry := time.Now().AddDate(0, 0, 1)
+	oldestRootCertTmpl := test.GenerateCertificateTemplate(oldestRootCertExpiry)
+	oldestRootCertTmpl.IsCA = true
+	oldestRootCertTmpl.SerialNumber = big.NewInt(3)
+	oldestRootCert, oldestRootCertPem := test.GenerateSelfSignedCertificateWithPrivateKey(oldestRootCertTmpl, rootPrivateKey)
+
+	serverCertExpiry := time.Now().AddDate(0, 0, 4)
+	serverCertTmpl := test.GenerateCertificateTemplate(serverCertExpiry)
+	serverCertTmpl.SerialNumber = big.NewInt(4)
+	serverCert, serverCertPem, serverKey := test.GenerateSignedCertificate(serverCertTmpl, olderRootCert, rootPrivateKey)
+
+	verifiedChains := [][]*x509.Certificate{
+		[]*x509.Certificate{
+			serverCert,
+			rootCert,
+		},
+		[]*x509.Certificate{
+			serverCert,
+			olderRootCert,
+		},
+		[]*x509.Certificate{
+			serverCert,
+			oldestRootCert,
+		},
+	}
+
+	caCertPem := bytes.Join([][]byte{oldestRootCertPem, olderRootCertPem, rootCertPem}, []byte(""))
+
+	server, caFile, teardown, err := test.SetupTCPServerWithCertAndKey(
+		caCertPem,
+		serverCertPem,
+		serverKey,
+	)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+	defer teardown()
+
+	server.StartTLS()
+	defer server.Close()
+
+	conf := &config.Config{
+		Modules: map[string]config.Module{
+			"tcp": config.Module{
+				Prober: "tcp",
+				TLSConfig: pconfig.TLSConfig{
+					CAFile: caFile,
+				},
+			},
+		},
+	}
+
+	rr, err := probe(server.Listener.Addr().String(), "tcp", conf)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	// Check verifiedNotAfter and verifiedNotBefore metrics
+	if err := checkVerifiedChainDates(verifiedChains, rr.Body.String()); err != nil {
+		t.Errorf(err.Error())
+	}
+}
+
 //  TestProbeHandlerTCPNoServer tests against a tcp server that doesn't exist
 func TestProbeHandlerTCPNoServer(t *testing.T) {
 	rr, err := probe("localhost:6666", "tcp", config.DefaultConfig)
@ -626,6 +796,44 @@ func checkDates(certPEM []byte, body string) error {
 	return nil
 }

+func checkVerifiedChainDates(verifiedChains [][]*x509.Certificate, body string) error {
+	for i, chain := range verifiedChains {
+		for _, cert := range chain {
+			notAfter := strconv.FormatFloat(float64(cert.NotAfter.UnixNano()/1e9), 'g', -1, 64)
+			notAfterMetric := "ssl_verified_cert_not_after{" + strings.Join([]string{
+				"chain_no=\"" + strconv.Itoa(i) + "\"",
+				"cn=\"example.ribbybibby.me\"",
+				"dnsnames=\",example.ribbybibby.me,example-2.ribbybibby.me,example-3.ribbybibby.me,\"",
+				"emails=\",me@ribbybibby.me,example@ribbybibby.me,\"",
+				"ips=\",127.0.0.1,::1,\"",
+				"issuer_cn=\"example.ribbybibby.me\"",
+				"ou=\",ribbybibbys org,\"",
+				"serial_no=\"" + cert.SerialNumber.String() + "\"",
+			}, ",") + "} " + notAfter
+			if ok := strings.Contains(body, notAfter); !ok {
+				return fmt.Errorf("expected `%s` in: %s", notAfterMetric, body)
+			}
+
+			notBefore := strconv.FormatFloat(float64(cert.NotBefore.UnixNano()/1e9), 'g', -1, 64)
+			notBeforeMetric := "ssl_verified_cert_not_before{" + strings.Join([]string{
+				"chain_no=\"" + strconv.Itoa(i) + "\"",
+				"cn=\"example.ribbybibby.me\"",
+				"dnsnames=\",example.ribbybibby.me,example-2.ribbybibby.me,example-3.ribbybibby.me,\"",
+				"emails=\",me@ribbybibby.me,example@ribbybibby.me,\"",
+				"ips=\",127.0.0.1,::1,\"",
+				"issuer_cn=\"example.ribbybibby.me\"",
+				"ou=\",ribbybibbys org,\"",
+				"serial_no=\"" + cert.SerialNumber.String() + "\"",
+			}, ",") + "} " + notBefore
+			if ok := strings.Contains(body, notBeforeMetric); !ok {
+				return fmt.Errorf("expected `%s` in: %s", notBeforeMetric, body)
+			}
+		}
+	}
+
+	return nil
+}
+
 func probe(target, module string, conf *config.Config) (*httptest.ResponseRecorder, error) {
 	uri := "/probe?target=" + target
 	if module != "" {
--- a/test/https.go
+++ b/test/https.go
@ -14,32 +14,41 @@ import (
 // SetupHTTPSServer sets up a server for testing with a generated cert and key
 // pair
 func SetupHTTPSServer() (*httptest.Server, []byte, []byte, string, func(), error) {
-	var teardown func()
-
 	testcertPEM, testkeyPEM := GenerateTestCertificate(time.Now().AddDate(0, 0, 1))

-	caFile, err := WriteFile("certfile.pem", testcertPEM)
+	server, caFile, teardown, err := SetupHTTPSServerWithCertAndKey(testcertPEM, testcertPEM, testkeyPEM)
 	if err != nil {
 		return nil, testcertPEM, testkeyPEM, caFile, teardown, err
 	}

+	return server, testcertPEM, testkeyPEM, caFile, teardown, nil
+}
+
+// SetupHTTPSServerWithCertAndKey sets up a server with a provided certs and key
+func SetupHTTPSServerWithCertAndKey(caPEM, certPEM, keyPEM []byte) (*httptest.Server, string, func(), error) {
+	var teardown func()
+
+	caFile, err := WriteFile("certfile.pem", caPEM)
+	if err != nil {
+		return nil, caFile, teardown, err
+	}
+
 	teardown = func() {
 		os.Remove(caFile)
 	}

-	// Create server
-	testcert, err := tls.X509KeyPair(testcertPEM, testkeyPEM)
+	testCert, err := tls.X509KeyPair(certPEM, keyPEM)
 	if err != nil {
-		return nil, testcertPEM, testkeyPEM, caFile, teardown, err
+		return nil, caFile, teardown, err
 	}
 	server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprintln(w, "Hello world")
 	}))
 	server.TLS = &tls.Config{
-		Certificates: []tls.Certificate{testcert},
+		Certificates: []tls.Certificate{testCert},
 	}

-	return server, testcertPEM, testkeyPEM, caFile, teardown, nil
+	return server, caFile, teardown, nil
 }

 // SetupHTTPProxyServer sets up a proxy server
--- a/test/tcp.go
+++ b/test/tcp.go
@ -145,35 +145,44 @@ func (t *TCPServer) Close() {
 // SetupTCPServer sets up a server for testing with a generated cert and key
 // pair
 func SetupTCPServer() (*TCPServer, []byte, []byte, string, func(), error) {
-	var teardown func()
-
 	testcertPEM, testkeyPEM := GenerateTestCertificate(time.Now().AddDate(0, 0, 1))

-	caFile, err := WriteFile("certfile.pem", testcertPEM)
+	server, caFile, teardown, err := SetupTCPServerWithCertAndKey(testcertPEM, testcertPEM, testkeyPEM)
 	if err != nil {
 		return nil, testcertPEM, testkeyPEM, caFile, teardown, err
 	}

+	return server, testcertPEM, testkeyPEM, caFile, teardown, nil
+}
+
+// SetupTCPServerWithCertAndKey sets up a server with the provided certs and key
+func SetupTCPServerWithCertAndKey(caPEM, certPEM, keyPEM []byte) (*TCPServer, string, func(), error) {
+	var teardown func()
+
+	caFile, err := WriteFile("certfile.pem", caPEM)
+	if err != nil {
+		return nil, caFile, teardown, err
+	}
+
 	teardown = func() {
 		os.Remove(caFile)
 	}

-	testcert, err := tls.X509KeyPair(testcertPEM, testkeyPEM)
+	testCert, err := tls.X509KeyPair(certPEM, keyPEM)
 	if err != nil {
-		panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err))
+		return nil, caFile, teardown, err
 	}

 	tlsConfig := &tls.Config{
 		ServerName:   "127.0.0.1",
-		Certificates: []tls.Certificate{testcert},
+		Certificates: []tls.Certificate{testCert},
 		MinVersion:   tls.VersionTLS13,
 		MaxVersion:   tls.VersionTLS13,
 	}

-	// Create server
 	ln, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
-		return nil, testcertPEM, testkeyPEM, caFile, teardown, err
+		return nil, caFile, teardown, err
 	}

 	server := &TCPServer{
@ -182,5 +191,5 @@ func SetupTCPServer() (*TCPServer, []byte, []byte, string, func(), error) {
 		stopCh:   make(chan (struct{})),
 	}

-	return server, testcertPEM, testkeyPEM, caFile, teardown, nil
+	return server, caFile, teardown, err
 }
--- a/test/test.go
+++ b/test/test.go
@ -15,38 +15,72 @@ import (

 // GenerateTestCertificate generates a test certificate with the given expiry date
 func GenerateTestCertificate(expiry time.Time) ([]byte, []byte) {
-	privatekey, err := rsa.GenerateKey(rand.Reader, 2048)
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
 	if err != nil {
 		panic(fmt.Sprintf("Error creating rsa key: %s", err))
 	}
-	publickey := &privatekey.PublicKey
+	pemKey := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)})

-	cert := x509.Certificate{
-		IsCA:                  true,
+	cert := GenerateCertificateTemplate(expiry)
+	cert.IsCA = true
+
+	_, pemCert := GenerateSelfSignedCertificateWithPrivateKey(cert, privateKey)
+
+	return pemCert, pemKey
+}
+
+func GenerateSignedCertificate(cert, parentCert *x509.Certificate, parentKey *rsa.PrivateKey) (*x509.Certificate, []byte, []byte) {
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		panic(fmt.Sprintf("Error creating rsa key: %s", err))
+	}
+
+	derCert, err := x509.CreateCertificate(rand.Reader, cert, parentCert, &privateKey.PublicKey, parentKey)
+	if err != nil {
+		panic(fmt.Sprintf("Error signing test-certificate: %s", err))
+	}
+
+	genCert, err := x509.ParseCertificate(derCert)
+	if err != nil {
+		panic(fmt.Sprintf("Error parsing test-certificate: %s", err))
+	}
+
+	return genCert,
+		pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derCert}),
+		pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)})
+}
+func GenerateSelfSignedCertificateWithPrivateKey(cert *x509.Certificate, privateKey *rsa.PrivateKey) (*x509.Certificate, []byte) {
+	derCert, err := x509.CreateCertificate(rand.Reader, cert, cert, &privateKey.PublicKey, privateKey)
+	if err != nil {
+		panic(fmt.Sprintf("Error signing test-certificate: %s", err))
+	}
+
+	genCert, err := x509.ParseCertificate(derCert)
+	if err != nil {
+		panic(fmt.Sprintf("Error parsing test-certificate: %s", err))
+	}
+
+	return genCert, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derCert})
+}
+
+func GenerateCertificateTemplate(expiry time.Time) *x509.Certificate {
+	return &x509.Certificate{
 		BasicConstraintsValid: true,
 		SubjectKeyId:          []byte{1},
 		SerialNumber:          big.NewInt(100),
+		NotBefore:             time.Now(),
+		NotAfter:              expiry,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		IPAddresses:           []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")},
 		Subject: pkix.Name{
 			CommonName:         "example.ribbybibby.me",
 			Organization:       []string{"ribbybibby"},
 			OrganizationalUnit: []string{"ribbybibbys org"},
 		},
 		EmailAddresses: []string{"me@ribbybibby.me", "example@ribbybibby.me"},
-		IPAddresses:    []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")},
 		DNSNames:       []string{"example.ribbybibby.me", "example-2.ribbybibby.me", "example-3.ribbybibby.me"},
-		NotBefore:      time.Now(),
-		NotAfter:       expiry,
-		ExtKeyUsage:    []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
-		KeyUsage:       x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
 	}
-
-	derCert, err := x509.CreateCertificate(rand.Reader, &cert, &cert, publickey, privatekey)
-	if err != nil {
-		panic(fmt.Sprintf("Error signing test-certificate: %s", err))
-	}
-	pemCert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derCert})
-	pemKey := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privatekey)})
-	return pemCert, pemKey
 }

 // WriteFile writes some content to a temporary file