From 526525177719aa7efd072613405fb1567beacf54 Mon Sep 17 00:00:00 2001 From: treydock Date: Fri, 2 Apr 2021 05:53:31 -0400 Subject: [PATCH] Support getting certificate information from a kubeconfig file (#61) * Support getting certificate information from a kubeconfig file * Support relative paths for cluster CA and user certificate in kubeconfig * Determine relative using filepath.IsAbs * Make relative path logic actually work, add test. Move all kubeconfig parsing into parsing specific function --- README.md | 37 ++++++- config/config.go | 3 + examples/ssl_exporter.yaml | 2 + prober/kubeconfig.go | 91 +++++++++++++++++ prober/kubeconfig_test.go | 195 +++++++++++++++++++++++++++++++++++++ prober/metrics.go | 100 +++++++++++++++++++ prober/prober.go | 1 + 7 files changed, 428 insertions(+), 1 deletion(-) create mode 100644 prober/kubeconfig.go create mode 100644 prober/kubeconfig_test.go diff --git a/README.md b/README.md index c50716b..264aa73 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ which allows for informational dashboards and flexible alert routing. - [HTTPS](#https) - [File](#file) - [Kubernetes](#kubernetes) + - [Kubeconfig](#kubeconfig) - [Configuration file](#configuration-file) - [<module>](#module) - [<tls_config>](#tls_config) @@ -86,6 +87,8 @@ Flags: | ssl_file_cert_not_before | The date before which a certificate found by the file prober is not valid. Expressed as a Unix Epoch Time. | file, serial_no, issuer_cn, cn, dnsnames, ips, emails, ou | file | | ssl_kubernetes_cert_not_after | The date after which a certificate found by the kubernetes prober expires. Expressed as a Unix Epoch Time. | namespace, secret, key, serial_no, issuer_cn, cn, dnsnames, ips, emails, ou | kubernetes | | ssl_kubernetes_cert_not_before | The date before which a certificate found by the kubernetes prober is not valid. Expressed as a Unix Epoch Time. | namespace, secret, key, serial_no, issuer_cn, cn, dnsnames, ips, emails, ou | kubernetes | +| ssl_kubeconfig_cert_not_after | The date after which a certificate found by the kubeconfig prober expires. Expressed as a Unix Epoch Time. | kubeconfig, name, type, serial_no, issuer_cn, cn, dnsnames, ips, emails, ou | kubeconfig | +| ssl_kubeconfig_cert_not_before | The date before which a certificate found by the kubeconfig prober is not valid. Expressed as a Unix Epoch Time. | kubeconfig, name, type, serial_no, issuer_cn, cn, dnsnames, ips, emails, ou | kubeconfig | | ssl_ocsp_response_next_update | The nextUpdate value in the OCSP response. Expressed as a Unix Epoch Time | | tcp, https | | ssl_ocsp_response_produced_at | The producedAt value in the OCSP response. Expressed as a Unix Epoch Time | | tcp, https | | ssl_ocsp_response_revoked_at | The revocationTime value in the OCSP response. Expressed as a Unix Epoch Time | | tcp, https | @@ -229,6 +232,38 @@ sources in the following order: - The default configuration file (`$HOME/.kube/config`) - The in-cluster environment, if running in a pod +### Kubeconfig + +The `kubeconfig` prober exports `ssl_kubeconfig_cert_not_after` and +`ssl_kubeconfig_cert_not_before` for PEM encoded certificates found in the specified kubeconfig file. + +Kubeconfigs local to the exporter can be scraped by providing them as the target +parameter: + +``` +curl "localhost:9219/probe?module=kubeconfig&target=/etc/kubernetes/admin.conf" +``` + +One specific usage of this prober could be to run the exporter as a DaemonSet in +Kubernetes and then scrape each instance to check the expiry of certificates on +each node: + +```yml +scrape_configs: + - job_name: "ssl-kubernetes-kubeconfig" + metrics_path: /probe + params: + module: ["kubeconfig"] + target: ["/etc/kubernetes/admin.conf"] + kubernetes_sd_configs: + - role: node + relabel_configs: + - source_labels: [__address__] + regex: ^(.*):(.*)$ + target_label: __address__ + replacement: ${1}:9219 +``` + ## Configuration file You can provide further module configuration by providing the path to a @@ -242,7 +277,7 @@ modules: [] ### \ ``` -# The type of probe (https, tcp, file, kubernetes) +# The type of probe (https, tcp, file, kubernetes, kubeconfig) prober: # How long the probe will wait before giving up. diff --git a/config/config.go b/config/config.go index 1630802..e1fa267 100644 --- a/config/config.go +++ b/config/config.go @@ -30,6 +30,9 @@ var ( "kubernetes": Module{ Prober: "kubernetes", }, + "kubeconfig": Module{ + Prober: "kubeconfig", + }, }, } ) diff --git a/examples/ssl_exporter.yaml b/examples/ssl_exporter.yaml index 5da5e2e..d54a540 100644 --- a/examples/ssl_exporter.yaml +++ b/examples/ssl_exporter.yaml @@ -36,3 +36,5 @@ modules: prober: kubernetes kubernetes: kubeconfig: /root/.kube/config + kubeconfig: + prober: kubeconfig diff --git a/prober/kubeconfig.go b/prober/kubeconfig.go new file mode 100644 index 0000000..0948d74 --- /dev/null +++ b/prober/kubeconfig.go @@ -0,0 +1,91 @@ +package prober + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/prometheus/client_golang/prometheus" + "github.com/ribbybibby/ssl_exporter/config" + "gopkg.in/yaml.v2" +) + +type KubeConfigCluster struct { + Name string + Cluster KubeConfigClusterCert +} + +type KubeConfigClusterCert struct { + CertificateAuthority string `yaml:"certificate-authority"` + CertificateAuthorityData string `yaml:"certificate-authority-data"` +} + +type KubeConfigUser struct { + Name string + User KubeConfigUserCert +} + +type KubeConfigUserCert struct { + ClientCertificate string `yaml:"client-certificate"` + ClientCertificateData string `yaml:"client-certificate-data"` +} + +type KubeConfig struct { + Path string + Clusters []KubeConfigCluster + Users []KubeConfigUser +} + +// ProbeKubeconfig collects certificate metrics from kubeconfig files +func ProbeKubeconfig(ctx context.Context, target string, module config.Module, registry *prometheus.Registry) error { + if _, err := os.Stat(target); err != nil { + return fmt.Errorf("kubeconfig not found: %s", target) + } + k, err := ParseKubeConfig(target) + if err != nil { + return err + } + err = collectKubeconfigMetrics(*k, registry) + if err != nil { + return err + } + return nil +} + +func ParseKubeConfig(file string) (*KubeConfig, error) { + k := &KubeConfig{} + + data, err := ioutil.ReadFile(file) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal([]byte(data), k) + if err != nil { + return nil, err + } + k.Path = file + clusters := []KubeConfigCluster{} + users := []KubeConfigUser{} + for _, c := range k.Clusters { + // Path is relative to kubeconfig path + if c.Cluster.CertificateAuthority != "" && !filepath.IsAbs(c.Cluster.CertificateAuthority) { + newPath := filepath.Join(filepath.Dir(k.Path), c.Cluster.CertificateAuthority) + c.Cluster.CertificateAuthority = newPath + } + clusters = append(clusters, c) + } + for _, u := range k.Users { + // Path is relative to kubeconfig path + if u.User.ClientCertificate != "" && !filepath.IsAbs(u.User.ClientCertificate) { + newPath := filepath.Join(filepath.Dir(k.Path), u.User.ClientCertificate) + u.User.ClientCertificate = newPath + } + users = append(users, u) + } + k.Clusters = clusters + k.Users = users + return k, nil +} diff --git a/prober/kubeconfig_test.go b/prober/kubeconfig_test.go new file mode 100644 index 0000000..46cff7b --- /dev/null +++ b/prober/kubeconfig_test.go @@ -0,0 +1,195 @@ +package prober + +import ( + "context" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/ribbybibby/ssl_exporter/config" + "github.com/ribbybibby/ssl_exporter/test" + + "github.com/prometheus/client_golang/prometheus" + "gopkg.in/yaml.v2" +) + +// TestProbeFile tests a file +func TestProbeKubeconfig(t *testing.T) { + cert, kubeconfig, err := createTestKubeconfig("", "kubeconfig") + if err != nil { + t.Fatalf(err.Error()) + } + defer os.Remove(kubeconfig) + + module := config.Module{} + + registry := prometheus.NewRegistry() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := ProbeKubeconfig(ctx, kubeconfig, module, registry); err != nil { + t.Fatalf("error: %s", err) + } + + checkKubeconfigMetrics(cert, kubeconfig, registry, t) +} + +func TestParseKubeConfigRelative(t *testing.T) { + tmpFile, err := ioutil.TempFile("", "kubeconfig") + if err != nil { + t.Fatalf("Unable to create Tempfile: %s", err.Error()) + } + defer os.Remove(tmpFile.Name()) + file := []byte(` +clusters: + - cluster: + certificate-authority: certs/example/ca.pem + server: https://master.example.com + name: example +users: + - user: + client-certificate: test/ca.pem + name: example`) + if _, err := tmpFile.Write(file); err != nil { + t.Fatalf("Unable to write Tempfile: %s", err.Error()) + } + expectedClusterPath := filepath.Join(filepath.Dir(tmpFile.Name()), "certs/example/ca.pem") + expectedUserPath := filepath.Join(filepath.Dir(tmpFile.Name()), "test/ca.pem") + k, err := ParseKubeConfig(tmpFile.Name()) + if err != nil { + t.Fatalf("Error parsing kubeconfig: %s", err.Error()) + } + if len(k.Clusters) != 1 { + t.Fatalf("Unexpected length for Clusters, got %d", len(k.Clusters)) + } + if k.Clusters[0].Cluster.CertificateAuthority != expectedClusterPath { + t.Errorf("Unexpected CertificateAuthority value\nExpected: %s\nGot: %s", expectedClusterPath, k.Clusters[0].Cluster.CertificateAuthority) + } + if len(k.Users) != 1 { + t.Fatalf("Unexpected length for Users, got %d", len(k.Users)) + } + if k.Users[0].User.ClientCertificate != expectedUserPath { + t.Errorf("Unexpected ClientCertificate value\nExpected: %s\nGot: %s", expectedUserPath, k.Users[0].User.ClientCertificate) + } +} + +// Create a certificate and write it to a file +func createTestKubeconfig(dir, filename string) (*x509.Certificate, string, error) { + certPEM, _ := test.GenerateTestCertificate(time.Now().Add(time.Hour * 1)) + clusterCert := KubeConfigClusterCert{CertificateAuthorityData: base64.StdEncoding.EncodeToString([]byte(certPEM))} + clusters := []KubeConfigCluster{KubeConfigCluster{Name: "kubernetes", Cluster: clusterCert}} + userCert := KubeConfigUserCert{ClientCertificateData: base64.StdEncoding.EncodeToString([]byte(certPEM))} + users := []KubeConfigUser{KubeConfigUser{Name: "kubernetes-admin", User: userCert}} + k := KubeConfig{ + Clusters: clusters, + Users: users, + } + block, _ := pem.Decode([]byte(certPEM)) + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, "", err + } + tmpFile, err := ioutil.TempFile(dir, filename) + if err != nil { + return nil, tmpFile.Name(), err + } + k.Path = tmpFile.Name() + d, err := yaml.Marshal(&k) + if err != nil { + return nil, tmpFile.Name(), err + } + if _, err := tmpFile.Write(d); err != nil { + return nil, tmpFile.Name(), err + } + if err := tmpFile.Close(); err != nil { + return nil, tmpFile.Name(), err + } + + return cert, tmpFile.Name(), nil +} + +// Check metrics +func checkKubeconfigMetrics(cert *x509.Certificate, kubeconfig string, registry *prometheus.Registry, t *testing.T) { + mfs, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + ips := "," + for _, ip := range cert.IPAddresses { + ips = ips + ip.String() + "," + } + expectedResults := []*registryResult{ + ®istryResult{ + Name: "ssl_kubeconfig_cert_not_after", + LabelValues: map[string]string{ + "kubeconfig": kubeconfig, + "name": "kubernetes", + "type": "cluster", + "serial_no": cert.SerialNumber.String(), + "issuer_cn": cert.Issuer.CommonName, + "cn": cert.Subject.CommonName, + "dnsnames": "," + strings.Join(cert.DNSNames, ",") + ",", + "ips": ips, + "emails": "," + strings.Join(cert.EmailAddresses, ",") + ",", + "ou": "," + strings.Join(cert.Subject.OrganizationalUnit, ",") + ",", + }, + Value: float64(cert.NotAfter.Unix()), + }, + ®istryResult{ + Name: "ssl_kubeconfig_cert_not_before", + LabelValues: map[string]string{ + "kubeconfig": kubeconfig, + "name": "kubernetes", + "type": "cluster", + "serial_no": cert.SerialNumber.String(), + "issuer_cn": cert.Issuer.CommonName, + "cn": cert.Subject.CommonName, + "dnsnames": "," + strings.Join(cert.DNSNames, ",") + ",", + "ips": ips, + "emails": "," + strings.Join(cert.EmailAddresses, ",") + ",", + "ou": "," + strings.Join(cert.Subject.OrganizationalUnit, ",") + ",", + }, + Value: float64(cert.NotBefore.Unix()), + }, + ®istryResult{ + Name: "ssl_kubeconfig_cert_not_after", + LabelValues: map[string]string{ + "kubeconfig": kubeconfig, + "name": "kubernetes-admin", + "type": "user", + "serial_no": cert.SerialNumber.String(), + "issuer_cn": cert.Issuer.CommonName, + "cn": cert.Subject.CommonName, + "dnsnames": "," + strings.Join(cert.DNSNames, ",") + ",", + "ips": ips, + "emails": "," + strings.Join(cert.EmailAddresses, ",") + ",", + "ou": "," + strings.Join(cert.Subject.OrganizationalUnit, ",") + ",", + }, + Value: float64(cert.NotAfter.Unix()), + }, + ®istryResult{ + Name: "ssl_kubeconfig_cert_not_before", + LabelValues: map[string]string{ + "kubeconfig": kubeconfig, + "name": "kubernetes-admin", + "type": "user", + "serial_no": cert.SerialNumber.String(), + "issuer_cn": cert.Issuer.CommonName, + "cn": cert.Subject.CommonName, + "dnsnames": "," + strings.Join(cert.DNSNames, ",") + ",", + "ips": ips, + "emails": "," + strings.Join(cert.EmailAddresses, ",") + ",", + "ou": "," + strings.Join(cert.Subject.OrganizationalUnit, ",") + ",", + }, + Value: float64(cert.NotBefore.Unix()), + }, + } + checkRegistryResults(expectedResults, mfs, t) +} diff --git a/prober/metrics.go b/prober/metrics.go index 1dbe7b6..5a5b59b 100644 --- a/prober/metrics.go +++ b/prober/metrics.go @@ -3,6 +3,7 @@ package prober import ( "crypto/tls" "crypto/x509" + "encoding/base64" "fmt" "io/ioutil" "sort" @@ -332,6 +333,105 @@ func collectKubernetesSecretMetrics(secrets []v1.Secret, registry *prometheus.Re return nil } +func collectKubeconfigMetrics(kubeconfig KubeConfig, registry *prometheus.Registry) error { + var ( + totalCerts []*x509.Certificate + kubeconfigNotAfter = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: prometheus.BuildFQName(namespace, "kubeconfig", "cert_not_after"), + Help: "NotAfter expressed as a Unix Epoch Time for a certificate found in a kubeconfig", + }, + []string{"kubeconfig", "name", "type", "serial_no", "issuer_cn", "cn", "dnsnames", "ips", "emails", "ou"}, + ) + kubeconfigNotBefore = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: prometheus.BuildFQName(namespace, "kubeconfig", "cert_not_before"), + Help: "NotBefore expressed as a Unix Epoch Time for a certificate found in a kubeconfig", + }, + []string{"kubeconfig", "name", "type", "serial_no", "issuer_cn", "cn", "dnsnames", "ips", "emails", "ou"}, + ) + ) + registry.MustRegister(kubeconfigNotAfter, kubeconfigNotBefore) + + for _, c := range kubeconfig.Clusters { + var data []byte + var err error + if c.Cluster.CertificateAuthorityData != "" { + data, err = base64.StdEncoding.DecodeString(c.Cluster.CertificateAuthorityData) + if err != nil { + return err + } + } else if c.Cluster.CertificateAuthority != "" { + data, err = ioutil.ReadFile(c.Cluster.CertificateAuthority) + if err != nil { + log.Debugf("Error reading file: %s error=%s", c.Cluster.CertificateAuthority, err) + return err + } + } + if data == nil { + continue + } + certs, err := decodeCertificates(data) + if err != nil { + return err + } + totalCerts = append(totalCerts, certs...) + for _, cert := range certs { + labels := append([]string{kubeconfig.Path, c.Name, "cluster"}, labelValues(cert)...) + + if !cert.NotAfter.IsZero() { + kubeconfigNotAfter.WithLabelValues(labels...).Set(float64(cert.NotAfter.Unix())) + } + + if !cert.NotBefore.IsZero() { + kubeconfigNotBefore.WithLabelValues(labels...).Set(float64(cert.NotBefore.Unix())) + } + } + } + + for _, u := range kubeconfig.Users { + var data []byte + var err error + if u.User.ClientCertificateData != "" { + data, err = base64.StdEncoding.DecodeString(u.User.ClientCertificateData) + if err != nil { + return err + } + } else if u.User.ClientCertificate != "" { + data, err = ioutil.ReadFile(u.User.ClientCertificate) + if err != nil { + log.Debugf("Error reading file: %s error=%s", u.User.ClientCertificate, err) + return err + } + } + if data == nil { + continue + } + certs, err := decodeCertificates(data) + if err != nil { + return err + } + totalCerts = append(totalCerts, certs...) + for _, cert := range certs { + labels := append([]string{kubeconfig.Path, u.Name, "user"}, labelValues(cert)...) + + if !cert.NotAfter.IsZero() { + kubeconfigNotAfter.WithLabelValues(labels...).Set(float64(cert.NotAfter.Unix())) + } + + if !cert.NotBefore.IsZero() { + kubeconfigNotBefore.WithLabelValues(labels...).Set(float64(cert.NotBefore.Unix())) + } + } + } + + if len(totalCerts) == 0 { + return fmt.Errorf("No certificates found") + } + + return nil +} + func labelValues(cert *x509.Certificate) []string { return []string{ cert.SerialNumber.String(), diff --git a/prober/prober.go b/prober/prober.go index f7c8086..5f2511e 100644 --- a/prober/prober.go +++ b/prober/prober.go @@ -15,6 +15,7 @@ var ( "tcp": ProbeTCP, "file": ProbeFile, "kubernetes": ProbeKubernetes, + "kubeconfig": ProbeKubeconfig, } )