From 515b990f520b8143314616de39d5820d6c125c46 Mon Sep 17 00:00:00 2001 From: jaroug Date: Sun, 28 Apr 2024 17:48:09 +0200 Subject: [PATCH] Add http_file prober (#144) * feat: add remote_file probe * fix: use tls module config * chore: write http/https tests for probing remote file * chore: get rid of useless lines * fix: get rid of useless file download, check body directly * fix: use checkCertificateMetrics to actually check values * Rename remote_file to http_file You can fetch remote content with a lot of different protocols, so I think it's worth being specific here. As part of this change I've fixed up some of the logic in the code. I've also created a separate `http_file` block in the module config. * Actually include renamed files --------- Co-authored-by: Anthony LE BERRE Co-authored-by: Rob Best --- README.md | 49 +++++++++++++- config/config.go | 21 ++++-- examples/example.prometheus.yml | 15 +++++ examples/ssl_exporter.yaml | 6 ++ prober/http_file.go | 59 +++++++++++++++++ prober/http_file_test.go | 112 ++++++++++++++++++++++++++++++++ prober/prober.go | 1 + 7 files changed, 256 insertions(+), 7 deletions(-) create mode 100644 prober/http_file.go create mode 100644 prober/http_file_test.go diff --git a/README.md b/README.md index d2148dd..da58d58 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ Exports metrics for certificates collected from various sources: - [TCP probes](#tcp) - [HTTPS probes](#https) - [PEM files](#file) +- [Remote PEM files](#http_file) - [Kubernetes secrets](#kubernetes) - [Kubeconfig files](#kubeconfig) @@ -130,7 +131,7 @@ scrape_configs: ``` This will use proxy servers discovered by the environment variables `HTTP_PROXY`, -`HTTPS_PROXY` and `ALL_PROXY`. Or, you can set the `proxy_url` option in the module +`HTTPS_PROXY` and `ALL_PROXY`. Or, you can set the `https.proxy_url` option in the module configuration. The latter takes precedence. @@ -175,6 +176,44 @@ scrape_configs: replacement: ${1}:9219 ``` +### HTTP File + +The `http_file` prober exports `ssl_cert_not_after` and +`ssl_cert_not_before` for PEM encoded certificates found at the +specified URL. + +``` +curl "localhost:9219/probe?module=http_file&target=https://www.paypalobjects.com/marketing/web/logos/paypal_com.pem" +``` + +Here's a sample Prometheus configuration: + +```yml +scrape_configs: + - job_name: 'ssl-http-files' + metrics_path: /probe + params: + module: ["http_file"] + static_configs: + - targets: + - 'https://www.paypalobjects.com/marketing/web/logos/paypal_com.pem' + - 'https://d3frv9g52qce38.cloudfront.net/amazondefault/amazon_web_services_inc_2024.pem' + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 127.0.0.1:9219 +``` + +For proxying to the target resource, this prober will use proxy servers +discovered in the environment variables `HTTP_PROXY`, `HTTPS_PROXY` and +`ALL_PROXY`. Or, you can set the `http_file.proxy_url` option in the module +configuration. + +The latter takes precedence. + ### Kubernetes The `kubernetes` prober exports `ssl_kubernetes_cert_not_after` and @@ -293,6 +332,7 @@ target: [ https: ] [ tcp: ] [ kubernetes: ] +[ http_file: ] ``` ### @@ -339,6 +379,13 @@ target: [ kubeconfig: ] ``` +### + +``` +# HTTP proxy server to use to connect to the targets. +[ proxy_url: ] +``` + ## Example Queries Certificates that expire within 7 days: diff --git a/config/config.go b/config/config.go index f7c4c53..15b8c86 100644 --- a/config/config.go +++ b/config/config.go @@ -17,22 +17,25 @@ var ( DefaultConfig = &Config{ DefaultModule: "tcp", Modules: map[string]Module{ - "tcp": Module{ + "tcp": { Prober: "tcp", }, - "http": Module{ + "http": { Prober: "https", }, - "https": Module{ + "https": { Prober: "https", }, - "file": Module{ + "file": { Prober: "file", }, - "kubernetes": Module{ + "http_file": { + Prober: "http_file", + }, + "kubernetes": { Prober: "kubernetes", }, - "kubeconfig": Module{ + "kubeconfig": { Prober: "kubeconfig", }, }, @@ -73,6 +76,7 @@ type Module struct { HTTPS HTTPSProbe `yaml:"https,omitempty"` TCP TCPProbe `yaml:"tcp,omitempty"` Kubernetes KubernetesProbe `yaml:"kubernetes,omitempty"` + HTTPFile HTTPFileProbe `yaml:"http_file,omitempty"` } // TLSConfig is a superset of config.TLSConfig that supports TLS renegotiation @@ -142,6 +146,11 @@ type KubernetesProbe struct { Kubeconfig string `yaml:"kubeconfig,omitempty"` } +// HTTPFileProbe configures a http_file probe +type HTTPFileProbe struct { + ProxyURL URL `yaml:"proxy_url,omitempty"` +} + // URL is a custom URL type that allows validation at configuration load time type URL struct { *url.URL diff --git a/examples/example.prometheus.yml b/examples/example.prometheus.yml index 02eb1c5..fd14912 100644 --- a/examples/example.prometheus.yml +++ b/examples/example.prometheus.yml @@ -34,3 +34,18 @@ scrape_configs: static_configs: - targets: - 127.0.0.1:9219 + - job_name: 'ssl-http-files' + metrics_path: /probe + params: + module: ["http_file"] + static_configs: + - targets: + - 'https://www.paypalobjects.com/marketing/web/logos/paypal_com.pem' + - 'https://d3frv9g52qce38.cloudfront.net/amazondefault/amazon_web_services_inc_2024.pem' + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 127.0.0.1:9219 diff --git a/examples/ssl_exporter.yaml b/examples/ssl_exporter.yaml index b5f491e..5d9919e 100644 --- a/examples/ssl_exporter.yaml +++ b/examples/ssl_exporter.yaml @@ -38,6 +38,12 @@ modules: file_ca_certificates: prober: file target: /etc/ssl/certs/ca-certificates.crt + http_file: + prober: http_file + http_file_proxy: + prober: http_file + http_file: + proxy_url: "socks5://localhost:8123" kubernetes: prober: kubernetes kubernetes_kubeconfig: diff --git a/prober/http_file.go b/prober/http_file.go new file mode 100644 index 0000000..1baccc4 --- /dev/null +++ b/prober/http_file.go @@ -0,0 +1,59 @@ +package prober + +import ( + "context" + "fmt" + "io" + "net/http" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" + "github.com/ribbybibby/ssl_exporter/v2/config" +) + +// ProbeHTTPFile collects certificate metrics from a remote file via http +func ProbeHTTPFile(ctx context.Context, logger log.Logger, target string, module config.Module, registry *prometheus.Registry) error { + proxy := http.ProxyFromEnvironment + if module.HTTPFile.ProxyURL.URL != nil { + proxy = http.ProxyURL(module.HTTPFile.ProxyURL.URL) + } + + tlsConfig, err := config.NewTLSConfig(&module.TLSConfig) + if err != nil { + return fmt.Errorf("creating TLS config: %w", err) + } + + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + Proxy: proxy, + DisableKeepAlives: true, + }, + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) + if err != nil { + return fmt.Errorf("creating http request: %w", err) + } + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("making http request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected response code: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response body: %w", err) + } + + certs, err := decodeCertificates(body) + if err != nil { + return fmt.Errorf("decoding certificates from response body: %w", err) + } + + return collectCertificateMetrics(certs, registry) +} diff --git a/prober/http_file_test.go b/prober/http_file_test.go new file mode 100644 index 0000000..c9c2e8a --- /dev/null +++ b/prober/http_file_test.go @@ -0,0 +1,112 @@ +package prober + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/ribbybibby/ssl_exporter/v2/config" + "github.com/ribbybibby/ssl_exporter/v2/test" +) + +func TestProbeHTTPFile(t *testing.T) { + testcertPEM, _ := test.GenerateTestCertificate(time.Now().AddDate(0, 0, 1)) + + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(testcertPEM) + })) + + server.Start() + defer server.Close() + + registry := prometheus.NewRegistry() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := ProbeHTTPFile(ctx, newTestLogger(), server.URL+"/file", config.Module{}, registry); err != nil { + t.Fatalf("error: %s", err) + } + + cert, err := newCertificate(testcertPEM) + if err != nil { + t.Fatal(err) + } + checkCertificateMetrics(cert, registry, t) +} + +func TestProbeHTTPFile_NotCertificate(t *testing.T) { + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("foobar")) + })) + + server.Start() + defer server.Close() + + registry := prometheus.NewRegistry() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := ProbeHTTPFile(ctx, newTestLogger(), server.URL+"/file", config.Module{}, registry); err == nil { + t.Errorf("expected error but got nil") + } +} + +func TestProbeHTTPFile_NotFound(t *testing.T) { + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + + server.Start() + defer server.Close() + + registry := prometheus.NewRegistry() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := ProbeHTTPFile(ctx, newTestLogger(), server.URL+"/file", config.Module{}, registry); err == nil { + t.Errorf("expected error but got nil") + } +} + +func TestProbeHTTPFileHTTPS(t *testing.T) { + server, certPEM, _, caFile, teardown, err := test.SetupHTTPSServer() + if err != nil { + t.Fatalf(err.Error()) + } + defer teardown() + + server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(certPEM) + }) + + server.StartTLS() + defer server.Close() + + module := config.Module{ + TLSConfig: config.TLSConfig{ + CAFile: caFile, + InsecureSkipVerify: false, + }, + } + + registry := prometheus.NewRegistry() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := ProbeHTTPFile(ctx, newTestLogger(), server.URL+"/file", module, registry); err != nil { + t.Fatalf("error: %s", err) + } + + cert, err := newCertificate(certPEM) + if err != nil { + t.Fatal(err) + } + checkCertificateMetrics(cert, registry, t) +} diff --git a/prober/prober.go b/prober/prober.go index d283b5e..d74086c 100644 --- a/prober/prober.go +++ b/prober/prober.go @@ -15,6 +15,7 @@ var ( "http": ProbeHTTPS, "tcp": ProbeTCP, "file": ProbeFile, + "http_file": ProbeHTTPFile, "kubernetes": ProbeKubernetes, "kubeconfig": ProbeKubeconfig, }