Add http_file prober (#144)

* feat: add remote_file probe

* fix: use tls module config

* chore: write http/https tests for probing remote file

* chore: get rid of useless lines

* fix: get rid of useless file download, check body directly

* fix: use checkCertificateMetrics to actually check values

* Rename remote_file to http_file

You can fetch remote content with a lot of different protocols, so I
think it's worth being specific here.

As part of this change I've fixed up some of the logic in the code. I've
also created a separate `http_file` block in the module config.

* Actually include renamed files

---------

Co-authored-by: Anthony LE BERRE <aleberre@veepee.com>
Co-authored-by: Rob Best <rob.best@jetstack.io>

README.md

@@ -4,6 +4,7 @@ Exports metrics for certificates collected from various sources:
 - [TCP probes](#tcp)
 - [HTTPS probes](#https)
 - [PEM files](#file)
+- [Remote PEM files](#http_file)
 - [Kubernetes secrets](#kubernetes)
 - [Kubeconfig files](#kubeconfig)
 
@@ -130,7 +131,7 @@ scrape_configs:
 ```
 
 This will use proxy servers discovered by the environment variables `HTTP_PROXY`,
-`HTTPS_PROXY` and `ALL_PROXY`. Or, you can set the `proxy_url` option in the module
+`HTTPS_PROXY` and `ALL_PROXY`. Or, you can set the `https.proxy_url` option in the module
 configuration.
 
 The latter takes precedence.
@@ -175,6 +176,44 @@ scrape_configs:
         replacement: ${1}:9219
 ```
 
+### HTTP File
+
+The `http_file` prober exports `ssl_cert_not_after` and
+`ssl_cert_not_before` for PEM encoded certificates found at the
+specified URL.
+
+```
+curl "localhost:9219/probe?module=http_file&target=https://www.paypalobjects.com/marketing/web/logos/paypal_com.pem"
+```
+
+Here's a sample Prometheus configuration:
+
+```yml
+scrape_configs:
+  - job_name: 'ssl-http-files'
+    metrics_path: /probe
+    params:
+      module: ["http_file"]
+    static_configs:
+      - targets:
+        - 'https://www.paypalobjects.com/marketing/web/logos/paypal_com.pem'
+        - 'https://d3frv9g52qce38.cloudfront.net/amazondefault/amazon_web_services_inc_2024.pem'
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 127.0.0.1:9219
+```
+
+For proxying to the target resource, this prober will use proxy servers
+discovered in the environment variables `HTTP_PROXY`, `HTTPS_PROXY` and
+`ALL_PROXY`. Or, you can set the `http_file.proxy_url` option in the module
+configuration.
+
+The latter takes precedence.
+
 ### Kubernetes
 
 The `kubernetes` prober exports `ssl_kubernetes_cert_not_after` and
@@ -293,6 +332,7 @@ target: <string>
 [ https: <https_probe> ]
 [ tcp: <tcp_probe> ]
 [ kubernetes: <kubernetes_probe> ]
+[ http_file: <http_file_probe> ]
 ```
 
 ### <tls_config>
@@ -339,6 +379,13 @@ target: <string>
 [ kubeconfig: <string> ]
 ```
 
+### <http_file_probe>
+
+```
+# HTTP proxy server to use to connect to the targets.
+[ proxy_url: <string> ]
+```
+
 ## Example Queries
 
 Certificates that expire within 7 days:

config/config.go

@@ -17,22 +17,25 @@ var (
 	DefaultConfig = &Config{
 		DefaultModule: "tcp",
 		Modules: map[string]Module{
-			"tcp": Module{
+			"tcp": {
 				Prober: "tcp",
 			},
-			"http": Module{
+			"http": {
 				Prober: "https",
 			},
-			"https": Module{
+			"https": {
 				Prober: "https",
 			},
-			"file": Module{
+			"file": {
 				Prober: "file",
 			},
-			"kubernetes": Module{
+			"http_file": {
+				Prober: "http_file",
+			},
+			"kubernetes": {
 				Prober: "kubernetes",
 			},
-			"kubeconfig": Module{
+			"kubeconfig": {
 				Prober: "kubeconfig",
 			},
 		},
@@ -73,6 +76,7 @@ type Module struct {
 	HTTPS      HTTPSProbe      `yaml:"https,omitempty"`
 	TCP        TCPProbe        `yaml:"tcp,omitempty"`
 	Kubernetes KubernetesProbe `yaml:"kubernetes,omitempty"`
+	HTTPFile   HTTPFileProbe   `yaml:"http_file,omitempty"`
 }
 
 // TLSConfig is a superset of config.TLSConfig that supports TLS renegotiation
@@ -142,6 +146,11 @@ type KubernetesProbe struct {
 	Kubeconfig string `yaml:"kubeconfig,omitempty"`
 }
 
+// HTTPFileProbe configures a http_file probe
+type HTTPFileProbe struct {
+	ProxyURL URL `yaml:"proxy_url,omitempty"`
+}
+
 // URL is a custom URL type that allows validation at configuration load time
 type URL struct {
 	*url.URL

examples/example.prometheus.yml

@@ -34,3 +34,18 @@ scrape_configs:
     static_configs:
       - targets:
           - 127.0.0.1:9219
+  - job_name: 'ssl-http-files'
+    metrics_path: /probe
+    params:
+      module: ["http_file"]
+    static_configs:
+      - targets:
+        - 'https://www.paypalobjects.com/marketing/web/logos/paypal_com.pem'
+        - 'https://d3frv9g52qce38.cloudfront.net/amazondefault/amazon_web_services_inc_2024.pem'
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 127.0.0.1:9219

examples/ssl_exporter.yaml

@@ -38,6 +38,12 @@ modules:
   file_ca_certificates:
     prober: file
     target: /etc/ssl/certs/ca-certificates.crt
+  http_file:
+    prober: http_file
+  http_file_proxy:
+    prober: http_file
+    http_file:
+      proxy_url: "socks5://localhost:8123"
   kubernetes:
     prober: kubernetes
   kubernetes_kubeconfig:

prober/http_file.go

@@ -0,0 +1,59 @@
+package prober
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/ribbybibby/ssl_exporter/v2/config"
+)
+
+// ProbeHTTPFile collects certificate metrics from a remote file via http
+func ProbeHTTPFile(ctx context.Context, logger log.Logger, target string, module config.Module, registry *prometheus.Registry) error {
+	proxy := http.ProxyFromEnvironment
+	if module.HTTPFile.ProxyURL.URL != nil {
+		proxy = http.ProxyURL(module.HTTPFile.ProxyURL.URL)
+	}
+
+	tlsConfig, err := config.NewTLSConfig(&module.TLSConfig)
+	if err != nil {
+		return fmt.Errorf("creating TLS config: %w", err)
+	}
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig:   tlsConfig,
+			Proxy:             proxy,
+			DisableKeepAlives: true,
+		},
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
+	if err != nil {
+		return fmt.Errorf("creating http request: %w", err)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("making http request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("unexpected response code: %d", resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("reading response body: %w", err)
+	}
+
+	certs, err := decodeCertificates(body)
+	if err != nil {
+		return fmt.Errorf("decoding certificates from response body: %w", err)
+	}
+
+	return collectCertificateMetrics(certs, registry)
+}

prober/http_file_test.go

@@ -0,0 +1,112 @@
+package prober
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/ribbybibby/ssl_exporter/v2/config"
+	"github.com/ribbybibby/ssl_exporter/v2/test"
+)
+
+func TestProbeHTTPFile(t *testing.T) {
+	testcertPEM, _ := test.GenerateTestCertificate(time.Now().AddDate(0, 0, 1))
+
+	server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write(testcertPEM)
+	}))
+
+	server.Start()
+	defer server.Close()
+
+	registry := prometheus.NewRegistry()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	if err := ProbeHTTPFile(ctx, newTestLogger(), server.URL+"/file", config.Module{}, registry); err != nil {
+		t.Fatalf("error: %s", err)
+	}
+
+	cert, err := newCertificate(testcertPEM)
+	if err != nil {
+		t.Fatal(err)
+	}
+	checkCertificateMetrics(cert, registry, t)
+}
+
+func TestProbeHTTPFile_NotCertificate(t *testing.T) {
+	server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("foobar"))
+	}))
+
+	server.Start()
+	defer server.Close()
+
+	registry := prometheus.NewRegistry()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	if err := ProbeHTTPFile(ctx, newTestLogger(), server.URL+"/file", config.Module{}, registry); err == nil {
+		t.Errorf("expected error but got nil")
+	}
+}
+
+func TestProbeHTTPFile_NotFound(t *testing.T) {
+	server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+
+	server.Start()
+	defer server.Close()
+
+	registry := prometheus.NewRegistry()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	if err := ProbeHTTPFile(ctx, newTestLogger(), server.URL+"/file", config.Module{}, registry); err == nil {
+		t.Errorf("expected error but got nil")
+	}
+}
+
+func TestProbeHTTPFileHTTPS(t *testing.T) {
+	server, certPEM, _, caFile, teardown, err := test.SetupHTTPSServer()
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+	defer teardown()
+
+	server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write(certPEM)
+	})
+
+	server.StartTLS()
+	defer server.Close()
+
+	module := config.Module{
+		TLSConfig: config.TLSConfig{
+			CAFile:             caFile,
+			InsecureSkipVerify: false,
+		},
+	}
+
+	registry := prometheus.NewRegistry()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	if err := ProbeHTTPFile(ctx, newTestLogger(), server.URL+"/file", module, registry); err != nil {
+		t.Fatalf("error: %s", err)
+	}
+
+	cert, err := newCertificate(certPEM)
+	if err != nil {
+		t.Fatal(err)
+	}
+	checkCertificateMetrics(cert, registry, t)
+}

prober/prober.go

@@ -15,6 +15,7 @@ var (
 		"http":       ProbeHTTPS,
 		"tcp":        ProbeTCP,
 		"file":       ProbeFile,
+		"http_file":  ProbeHTTPFile,
 		"kubernetes": ProbeKubernetes,
 		"kubeconfig": ProbeKubeconfig,
 	}