Add workers and workers_utilization metrics

CHANGELOG.md

@@ -6,6 +6,7 @@
 - Add `imgproxy.source_image_url` and `imgproxy.source_image_origin` attributes to `downloading_image` spans in New Relic, DataDog, and OpenTelemetry traces.
 - Add `imgproxy.processing_options` attribute to `processing_image` spans in New Relic, DataDog, and OpenTelemetry traces.
 - Add `Source Image Origin` attribute to error reports.
+- Add `workers` and `workers_utilization` metrics to all metrics services.
 - (pro) Add [crop_aspect_ratio](https://docs.imgproxy.net/latest/usage/processing#crop-aspect-ratio) processing option.
 - (pro) Add [IMGPROXY_PDF_NO_BACKGROUND](https://docs.imgproxy.net/latest/configuration/options#IMGPROXY_PDF_NO_BACKGROUND) config.
 

metrics/cloudwatch/cloudwatch.go

@@ -220,6 +220,13 @@ func runMetricsCollector() {
 				}
 			}()
 
+			metrics = append(metrics, cloudwatchTypes.MetricDatum{
+				Dimensions: []cloudwatchTypes.Dimension{dimension},
+				MetricName: aws.String("Workers"),
+				Unit:       cloudwatchTypes.StandardUnitCount,
+				Value:      aws.Float64(float64(config.Workers)),
+			})
+
 			metrics = append(metrics, cloudwatchTypes.MetricDatum{
 				Dimensions: []cloudwatchTypes.Dimension{dimension},
 				MetricName: aws.String("RequestsInProgress"),
@@ -239,7 +246,7 @@ func runMetricsCollector() {
 				MetricName: aws.String("ConcurrencyUtilization"),
 				Unit:       cloudwatchTypes.StandardUnitPercent,
 				Value: aws.Float64(
-					stats.RequestsInProgress() / float64(config.Workers) * 100.0,
+					stats.WorkersUtilization(),
 				),
 			})
 
@@ -248,7 +255,7 @@ func runMetricsCollector() {
 				MetricName: aws.String("WorkersUtilization"),
 				Unit:       cloudwatchTypes.StandardUnitPercent,
 				Value: aws.Float64(
-					stats.RequestsInProgress() / float64(config.Workers) * 100.0,
+					stats.WorkersUtilization(),
 				),
 			})
 

metrics/datadog/datadog.go

@@ -218,8 +218,10 @@ func runMetricsCollector() {
 				}
 			}()
 
+			statsdClient.Gauge("imgproxy.workers", float64(config.Workers), nil, 1)
 			statsdClient.Gauge("imgproxy.requests_in_progress", stats.RequestsInProgress(), nil, 1)
 			statsdClient.Gauge("imgproxy.images_in_progress", stats.ImagesInProgress(), nil, 1)
+			statsdClient.Gauge("imgproxy.workers_utilization", stats.WorkersUtilization(), nil, 1)
 		case <-statsdClientStop:
 			return
 		}

metrics/newrelic/newrelic.go

@@ -291,6 +291,12 @@ func runMetricsCollector() {
 				}
 			}()
 
+			harvester.RecordMetric(telemetry.Gauge{
+				Name:      "imgproxy.workers",
+				Value:     float64(config.Workers),
+				Timestamp: time.Now(),
+			})
+
 			harvester.RecordMetric(telemetry.Gauge{
 				Name:      "imgproxy.requests_in_progress",
 				Value:     stats.RequestsInProgress(),
@@ -303,6 +309,12 @@ func runMetricsCollector() {
 				Timestamp: time.Now(),
 			})
 
+			harvester.RecordMetric(telemetry.Gauge{
+				Name:      "imgproxy.workers_utilization",
+				Value:     stats.WorkersUtilization(),
+				Timestamp: time.Now(),
+			})
+
 			harvester.HarvestNow(harvesterCtx)
 		case <-harvesterCtx.Done():
 			return

metrics/otel/otel.go

@@ -575,6 +575,15 @@ func addDefaultMetrics() error {
 		return fmt.Errorf("Can't add go_threads gauge to OpenTelemetry: %s", err)
 	}
 
+	workersGauge, err := meter.Int64ObservableGauge(
+		"workers",
+		metric.WithUnit("1"),
+		metric.WithDescription("A gauge of the number of running workers."),
+	)
+	if err != nil {
+		return fmt.Errorf("Can't add workets gauge to OpenTelemetry: %s", err)
+	}
+
 	requestsInProgressGauge, err := meter.Float64ObservableGauge(
 		"requests_in_progress",
 		metric.WithUnit("1"),
@@ -593,6 +602,15 @@ func addDefaultMetrics() error {
 		return fmt.Errorf("Can't add images_in_progress gauge to OpenTelemetry: %s", err)
 	}
 
+	workersUtilizationGauge, err := meter.Float64ObservableGauge(
+		"workers_utilization",
+		metric.WithUnit("%"),
+		metric.WithDescription("A gauge of the workers utilization in percents."),
+	)
+	if err != nil {
+		return fmt.Errorf("Can't add workers_utilization gauge to OpenTelemetry: %s", err)
+	}
+
 	bufferDefaultSizeGauge, err := meter.Int64ObservableGauge(
 		"buffer_default_size_bytes",
 		metric.WithUnit("By"),
@@ -632,8 +650,10 @@ func addDefaultMetrics() error {
 			o.ObserveInt64(goGoroutines, int64(runtime.NumGoroutine()))
 			o.ObserveInt64(goThreads, int64(threadsNum))
 
+			o.ObserveInt64(workersGauge, int64(config.Workers))
 			o.ObserveFloat64(requestsInProgressGauge, stats.RequestsInProgress())
 			o.ObserveFloat64(imagesInProgressGauge, stats.ImagesInProgress())
+			o.ObserveFloat64(workersUtilizationGauge, stats.WorkersUtilization())
 
 			bufferStatsMutex.Lock()
 			defer bufferStatsMutex.Unlock()
@@ -653,8 +673,10 @@ func addDefaultMetrics() error {
 		goMemstatsHeapInuse,
 		goGoroutines,
 		goThreads,
+		workersGauge,
 		requestsInProgressGauge,
 		imagesInProgressGauge,
+		workersUtilizationGauge,
 		bufferDefaultSizeGauge,
 		bufferMaxSizeGauge,
 	)

metrics/prometheus/prometheus.go

@@ -33,8 +33,10 @@ var (
 	bufferDefaultSize *prometheus.GaugeVec
 	bufferMaxSize     *prometheus.GaugeVec
 
+	workers            prometheus.Gauge
 	requestsInProgress prometheus.GaugeFunc
 	imagesInProgress   prometheus.GaugeFunc
+	workersUtilization prometheus.GaugeFunc
 )
 
 func Init() {
@@ -103,6 +105,13 @@ func Init() {
 		Help:      "A gauge of the buffer max size in bytes.",
 	}, []string{"type"})
 
+	workers = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: config.PrometheusNamespace,
+		Name:      "workers",
+		Help:      "A gauge of the number of running workers.",
+	})
+	workers.Set(float64(config.Workers))
+
 	requestsInProgress = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
 		Namespace: config.PrometheusNamespace,
 		Name:      "requests_in_progress",
@@ -115,6 +124,12 @@ func Init() {
 		Help:      "A gauge of the number of images currently being in progress.",
 	}, stats.ImagesInProgress)
 
+	workersUtilization = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Namespace: config.PrometheusNamespace,
+		Name:      "workers_utilization",
+		Help:      "A gauge of the workers utilization in percents.",
+	}, stats.WorkersUtilization)
+
 	prometheus.MustRegister(
 		requestsTotal,
 		statusCodesTotal,
@@ -126,8 +141,10 @@ func Init() {
 		bufferSize,
 		bufferDefaultSize,
 		bufferMaxSize,
+		workers,
 		requestsInProgress,
 		imagesInProgress,
+		workersUtilization,
 	)
 
 	enabled = true

metrics/stats/stats.go

@@ -1,6 +1,10 @@
 package stats
 
-import "sync/atomic"
+import (
+	"sync/atomic"
+
+	"github.com/imgproxy/imgproxy/v3/config"
+)
 
 var (
 	requestsInProgress int64
@@ -30,3 +34,7 @@ func IncImagesInProgress() {
 func DecImagesInProgress() {
 	atomic.AddInt64(&imagesInProgress, -1)
 }
+
+func WorkersUtilization() float64 {
+	return RequestsInProgress() / float64(config.Workers) * 100.0
+}