From 2661db100031e5c8e66eb5caec488561ae08ede0 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Thu, 7 Jul 2022 19:13:30 +0600 Subject: [PATCH] Additional metrics for New Relic --- CHANGELOG.md | 1 + docs/new_relic.md | 9 ++ go.mod | 1 + go.sum | 2 + metrics/metrics.go | 4 + metrics/newrelic/newrelic.go | 174 +++++++++++++++++++++++++++++++++-- vips/vips.go | 5 + 7 files changed, 190 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b6c06ae..9b693c48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Add `IMGPROXY_PREFERRED_FORMATS` config. - Add `IMGPROXY_REQUESTS_QUEUE_SIZE` config. - Add sending additional metrics to Datadog and `IMGPROXY_DATADOG_ENABLE_ADDITIONAL_METRICS` config. +- Add sending additional metrics to New Relic. ### Change - Change `IMGPROXY_MAX_CLIENTS` default value to 2048. diff --git a/docs/new_relic.md b/docs/new_relic.md index 6cdf1780..368f2383 100644 --- a/docs/new_relic.md +++ b/docs/new_relic.md @@ -14,3 +14,12 @@ imgproxy will send the following info to New Relic: * Image downloading time * Image processing time * Errors that occurred while downloading and processing an image + +Additionally, imgproxy sends the following metrics over [Metrics API](https://docs.newrelic.com/docs/data-apis/ingest-apis/metric-api/introduction-metric-api/): + +* `imgproxy.buffer.size`: a summary of the download/gzip buffers sizes (in bytes) +* `imgproxy.buffer.default_size`: calibrated default buffer size (in bytes) +* `imgproxy.buffer.max_size`: calibrated maximum buffer size (in bytes) +* `imgproxy.vips.memory`: libvips memory usage (in bytes) +* `imgproxy.vips.max_memory`: libvips maximum memory usage (in bytes) +* `imgproxy.vips.allocs`: the number of active vips allocations diff --git a/go.mod b/go.mod index 1d23b302..f49b65f0 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/matoous/go-nanoid/v2 v2.0.0 github.com/ncw/swift/v2 v2.0.1 github.com/newrelic/go-agent/v3 v3.16.1 + github.com/newrelic/newrelic-telemetry-sdk-go v0.8.1 // indirect github.com/onsi/ginkgo v1.16.5 // indirect github.com/prometheus/client_golang v1.12.2 github.com/sirupsen/logrus v1.8.1 diff --git a/go.sum b/go.sum index f9899654..d35c853c 100644 --- a/go.sum +++ b/go.sum @@ -884,6 +884,8 @@ github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0= github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= github.com/newrelic/go-agent/v3 v3.16.1 h1:gH053irA4rIAySGSvMc2grKiKNhrM4gCzc+p3M+rqAE= github.com/newrelic/go-agent/v3 v3.16.1/go.mod h1:BFJOlbZWRlPTXKYIC1TTTtQKTnYntEJaU0VU507hDc0= +github.com/newrelic/newrelic-telemetry-sdk-go v0.8.1 h1:6OX5VXMuj2salqNBc41eXKz6K+nV6OB/hhlGnAKCbwU= +github.com/newrelic/newrelic-telemetry-sdk-go v0.8.1/go.mod h1:2kY6OeOxrJ+RIQlVjWDc/pZlT3MIf30prs6drzMfJ6E= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= diff --git a/metrics/metrics.go b/metrics/metrics.go index c9986b1a..5c429c6d 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -22,6 +22,7 @@ func Init() error { } func Stop() { + newrelic.Stop() datadog.Stop() } @@ -81,15 +82,18 @@ func SendError(ctx context.Context, errType string, err error) { func ObserveBufferSize(t string, size int) { prometheus.ObserveBufferSize(t, size) + newrelic.ObserveBufferSize(t, size) datadog.ObserveBufferSize(t, size) } func SetBufferDefaultSize(t string, size int) { prometheus.SetBufferDefaultSize(t, size) + newrelic.SetBufferDefaultSize(t, size) datadog.SetBufferDefaultSize(t, size) } func SetBufferMaxSize(t string, size int) { prometheus.SetBufferMaxSize(t, size) + newrelic.SetBufferMaxSize(t, size) datadog.SetBufferMaxSize(t, size) } diff --git a/metrics/newrelic/newrelic.go b/metrics/newrelic/newrelic.go index 5e167366..8404f873 100644 --- a/metrics/newrelic/newrelic.go +++ b/metrics/newrelic/newrelic.go @@ -3,19 +3,48 @@ package newrelic import ( "context" "fmt" + "math" "net/http" + "regexp" + "sync" + "time" + + "github.com/newrelic/go-agent/v3/newrelic" + "github.com/newrelic/newrelic-telemetry-sdk-go/telemetry" + log "github.com/sirupsen/logrus" "github.com/imgproxy/imgproxy/v3/config" "github.com/imgproxy/imgproxy/v3/metrics/errformat" - "github.com/newrelic/go-agent/v3/newrelic" ) type transactionCtxKey struct{} -var ( - enabled = false +type GaugeFunc func() float64 - newRelicApp *newrelic.Application +const ( + defaultMetricURL = "https://metric-api.newrelic.com/metric/v1" + euMetricURL = "https://metric-api.eu.newrelic.com/metric/v1" +) + +var ( + enabled = false + enabledHarvester = false + + app *newrelic.Application + harvester *telemetry.Harvester + + harvesterCtx context.Context + harvesterCtxCancel context.CancelFunc + + gaugeFuncs = make(map[string]GaugeFunc) + gaugeFuncsMutex sync.RWMutex + + bufferSummaries = make(map[string]*telemetry.Summary) + bufferSummariesMutex sync.RWMutex + + interval = 10 * time.Second + + licenseEuRegex = regexp.MustCompile(`(^eu.+?)x`) ) func Init() error { @@ -30,7 +59,7 @@ func Init() error { var err error - newRelicApp, err = newrelic.NewApplication( + app, err = newrelic.NewApplication( newrelic.ConfigAppName(name), newrelic.ConfigLicense(config.NewRelicKey), func(c *newrelic.Config) { @@ -44,11 +73,47 @@ func Init() error { return fmt.Errorf("Can't init New Relic agent: %s", err) } + harvesterAttributes := map[string]interface{}{"appName": name} + for k, v := range config.NewRelicLabels { + harvesterAttributes[k] = v + } + + metricsURL := defaultMetricURL + if licenseEuRegex.MatchString(config.NewRelicKey) { + metricsURL = euMetricURL + } + + harvester, err = telemetry.NewHarvester( + telemetry.ConfigAPIKey(config.NewRelicKey), + telemetry.ConfigCommonAttributes(harvesterAttributes), + telemetry.ConfigHarvestPeriod(0), // Don't harvest automatically + telemetry.ConfigMetricsURLOverride(metricsURL), + telemetry.ConfigBasicErrorLogger(log.StandardLogger().WithField("from", "newrelic").WriterLevel(log.WarnLevel)), + ) + if err == nil { + harvesterCtx, harvesterCtxCancel = context.WithCancel(context.Background()) + enabledHarvester = true + go runMetricsCollector() + } else { + log.Warnf("Can't init New Relic telemetry harvester: %s", err) + } + enabled = true return nil } +func Stop() { + if enabled { + app.Shutdown(5 * time.Second) + + if enabledHarvester { + harvesterCtxCancel() + harvester.HarvestNow(context.Background()) + } + } +} + func Enabled() bool { return enabled } @@ -58,7 +123,7 @@ func StartTransaction(ctx context.Context, rw http.ResponseWriter, r *http.Reque return ctx, func() {}, rw } - txn := newRelicApp.StartTransaction("request") + txn := app.StartTransaction("request") txn.SetWebRequestHTTP(r) newRw := txn.SetWebResponse(rw) cancel := func() { txn.End() } @@ -90,3 +155,100 @@ func SendError(ctx context.Context, errType string, err error) { }) } } + +func AddGaugeFunc(name string, f GaugeFunc) { + gaugeFuncsMutex.Lock() + defer gaugeFuncsMutex.Unlock() + + gaugeFuncs["imgproxy."+name] = f +} + +func ObserveBufferSize(t string, size int) { + if enabledHarvester { + bufferSummariesMutex.Lock() + defer bufferSummariesMutex.Unlock() + + summary, ok := bufferSummaries[t] + if !ok { + summary = &telemetry.Summary{ + Name: "imgproxy.buffer.size", + Attributes: map[string]interface{}{"buffer_type": t}, + Timestamp: time.Now(), + } + bufferSummaries[t] = summary + } + + sizef := float64(size) + + summary.Count += 1 + summary.Sum += sizef + summary.Min = math.Min(summary.Min, sizef) + summary.Max = math.Max(summary.Max, sizef) + } +} + +func SetBufferDefaultSize(t string, size int) { + if enabledHarvester { + harvester.RecordMetric(telemetry.Gauge{ + Name: "imgproxy.buffer.default_size", + Value: float64(size), + Attributes: map[string]interface{}{"buffer_type": t}, + Timestamp: time.Now(), + }) + } +} + +func SetBufferMaxSize(t string, size int) { + if enabledHarvester { + harvester.RecordMetric(telemetry.Gauge{ + Name: "imgproxy.buffer.max_size", + Value: float64(size), + Attributes: map[string]interface{}{"buffer_type": t}, + Timestamp: time.Now(), + }) + } +} + +func runMetricsCollector() { + tick := time.NewTicker(interval) + defer tick.Stop() + for { + select { + case <-tick.C: + func() { + gaugeFuncsMutex.RLock() + defer gaugeFuncsMutex.RUnlock() + + for name, f := range gaugeFuncs { + harvester.RecordMetric(telemetry.Gauge{ + Name: name, + Value: f(), + Timestamp: time.Now(), + }) + } + }() + + func() { + bufferSummariesMutex.RLock() + defer bufferSummariesMutex.RUnlock() + + now := time.Now() + + for _, summary := range bufferSummaries { + summary.Interval = now.Sub(summary.Timestamp) + harvester.RecordMetric(*summary) + + summary.Timestamp = now + summary.Count = 0 + summary.Sum = 0 + summary.Min = 0 + summary.Max = 0 + } + }() + + harvester.HarvestNow(harvesterCtx) + case <-harvesterCtx.Done(): + return + } + } +} diff --git a/vips/vips.go b/vips/vips.go index 9785d3e2..91b8611e 100644 --- a/vips/vips.go +++ b/vips/vips.go @@ -22,6 +22,7 @@ import ( "github.com/imgproxy/imgproxy/v3/imagedata" "github.com/imgproxy/imgproxy/v3/imagetype" "github.com/imgproxy/imgproxy/v3/metrics/datadog" + "github.com/imgproxy/imgproxy/v3/metrics/newrelic" "github.com/imgproxy/imgproxy/v3/metrics/prometheus" ) @@ -100,6 +101,10 @@ func Init() error { datadog.AddGaugeFunc("vips.max_memory", GetMemHighwater) datadog.AddGaugeFunc("vips.allocs", GetAllocs) + newrelic.AddGaugeFunc("vips.memory", GetMem) + newrelic.AddGaugeFunc("vips.max_memory", GetMemHighwater) + newrelic.AddGaugeFunc("vips.allocs", GetAllocs) + return nil }