1
0
mirror of https://github.com/open-telemetry/opentelemetry-go.git synced 2025-01-28 03:57:09 +02:00

Add utf8 support to the prometheus exporter (#5755)

### Changes

Disable sanitization when the UTF-8 support is enabled in the Prometheus
library.

### Usage

To enable UTF-8 support for the Prometheus exporter after this change,
set the following in your application:

```golang
import "github.com/prometheus/common/model"

func init() {
    model.NameValidationScheme = model.UTF8Validation
}
```

See `exporters/prometheus/testdata/counter_utf8.txt` for an example of
the text exposition format including names/labels with dots.
This commit is contained in:
David Ashpole 2024-09-09 11:08:14 -04:00 committed by GitHub
parent 506a9baf5e
commit 71b341f0a0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 111 additions and 129 deletions

View File

@ -13,6 +13,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
- Support `OTEL_EXPORTER_OTLP_LOGS_INSECURE` and `OTEL_EXPORTER_OTLP_INSECURE` environments in `go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc`. (#5739)
- The `WithResource` option for `NewMeterProvider` now merges the provided resources with the ones from environment variables. (#5773)
- The `WithResource` option for `NewLoggerProvider` now merges the provided resources with the ones from environment variables. (#5773)
- Add UTF-8 support to `go.opentelemetry.io/otel/exporters/prometheus`. (#5755)
### Fixed

View File

@ -7,6 +7,7 @@ import (
"strings"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/model"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/sdk/metric"
@ -131,7 +132,10 @@ func WithoutScopeInfo() Option {
// have special behavior based on their name.
func WithNamespace(ns string) Option {
return optionFunc(func(cfg config) config {
ns = sanitizeName(ns)
if model.NameValidationScheme != model.UTF8Validation {
// Only sanitize if prometheus does not support UTF-8.
ns = model.EscapeName(ns, model.NameEscapingScheme)
}
if !strings.HasSuffix(ns, "_") {
// namespace and metric names should be separated with an underscore,
// adds a trailing underscore if there is not one already.

View File

@ -11,11 +11,10 @@ import (
"slices"
"strings"
"sync"
"unicode"
"unicode/utf8"
"github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/common/model"
"google.golang.org/protobuf/proto"
"go.opentelemetry.io/otel"
@ -298,28 +297,38 @@ func addGaugeMetric[N int64 | float64](ch chan<- prometheus.Metric, gauge metric
}
// getAttrs parses the attribute.Set to two lists of matching Prometheus-style
// keys and values. It sanitizes invalid characters and handles duplicate keys
// (due to sanitization) by sorting and concatenating the values following the spec.
// keys and values.
func getAttrs(attrs attribute.Set, ks, vs [2]string, resourceKV keyVals) ([]string, []string) {
keysMap := make(map[string][]string)
itr := attrs.Iter()
for itr.Next() {
kv := itr.Attribute()
key := strings.Map(sanitizeRune, string(kv.Key))
if _, ok := keysMap[key]; !ok {
keysMap[key] = []string{kv.Value.Emit()}
} else {
// if the sanitized key is a duplicate, append to the list of keys
keysMap[key] = append(keysMap[key], kv.Value.Emit())
}
}
keys := make([]string, 0, attrs.Len())
values := make([]string, 0, attrs.Len())
for key, vals := range keysMap {
keys = append(keys, key)
slices.Sort(vals)
values = append(values, strings.Join(vals, ";"))
itr := attrs.Iter()
if model.NameValidationScheme == model.UTF8Validation {
// Do not perform sanitization if prometheus supports UTF-8.
for itr.Next() {
kv := itr.Attribute()
keys = append(keys, string(kv.Key))
values = append(values, kv.Value.Emit())
}
} else {
// It sanitizes invalid characters and handles duplicate keys
// (due to sanitization) by sorting and concatenating the values following the spec.
keysMap := make(map[string][]string)
for itr.Next() {
kv := itr.Attribute()
key := model.EscapeName(string(kv.Key), model.NameEscapingScheme)
if _, ok := keysMap[key]; !ok {
keysMap[key] = []string{kv.Value.Emit()}
} else {
// if the sanitized key is a duplicate, append to the list of keys
keysMap[key] = append(keysMap[key], kv.Value.Emit())
}
}
for key, vals := range keysMap {
keys = append(keys, key)
slices.Sort(vals)
values = append(values, strings.Join(vals, ";"))
}
}
if ks[0] != "" {
@ -347,13 +356,6 @@ func createScopeInfoMetric(scope instrumentation.Scope) (prometheus.Metric, erro
return prometheus.NewConstMetric(desc, prometheus.GaugeValue, float64(1), scope.Name, scope.Version)
}
func sanitizeRune(r rune) rune {
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == ':' || r == '_' {
return r
}
return '_'
}
var unitSuffixes = map[string]string{
// Time
"d": "_days",
@ -392,7 +394,11 @@ var unitSuffixes = map[string]string{
// getName returns the sanitized name, prefixed with the namespace and suffixed with unit.
func (c *collector) getName(m metricdata.Metrics, typ *dto.MetricType) string {
name := sanitizeName(m.Name)
name := m.Name
if model.NameValidationScheme != model.UTF8Validation {
// Only sanitize if prometheus does not support UTF-8.
name = model.EscapeName(name, model.NameEscapingScheme)
}
addCounterSuffix := !c.withoutCounterSuffixes && *typ == dto.MetricType_COUNTER
if addCounterSuffix {
// Remove the _total suffix here, as we will re-add the total suffix
@ -411,59 +417,6 @@ func (c *collector) getName(m metricdata.Metrics, typ *dto.MetricType) string {
return name
}
func sanitizeName(n string) string {
// This algorithm is based on strings.Map from Go 1.19.
const replacement = '_'
valid := func(i int, r rune) bool {
// Taken from
// https://github.com/prometheus/common/blob/dfbc25bd00225c70aca0d94c3c4bb7744f28ace0/model/metric.go#L92-L102
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || r == '_' || r == ':' || (r >= '0' && r <= '9' && i > 0) {
return true
}
return false
}
// This output buffer b is initialized on demand, the first time a
// character needs to be replaced.
var b strings.Builder
for i, c := range n {
if valid(i, c) {
continue
}
if i == 0 && c >= '0' && c <= '9' {
// Prefix leading number with replacement character.
b.Grow(len(n) + 1)
_ = b.WriteByte(byte(replacement))
break
}
b.Grow(len(n))
_, _ = b.WriteString(n[:i])
_ = b.WriteByte(byte(replacement))
width := utf8.RuneLen(c)
n = n[i+width:]
break
}
// Fast path for unchanged input.
if b.Cap() == 0 { // b.Grow was not called above.
return n
}
for _, c := range n {
// Due to inlining, it is more performant to invoke WriteByte rather then
// WriteRune.
if valid(1, c) { // We are guaranteed to not be at the start.
_ = b.WriteByte(byte(c))
} else {
_ = b.WriteByte(byte(replacement))
}
}
return b.String()
}
func (c *collector) metricType(m metricdata.Metrics) *dto.MetricType {
switch v := m.Data.(type) {
case metricdata.Histogram[int64], metricdata.Histogram[float64]:

View File

@ -14,6 +14,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/common/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -34,6 +35,7 @@ func TestPrometheusExporter(t *testing.T) {
recordMetrics func(ctx context.Context, meter otelmetric.Meter)
options []Option
expectedFile string
enableUTF8 bool
}{
{
name: "counter",
@ -399,10 +401,47 @@ func TestPrometheusExporter(t *testing.T) {
counter.Add(ctx, 5.3, opt)
},
},
{
name: "counter utf-8",
expectedFile: "testdata/counter_utf8.txt",
enableUTF8: true,
recordMetrics: func(ctx context.Context, meter otelmetric.Meter) {
opt := otelmetric.WithAttributes(
attribute.Key("A.G").String("B"),
attribute.Key("C.H").String("D"),
attribute.Key("E.I").Bool(true),
attribute.Key("F.J").Int(42),
)
counter, err := meter.Float64Counter(
"foo.things",
otelmetric.WithDescription("a simple counter"),
otelmetric.WithUnit("s"),
)
require.NoError(t, err)
counter.Add(ctx, 5, opt)
counter.Add(ctx, 10.3, opt)
counter.Add(ctx, 9, opt)
attrs2 := attribute.NewSet(
attribute.Key("A.G").String("D"),
attribute.Key("C.H").String("B"),
attribute.Key("E.I").Bool(true),
attribute.Key("F.J").Int(42),
)
counter.Add(ctx, 5, otelmetric.WithAttributeSet(attrs2))
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
if tc.enableUTF8 {
model.NameValidationScheme = model.UTF8Validation
defer func() {
// Reset to defaults
model.NameValidationScheme = model.LegacyValidation
}()
}
ctx := context.Background()
registry := prometheus.NewRegistry()
exporter, err := New(append(tc.options, WithRegisterer(registry))...)
@ -452,36 +491,6 @@ func TestPrometheusExporter(t *testing.T) {
}
}
func TestSantitizeName(t *testing.T) {
tests := []struct {
input string
want string
}{
{"name€_with_4_width_rune", "name__with_4_width_rune"},
{"`", "_"},
{
`! "#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWKYZ[]\^_abcdefghijklmnopqrstuvwkyz{|}~`,
`________________0123456789:______ABCDEFGHIJKLMNOPQRSTUVWKYZ_____abcdefghijklmnopqrstuvwkyz____`,
},
// Test cases taken from
// https://github.com/prometheus/common/blob/dfbc25bd00225c70aca0d94c3c4bb7744f28ace0/model/metric_test.go#L85-L136
{"Avalid_23name", "Avalid_23name"},
{"_Avalid_23name", "_Avalid_23name"},
{"1valid_23name", "_1valid_23name"},
{"avalid_23name", "avalid_23name"},
{"Ava:lid_23name", "Ava:lid_23name"},
{"a lid_23name", "a_lid_23name"},
{":leading_colon", ":leading_colon"},
{"colon:in:the:middle", "colon:in:the:middle"},
{"", ""},
}
for _, test := range tests {
require.Equalf(t, test.want, sanitizeName(test.input), "input: %q", test.input)
}
}
func TestMultiScopes(t *testing.T) {
ctx := context.Background()
registry := prometheus.NewRegistry()

View File

@ -5,6 +5,7 @@ go 1.22
require (
github.com/prometheus/client_golang v1.20.3
github.com/prometheus/client_model v0.6.1
github.com/prometheus/common v0.59.1
github.com/stretchr/testify v1.9.0
go.opentelemetry.io/otel v1.29.0
go.opentelemetry.io/otel/metric v1.29.0
@ -25,7 +26,6 @@ require (
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/common v0.59.1 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
golang.org/x/sys v0.25.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect

View File

@ -0,0 +1,10 @@
# HELP "foo.things_seconds_total" a simple counter
# TYPE "foo.things_seconds_total" counter
{"foo.things_seconds_total","A.G"="B","C.H"="D","E.I"="true","F.J"="42",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 24.3
{"foo.things_seconds_total","A.G"="D","C.H"="B","E.I"="true","F.J"="42",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 5
# HELP otel_scope_info Instrumentation Scope metadata
# TYPE otel_scope_info gauge
otel_scope_info{otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 1
# HELP target_info Target metadata
# TYPE target_info gauge
target_info{"service.name"="prometheus_test","telemetry.sdk.language"="go","telemetry.sdk.name"="opentelemetry","telemetry.sdk.version"="latest"} 1

View File

@ -1,9 +1,9 @@
# HELP bar a fun little gauge
# TYPE bar gauge
bar{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 75
# HELP _0invalid_counter_name_total a counter with an invalid name
# TYPE _0invalid_counter_name_total counter
_0invalid_counter_name_total{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 100
# HELP _invalid_counter_name_total a counter with an invalid name
# TYPE _invalid_counter_name_total counter
_invalid_counter_name_total{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 100
# HELP invalid_gauge_name a gauge with an invalid name
# TYPE invalid_gauge_name gauge
invalid_gauge_name{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 100

View File

@ -51,7 +51,7 @@ require (
github.com/butuzov/mirror v1.2.0 // indirect
github.com/catenacyber/perfsprint v0.7.1 // indirect
github.com/ccojocar/zxcvbn-go v1.0.2 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/charithe/durationcheck v0.0.10 // indirect
github.com/chavacava/garif v0.1.0 // indirect
github.com/ckaznocha/intrange v0.1.2 // indirect
@ -133,6 +133,7 @@ require (
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moricho/tparallel v0.3.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nakabonne/nestif v0.3.1 // indirect
github.com/nishanths/exhaustive v0.12.0 // indirect
github.com/nishanths/predeclared v0.2.2 // indirect
@ -142,10 +143,10 @@ require (
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/polyfloyd/go-errorlint v1.6.0 // indirect
github.com/prometheus/client_golang v1.19.0 // indirect
github.com/prometheus/client_golang v1.20.2 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.48.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/prometheus/common v0.57.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/quasilyte/go-ruleguard v0.4.2 // indirect
github.com/quasilyte/go-ruleguard/dsl v0.3.22 // indirect
github.com/quasilyte/gogrep v0.5.0 // indirect

View File

@ -72,8 +72,8 @@ github.com/catenacyber/perfsprint v0.7.1 h1:PGW5G/Kxn+YrN04cRAZKC+ZuvlVwolYMrIyy
github.com/catenacyber/perfsprint v0.7.1/go.mod h1:/wclWYompEyjUD2FuIIDVKNkqz7IgBIWXIH3V0Zol50=
github.com/ccojocar/zxcvbn-go v1.0.2 h1:na/czXU8RrhXO4EZme6eQJLR4PzcGsahsBOAwU6I3Vg=
github.com/ccojocar/zxcvbn-go v1.0.2/go.mod h1:g1qkXtUSvHP8lhHp5GrSmTz6uWALGRMQdw6Qnz/hi60=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/charithe/durationcheck v0.0.10 h1:wgw73BiocdBDQPik+zcEoBG/ob8uyBHf2iyoHGPf5w4=
github.com/charithe/durationcheck v0.0.10/go.mod h1:bCWXb7gYRysD1CU3C+u4ceO49LoGOY1C1L6uouGNreQ=
github.com/chavacava/garif v0.1.0 h1:2JHa3hbYf5D9dsgseMKAmc/MZ109otzgNFk5s87H9Pc=
@ -247,6 +247,8 @@ github.com/kisielk/errcheck v1.7.0/go.mod h1:1kLL+jV4e+CFfueBmI1dSK2ADDyQnlrnrY/
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kkHAIKE/contextcheck v1.1.5 h1:CdnJh63tcDe53vG+RebdpdXJTc9atMgGqdx8LXxiilg=
github.com/kkHAIKE/contextcheck v1.1.5/go.mod h1:O930cpht4xb1YQpK+1+AgoM3mFsvxr7uyFptcnWTYUA=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@ -298,6 +300,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/moricho/tparallel v0.3.2 h1:odr8aZVFA3NZrNybggMkYO3rgPRcqjeQUlBBFVxKHTI=
github.com/moricho/tparallel v0.3.2/go.mod h1:OQ+K3b4Ln3l2TZveGCywybl68glfLEwFGqvnjok8b+U=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81U=
github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4Ngq6aY7OE=
github.com/nishanths/exhaustive v0.12.0 h1:vIY9sALmw6T/yxiASewa4TQcFsVYZQQRUQJhKRf3Swg=
@ -332,14 +336,14 @@ github.com/polyfloyd/go-errorlint v1.6.0 h1:tftWV9DE7txiFzPpztTAwyoRLKNj9gpVm2cg
github.com/polyfloyd/go-errorlint v1.6.0/go.mod h1:HR7u8wuP1kb1NeN1zqTd1ZMlqUKPPHF+Id4vIPvDqVw=
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg=
github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVhoNcY=
github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/quasilyte/go-ruleguard v0.4.2 h1:htXcXDK6/rO12kiTHKfHuqR4kr3Y4M0J0rOL6CH/BYs=
github.com/quasilyte/go-ruleguard v0.4.2/go.mod h1:GJLgqsLeo4qgavUoL8JeGFNS7qcisx3awV/w9eWTmNI=
github.com/quasilyte/go-ruleguard/dsl v0.3.22 h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE=