From e016a78c9f5b24a1c2beeaad47686c2f2213f49a Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Tue, 26 Nov 2024 00:41:55 -0800 Subject: [PATCH] Fix attribute value truncation (#5997) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix #5996 ### Correctness From the [OTel specification](https://github.com/open-telemetry/opentelemetry-specification/blob/88bffeac48aa5eb37ac8044e931af63a0b55fe00/specification/common/README.md#attribute-limits): > - set an attribute value length limit such that for each attribute value: > - if it is a string, if it exceeds that limit (counting any character in it as 1), SDKs MUST truncate that value, so that its length is at most equal to the limit... Our current implementation truncates on number of bytes not characters. Unit tests are added/updated to validate this fix and prevent regressions. ### Performance ``` goos: linux goarch: amd64 pkg: go.opentelemetry.io/otel/sdk/trace cpu: Intel(R) Core(TM) i7-8550U CPU @ 1.80GHz │ commit-b6264913(old).txt │ commit-54c61ac2(new).txt │ │ sec/op │ sec/op vs base │ Truncate/Unlimited-8 1.2300n ± 7% 0.8757n ± 3% -28.80% (p=0.000 n=10) Truncate/Zero-8 2.341n ± 2% 1.550n ± 9% -33.77% (p=0.000 n=10) Truncate/Short-8 31.6800n ± 3% 0.9960n ± 4% -96.86% (p=0.000 n=10) Truncate/ASCII-8 8.821n ± 1% 3.567n ± 3% -59.57% (p=0.000 n=10) Truncate/ValidUTF-8-8 11.960n ± 1% 7.163n ± 1% -40.10% (p=0.000 n=10) Truncate/InvalidUTF-8-8 56.35n ± 0% 37.34n ± 18% -33.74% (p=0.000 n=10) Truncate/MixedUTF-8-8 81.83n ± 1% 50.00n ± 1% -38.90% (p=0.000 n=10) geomean 12.37n 4.865n -60.68% │ commit-b6264913(old).txt │ commit-54c61ac2(new).txt │ │ B/op │ B/op vs base │ Truncate/Unlimited-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Truncate/Zero-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Truncate/Short-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Truncate/ASCII-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Truncate/ValidUTF-8-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Truncate/InvalidUTF-8-8 16.00 ± 0% 16.00 ± 0% ~ (p=1.000 n=10) ¹ Truncate/MixedUTF-8-8 32.00 ± 0% 32.00 ± 0% ~ (p=1.000 n=10) ¹ geomean ² +0.00% ² ¹ all samples are equal ² summaries must be >0 to compute geomean │ commit-b6264913(old).txt │ commit-54c61ac2(new).txt │ │ allocs/op │ allocs/op vs base │ Truncate/Unlimited-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Truncate/Zero-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Truncate/Short-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Truncate/ASCII-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Truncate/ValidUTF-8-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Truncate/InvalidUTF-8-8 1.000 ± 0% 1.000 ± 0% ~ (p=1.000 n=10) ¹ Truncate/MixedUTF-8-8 1.000 ± 0% 1.000 ± 0% ~ (p=1.000 n=10) ¹ geomean ² +0.00% ² ¹ all samples are equal ² summaries must be >0 to compute geomean ``` #### Values shorter than limit This is the default code path. Most attribute values will be shorter than the default 128 limit that users will not modify. The current code, `safeTruncate` requires a full iteration of the value to determine it is valid and under the limit. The replacement, `truncate`, first checks if the number of bytes in the value are less than or equal to the limit (which guarantees the number of characters are less than or equal to the limit) and returns immediately. This will mean that invalid encoding less than the limit is not changed, which meets the specification requirements. #### Values longer than the limit For values who's number of bytes exceeds the limit, they are iterated only once with the replacement, `truncate`. In comparison, the current code, `safeTruncate`, can iterate the string up to three separate times when the string contains invalid characters. --- CHANGELOG.md | 1 + sdk/trace/span.go | 97 ++++++++++++++------ sdk/trace/span_limits_test.go | 2 +- sdk/trace/span_test.go | 168 +++++++++++++++++++++++++++++----- 4 files changed, 217 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e573f61e5..a19036cae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - Fix inconsistent request body closing in `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp`. (#5954) - Fix inconsistent request body closing in `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp`. (#5954) - Fix invalid exemplar keys in `go.opentelemetry.io/otel/exporters/prometheus`. (#5995) +- Fix attribute value truncation in `go.opentelemetry.io/otel/sdk/trace`. (#5997) diff --git a/sdk/trace/span.go b/sdk/trace/span.go index 17f883c2c..15cd16a32 100644 --- a/sdk/trace/span.go +++ b/sdk/trace/span.go @@ -347,47 +347,92 @@ func truncateAttr(limit int, attr attribute.KeyValue) attribute.KeyValue { } switch attr.Value.Type() { case attribute.STRING: - if v := attr.Value.AsString(); len(v) > limit { - return attr.Key.String(safeTruncate(v, limit)) - } + v := attr.Value.AsString() + return attr.Key.String(truncate(limit, v)) case attribute.STRINGSLICE: v := attr.Value.AsStringSlice() for i := range v { - if len(v[i]) > limit { - v[i] = safeTruncate(v[i], limit) - } + v[i] = truncate(limit, v[i]) } return attr.Key.StringSlice(v) } return attr } -// safeTruncate truncates the string and guarantees valid UTF-8 is returned. -func safeTruncate(input string, limit int) string { - if trunc, ok := safeTruncateValidUTF8(input, limit); ok { - return trunc +// truncate returns a truncated version of s such that it contains less than +// the limit number of characters. Truncation is applied by returning the limit +// number of valid characters contained in s. +// +// If limit is negative, it returns the original string. +// +// UTF-8 is supported. When truncating, all invalid characters are dropped +// before applying truncation. +// +// If s already contains less than the limit number of bytes, it is returned +// unchanged. No invalid characters are removed. +func truncate(limit int, s string) string { + // This prioritize performance in the following order based on the most + // common expected use-cases. + // + // - Short values less than the default limit (128). + // - Strings with valid encodings that exceed the limit. + // - No limit. + // - Strings with invalid encodings that exceed the limit. + if limit < 0 || len(s) <= limit { + return s } - trunc, _ := safeTruncateValidUTF8(strings.ToValidUTF8(input, ""), limit) - return trunc -} -// safeTruncateValidUTF8 returns a copy of the input string safely truncated to -// limit. The truncation is ensured to occur at the bounds of complete UTF-8 -// characters. If invalid encoding of UTF-8 is encountered, input is returned -// with false, otherwise, the truncated input will be returned with true. -func safeTruncateValidUTF8(input string, limit int) (string, bool) { - for cnt := 0; cnt <= limit; { - r, size := utf8.DecodeRuneInString(input[cnt:]) - if r == utf8.RuneError { - return input, false + // Optimistically, assume all valid UTF-8. + var b strings.Builder + count := 0 + for i, c := range s { + if c != utf8.RuneError { + count++ + if count > limit { + return s[:i] + } + continue } - if cnt+size > limit { - return input[:cnt], true + _, size := utf8.DecodeRuneInString(s[i:]) + if size == 1 { + // Invalid encoding. + b.Grow(len(s) - 1) + _, _ = b.WriteString(s[:i]) + s = s[i:] + break } - cnt += size } - return input, true + + // Fast-path, no invalid input. + if b.Cap() == 0 { + return s + } + + // Truncate while validating UTF-8. + for i := 0; i < len(s) && count < limit; { + c := s[i] + if c < utf8.RuneSelf { + // Optimization for single byte runes (common case). + _ = b.WriteByte(c) + i++ + count++ + continue + } + + _, size := utf8.DecodeRuneInString(s[i:]) + if size == 1 { + // We checked for all 1-byte runes above, this is a RuneError. + i++ + continue + } + + _, _ = b.WriteString(s[i : i+size]) + i += size + count++ + } + + return b.String() } // End ends the span. This method does nothing if the span is already ended or diff --git a/sdk/trace/span_limits_test.go b/sdk/trace/span_limits_test.go index bd57fa662..74f8997c8 100644 --- a/sdk/trace/span_limits_test.go +++ b/sdk/trace/span_limits_test.go @@ -183,7 +183,7 @@ func TestSpanLimits(t *testing.T) { // Ensure string and string slice attributes are truncated. assert.Contains(t, attrs, attribute.String("string", "ab")) assert.Contains(t, attrs, attribute.StringSlice("stringSlice", []string{"ab", "de"})) - assert.Contains(t, attrs, attribute.String("euro", "")) + assert.Contains(t, attrs, attribute.String("euro", "€")) limits.AttributeValueLengthLimit = 0 attrs = testSpanLimits(t, limits).Attributes() diff --git a/sdk/trace/span_test.go b/sdk/trace/span_test.go index 38bf2b2e8..5a0355c0b 100644 --- a/sdk/trace/span_test.go +++ b/sdk/trace/span_test.go @@ -201,30 +201,6 @@ func TestTruncateAttr(t *testing.T) { attr: strSliceAttr, want: strSliceAttr, }, - { - // This tests the ordinary safeTruncate(). - limit: 10, - attr: attribute.String(key, "€€€€"), // 3 bytes each - want: attribute.String(key, "€€€"), - }, - { - // This tests truncation with an invalid UTF-8 input. - // - // Note that after removing the invalid rune, - // the string is over length and still has to - // be truncated on a code point boundary. - limit: 10, - attr: attribute.String(key, "€"[0:2]+"hello€€"), // corrupted first rune, then over limit - want: attribute.String(key, "hello€"), - }, - { - // This tests the fallback to invalidTruncate() - // where after validation the string does not require - // truncation. - limit: 6, - attr: attribute.String(key, "€"[0:2]+"hello"), // corrupted first rune, then not over limit - want: attribute.String(key, "hello"), - }, } for _, test := range tests { @@ -235,6 +211,150 @@ func TestTruncateAttr(t *testing.T) { } } +func TestTruncate(t *testing.T) { + type group struct { + limit int + input string + expected string + } + + tests := []struct { + name string + groups []group + }{ + // Edge case: limit is negative, no truncation should occur + { + name: "NoTruncation", + groups: []group{ + {-1, "No truncation!", "No truncation!"}, + }, + }, + + // Edge case: string is already shorter than the limit, no truncation + // should occur + { + name: "ShortText", + groups: []group{ + {10, "Short text", "Short text"}, + {15, "Short text", "Short text"}, + {100, "Short text", "Short text"}, + }, + }, + + // Edge case: truncation happens with ASCII characters only + { + name: "ASCIIOnly", + groups: []group{ + {1, "Hello World!", "H"}, + {5, "Hello World!", "Hello"}, + {12, "Hello World!", "Hello World!"}, + }, + }, + + // Truncation including multi-byte characters (UTF-8) + { + name: "ValidUTF-8", + groups: []group{ + {7, "Hello, 世界", "Hello, "}, + {8, "Hello, 世界", "Hello, 世"}, + {2, "こんにちは", "こん"}, + {3, "こんにちは", "こんに"}, + {5, "こんにちは", "こんにちは"}, + {12, "こんにちは", "こんにちは"}, + }, + }, + + // Truncation with invalid UTF-8 characters + { + name: "InvalidUTF-8", + groups: []group{ + {11, "Invalid\x80text", "Invalidtext"}, + // Do not modify invalid text if equal to limit. + {11, "Valid text\x80", "Valid text\x80"}, + // Do not modify invalid text if under limit. + {15, "Valid text\x80", "Valid text\x80"}, + {5, "Hello\x80World", "Hello"}, + {11, "Hello\x80World\x80!", "HelloWorld!"}, + {15, "Hello\x80World\x80Test", "HelloWorldTest"}, + {15, "Hello\x80\x80\x80World\x80Test", "HelloWorldTest"}, + {15, "\x80\x80\x80Hello\x80\x80\x80World\x80Test\x80\x80", "HelloWorldTest"}, + }, + }, + + // Truncation with mixed validn and invalid UTF-8 characters + { + name: "MixedUTF-8", + groups: []group{ + {6, "€"[0:2] + "hello€€", "hello€"}, + {6, "€" + "€"[0:2] + "hello", "€hello"}, + {11, "Valid text\x80📜", "Valid text📜"}, + {11, "Valid text📜\x80", "Valid text📜"}, + {14, "😊 Hello\x80World🌍🚀", "😊 HelloWorld🌍🚀"}, + {14, "😊\x80 Hello\x80World🌍🚀", "😊 HelloWorld🌍🚀"}, + {14, "😊\x80 Hello\x80World🌍\x80🚀", "😊 HelloWorld🌍🚀"}, + {14, "😊\x80 Hello\x80World🌍\x80🚀\x80", "😊 HelloWorld🌍🚀"}, + {14, "\x80😊\x80 Hello\x80World🌍\x80🚀\x80", "😊 HelloWorld🌍🚀"}, + }, + }, + + // Edge case: empty string, should return empty string + { + name: "Empty", + groups: []group{ + {5, "", ""}, + }, + }, + + // Edge case: limit is 0, should return an empty string + { + name: "Zero", + groups: []group{ + {0, "Some text", ""}, + {0, "", ""}, + }, + }, + } + + for _, tt := range tests { + for _, g := range tt.groups { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := truncate(g.limit, g.input) + assert.Equalf( + t, g.expected, got, + "input: %q([]rune%v))\ngot: %q([]rune%v)\nwant %q([]rune%v)", + g.input, []rune(g.input), + got, []rune(got), + g.expected, []rune(g.expected), + ) + }) + } + } +} + +func BenchmarkTruncate(b *testing.B) { + run := func(limit int, input string) func(b *testing.B) { + return func(b *testing.B) { + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + var out string + for pb.Next() { + out = truncate(limit, input) + } + _ = out + }) + } + } + b.Run("Unlimited", run(-1, "hello 😊 world 🌍🚀")) + b.Run("Zero", run(0, "Some text")) + b.Run("Short", run(10, "Short Text")) + b.Run("ASCII", run(5, "Hello, World!")) + b.Run("ValidUTF-8", run(10, "hello 😊 world 🌍🚀")) + b.Run("InvalidUTF-8", run(6, "€"[0:2]+"hello€€")) + b.Run("MixedUTF-8", run(14, "\x80😊\x80 Hello\x80World🌍\x80🚀\x80")) +} + func TestLogDropAttrs(t *testing.T) { orig := logDropAttrs t.Cleanup(func() { logDropAttrs = orig })