// Copyright The OpenTelemetry Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // This test is too large for the race detector. This SDK uses no locks // that the race detector would help with, anyway. // +build !race package metric import ( "context" "fmt" "math" "math/rand" "runtime" "sort" "strings" "sync" "sync/atomic" "testing" "time" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/label" export "go.opentelemetry.io/otel/sdk/export/metric" "go.opentelemetry.io/otel/sdk/export/metric/aggregation" "go.opentelemetry.io/otel/sdk/metric/processor/processortest" ) const ( concurrencyPerCPU = 100 reclaimPeriod = time.Millisecond * 100 testRun = 5 * time.Second epsilon = 1e-10 ) var Must = otel.Must type ( testFixture struct { // stop has to be aligned for 64-bit atomic operations. stop int64 expected sync.Map received sync.Map // Note: doesn't require synchronization wg sync.WaitGroup impl testImpl T *testing.T export.AggregatorSelector lock sync.Mutex lused map[string]bool dupCheck map[testKey]int totalDups int64 } testKey struct { labels string descriptor *otel.Descriptor } testImpl struct { newInstrument func(meter otel.Meter, name string) SyncImpler getUpdateValue func() otel.Number operate func(interface{}, context.Context, otel.Number, []label.KeyValue) newStore func() interface{} // storeCollect and storeExpect are the same for // counters, different for lastValues, to ensure we are // testing the timestamps correctly. storeCollect func(store interface{}, value otel.Number, ts time.Time) storeExpect func(store interface{}, value otel.Number) readStore func(store interface{}) otel.Number equalValues func(a, b otel.Number) bool } SyncImpler interface { SyncImpl() otel.SyncImpl } // lastValueState supports merging lastValue values, for the case // where a race condition causes duplicate records. We always // take the later timestamp. lastValueState struct { // raw has to be aligned for 64-bit atomic operations. raw otel.Number ts time.Time } ) func concurrency() int { return concurrencyPerCPU * runtime.NumCPU() } func canonicalizeLabels(ls []label.KeyValue) string { copy := append(ls[0:0:0], ls...) sort.SliceStable(copy, func(i, j int) bool { return copy[i].Key < copy[j].Key }) var b strings.Builder for _, kv := range copy { b.WriteString(string(kv.Key)) b.WriteString("=") b.WriteString(kv.Value.Emit()) b.WriteString("$") } return b.String() } func getPeriod() time.Duration { dur := math.Max( float64(reclaimPeriod)/10, float64(reclaimPeriod)*(1+0.1*rand.NormFloat64()), ) return time.Duration(dur) } func (f *testFixture) someLabels() []label.KeyValue { n := 1 + rand.Intn(3) l := make([]label.KeyValue, n) for { oused := map[string]bool{} for i := 0; i < n; i++ { var k string for { k = fmt.Sprint("k", rand.Intn(1000000000)) if !oused[k] { oused[k] = true break } } l[i] = label.String(k, fmt.Sprint("v", rand.Intn(1000000000))) } lc := canonicalizeLabels(l) f.lock.Lock() avail := !f.lused[lc] if avail { f.lused[lc] = true f.lock.Unlock() return l } f.lock.Unlock() } } func (f *testFixture) startWorker(impl *Accumulator, meter otel.Meter, wg *sync.WaitGroup, i int) { ctx := context.Background() name := fmt.Sprint("test_", i) instrument := f.impl.newInstrument(meter, name) var descriptor *otel.Descriptor if ii, ok := instrument.SyncImpl().(*syncInstrument); ok { descriptor = &ii.descriptor } kvs := f.someLabels() clabs := canonicalizeLabels(kvs) dur := getPeriod() key := testKey{ labels: clabs, descriptor: descriptor, } for { sleep := time.Duration(rand.ExpFloat64() * float64(dur)) time.Sleep(sleep) value := f.impl.getUpdateValue() f.impl.operate(instrument, ctx, value, kvs) actual, _ := f.expected.LoadOrStore(key, f.impl.newStore()) f.impl.storeExpect(actual, value) if atomic.LoadInt64(&f.stop) != 0 { wg.Done() return } } } func (f *testFixture) assertTest(numCollect int) { var allErrs []func() csize := 0 f.received.Range(func(key, gstore interface{}) bool { csize++ gvalue := f.impl.readStore(gstore) estore, loaded := f.expected.Load(key) if !loaded { allErrs = append(allErrs, func() { f.T.Error("Could not locate expected key: ", key) }) return true } evalue := f.impl.readStore(estore) if !f.impl.equalValues(evalue, gvalue) { allErrs = append(allErrs, func() { f.T.Error("Expected value mismatch: ", evalue, "!=", gvalue, " for ", key) }) } return true }) rsize := 0 f.expected.Range(func(key, value interface{}) bool { rsize++ if _, loaded := f.received.Load(key); !loaded { allErrs = append(allErrs, func() { f.T.Error("Did not receive expected key: ", key) }) } return true }) if rsize != csize { f.T.Error("Did not receive the correct set of metrics: Received != Expected", rsize, csize) } for _, anErr := range allErrs { anErr() } // Note: It's useful to know the test triggers this condition, // but we can't assert it. Infrequently no duplicates are // found, and we can't really force a race to happen. // // fmt.Printf("Test duplicate records seen: %.1f%%\n", // float64(100*f.totalDups/int64(numCollect*concurrency()))) } func (f *testFixture) preCollect() { // Collect calls Process in a single-threaded context. No need // to lock this struct. f.dupCheck = map[testKey]int{} } func (*testFixture) CheckpointSet() export.CheckpointSet { return nil } func (f *testFixture) Process(accumulation export.Accumulation) error { labels := accumulation.Labels().ToSlice() key := testKey{ labels: canonicalizeLabels(labels), descriptor: accumulation.Descriptor(), } if f.dupCheck[key] == 0 { f.dupCheck[key]++ } else { f.totalDups++ } actual, _ := f.received.LoadOrStore(key, f.impl.newStore()) agg := accumulation.Aggregator() switch accumulation.Descriptor().InstrumentKind() { case otel.CounterInstrumentKind: sum, err := agg.(aggregation.Sum).Sum() if err != nil { f.T.Fatal("Sum error: ", err) } f.impl.storeCollect(actual, sum, time.Time{}) case otel.ValueRecorderInstrumentKind: lv, ts, err := agg.(aggregation.LastValue).LastValue() if err != nil && err != aggregation.ErrNoData { f.T.Fatal("Last value error: ", err) } f.impl.storeCollect(actual, lv, ts) default: panic("Not used in this test") } return nil } func stressTest(t *testing.T, impl testImpl) { ctx := context.Background() t.Parallel() fixture := &testFixture{ T: t, impl: impl, lused: map[string]bool{}, AggregatorSelector: processortest.AggregatorSelector(), } cc := concurrency() sdk := NewAccumulator(fixture, nil) meter := otel.WrapMeterImpl(sdk, "stress_test") fixture.wg.Add(cc + 1) for i := 0; i < cc; i++ { go fixture.startWorker(sdk, meter, &fixture.wg, i) } numCollect := 0 go func() { for { time.Sleep(reclaimPeriod) fixture.preCollect() sdk.Collect(ctx) numCollect++ if atomic.LoadInt64(&fixture.stop) != 0 { fixture.wg.Done() return } } }() time.Sleep(testRun) atomic.StoreInt64(&fixture.stop, 1) fixture.wg.Wait() fixture.preCollect() sdk.Collect(ctx) numCollect++ fixture.assertTest(numCollect) } func int64sEqual(a, b otel.Number) bool { return a.AsInt64() == b.AsInt64() } func float64sEqual(a, b otel.Number) bool { diff := math.Abs(a.AsFloat64() - b.AsFloat64()) return diff < math.Abs(a.AsFloat64())*epsilon } // Counters func intCounterTestImpl() testImpl { return testImpl{ newInstrument: func(meter otel.Meter, name string) SyncImpler { return Must(meter).NewInt64Counter(name + ".sum") }, getUpdateValue: func() otel.Number { for { x := int64(rand.Intn(100)) if x != 0 { return otel.NewInt64Number(x) } } }, operate: func(inst interface{}, ctx context.Context, value otel.Number, labels []label.KeyValue) { counter := inst.(otel.Int64Counter) counter.Add(ctx, value.AsInt64(), labels...) }, newStore: func() interface{} { n := otel.NewInt64Number(0) return &n }, storeCollect: func(store interface{}, value otel.Number, _ time.Time) { store.(*otel.Number).AddInt64Atomic(value.AsInt64()) }, storeExpect: func(store interface{}, value otel.Number) { store.(*otel.Number).AddInt64Atomic(value.AsInt64()) }, readStore: func(store interface{}) otel.Number { return store.(*otel.Number).AsNumberAtomic() }, equalValues: int64sEqual, } } func TestStressInt64Counter(t *testing.T) { stressTest(t, intCounterTestImpl()) } func floatCounterTestImpl() testImpl { return testImpl{ newInstrument: func(meter otel.Meter, name string) SyncImpler { return Must(meter).NewFloat64Counter(name + ".sum") }, getUpdateValue: func() otel.Number { for { x := rand.Float64() if x != 0 { return otel.NewFloat64Number(x) } } }, operate: func(inst interface{}, ctx context.Context, value otel.Number, labels []label.KeyValue) { counter := inst.(otel.Float64Counter) counter.Add(ctx, value.AsFloat64(), labels...) }, newStore: func() interface{} { n := otel.NewFloat64Number(0.0) return &n }, storeCollect: func(store interface{}, value otel.Number, _ time.Time) { store.(*otel.Number).AddFloat64Atomic(value.AsFloat64()) }, storeExpect: func(store interface{}, value otel.Number) { store.(*otel.Number).AddFloat64Atomic(value.AsFloat64()) }, readStore: func(store interface{}) otel.Number { return store.(*otel.Number).AsNumberAtomic() }, equalValues: float64sEqual, } } func TestStressFloat64Counter(t *testing.T) { stressTest(t, floatCounterTestImpl()) } // LastValue func intLastValueTestImpl() testImpl { return testImpl{ newInstrument: func(meter otel.Meter, name string) SyncImpler { return Must(meter).NewInt64ValueRecorder(name + ".lastvalue") }, getUpdateValue: func() otel.Number { r1 := rand.Int63() return otel.NewInt64Number(rand.Int63() - r1) }, operate: func(inst interface{}, ctx context.Context, value otel.Number, labels []label.KeyValue) { valuerecorder := inst.(otel.Int64ValueRecorder) valuerecorder.Record(ctx, value.AsInt64(), labels...) }, newStore: func() interface{} { return &lastValueState{ raw: otel.NewInt64Number(0), } }, storeCollect: func(store interface{}, value otel.Number, ts time.Time) { gs := store.(*lastValueState) if !ts.Before(gs.ts) { gs.ts = ts gs.raw.SetInt64Atomic(value.AsInt64()) } }, storeExpect: func(store interface{}, value otel.Number) { gs := store.(*lastValueState) gs.raw.SetInt64Atomic(value.AsInt64()) }, readStore: func(store interface{}) otel.Number { gs := store.(*lastValueState) return gs.raw.AsNumberAtomic() }, equalValues: int64sEqual, } } func TestStressInt64LastValue(t *testing.T) { stressTest(t, intLastValueTestImpl()) } func floatLastValueTestImpl() testImpl { return testImpl{ newInstrument: func(meter otel.Meter, name string) SyncImpler { return Must(meter).NewFloat64ValueRecorder(name + ".lastvalue") }, getUpdateValue: func() otel.Number { return otel.NewFloat64Number((-0.5 + rand.Float64()) * 100000) }, operate: func(inst interface{}, ctx context.Context, value otel.Number, labels []label.KeyValue) { valuerecorder := inst.(otel.Float64ValueRecorder) valuerecorder.Record(ctx, value.AsFloat64(), labels...) }, newStore: func() interface{} { return &lastValueState{ raw: otel.NewFloat64Number(0), } }, storeCollect: func(store interface{}, value otel.Number, ts time.Time) { gs := store.(*lastValueState) if !ts.Before(gs.ts) { gs.ts = ts gs.raw.SetFloat64Atomic(value.AsFloat64()) } }, storeExpect: func(store interface{}, value otel.Number) { gs := store.(*lastValueState) gs.raw.SetFloat64Atomic(value.AsFloat64()) }, readStore: func(store interface{}) otel.Number { gs := store.(*lastValueState) return gs.raw.AsNumberAtomic() }, equalValues: float64sEqual, } } func TestStressFloat64LastValue(t *testing.T) { stressTest(t, floatLastValueTestImpl()) }