1
0
mirror of https://github.com/open-telemetry/opentelemetry-go.git synced 2024-12-16 10:19:23 +02:00
opentelemetry-go/exporters/otlp/otlptrace/otlptracegrpc/mock_collector_test.go
Tyler Yahn 7d92434295
Fix goroutine leaks in otlptracegrpc testing (#2409)
* Fail in RunEndToEndTest if collector stop fails

* Use testing T.Cleanup to check shut downs

* Add goroutine leak detection

* Fix TestExporterShutdown go leak

The shutdown tests checking if a context error is honored did not
completely clean up the resources used by the client after the error was
evaluated. Update the connection client to handle multiple calls to
shutdown and make a second call to these clients that must succeed so
the test does not have abandoned goroutines.

* Fix leak in TestNew_WithTimeout

The mockTraceService did not delay with its lock being held. This
resulted in the mockCollector stopping and being able to acquire the
lock. It was assumed that no export was taking place because of this and
the mockTraceService was abandoned without cleaning up resources it held
and goroutines it had spawned. This reworks the export blocking logic to
block on a channel read. This will make the block more deterministic and
not depend on the scheduler timing. Additionally, this blocking is moved
inside the lock acquire. Meaning code will deadlock if the block is not
released before a shutdown (something the developer will immediately be
aware of when they submit a bad patch), and will ensure all resources
are released before shutdown.

Replace TestNew_WithTimeout with TestExportSpansTimeoutHonored which
directly tests if a span export errors when the timeout is reached. This
is the only unique thing that TestNew_WithTimeout, but it also tests the
non-error path. That non-error path is tested in many other tests.

* Guard otlptracehttp client stopCh when stopping

In normal operations the exporter is guaranteed to only ever call the
client Stop method once. However in testing we need to call this
multiple times when checking it returns an error in particular context.
Add a lightweight sync.Once to the closing of the stopCh to ensure tests
do not panic when cleaning up.

* Release export block after export

Prevent deadlock in TestExportSpansTimeoutHonored.
2021-11-22 07:54:32 -08:00

241 lines
5.9 KiB
Go

// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package otlptracegrpc_test
import (
"context"
"fmt"
"net"
"runtime"
"strings"
"sync"
"testing"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/internal/otlptracetest"
collectortracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
)
func makeMockCollector(t *testing.T, mockConfig *mockConfig) *mockCollector {
return &mockCollector{
t: t,
traceSvc: &mockTraceService{
storage: otlptracetest.NewSpansStorage(),
errors: mockConfig.errors,
},
}
}
type mockTraceService struct {
collectortracepb.UnimplementedTraceServiceServer
errors []error
requests int
mu sync.RWMutex
storage otlptracetest.SpansStorage
headers metadata.MD
exportBlock chan struct{}
}
func (mts *mockTraceService) getHeaders() metadata.MD {
mts.mu.RLock()
defer mts.mu.RUnlock()
return mts.headers
}
func (mts *mockTraceService) getSpans() []*tracepb.Span {
mts.mu.RLock()
defer mts.mu.RUnlock()
return mts.storage.GetSpans()
}
func (mts *mockTraceService) getResourceSpans() []*tracepb.ResourceSpans {
mts.mu.RLock()
defer mts.mu.RUnlock()
return mts.storage.GetResourceSpans()
}
func (mts *mockTraceService) Export(ctx context.Context, exp *collectortracepb.ExportTraceServiceRequest) (*collectortracepb.ExportTraceServiceResponse, error) {
mts.mu.Lock()
defer func() {
mts.requests++
mts.mu.Unlock()
}()
if mts.exportBlock != nil {
// Do this with the lock held so the mockCollector.Stop does not
// abandon cleaning up resources.
<-mts.exportBlock
}
reply := &collectortracepb.ExportTraceServiceResponse{}
if mts.requests < len(mts.errors) {
idx := mts.requests
return reply, mts.errors[idx]
}
mts.headers, _ = metadata.FromIncomingContext(ctx)
mts.storage.AddSpans(exp)
return reply, nil
}
type mockCollector struct {
t *testing.T
traceSvc *mockTraceService
endpoint string
ln *listener
stopFunc func()
stopOnce sync.Once
}
type mockConfig struct {
errors []error
endpoint string
}
var _ collectortracepb.TraceServiceServer = (*mockTraceService)(nil)
var errAlreadyStopped = fmt.Errorf("already stopped")
func (mc *mockCollector) stop() error {
var err = errAlreadyStopped
mc.stopOnce.Do(func() {
err = nil
if mc.stopFunc != nil {
mc.stopFunc()
}
})
// Give it sometime to shutdown.
<-time.After(160 * time.Millisecond)
// Getting the lock ensures the traceSvc is done flushing.
mc.traceSvc.mu.Lock()
defer mc.traceSvc.mu.Unlock()
return err
}
func (mc *mockCollector) Stop() error {
return mc.stop()
}
func (mc *mockCollector) getSpans() []*tracepb.Span {
return mc.traceSvc.getSpans()
}
func (mc *mockCollector) getResourceSpans() []*tracepb.ResourceSpans {
return mc.traceSvc.getResourceSpans()
}
func (mc *mockCollector) GetResourceSpans() []*tracepb.ResourceSpans {
return mc.getResourceSpans()
}
func (mc *mockCollector) getHeaders() metadata.MD {
return mc.traceSvc.getHeaders()
}
// runMockCollector is a helper function to create a mock Collector
func runMockCollector(t *testing.T) *mockCollector {
return runMockCollectorAtEndpoint(t, "localhost:0")
}
func runMockCollectorAtEndpoint(t *testing.T, endpoint string) *mockCollector {
return runMockCollectorWithConfig(t, &mockConfig{endpoint: endpoint})
}
func runMockCollectorWithConfig(t *testing.T, mockConfig *mockConfig) *mockCollector {
ln, err := net.Listen("tcp", mockConfig.endpoint)
if err != nil {
t.Fatalf("Failed to get an endpoint: %v", err)
}
srv := grpc.NewServer()
mc := makeMockCollector(t, mockConfig)
collectortracepb.RegisterTraceServiceServer(srv, mc.traceSvc)
mc.ln = newListener(ln)
go func() {
_ = srv.Serve((net.Listener)(mc.ln))
}()
mc.endpoint = ln.Addr().String()
// srv.Stop calls Close on mc.ln.
mc.stopFunc = srv.Stop
return mc
}
type listener struct {
closeOnce sync.Once
wrapped net.Listener
C chan struct{}
}
func newListener(wrapped net.Listener) *listener {
return &listener{
wrapped: wrapped,
C: make(chan struct{}, 1),
}
}
func (l *listener) Close() error { return l.wrapped.Close() }
func (l *listener) Addr() net.Addr { return l.wrapped.Addr() }
// Accept waits for and returns the next connection to the listener. It will
// send a signal on l.C that a connection has been made before returning.
func (l *listener) Accept() (net.Conn, error) {
conn, err := l.wrapped.Accept()
if err != nil {
// Go 1.16 exported net.ErrClosed that could clean up this check, but to
// remain backwards compatible with previous versions of Go that we
// support the following string evaluation is used instead to keep in line
// with the previously recommended way to check this:
// https://github.com/golang/go/issues/4373#issuecomment-353076799
if strings.Contains(err.Error(), "use of closed network connection") {
// If the listener has been closed, do not allow callers of
// WaitForConn to wait for a connection that will never come.
l.closeOnce.Do(func() { close(l.C) })
}
return conn, err
}
select {
case l.C <- struct{}{}:
default:
// If C is full, assume nobody is listening and move on.
}
return conn, nil
}
// WaitForConn will wait indefintely for a connection to be estabilished with
// the listener before returning.
func (l *listener) WaitForConn() {
for {
select {
case <-l.C:
return
default:
runtime.Gosched()
}
}
}