TracerProvider allows calling Tracer() while it's shutting down (#3924)

sdk/trace/provider.go

@@ -76,7 +76,8 @@ type TracerProvider struct {
 	mu             sync.Mutex
 	namedTracer    map[instrumentation.Scope]*tracer
 	spanProcessors atomic.Pointer[spanProcessorStates]
-	isShutdown     bool
+
+	isShutdown atomic.Bool
 
 	// These fields are not protected by the lock mu. They are assumed to be
 	// immutable after creation of the TracerProvider.
@@ -136,10 +137,11 @@ func NewTracerProvider(opts ...TracerProviderOption) *TracerProvider {
 //
 // This method is safe to be called concurrently.
 func (p *TracerProvider) Tracer(name string, opts ...trace.TracerOption) trace.Tracer {
+	// This check happens before the mutex is acquired to avoid deadlocking if Tracer() is called from within Shutdown().
+	if p.isShutdown.Load() {
+		return trace.NewNoopTracerProvider().Tracer(name, opts...)
+	}
 	c := trace.NewTracerConfig(opts...)
-
-	p.mu.Lock()
-	defer p.mu.Unlock()
 	if name == "" {
 		name = defaultTracerName
 	}
@@ -148,23 +150,46 @@ func (p *TracerProvider) Tracer(name string, opts ...trace.TracerOption) trace.T
 		Version:   c.InstrumentationVersion(),
 		SchemaURL: c.SchemaURL(),
 	}
-	t, ok := p.namedTracer[is]
-	if !ok {
-		t = &tracer{
-			provider:             p,
-			instrumentationScope: is,
+
+	t, ok := func() (trace.Tracer, bool) {
+		p.mu.Lock()
+		defer p.mu.Unlock()
+		// Must check the flag after acquiring the mutex to avoid returning a valid tracer if Shutdown() ran
+		// after the first check above but before we acquired the mutex.
+		if p.isShutdown.Load() {
+			return trace.NewNoopTracerProvider().Tracer(name, opts...), true
 		}
-		p.namedTracer[is] = t
-		global.Info("Tracer created", "name", name, "version", c.InstrumentationVersion(), "schemaURL", c.SchemaURL())
+		t, ok := p.namedTracer[is]
+		if !ok {
+			t = &tracer{
+				provider:             p,
+				instrumentationScope: is,
+			}
+			p.namedTracer[is] = t
+		}
+		return t, ok
+	}()
+	if !ok {
+		// This code is outside the mutex to not hold the lock while calling third party logging code:
+		// - That code may do slow things like I/O, which would prolong the duration the lock is held,
+		//   slowing down all tracing consumers.
+		// - Logging code may be instrumented with tracing and deadlock because it could try
+		//   acquiring the same non-reentrant mutex.
+		global.Info("Tracer created", "name", name, "version", is.Version, "schemaURL", is.SchemaURL)
 	}
 	return t
 }
 
 // RegisterSpanProcessor adds the given SpanProcessor to the list of SpanProcessors.
 func (p *TracerProvider) RegisterSpanProcessor(sp SpanProcessor) {
+	// This check prevents calls during a shutdown.
+	if p.isShutdown.Load() {
+		return
+	}
 	p.mu.Lock()
 	defer p.mu.Unlock()
-	if p.isShutdown {
+	// This check prevents calls after a shutdown.
+	if p.isShutdown.Load() {
 		return
 	}
 	newSPS := spanProcessorStates{}
@@ -175,9 +200,14 @@ func (p *TracerProvider) RegisterSpanProcessor(sp SpanProcessor) {
 
 // UnregisterSpanProcessor removes the given SpanProcessor from the list of SpanProcessors.
 func (p *TracerProvider) UnregisterSpanProcessor(sp SpanProcessor) {
+	// This check prevents calls during a shutdown.
+	if p.isShutdown.Load() {
+		return
+	}
 	p.mu.Lock()
 	defer p.mu.Unlock()
-	if p.isShutdown {
+	// This check prevents calls after a shutdown.
+	if p.isShutdown.Load() {
 		return
 	}
 	old := *(p.spanProcessors.Load())
@@ -236,13 +266,18 @@ func (p *TracerProvider) ForceFlush(ctx context.Context) error {
 
 // Shutdown shuts down TracerProvider. All registered span processors are shut down
 // in the order they were registered and any held computational resources are released.
+// After Shutdown is called, all methods are no-ops.
 func (p *TracerProvider) Shutdown(ctx context.Context) error {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-	if p.isShutdown {
+	// This check prevents deadlocks in case of recursive shutdown.
+	if p.isShutdown.Load() {
+		return nil
+	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	// This check prevents calls after a shutdown has already been done concurrently.
+	if !p.isShutdown.CompareAndSwap(false, true) { // did toggle?
 		return nil
 	}
-	p.isShutdown = true
 	spss := *(p.spanProcessors.Load())
 
 	var retErr error