Repository: grpc/grpc-go

Issue:
cleanup: Remove grpcsync.OnceFunc
The [`OnceFunc`](https://pkg.go.dev/sync#OnceFunc) function in the standard library was introduced in go1.21. The [min Go version for grpc-go](https://github.com/grpc/grpc-go/blob/c879198e3bb82dba601bb3fd132e865dcaec2893/go.mod#L3) is go1.22. We can remove [grpc-go's copy of OnceFuc](https://github.com/grpc/grpc-go/blob/c879198e3bb82dba601bb3fd132e865dcaec2893/internal/grpcsync/oncefunc.go#L1) and move all usages to the std lib implementation.

Hints from issue comments:

Solution patch:
diff --git a/balancer_wrapper.go b/balancer_wrapper.go
index c2688376..d08bf82f 100644
--- a/balancer_wrapper.go
+++ b/balancer_wrapper.go
@@ -415,7 +415,7 @@ func (acbw *acBalancerWrapper) GetOrBuildProducer(pb balancer.ProducerBuilder) (
 		}
 		acbw.producersMu.Unlock()
 	}
-	return pData.producer, grpcsync.OnceFunc(unref)
+	return pData.producer, sync.OnceFunc(unref)
 }
 
 func (acbw *acBalancerWrapper) closeProducers() {
diff --git a/experimental/opentelemetry/trace_options.go b/experimental/opentelemetry/trace_options.go
new file mode 100644
index 00000000..9dc0079a
--- /dev/null
+++ b/experimental/opentelemetry/trace_options.go
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package opentelemetry is EXPERIMENTAL and will be moved to stats/opentelemetry
+// package in a later release.
+package opentelemetry
+
+import (
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// TraceOptions contains the tracing settings for OpenTelemetry instrumentation.
+type TraceOptions struct {
+	// TracerProvider is the OpenTelemetry tracer which is required to
+	// record traces/trace spans for instrumentation.  If unset, tracing
+	// will not be recorded.
+	TracerProvider trace.TracerProvider
+
+	// TextMapPropagator propagates span context through text map carrier.
+	// If unset, tracing will not be recorded.
+	TextMapPropagator propagation.TextMapPropagator
+}
diff --git a/internal/grpcsync/oncefunc.go b/internal/grpcsync/oncefunc.go
deleted file mode 100644
index 6635f7bc..00000000
--- a/internal/grpcsync/oncefunc.go
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- *
- * Copyright 2022 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package grpcsync
-
-import (
-	"sync"
-)
-
-// OnceFunc returns a function wrapping f which ensures f is only executed
-// once even if the returned function is executed multiple times.
-func OnceFunc(f func()) func() {
-	var once sync.Once
-	return func() {
-		once.Do(f)
-	}
-}
diff --git a/internal/xds/bootstrap/bootstrap.go b/internal/xds/bootstrap/bootstrap.go
index 35aeea70..f3211c1e 100644
--- a/internal/xds/bootstrap/bootstrap.go
+++ b/internal/xds/bootstrap/bootstrap.go
@@ -179,9 +179,9 @@ type ServerConfig struct {
 	// As part of unmarshalling the JSON config into this struct, we ensure that
 	// the credentials config is valid by building an instance of the specified
 	// credentials and store it here for easy access.
-	selectedCreds   ChannelCreds
-	credsDialOption grpc.DialOption
-	dialerOption    grpc.DialOption
+	selectedCreds    ChannelCreds
+	credsDialOption  grpc.DialOption
+	extraDialOptions []grpc.DialOption
 
 	cleanups []func()
 }
@@ -224,8 +224,8 @@ func (sc *ServerConfig) ServerFeaturesIgnoreResourceDeletion() bool {
 // server.
 func (sc *ServerConfig) DialOptions() []grpc.DialOption {
 	dopts := []grpc.DialOption{sc.credsDialOption}
-	if sc.dialerOption != nil {
-		dopts = append(dopts, sc.dialerOption)
+	if sc.extraDialOptions != nil {
+		dopts = append(dopts, sc.extraDialOptions...)
 	}
 	return dopts
 }
@@ -283,11 +283,18 @@ func (sc *ServerConfig) MarshalJSON() ([]byte, error) {
 }
 
 // dialer captures the Dialer method specified via the credentials bundle.
+// Deprecated: use extradDialOptions. Will take precedence over this.
 type dialer interface {
 	// Dialer specifies how to dial the xDS server.
 	Dialer(context.Context, string) (net.Conn, error)
 }
 
+// extraDialOptions captures custom dial options specified via
+// credentials.Bundle.
+type extraDialOptions interface {
+	DialOptions() []grpc.DialOption
+}
+
 // UnmarshalJSON takes the json data (a server) and unmarshals it to the struct.
 func (sc *ServerConfig) UnmarshalJSON(data []byte) error {
 	server := serverConfigJSON{}
@@ -311,8 +318,10 @@ func (sc *ServerConfig) UnmarshalJSON(data []byte) error {
 		}
 		sc.selectedCreds = cc
 		sc.credsDialOption = grpc.WithCredentialsBundle(bundle)
-		if d, ok := bundle.(dialer); ok {
-			sc.dialerOption = grpc.WithContextDialer(d.Dialer)
+		if d, ok := bundle.(extraDialOptions); ok {
+			sc.extraDialOptions = d.DialOptions()
+		} else if d, ok := bundle.(dialer); ok {
+			sc.extraDialOptions = []grpc.DialOption{grpc.WithContextDialer(d.Dialer)}
 		}
 		sc.cleanups = append(sc.cleanups, cancel)
 		break
diff --git a/internal/xds/bootstrap/tlscreds/bundle.go b/internal/xds/bootstrap/tlscreds/bundle.go
index 02da3dbf..ed90720b 100644
--- a/internal/xds/bootstrap/tlscreds/bundle.go
+++ b/internal/xds/bootstrap/tlscreds/bundle.go
@@ -27,11 +27,11 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"sync"
 
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
-	"google.golang.org/grpc/internal/grpcsync"
 )
 
 // bundle is an implementation of credentials.Bundle which implements mTLS
@@ -81,7 +81,7 @@ func NewBundle(jd json.RawMessage) (credentials.Bundle, func(), error) {
 	}
 	return &bundle{
 		transportCredentials: &reloadingCreds{provider: provider},
-	}, grpcsync.OnceFunc(func() { provider.Close() }), nil
+	}, sync.OnceFunc(func() { provider.Close() }), nil
 }
 
 func (t *bundle) TransportCredentials() credentials.TransportCredentials {
diff --git a/orca/producer.go b/orca/producer.go
index 4d370310..87fd374e 100644
--- a/orca/producer.go
+++ b/orca/producer.go
@@ -25,7 +25,6 @@ import (
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/internal/backoff"
-	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/orca/internal"
 	"google.golang.org/grpc/status"
 
@@ -85,7 +84,7 @@ func RegisterOOBListener(sc balancer.SubConn, l OOBListener, opts OOBListenerOpt
 
 	// If stop is called multiple times, prevent it from having any effect on
 	// subsequent calls.
-	return grpcsync.OnceFunc(func() {
+	return sync.OnceFunc(func() {
 		p.unregisterListener(l, opts.ReportInterval)
 		closeFn()
 	})
diff --git a/server.go b/server.go
index c025de56..1e770197 100644
--- a/server.go
+++ b/server.go
@@ -643,7 +643,7 @@ func (s *Server) serverWorker() {
 // connections to reduce the time spent overall on runtime.morestack.
 func (s *Server) initServerWorkers() {
 	s.serverWorkerChannel = make(chan func())
-	s.serverWorkerChannelClose = grpcsync.OnceFunc(func() {
+	s.serverWorkerChannelClose = sync.OnceFunc(func() {
 		close(s.serverWorkerChannel)
 	})
 	for i := uint32(0); i < s.opts.numServerWorkers; i++ {
@@ -1930,7 +1930,7 @@ func (s *Server) stop(graceful bool) {
 	s.conns = nil
 
 	if s.opts.numServerWorkers > 0 {
-		// Closing the channel (only once, via grpcsync.OnceFunc) after all the
+		// Closing the channel (only once, via sync.OnceFunc) after all the
 		// connections have been closed above ensures that there are no
 		// goroutines executing the callback passed to st.HandleStreams (where
 		// the channel is written to).
diff --git a/stats/opentelemetry/client_metrics.go b/stats/opentelemetry/client_metrics.go
index 265791e5..4fffba60 100644
--- a/stats/opentelemetry/client_metrics.go
+++ b/stats/opentelemetry/client_metrics.go
@@ -21,7 +21,10 @@ import (
 	"sync/atomic"
 	"time"
 
+	otelcodes "go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 	"google.golang.org/grpc"
+	grpccodes "google.golang.org/grpc/codes"
 	estats "google.golang.org/grpc/experimental/stats"
 	istats "google.golang.org/grpc/internal/stats"
 	"google.golang.org/grpc/metadata"
@@ -85,8 +88,12 @@ func (h *clientStatsHandler) unaryInterceptor(ctx context.Context, method string
 	}
 
 	startTime := time.Now()
+	var span trace.Span
+	if h.options.isTracingEnabled() {
+		ctx, span = h.createCallTraceSpan(ctx, method)
+	}
 	err := invoker(ctx, method, req, reply, cc, opts...)
-	h.perCallMetrics(ctx, err, startTime, ci)
+	h.perCallTracesAndMetrics(ctx, err, startTime, ci, span)
 	return err
 }
 
@@ -119,22 +126,37 @@ func (h *clientStatsHandler) streamInterceptor(ctx context.Context, desc *grpc.S
 	}
 
 	startTime := time.Now()
-
+	var span trace.Span
+	if h.options.isTracingEnabled() {
+		ctx, span = h.createCallTraceSpan(ctx, method)
+	}
 	callback := func(err error) {
-		h.perCallMetrics(ctx, err, startTime, ci)
+		h.perCallTracesAndMetrics(ctx, err, startTime, ci, span)
 	}
 	opts = append([]grpc.CallOption{grpc.OnFinish(callback)}, opts...)
 	return streamer(ctx, desc, cc, method, opts...)
 }
 
-func (h *clientStatsHandler) perCallMetrics(ctx context.Context, err error, startTime time.Time, ci *callInfo) {
-	callLatency := float64(time.Since(startTime)) / float64(time.Second) // calculate ASAP
-	attrs := otelmetric.WithAttributeSet(otelattribute.NewSet(
-		otelattribute.String("grpc.method", ci.method),
-		otelattribute.String("grpc.target", ci.target),
-		otelattribute.String("grpc.status", canonicalString(status.Code(err))),
-	))
-	h.clientMetrics.callDuration.Record(ctx, callLatency, attrs)
+// perCallTracesAndMetrics records per call trace spans and metrics.
+func (h *clientStatsHandler) perCallTracesAndMetrics(ctx context.Context, err error, startTime time.Time, ci *callInfo, ts trace.Span) {
+	if h.options.isTracingEnabled() {
+		s := status.Convert(err)
+		if s.Code() == grpccodes.OK {
+			ts.SetStatus(otelcodes.Ok, s.Message())
+		} else {
+			ts.SetStatus(otelcodes.Error, s.Message())
+		}
+		ts.End()
+	}
+	if h.options.isMetricsEnabled() {
+		callLatency := float64(time.Since(startTime)) / float64(time.Second)
+		attrs := otelmetric.WithAttributeSet(otelattribute.NewSet(
+			otelattribute.String("grpc.method", ci.method),
+			otelattribute.String("grpc.target", ci.target),
+			otelattribute.String("grpc.status", canonicalString(status.Code(err))),
+		))
+		h.clientMetrics.callDuration.Record(ctx, callLatency, attrs)
+	}
 }
 
 // TagConn exists to satisfy stats.Handler.
@@ -163,15 +185,17 @@ func (h *clientStatsHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo)
 		}
 		ctx = istats.SetLabels(ctx, labels)
 	}
-	ai := &attemptInfo{ // populates information about RPC start.
+	ai := &attemptInfo{
 		startTime: time.Now(),
 		xdsLabels: labels.TelemetryLabels,
-		method:    info.FullMethodName,
+		method:    removeLeadingSlash(info.FullMethodName),
 	}
-	ri := &rpcInfo{
-		ai: ai,
+	if h.options.isTracingEnabled() {
+		ctx, ai = h.traceTagRPC(ctx, ai)
 	}
-	return setRPCInfo(ctx, ri)
+	return setRPCInfo(ctx, &rpcInfo{
+		ai: ai,
+	})
 }
 
 func (h *clientStatsHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) {
@@ -180,7 +204,12 @@ func (h *clientStatsHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) {
 		logger.Error("ctx passed into client side stats handler metrics event handling has no client attempt data present")
 		return
 	}
-	h.processRPCEvent(ctx, rs, ri.ai)
+	if h.options.isMetricsEnabled() {
+		h.processRPCEvent(ctx, rs, ri.ai)
+	}
+	if h.options.isTracingEnabled() {
+		populateSpan(rs, ri.ai)
+	}
 }
 
 func (h *clientStatsHandler) processRPCEvent(ctx context.Context, s stats.RPCStats, ai *attemptInfo) {
diff --git a/stats/opentelemetry/client_tracing.go b/stats/opentelemetry/client_tracing.go
new file mode 100644
index 00000000..075f4015
--- /dev/null
+++ b/stats/opentelemetry/client_tracing.go
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opentelemetry
+
+import (
+	"context"
+	"strings"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/trace"
+	otelinternaltracing "google.golang.org/grpc/stats/opentelemetry/internal/tracing"
+)
+
+// traceTagRPC populates provided context with a new span using the
+// TextMapPropagator supplied in trace options and internal itracing.carrier.
+// It creates a new outgoing carrier which serializes information about this
+// span into gRPC Metadata, if TextMapPropagator is provided in the trace
+// options. if TextMapPropagator is not provided, it returns the context as is.
+func (h *clientStatsHandler) traceTagRPC(ctx context.Context, ai *attemptInfo) (context.Context, *attemptInfo) {
+	mn := "Attempt." + strings.Replace(ai.method, "/", ".", -1)
+	tracer := otel.Tracer("grpc-open-telemetry")
+	ctx, span := tracer.Start(ctx, mn)
+	carrier := otelinternaltracing.NewOutgoingCarrier(ctx)
+	otel.GetTextMapPropagator().Inject(ctx, carrier)
+	ai.traceSpan = span
+	return carrier.Context(), ai
+}
+
+// createCallTraceSpan creates a call span to put in the provided context using
+// provided TraceProvider. If TraceProvider is nil, it returns context as is.
+func (h *clientStatsHandler) createCallTraceSpan(ctx context.Context, method string) (context.Context, trace.Span) {
+	if h.options.TraceOptions.TracerProvider == nil {
+		logger.Error("TraceProvider is not provided in trace options")
+		return ctx, nil
+	}
+	mn := strings.Replace(removeLeadingSlash(method), "/", ".", -1)
+	tracer := otel.Tracer("grpc-open-telemetry")
+	ctx, span := tracer.Start(ctx, mn, trace.WithSpanKind(trace.SpanKindClient))
+	return ctx, span
+}
diff --git a/stats/opentelemetry/opentelemetry.go b/stats/opentelemetry/opentelemetry.go
index dcc42477..d99169e2 100644
--- a/stats/opentelemetry/opentelemetry.go
+++ b/stats/opentelemetry/opentelemetry.go
@@ -27,35 +27,50 @@ import (
 	"strings"
 	"time"
 
+	otelattribute "go.opentelemetry.io/otel/attribute"
+	otelmetric "go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/metric/noop"
+	"go.opentelemetry.io/otel/trace"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
+	experimental "google.golang.org/grpc/experimental/opentelemetry"
 	estats "google.golang.org/grpc/experimental/stats"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/stats"
 	otelinternal "google.golang.org/grpc/stats/opentelemetry/internal"
-
-	otelattribute "go.opentelemetry.io/otel/attribute"
-	otelmetric "go.opentelemetry.io/otel/metric"
-	"go.opentelemetry.io/otel/metric/noop"
 )
 
 func init() {
 	otelinternal.SetPluginOption = func(o *Options, po otelinternal.PluginOption) {
 		o.MetricsOptions.pluginOption = po
+		// Log an error if one of the options is missing.
+		if (o.TraceOptions.TextMapPropagator == nil) != (o.TraceOptions.TracerProvider == nil) {
+			logger.Warning("Tracing will not be recorded because traceOptions are not set properly: one of TextMapPropagator or TracerProvider is missing")
+		}
 	}
 }
 
-var logger = grpclog.Component("otel-plugin")
-
-var canonicalString = internal.CanonicalString.(func(codes.Code) string)
-
-var joinDialOptions = internal.JoinDialOptions.(func(...grpc.DialOption) grpc.DialOption)
+var (
+	logger          = grpclog.Component("otel-plugin")
+	canonicalString = internal.CanonicalString.(func(codes.Code) string)
+	joinDialOptions = internal.JoinDialOptions.(func(...grpc.DialOption) grpc.DialOption)
+)
 
 // Options are the options for OpenTelemetry instrumentation.
 type Options struct {
 	// MetricsOptions are the metrics options for OpenTelemetry instrumentation.
 	MetricsOptions MetricsOptions
+	// TraceOptions are the tracing options for OpenTelemetry instrumentation.
+	TraceOptions experimental.TraceOptions
+}
+
+func (o *Options) isMetricsEnabled() bool {
+	return o.MetricsOptions.MeterProvider != nil
+}
+
+func (o *Options) isTracingEnabled() bool {
+	return o.TraceOptions.TracerProvider != nil
 }
 
 // MetricsOptions are the metrics options for OpenTelemetry instrumentation.
@@ -187,6 +202,15 @@ type attemptInfo struct {
 
 	pluginOptionLabels map[string]string // pluginOptionLabels to attach to metrics emitted
 	xdsLabels          map[string]string
+
+	// traceSpan is data used for recording traces.
+	traceSpan trace.Span
+	// message counters for sent and received messages (used for
+	// generating message IDs), and the number of previous RPC attempts for the
+	// associated call.
+	countSentMsg        uint32
+	countRecvMsg        uint32
+	previousRPCAttempts uint32
 }
 
 type clientMetrics struct {
diff --git a/stats/opentelemetry/server_metrics.go b/stats/opentelemetry/server_metrics.go
index 4765afa8..da3f60a9 100644
--- a/stats/opentelemetry/server_metrics.go
+++ b/stats/opentelemetry/server_metrics.go
@@ -201,10 +201,12 @@ func (h *serverStatsHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo)
 		startTime: time.Now(),
 		method:    removeLeadingSlash(method),
 	}
-	ri := &rpcInfo{
-		ai: ai,
+	if h.options.isTracingEnabled() {
+		ctx, ai = h.traceTagRPC(ctx, ai)
 	}
-	return setRPCInfo(ctx, ri)
+	return setRPCInfo(ctx, &rpcInfo{
+		ai: ai,
+	})
 }
 
 // HandleRPC implements per RPC tracing and stats implementation.
@@ -214,7 +216,12 @@ func (h *serverStatsHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) {
 		logger.Error("ctx passed into server side stats handler metrics event handling has no server call data present")
 		return
 	}
-	h.processRPCData(ctx, rs, ri.ai)
+	if h.options.isTracingEnabled() {
+		populateSpan(rs, ri.ai)
+	}
+	if h.options.isMetricsEnabled() {
+		h.processRPCData(ctx, rs, ri.ai)
+	}
 }
 
 func (h *serverStatsHandler) processRPCData(ctx context.Context, s stats.RPCStats, ai *attemptInfo) {
diff --git a/stats/opentelemetry/server_tracing.go b/stats/opentelemetry/server_tracing.go
new file mode 100644
index 00000000..c55e03dc
--- /dev/null
+++ b/stats/opentelemetry/server_tracing.go
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opentelemetry
+
+import (
+	"context"
+	"strings"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/trace"
+	otelinternaltracing "google.golang.org/grpc/stats/opentelemetry/internal/tracing"
+)
+
+// traceTagRPC populates context with new span data using the TextMapPropagator
+// supplied in trace options and internal itracing.Carrier. It creates a new
+// incoming carrier which extracts an existing span context (if present) by
+// deserializing from provided context. If valid span context is extracted, it
+// is set as parent of the new span otherwise new span remains the root span.
+// If TextMapPropagator is not provided in the trace options, it returns context
+// as is.
+func (h *serverStatsHandler) traceTagRPC(ctx context.Context, ai *attemptInfo) (context.Context, *attemptInfo) {
+	mn := strings.Replace(ai.method, "/", ".", -1)
+	var span trace.Span
+	tracer := otel.Tracer("grpc-open-telemetry")
+	ctx = otel.GetTextMapPropagator().Extract(ctx, otelinternaltracing.NewIncomingCarrier(ctx))
+	// If the context.Context provided in `ctx` to tracer.Start(), contains a
+	// span then the newly-created Span will be a child of that span,
+	// otherwise it will be a root span.
+	ctx, span = tracer.Start(ctx, mn, trace.WithSpanKind(trace.SpanKindServer))
+	ai.traceSpan = span
+	return ctx, ai
+}
diff --git a/stats/opentelemetry/trace.go b/stats/opentelemetry/trace.go
new file mode 100644
index 00000000..cd5c23cd
--- /dev/null
+++ b/stats/opentelemetry/trace.go
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opentelemetry
+
+import (
+	"sync/atomic"
+
+	"go.opentelemetry.io/otel/attribute"
+	otelcodes "go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+	"google.golang.org/grpc/stats"
+	"google.golang.org/grpc/status"
+)
+
+// populateSpan populates span information based on stats passed in, representing
+// invariants of the RPC lifecycle. It ends the span, triggering its export.
+// This function handles attempt spans on the client-side and call spans on the
+// server-side.
+func populateSpan(rs stats.RPCStats, ai *attemptInfo) {
+	if ai == nil || ai.traceSpan == nil {
+		// Shouldn't happen, tagRPC call comes before this function gets called
+		// which populates this information.
+		logger.Error("ctx passed into stats handler tracing event handling has no traceSpan present")
+		return
+	}
+	span := ai.traceSpan
+
+	switch rs := rs.(type) {
+	case *stats.Begin:
+		// Note: Go always added Client and FailFast attributes even though they are not
+		// defined by the OpenCensus gRPC spec. Thus, they are unimportant for
+		// correctness.
+		span.SetAttributes(
+			attribute.Bool("Client", rs.Client),
+			attribute.Bool("FailFast", rs.Client),
+			attribute.Int64("previous-rpc-attempts", int64(ai.previousRPCAttempts)),
+			attribute.Bool("transparent-retry", rs.IsTransparentRetryAttempt),
+		)
+		// increment previous rpc attempts applicable for next attempt
+		atomic.AddUint32(&ai.previousRPCAttempts, 1)
+	case *stats.PickerUpdated:
+		span.AddEvent("Delayed LB pick complete")
+	case *stats.InPayload:
+		// message id - "must be calculated as two different counters starting
+		// from one for sent messages and one for received messages."
+		ai.countRecvMsg++
+		span.AddEvent("Inbound compressed message", trace.WithAttributes(
+			attribute.Int64("sequence-number", int64(ai.countRecvMsg)),
+			attribute.Int64("message-size", int64(rs.Length)),
+			attribute.Int64("message-size-compressed", int64(rs.CompressedLength)),
+		))
+	case *stats.OutPayload:
+		ai.countSentMsg++
+		span.AddEvent("Outbound compressed message", trace.WithAttributes(
+			attribute.Int64("sequence-number", int64(ai.countSentMsg)),
+			attribute.Int64("message-size", int64(rs.Length)),
+			attribute.Int64("message-size-compressed", int64(rs.CompressedLength)),
+		))
+	case *stats.End:
+		if rs.Error != nil {
+			s := status.Convert(rs.Error)
+			span.SetStatus(otelcodes.Error, s.Message())
+		} else {
+			span.SetStatus(otelcodes.Ok, "Ok")
+		}
+		span.End()
+	}
+}
diff --git a/xds/internal/xdsclient/authority.go b/xds/internal/xdsclient/authority.go
index f81685a4..67b3d769 100644
--- a/xds/internal/xdsclient/authority.go
+++ b/xds/internal/xdsclient/authority.go
@@ -20,6 +20,7 @@ package xdsclient
 import (
 	"context"
 	"fmt"
+	"sync"
 	"sync/atomic"
 
 	"google.golang.org/grpc/grpclog"
@@ -674,7 +675,7 @@ func (a *authority) watchResource(rType xdsresource.Type, resourceName string, w
 }
 
 func (a *authority) unwatchResource(rType xdsresource.Type, resourceName string, watcher xdsresource.ResourceWatcher) func() {
-	return grpcsync.OnceFunc(func() {
+	return sync.OnceFunc(func() {
 		done := make(chan struct{})
 		a.xdsClientSerializer.ScheduleOr(func(context.Context) {
 			defer close(done)
diff --git a/xds/internal/xdsclient/clientimpl.go b/xds/internal/xdsclient/clientimpl.go
index c1f77cd0..33c9befa 100644
--- a/xds/internal/xdsclient/clientimpl.go
+++ b/xds/internal/xdsclient/clientimpl.go
@@ -342,7 +342,7 @@ func (c *clientImpl) getOrCreateChannel(serverConfig *bootstrap.ServerConfig, in
 // reference to the xdsChannel. This returned function is idempotent, meaning
 // it can be called multiple times without any additional effect.
 func (c *clientImpl) releaseChannel(serverConfig *bootstrap.ServerConfig, state *channelState, deInitLocked func(*channelState)) func() {
-	return grpcsync.OnceFunc(func() {
+	return sync.OnceFunc(func() {
 		c.channelsMu.Lock()
 
 		if c.logger.V(2) {
diff --git a/xds/internal/xdsclient/pool.go b/xds/internal/xdsclient/pool.go
index 37dd3636..f73bffb6 100644
--- a/xds/internal/xdsclient/pool.go
+++ b/xds/internal/xdsclient/pool.go
@@ -25,7 +25,6 @@ import (
 
 	v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
 	"google.golang.org/grpc/internal/backoff"
-	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/internal/xds/bootstrap"
 )
 
@@ -131,7 +130,7 @@ func (p *Pool) GetClientForTesting(name string) (XDSClient, func(), error) {
 		return nil, nil, fmt.Errorf("xds:: xDS client with name %q not found", name)
 	}
 	c.incrRef()
-	return c, grpcsync.OnceFunc(func() { p.clientRefCountedClose(name) }), nil
+	return c, sync.OnceFunc(func() { p.clientRefCountedClose(name) }), nil
 }
 
 // SetFallbackBootstrapConfig is used to specify a bootstrap configuration
@@ -193,7 +192,7 @@ func (p *Pool) newRefCounted(name string, watchExpiryTimeout time.Duration, stre
 
 	if c := p.clients[name]; c != nil {
 		c.incrRef()
-		return c, grpcsync.OnceFunc(func() { p.clientRefCountedClose(name) }), nil
+		return c, sync.OnceFunc(func() { p.clientRefCountedClose(name) }), nil
 	}
 
 	c, err := newClientImpl(p.config, watchExpiryTimeout, streamBackoff)
@@ -208,5 +207,5 @@ func (p *Pool) newRefCounted(name string, watchExpiryTimeout time.Duration, stre
 	xdsClientImplCreateHook(name)
 
 	logger.Infof("xDS node ID: %s", p.config.Node().GetId())
-	return client, grpcsync.OnceFunc(func() { p.clientRefCountedClose(name) }), nil
+	return client, sync.OnceFunc(func() { p.clientRefCountedClose(name) }), nil
 }
diff --git a/xds/internal/xdsclient/transport/lrs/lrs_stream.go b/xds/internal/xdsclient/transport/lrs/lrs_stream.go
index bab92bcb..e33ac694 100644
--- a/xds/internal/xdsclient/transport/lrs/lrs_stream.go
+++ b/xds/internal/xdsclient/transport/lrs/lrs_stream.go
@@ -29,7 +29,6 @@ import (
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/backoff"
 	igrpclog "google.golang.org/grpc/internal/grpclog"
-	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/internal/pretty"
 	"google.golang.org/grpc/xds/internal"
 	"google.golang.org/grpc/xds/internal/xdsclient/load"
@@ -107,7 +106,7 @@ func (lrs *StreamImpl) ReportLoad() (*load.Store, func()) {
 	lrs.mu.Lock()
 	defer lrs.mu.Unlock()
 
-	cleanup := grpcsync.OnceFunc(func() {
+	cleanup := sync.OnceFunc(func() {
 		lrs.mu.Lock()
 		defer lrs.mu.Unlock()
 



Test patch:
diff --git a/internal/grpcsync/oncefunc_test.go b/internal/grpcsync/oncefunc_test.go
deleted file mode 100644
index 2b0db8d3..00000000
--- a/internal/grpcsync/oncefunc_test.go
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- *
- * Copyright 2022 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package grpcsync
-
-import (
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-)
-
-// TestOnceFunc tests that a OnceFunc is executed only once even with multiple
-// simultaneous callers of it.
-func (s) TestOnceFunc(t *testing.T) {
-	var v int32
-	inc := OnceFunc(func() { atomic.AddInt32(&v, 1) })
-
-	const numWorkers = 100
-	var wg sync.WaitGroup // Blocks until all workers have called inc.
-	wg.Add(numWorkers)
-
-	block := NewEvent() // Signal to worker goroutines to call inc
-
-	for i := 0; i < numWorkers; i++ {
-		go func() {
-			<-block.Done() // Wait for a signal.
-			inc()          // Call the OnceFunc.
-			wg.Done()
-		}()
-	}
-	time.Sleep(time.Millisecond) // Allow goroutines to get to the block.
-	block.Fire()                 // Unblock them.
-	wg.Wait()                    // Wait for them to complete.
-	if v != 1 {
-		t.Fatalf("OnceFunc() called %v times; want 1", v)
-	}
-}
diff --git a/stats/opentelemetry/e2e_test.go b/stats/opentelemetry/e2e_test.go
index ac671e29..1250237b 100644
--- a/stats/opentelemetry/e2e_test.go
+++ b/stats/opentelemetry/e2e_test.go
@@ -19,11 +19,14 @@ package opentelemetry_test
 import (
 	"context"
 	"fmt"
-
 	"io"
 	"testing"
 	"time"
 
+	"go.opentelemetry.io/otel"
+	otelcodes "go.opentelemetry.io/otel/codes"
+	oteltrace "go.opentelemetry.io/otel/trace"
+
 	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
@@ -35,9 +38,17 @@ import (
 	"google.golang.org/protobuf/types/known/durationpb"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest"
+	"go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/encoding/gzip"
+	experimental "google.golang.org/grpc/experimental/opentelemetry"
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/stubserver"
@@ -49,11 +60,6 @@ import (
 	"google.golang.org/grpc/orca"
 	"google.golang.org/grpc/stats/opentelemetry"
 	"google.golang.org/grpc/stats/opentelemetry/internal/testutils"
-
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/sdk/metric"
-	"go.opentelemetry.io/otel/sdk/metric/metricdata"
-	"go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest"
 )
 
 var defaultTestTimeout = 5 * time.Second
@@ -66,14 +72,48 @@ func Test(t *testing.T) {
 	grpctest.RunSubTests(t, s{})
 }
 
-// setupStubServer creates a stub server with OpenTelemetry component configured on client
-// and server side. It returns a reader for metrics emitted from OpenTelemetry
-// component and the server.
-func setupStubServer(t *testing.T, methodAttributeFilter func(string) bool) (*metric.ManualReader, *stubserver.StubServer) {
+// traceSpanInfo is the information received about the trace span. It contains
+// subset of information that is needed to verify if correct trace is being
+// attributed to the rpc.
+type traceSpanInfo struct {
+	spanKind   string
+	name       string
+	events     []trace.Event
+	attributes []attribute.KeyValue
+}
+
+// defaultMetricsOptions creates default metrics options
+func defaultMetricsOptions(_ *testing.T, methodAttributeFilter func(string) bool) (*opentelemetry.MetricsOptions, *metric.ManualReader) {
 	reader := metric.NewManualReader()
 	provider := metric.NewMeterProvider(metric.WithReader(reader))
+	metricsOptions := &opentelemetry.MetricsOptions{
+		MeterProvider:         provider,
+		Metrics:               opentelemetry.DefaultMetrics(),
+		MethodAttributeFilter: methodAttributeFilter,
+	}
+	return metricsOptions, reader
+}
+
+// defaultTraceOptions function to create default trace options
+func defaultTraceOptions(_ *testing.T) (*experimental.TraceOptions, *tracetest.InMemoryExporter) {
+	spanExporter := tracetest.NewInMemoryExporter()
+	spanProcessor := trace.NewSimpleSpanProcessor(spanExporter)
+	tracerProvider := trace.NewTracerProvider(trace.WithSpanProcessor(spanProcessor))
+	textMapPropagator := propagation.NewCompositeTextMapPropagator(opentelemetry.GRPCTraceBinPropagator{})
+	otel.SetTextMapPropagator(textMapPropagator)
+	otel.SetTracerProvider(tracerProvider)
+	traceOptions := &experimental.TraceOptions{
+		TracerProvider:    tracerProvider,
+		TextMapPropagator: textMapPropagator,
+	}
+	return traceOptions, spanExporter
+}
+
+// setupStubServer creates a stub server with OpenTelemetry component configured on client
+// and server side and returns the server.
+func setupStubServer(t *testing.T, metricsOptions *opentelemetry.MetricsOptions, traceOptions *experimental.TraceOptions) *stubserver.StubServer {
 	ss := &stubserver.StubServer{
-		UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+		UnaryCallF: func(_ context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
 			return &testpb.SimpleResponse{Payload: &testpb.Payload{
 				Body: make([]byte, len(in.GetPayload().GetBody())),
 			}}, nil
@@ -88,20 +128,19 @@ func setupStubServer(t *testing.T, methodAttributeFilter func(string) bool) (*me
 		},
 	}
 
-	if err := ss.Start([]grpc.ServerOption{opentelemetry.ServerOption(opentelemetry.Options{
-		MetricsOptions: opentelemetry.MetricsOptions{
-			MeterProvider:         provider,
-			Metrics:               opentelemetry.DefaultMetrics(),
-			MethodAttributeFilter: methodAttributeFilter,
-		}})}, opentelemetry.DialOption(opentelemetry.Options{
-		MetricsOptions: opentelemetry.MetricsOptions{
-			MeterProvider: provider,
-			Metrics:       opentelemetry.DefaultMetrics(),
-		},
-	})); err != nil {
+	otelOptions := opentelemetry.Options{}
+	if metricsOptions != nil {
+		otelOptions.MetricsOptions = *metricsOptions
+	}
+	if traceOptions != nil {
+		otelOptions.TraceOptions = *traceOptions
+	}
+
+	if err := ss.Start([]grpc.ServerOption{opentelemetry.ServerOption(otelOptions)},
+		opentelemetry.DialOption(otelOptions)); err != nil {
 		t.Fatalf("Error starting endpoint server: %v", err)
 	}
-	return reader, ss
+	return ss
 }
 
 // TestMethodAttributeFilter tests the method attribute filter. The method
@@ -112,7 +151,8 @@ func (s) TestMethodAttributeFilter(t *testing.T) {
 		// Will allow duplex/any other type of RPC.
 		return str != testgrpc.TestService_UnaryCall_FullMethodName
 	}
-	reader, ss := setupStubServer(t, maf)
+	mo, reader := defaultMetricsOptions(t, maf)
+	ss := setupStubServer(t, mo, nil)
 	defer ss.Stop()
 
 	// Make a Unary and Streaming RPC. The Unary RPC should be filtered by the
@@ -197,13 +237,14 @@ func (s) TestMethodAttributeFilter(t *testing.T) {
 // on the Client (no StaticMethodCallOption set) and Server. The method
 // attribute on subsequent metrics should be bucketed in "other".
 func (s) TestAllMetricsOneFunction(t *testing.T) {
-	reader, ss := setupStubServer(t, nil)
+	mo, reader := defaultMetricsOptions(t, nil)
+	ss := setupStubServer(t, mo, nil)
 	defer ss.Stop()
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	// Make two RPC's, a unary RPC and a streaming RPC. These should cause
-	// certain metrics to be emitted, which should be able to be observed
-	// through the Metric Reader.
+	// certain metrics to be emitted, which should be observed through the
+	// Metric Reader.
 	if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{
 		Body: make([]byte, 10000),
 	}}, grpc.UseCompressor(gzip.Name)); err != nil { // Deterministic compression.
@@ -362,7 +403,7 @@ func metricsDataFromReader(ctx context.Context, reader *metric.ManualReader) map
 func (s) TestWRRMetrics(t *testing.T) {
 	cmr := orca.NewServerMetricsRecorder().(orca.CallMetricsRecorder)
 	backend1 := stubserver.StartTestService(t, &stubserver.StubServer{
-		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
 			if r := orca.CallMetricsRecorderFromContext(ctx); r != nil {
 				// Copy metrics from what the test set in cmr into r.
 				sm := cmr.(orca.ServerMetricsProvider).ServerMetrics()
@@ -380,7 +421,7 @@ func (s) TestWRRMetrics(t *testing.T) {
 	cmr.SetApplicationUtilization(1.0)
 
 	backend2 := stubserver.StartTestService(t, &stubserver.StubServer{
-		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
 			if r := orca.CallMetricsRecorderFromContext(ctx); r != nil {
 				// Copy metrics from what the test set in cmr into r.
 				sm := cmr.(orca.ServerMetricsProvider).ServerMetrics()
@@ -582,3 +623,959 @@ func pollForWantMetrics(ctx context.Context, t *testing.T, reader *metric.Manual
 
 	return fmt.Errorf("error waiting for metrics %v: %v", wantMetrics, ctx.Err())
 }
+
+// TestMetricsAndTracesOptionEnabled verifies the integration of metrics and traces
+// emitted by the OpenTelemetry instrumentation in a gRPC environment. It sets up a
+// stub server with both metrics and traces enabled, and tests the correct emission
+// of metrics and traces during a Unary RPC and a Streaming RPC. The test ensures
+// that the emitted metrics reflect the operations performed, including the size of
+// the compressed message, and verifies that tracing information is correctly recorded.
+func (s) TestMetricsAndTracesOptionEnabled(t *testing.T) {
+	// Create default metrics options
+	mo, reader := defaultMetricsOptions(t, nil)
+	// Create default trace options
+	to, exporter := defaultTraceOptions(t)
+
+	ss := setupStubServer(t, mo, to)
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout*2)
+	defer cancel()
+
+	// Make two RPC's, a unary RPC and a streaming RPC. These should cause
+	// certain metrics and traces to be emitted which should be observed
+	// through metrics reader and span exporter respectively.
+	if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{
+		Body: make([]byte, 10000),
+	}}, grpc.UseCompressor(gzip.Name)); err != nil { // Deterministic compression.
+		t.Fatalf("Unexpected error from UnaryCall: %v", err)
+	}
+	stream, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("ss.Client.FullDuplexCall failed: %f", err)
+	}
+
+	stream.CloseSend()
+	if _, err = stream.Recv(); err != io.EOF {
+		t.Fatalf("stream.Recv received an unexpected error: %v, expected an EOF error", err)
+	}
+
+	// Verify metrics
+	rm := &metricdata.ResourceMetrics{}
+	reader.Collect(ctx, rm)
+
+	gotMetrics := map[string]metricdata.Metrics{}
+	for _, sm := range rm.ScopeMetrics {
+		for _, m := range sm.Metrics {
+			gotMetrics[m.Name] = m
+		}
+	}
+
+	wantMetrics := testutils.MetricData(testutils.MetricDataOptions{
+		Target:                     ss.Target,
+		UnaryCompressedMessageSize: float64(57),
+	})
+	testutils.CompareMetrics(ctx, t, reader, gotMetrics, wantMetrics)
+
+	// Verify traces
+	spans := exporter.GetSpans()
+	if got, want := len(spans), 6; got != want {
+		t.Fatalf("got %d spans, want %d", got, want)
+	}
+
+	wantSI := []traceSpanInfo{
+		{
+			name:     "grpc.testing.TestService.UnaryCall",
+			spanKind: oteltrace.SpanKindServer.String(),
+			attributes: []attribute.KeyValue{
+				{
+					Key:   "Client",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "FailFast",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "previous-rpc-attempts",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "transparent-retry",
+					Value: attribute.IntValue(0),
+				},
+			},
+			events: []trace.Event{
+				{
+					Name: "Inbound compressed message",
+					Attributes: []attribute.KeyValue{
+						{
+							Key:   "sequence-number",
+							Value: attribute.IntValue(1),
+						},
+						{
+							Key:   "message-size",
+							Value: attribute.IntValue(10006),
+						},
+						{
+							Key:   "message-size-compressed",
+							Value: attribute.IntValue(57),
+						},
+					},
+				},
+				{
+					Name: "Outbound compressed message",
+					Attributes: []attribute.KeyValue{
+						{
+							Key:   "sequence-number",
+							Value: attribute.IntValue(1),
+						},
+						{
+							Key:   "message-size",
+							Value: attribute.IntValue(10006),
+						},
+						{
+							Key:   "message-size-compressed",
+							Value: attribute.IntValue(57),
+						},
+					},
+				},
+			},
+		},
+		{
+			name:     "Attempt.grpc.testing.TestService.UnaryCall",
+			spanKind: oteltrace.SpanKindInternal.String(),
+			attributes: []attribute.KeyValue{
+				{
+					Key:   "Client",
+					Value: attribute.IntValue(1),
+				},
+				{
+					Key:   "FailFast",
+					Value: attribute.IntValue(1),
+				},
+				{
+					Key:   "previous-rpc-attempts",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "transparent-retry",
+					Value: attribute.IntValue(0),
+				},
+			},
+			events: []trace.Event{
+				{
+					Name: "Outbound compressed message",
+					Attributes: []attribute.KeyValue{
+						{
+							Key:   "sequence-number",
+							Value: attribute.IntValue(1),
+						},
+						{
+							Key:   "message-size",
+							Value: attribute.IntValue(10006),
+						},
+						{
+							Key:   "message-size-compressed",
+							Value: attribute.IntValue(57),
+						},
+					},
+				},
+				{
+					Name: "Inbound compressed message",
+					Attributes: []attribute.KeyValue{
+						{
+							Key:   "sequence-number",
+							Value: attribute.IntValue(1),
+						},
+						{
+							Key:   "message-size",
+							Value: attribute.IntValue(10006),
+						},
+						{
+							Key:   "message-size-compressed",
+							Value: attribute.IntValue(57),
+						},
+					},
+				},
+			},
+		},
+		{
+			name:       "grpc.testing.TestService.UnaryCall",
+			spanKind:   oteltrace.SpanKindClient.String(),
+			attributes: []attribute.KeyValue{},
+			events:     []trace.Event{},
+		},
+		{
+			name:     "grpc.testing.TestService.FullDuplexCall",
+			spanKind: oteltrace.SpanKindServer.String(),
+			attributes: []attribute.KeyValue{
+				{
+					Key:   "Client",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "FailFast",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "previous-rpc-attempts",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "transparent-retry",
+					Value: attribute.IntValue(0),
+				},
+			},
+			events: []trace.Event{},
+		},
+		{
+			name:       "grpc.testing.TestService.FullDuplexCall",
+			spanKind:   oteltrace.SpanKindClient.String(),
+			attributes: []attribute.KeyValue{},
+			events:     []trace.Event{},
+		},
+		{
+			name:     "Attempt.grpc.testing.TestService.FullDuplexCall",
+			spanKind: oteltrace.SpanKindInternal.String(),
+			attributes: []attribute.KeyValue{
+				{
+					Key:   "Client",
+					Value: attribute.IntValue(1),
+				},
+				{
+					Key:   "FailFast",
+					Value: attribute.IntValue(1),
+				},
+				{
+					Key:   "previous-rpc-attempts",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "transparent-retry",
+					Value: attribute.IntValue(0),
+				},
+			},
+			events: []trace.Event{},
+		},
+	}
+
+	// Check that same traceID is used in client and server for unary RPC call.
+	if got, want := spans[0].SpanContext.TraceID(), spans[2].SpanContext.TraceID(); got != want {
+		t.Fatal("TraceID mismatch in client span and server span.")
+	}
+	// Check that the attempt span id of client matches the span id of server
+	// SpanContext.
+	if got, want := spans[0].Parent.SpanID(), spans[1].SpanContext.SpanID(); got != want {
+		t.Fatal("SpanID mismatch in client span and server span.")
+	}
+
+	// Check that same traceID is used in client and server for streaming RPC call.
+	if got, want := spans[3].SpanContext.TraceID(), spans[4].SpanContext.TraceID(); got != want {
+		t.Fatal("TraceID mismatch in client span and server span.")
+	}
+	// Check that the attempt span id of client matches the span id of server
+	// SpanContext.
+	if got, want := spans[3].Parent.SpanID(), spans[5].SpanContext.SpanID(); got != want {
+		t.Fatal("SpanID mismatch in client span and server span.")
+	}
+
+	for index, span := range spans {
+		// Check that the attempt span has the correct status
+		if got, want := spans[index].Status.Code, otelcodes.Ok; got != want {
+			t.Errorf("Got status code %v, want %v", got, want)
+		}
+		// name
+		if got, want := span.Name, wantSI[index].name; got != want {
+			t.Errorf("Span name is %q, want %q", got, want)
+		}
+		// spanKind
+		if got, want := span.SpanKind.String(), wantSI[index].spanKind; got != want {
+			t.Errorf("Got span kind %q, want %q", got, want)
+		}
+		// attributes
+		if got, want := len(span.Attributes), len(wantSI[index].attributes); got != want {
+			t.Errorf("Got attributes list of size %q, want %q", got, want)
+		}
+		for idx, att := range span.Attributes {
+			if got, want := att.Key, wantSI[index].attributes[idx].Key; got != want {
+				t.Errorf("Got attribute key for span name %v as %v, want %v", span.Name, got, want)
+			}
+		}
+		// events
+		if got, want := len(span.Events), len(wantSI[index].events); got != want {
+			t.Errorf("Event length is %q, want %q", got, want)
+		}
+		for eventIdx, event := range span.Events {
+			if got, want := event.Name, wantSI[index].events[eventIdx].Name; got != want {
+				t.Errorf("Got event name for span name %q as %q, want %q", span.Name, got, want)
+			}
+			for idx, att := range event.Attributes {
+				if got, want := att.Key, wantSI[index].events[eventIdx].Attributes[idx].Key; got != want {
+					t.Errorf("Got attribute key for span name %q with event name %v, as %v, want %v", span.Name, event.Name, got, want)
+				}
+				if got, want := att.Value, wantSI[index].events[eventIdx].Attributes[idx].Value; got != want {
+					t.Errorf("Got attribute value for span name %v with event name %v, as %v, want %v", span.Name, event.Name, got, want)
+				}
+			}
+		}
+	}
+}
+
+// TestSpan verifies that the gRPC Trace Binary propagator correctly
+// propagates span context between a client and server using the grpc-
+// trace-bin header. It sets up a stub server with OpenTelemetry tracing
+// enabled, makes a unary RPC, and streaming RPC as well.
+//
+// Verification:
+//   - Verifies that the span context is correctly propagated from the client
+//     to the server, including the trace ID and span ID.
+//   - Verifies that the server can access the span context and create
+//     child spans as expected during the RPC calls.
+//   - Verifies that the tracing information is recorded accurately in
+//     the OpenTelemetry backend.
+func (s) TestSpan(t *testing.T) {
+	mo, _ := defaultMetricsOptions(t, nil)
+	// Using defaultTraceOptions to set up OpenTelemetry with an in-memory exporter.
+	to, spanExporter := defaultTraceOptions(t)
+	// Start the server with trace options.
+	ss := setupStubServer(t, mo, to)
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Make two RPC's, a unary RPC and a streaming RPC. These should cause
+	// certain traces to be emitted, which should be observed through the
+	// span exporter.
+	if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{
+		Body: make([]byte, 10000),
+	}}); err != nil {
+		t.Fatalf("Unexpected error from UnaryCall: %v", err)
+	}
+	stream, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("ss.Client.FullDuplexCall failed: %f", err)
+	}
+	stream.CloseSend()
+	if _, err = stream.Recv(); err != io.EOF {
+		t.Fatalf("stream.Recv received an unexpected error: %v, expected an EOF error", err)
+	}
+
+	// Get the spans from the exporter
+	spans := spanExporter.GetSpans()
+	if got, want := len(spans), 6; got != want {
+		t.Fatalf("got %d spans, want %d", got, want)
+	}
+
+	wantSI := []traceSpanInfo{
+		{
+			name:     "grpc.testing.TestService.UnaryCall",
+			spanKind: oteltrace.SpanKindServer.String(),
+			attributes: []attribute.KeyValue{
+				{
+					Key:   "Client",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "FailFast",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "previous-rpc-attempts",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "transparent-retry",
+					Value: attribute.IntValue(0),
+				},
+			},
+			events: []trace.Event{
+				{
+					Name: "Inbound compressed message",
+					Attributes: []attribute.KeyValue{
+						{
+							Key:   "sequence-number",
+							Value: attribute.IntValue(1),
+						},
+						{
+							Key:   "message-size",
+							Value: attribute.IntValue(10006),
+						},
+						{
+							Key:   "message-size-compressed",
+							Value: attribute.IntValue(10006),
+						},
+					},
+				},
+				{
+					Name: "Outbound compressed message",
+					Attributes: []attribute.KeyValue{
+						{
+							Key:   "sequence-number",
+							Value: attribute.IntValue(1),
+						},
+						{
+							Key:   "message-size",
+							Value: attribute.IntValue(10006),
+						},
+						{
+							Key:   "message-size-compressed",
+							Value: attribute.IntValue(10006),
+						},
+					},
+				},
+			},
+		},
+		{
+			name:     "Attempt.grpc.testing.TestService.UnaryCall",
+			spanKind: oteltrace.SpanKindInternal.String(),
+			attributes: []attribute.KeyValue{
+				{
+					Key:   "Client",
+					Value: attribute.IntValue(1),
+				},
+				{
+					Key:   "FailFast",
+					Value: attribute.IntValue(1),
+				},
+				{
+					Key:   "previous-rpc-attempts",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "transparent-retry",
+					Value: attribute.IntValue(0),
+				},
+			},
+			events: []trace.Event{
+				{
+					Name: "Outbound compressed message",
+					Attributes: []attribute.KeyValue{
+						{
+							Key:   "sequence-number",
+							Value: attribute.IntValue(1),
+						},
+						{
+							Key:   "message-size",
+							Value: attribute.IntValue(10006),
+						},
+						{
+							Key:   "message-size-compressed",
+							Value: attribute.IntValue(10006),
+						},
+					},
+				},
+				{
+					Name: "Inbound compressed message",
+					Attributes: []attribute.KeyValue{
+						{
+							Key:   "sequence-number",
+							Value: attribute.IntValue(1),
+						},
+						{
+							Key:   "message-size",
+							Value: attribute.IntValue(10006),
+						},
+						{
+							Key:   "message-size-compressed",
+							Value: attribute.IntValue(10006),
+						},
+					},
+				},
+			},
+		},
+		{
+			name:       "grpc.testing.TestService.UnaryCall",
+			spanKind:   oteltrace.SpanKindClient.String(),
+			attributes: []attribute.KeyValue{},
+			events:     []trace.Event{},
+		},
+		{
+			name:     "grpc.testing.TestService.FullDuplexCall",
+			spanKind: oteltrace.SpanKindServer.String(),
+			attributes: []attribute.KeyValue{
+				{
+					Key:   "Client",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "FailFast",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "previous-rpc-attempts",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "transparent-retry",
+					Value: attribute.IntValue(0),
+				},
+			},
+			events: []trace.Event{},
+		},
+		{
+			name:       "grpc.testing.TestService.FullDuplexCall",
+			spanKind:   oteltrace.SpanKindClient.String(),
+			attributes: []attribute.KeyValue{},
+			events:     []trace.Event{},
+		},
+		{
+			name:     "Attempt.grpc.testing.TestService.FullDuplexCall",
+			spanKind: oteltrace.SpanKindInternal.String(),
+			attributes: []attribute.KeyValue{
+				{
+					Key:   "Client",
+					Value: attribute.IntValue(1),
+				},
+				{
+					Key:   "FailFast",
+					Value: attribute.IntValue(1),
+				},
+				{
+					Key:   "previous-rpc-attempts",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "transparent-retry",
+					Value: attribute.IntValue(0),
+				},
+			},
+			events: []trace.Event{},
+		},
+	}
+
+	// Check that same traceID is used in client and server for unary RPC call.
+	if got, want := spans[0].SpanContext.TraceID(), spans[2].SpanContext.TraceID(); got != want {
+		t.Fatal("TraceID mismatch in client span and server span.")
+	}
+	// Check that the attempt span id of client matches the span id of server
+	// SpanContext.
+	if got, want := spans[0].Parent.SpanID(), spans[1].SpanContext.SpanID(); got != want {
+		t.Fatal("SpanID mismatch in client span and server span.")
+	}
+
+	// Check that same traceID is used in client and server for streaming RPC call.
+	if got, want := spans[3].SpanContext.TraceID(), spans[4].SpanContext.TraceID(); got != want {
+		t.Fatal("TraceID mismatch in client span and server span.")
+	}
+	// Check that the attempt span id of client matches the span id of server
+	// SpanContext.
+	if got, want := spans[3].Parent.SpanID(), spans[5].SpanContext.SpanID(); got != want {
+		t.Fatal("SpanID mismatch in client span and server span.")
+	}
+
+	for index, span := range spans {
+		// Check that the attempt span has the correct status
+		if got, want := spans[index].Status.Code, otelcodes.Ok; got != want {
+			t.Errorf("Got status code %v, want %v", got, want)
+		}
+		// name
+		if got, want := span.Name, wantSI[index].name; got != want {
+			t.Errorf("Span name is %q, want %q", got, want)
+		}
+		// spanKind
+		if got, want := span.SpanKind.String(), wantSI[index].spanKind; got != want {
+			t.Errorf("Got span kind %q, want %q", got, want)
+		}
+		// attributes
+		if got, want := len(span.Attributes), len(wantSI[index].attributes); got != want {
+			t.Errorf("Got attributes list of size %q, want %q", got, want)
+		}
+		for idx, att := range span.Attributes {
+			if got, want := att.Key, wantSI[index].attributes[idx].Key; got != want {
+				t.Errorf("Got attribute key for span name %v as %v, want %v", span.Name, got, want)
+			}
+		}
+		// events
+		if got, want := len(span.Events), len(wantSI[index].events); got != want {
+			t.Errorf("Event length is %q, want %q", got, want)
+		}
+		for eventIdx, event := range span.Events {
+			if got, want := event.Name, wantSI[index].events[eventIdx].Name; got != want {
+				t.Errorf("Got event name for span name %q as %q, want %q", span.Name, got, want)
+			}
+			for idx, att := range event.Attributes {
+				if got, want := att.Key, wantSI[index].events[eventIdx].Attributes[idx].Key; got != want {
+					t.Errorf("Got attribute key for span name %q with event name %v, as %v, want %v", span.Name, event.Name, got, want)
+				}
+				if got, want := att.Value, wantSI[index].events[eventIdx].Attributes[idx].Value; got != want {
+					t.Errorf("Got attribute value for span name %v with event name %v, as %v, want %v", span.Name, event.Name, got, want)
+				}
+			}
+		}
+	}
+}
+
+// TestSpan_WithW3CContextPropagator sets up a stub server with OpenTelemetry tracing
+// enabled, makes a unary and a streaming RPC, and then asserts that the correct
+// number of spans are created with the expected spans.
+//
+// Verification:
+//   - Verifies that the correct number of spans are created for both unary and
+//     streaming RPCs.
+//   - Verifies that the spans have the expected names and attributes, ensuring
+//     they accurately reflect the operations performed.
+//   - Verifies that the trace ID and span ID are correctly assigned and accessible
+//     in the OpenTelemetry backend.
+func (s) TestSpan_WithW3CContextPropagator(t *testing.T) {
+	mo, _ := defaultMetricsOptions(t, nil)
+	// Using defaultTraceOptions to set up OpenTelemetry with an in-memory exporter
+	to, spanExporter := defaultTraceOptions(t)
+	// Set the W3CContextPropagator as part of TracingOptions.
+	to.TextMapPropagator = propagation.NewCompositeTextMapPropagator(propagation.TraceContext{})
+	// Start the server with OpenTelemetry options
+	ss := setupStubServer(t, mo, to)
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Make two RPC's, a unary RPC and a streaming RPC. These should cause
+	// certain traces to be emitted, which should be observed through the
+	// span exporter.
+	if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{
+		Body: make([]byte, 10000),
+	}}); err != nil {
+		t.Fatalf("Unexpected error from UnaryCall: %v", err)
+	}
+	stream, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("ss.Client.FullDuplexCall failed: %f", err)
+	}
+
+	stream.CloseSend()
+	if _, err = stream.Recv(); err != io.EOF {
+		t.Fatalf("stream.Recv received an unexpected error: %v, expected an EOF error", err)
+	}
+	// Get the spans from the exporter
+	spans := spanExporter.GetSpans()
+	if got, want := len(spans), 6; got != want {
+		t.Fatalf("Got %d spans, want %d", got, want)
+	}
+
+	wantSI := []traceSpanInfo{
+		{
+			name:     "grpc.testing.TestService.UnaryCall",
+			spanKind: oteltrace.SpanKindServer.String(),
+			attributes: []attribute.KeyValue{
+				{
+					Key:   "Client",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "FailFast",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "previous-rpc-attempts",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "transparent-retry",
+					Value: attribute.IntValue(0),
+				},
+			},
+			events: []trace.Event{
+				{
+					Name: "Inbound compressed message",
+					Attributes: []attribute.KeyValue{
+						{
+							Key:   "sequence-number",
+							Value: attribute.IntValue(1),
+						},
+						{
+							Key:   "message-size",
+							Value: attribute.IntValue(10006),
+						},
+						{
+							Key:   "message-size-compressed",
+							Value: attribute.IntValue(10006),
+						},
+					},
+				},
+				{
+					Name: "Outbound compressed message",
+					Attributes: []attribute.KeyValue{
+						{
+							Key:   "sequence-number",
+							Value: attribute.IntValue(1),
+						},
+						{
+							Key:   "message-size",
+							Value: attribute.IntValue(10006),
+						},
+						{
+							Key:   "message-size-compressed",
+							Value: attribute.IntValue(10006),
+						},
+					},
+				},
+			},
+		},
+		{
+			name:     "Attempt.grpc.testing.TestService.UnaryCall",
+			spanKind: oteltrace.SpanKindInternal.String(),
+			attributes: []attribute.KeyValue{
+				{
+					Key:   "Client",
+					Value: attribute.IntValue(1),
+				},
+				{
+					Key:   "FailFast",
+					Value: attribute.IntValue(1),
+				},
+				{
+					Key:   "previous-rpc-attempts",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "transparent-retry",
+					Value: attribute.IntValue(0),
+				},
+			},
+			events: []trace.Event{
+				{
+					Name: "Outbound compressed message",
+					Attributes: []attribute.KeyValue{
+						{
+							Key:   "sequence-number",
+							Value: attribute.IntValue(1),
+						},
+						{
+							Key:   "message-size",
+							Value: attribute.IntValue(10006),
+						},
+						{
+							Key:   "message-size-compressed",
+							Value: attribute.IntValue(10006),
+						},
+					},
+				},
+				{
+					Name: "Inbound compressed message",
+					Attributes: []attribute.KeyValue{
+						{
+							Key:   "sequence-number",
+							Value: attribute.IntValue(1),
+						},
+						{
+							Key:   "message-size",
+							Value: attribute.IntValue(10006),
+						},
+						{
+							Key:   "message-size-compressed",
+							Value: attribute.IntValue(10006),
+						},
+					},
+				},
+			},
+		},
+		{
+			name:       "grpc.testing.TestService.UnaryCall",
+			spanKind:   oteltrace.SpanKindClient.String(),
+			attributes: []attribute.KeyValue{},
+			events:     []trace.Event{},
+		},
+		{
+			name:     "grpc.testing.TestService.FullDuplexCall",
+			spanKind: oteltrace.SpanKindServer.String(),
+			attributes: []attribute.KeyValue{
+				{
+					Key:   "Client",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "FailFast",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "previous-rpc-attempts",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "transparent-retry",
+					Value: attribute.IntValue(0),
+				},
+			},
+			events: []trace.Event{},
+		},
+		{
+			name:       "grpc.testing.TestService.FullDuplexCall",
+			spanKind:   oteltrace.SpanKindClient.String(),
+			attributes: []attribute.KeyValue{},
+			events:     []trace.Event{},
+		},
+		{
+			name:     "Attempt.grpc.testing.TestService.FullDuplexCall",
+			spanKind: oteltrace.SpanKindInternal.String(),
+			attributes: []attribute.KeyValue{
+				{
+					Key:   "Client",
+					Value: attribute.IntValue(1),
+				},
+				{
+					Key:   "FailFast",
+					Value: attribute.IntValue(1),
+				},
+				{
+					Key:   "previous-rpc-attempts",
+					Value: attribute.IntValue(0),
+				},
+				{
+					Key:   "transparent-retry",
+					Value: attribute.IntValue(0),
+				},
+			},
+			events: []trace.Event{},
+		},
+	}
+
+	// Check that same traceID is used in client and server.
+	if got, want := spans[0].SpanContext.TraceID(), spans[2].SpanContext.TraceID(); got != want {
+		t.Fatal("TraceID mismatch in client span and server span.")
+	}
+	// Check that the attempt span id of client matches the span id of server
+	// SpanContext.
+	if got, want := spans[0].Parent.SpanID(), spans[1].SpanContext.SpanID(); got != want {
+		t.Fatal("SpanID mismatch in client span and server span.")
+	}
+
+	// Check that same traceID is used in client and server.
+	if got, want := spans[3].SpanContext.TraceID(), spans[4].SpanContext.TraceID(); got != want {
+		t.Fatal("TraceID mismatch in client span and server span.")
+	}
+	// Check that the attempt span id of client matches the span id of server
+	// SpanContext.
+	if got, want := spans[3].Parent.SpanID(), spans[5].SpanContext.SpanID(); got != want {
+		t.Fatal("SpanID mismatch in client span and server span.")
+	}
+	for index, span := range spans {
+		// Check that the attempt span has the correct status
+		if got, want := spans[index].Status.Code, otelcodes.Ok; got != want {
+			t.Errorf("Got status code %v, want %v", got, want)
+		}
+		// name
+		if got, want := span.Name, wantSI[index].name; got != want {
+			t.Errorf("Span name is %q, want %q", got, want)
+		}
+		// spanKind
+		if got, want := span.SpanKind.String(), wantSI[index].spanKind; got != want {
+			t.Errorf("Got span kind %q, want %q", got, want)
+		}
+		// attributes
+		if got, want := len(span.Attributes), len(wantSI[index].attributes); got != want {
+			t.Errorf("Got attributes list of size %q, want %q", got, want)
+		}
+		for idx, att := range span.Attributes {
+			if got, want := att.Key, wantSI[index].attributes[idx].Key; got != want {
+				t.Errorf("Got attribute key for span name %v as %v, want %v", span.Name, got, want)
+			}
+		}
+		// events
+		if got, want := len(span.Events), len(wantSI[index].events); got != want {
+			t.Errorf("Event length is %q, want %q", got, want)
+		}
+		for eventIdx, event := range span.Events {
+			if got, want := event.Name, wantSI[index].events[eventIdx].Name; got != want {
+				t.Errorf("Got event name for span name %q as %q, want %q", span.Name, got, want)
+			}
+			for idx, att := range event.Attributes {
+				if got, want := att.Key, wantSI[index].events[eventIdx].Attributes[idx].Key; got != want {
+					t.Errorf("Got attribute key for span name %q with event name %v, as %v, want %v", span.Name, event.Name, got, want)
+				}
+				if got, want := att.Value, wantSI[index].events[eventIdx].Attributes[idx].Value; got != want {
+					t.Errorf("Got attribute value for span name %v with event name %v, as %v, want %v", span.Name, event.Name, got, want)
+				}
+			}
+		}
+	}
+}
+
+// TestMetricsAndTracesDisabled verifies that RPCs call succeed as expected
+// when metrics and traces are disabled in the OpenTelemetry instrumentation.
+func (s) TestMetricsAndTracesDisabled(t *testing.T) {
+	ss := &stubserver.StubServer{
+		UnaryCallF: func(_ context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			return &testpb.SimpleResponse{Payload: &testpb.Payload{
+				Body: make([]byte, len(in.GetPayload().GetBody())),
+			}}, nil
+		},
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			for {
+				_, err := stream.Recv()
+				if err == io.EOF {
+					return nil
+				}
+			}
+		},
+	}
+
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Make two RPCs, a unary RPC and a streaming RPC.
+	if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{
+		Body: make([]byte, 10000),
+	}}); err != nil {
+		t.Fatalf("Unexpected error from UnaryCall: %v", err)
+	}
+	stream, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("ss.Client.FullDuplexCall failed: %v", err)
+	}
+
+	stream.CloseSend()
+	if _, err = stream.Recv(); err != io.EOF {
+		t.Fatalf("stream.Recv received an unexpected error: %v, expected an EOF error", err)
+	}
+}
+
+// TestRPCSpanErrorStatus verifies that errors during RPC calls are correctly
+// reflected in the span status. It simulates a unary RPC that returns an error
+// and checks that the span's status is set to error with the appropriate message.
+func (s) TestRPCSpanErrorStatus(t *testing.T) {
+	mo, _ := defaultMetricsOptions(t, nil)
+	// Using defaultTraceOptions to set up OpenTelemetry with an in-memory exporter
+	to, exporter := defaultTraceOptions(t)
+	const rpcErrorMsg = "unary call: internal server error"
+	ss := &stubserver.StubServer{
+		UnaryCallF: func(_ context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			return nil, fmt.Errorf("%v", rpcErrorMsg)
+		},
+	}
+
+	otelOptions := opentelemetry.Options{
+		MetricsOptions: *mo,
+		TraceOptions:   *to,
+	}
+
+	if err := ss.Start([]grpc.ServerOption{opentelemetry.ServerOption(otelOptions)},
+		opentelemetry.DialOption(otelOptions)); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{
+		Body: make([]byte, 10000),
+	}})
+
+	// Verify traces
+	spans := exporter.GetSpans()
+	if got, want := len(spans), 3; got != want {
+		t.Fatalf("got %d spans, want %d", got, want)
+	}
+
+	// Verify spans has error status with rpcErrorMsg as error message.
+	if got, want := spans[0].Status.Description, rpcErrorMsg; got != want {
+		t.Fatalf("got rpc error %s, want %s", spans[0].Status.Description, rpcErrorMsg)
+	}
+}
diff --git a/test/goaway_test.go b/test/goaway_test.go
index 65d2cc02..84efde7d 100644
--- a/test/goaway_test.go
+++ b/test/goaway_test.go
@@ -56,22 +56,19 @@ import (
 func (s) TestGracefulClientOnGoAway(t *testing.T) {
 	const maxConnAge = 100 * time.Millisecond
 	const testTime = maxConnAge * 10
-
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("Failed to create listener: %v", err)
+	}
 	ss := &stubserver.StubServer{
+		Listener: lis,
 		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
 			return &testpb.Empty{}, nil
 		},
+		S: grpc.NewServer(grpc.KeepaliveParams(keepalive.ServerParameters{MaxConnectionAge: maxConnAge})),
 	}
-
-	s := grpc.NewServer(grpc.KeepaliveParams(keepalive.ServerParameters{MaxConnectionAge: maxConnAge}))
-	defer s.Stop()
-	testgrpc.RegisterTestServiceServer(s, ss)
-
-	lis, err := net.Listen("tcp", "localhost:0")
-	if err != nil {
-		t.Fatalf("Failed to create listener: %v", err)
-	}
-	go s.Serve(lis)
+	stubserver.StartTestService(t, ss)
+	defer ss.S.Stop()
 
 	cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
@@ -82,7 +79,6 @@ func (s) TestGracefulClientOnGoAway(t *testing.T) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
-
 	endTime := time.Now().Add(testTime)
 	for time.Now().Before(endTime) {
 		if _, err := c.EmptyCall(ctx, &testpb.Empty{}); err != nil {
@@ -547,40 +543,48 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 
-	lis1, err := net.Listen("tcp", "localhost:0")
+	lis1, err := testutils.LocalTCPListener()
 	if err != nil {
 		t.Fatalf("Error while listening. Err: %v", err)
 	}
-	s1 := grpc.NewServer()
-	defer s1.Stop()
-	ts := &funcServer{
-		unaryCall: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
-			return &testpb.SimpleResponse{}, nil
-		},
-		fullDuplexCall: func(stream testgrpc.TestService_FullDuplexCallServer) error {
-			if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil {
-				t.Errorf("unexpected error from send: %v", err)
-				return err
-			}
-			// Wait forever.
-			_, err := stream.Recv()
-			if err == nil {
-				t.Error("expected to never receive any message")
-			}
+
+	unaryCallF := func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+		return &testpb.SimpleResponse{}, nil
+	}
+	fullDuplexCallF := func(stream testgrpc.TestService_FullDuplexCallServer) error {
+		if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil {
+			t.Errorf("Unexpected error from send: %v", err)
 			return err
-		},
+		}
+		// Wait until a message is received from client
+		_, err := stream.Recv()
+		if err == nil {
+			t.Error("Expected to never receive any message")
+		}
+		return err
 	}
-	testgrpc.RegisterTestServiceServer(s1, ts)
-	go s1.Serve(lis1)
+	ss1 := &stubserver.StubServer{
+		Listener:        lis1,
+		UnaryCallF:      unaryCallF,
+		FullDuplexCallF: fullDuplexCallF,
+		S:               grpc.NewServer(),
+	}
+	stubserver.StartTestService(t, ss1)
+	defer ss1.S.Stop()
 
 	conn2Established := grpcsync.NewEvent()
 	lis2, err := listenWithNotifyingListener("tcp", "localhost:0", conn2Established)
 	if err != nil {
 		t.Fatalf("Error while listening. Err: %v", err)
 	}
-	s2 := grpc.NewServer()
-	defer s2.Stop()
-	testgrpc.RegisterTestServiceServer(s2, ts)
+	ss2 := &stubserver.StubServer{
+		Listener:        lis2,
+		UnaryCallF:      unaryCallF,
+		FullDuplexCallF: fullDuplexCallF,
+		S:               grpc.NewServer(),
+	}
+	stubserver.StartTestService(t, ss2)
+	defer ss2.S.Stop()
 
 	r := manual.NewBuilderWithScheme("whatever")
 	r.InitialState(resolver.State{Addresses: []resolver.Address{
@@ -613,10 +617,8 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 		t.Fatalf("unexpected error from first recv: %v", err)
 	}
 
-	go s2.Serve(lis2)
-
 	t.Log("Gracefully stopping server 1.")
-	go s1.GracefulStop()
+	go ss1.S.GracefulStop()
 
 	t.Log("Waiting for the ClientConn to enter IDLE state.")
 	testutils.AwaitState(ctx, t, cc, connectivity.Idle)
@@ -637,7 +639,7 @@ func (s) TestGoAwayThenClose(t *testing.T) {
 	lis2.Close()
 
 	t.Log("Hard closing connection 1.")
-	s1.Stop()
+	ss1.S.Stop()
 
 	t.Log("Waiting for the first stream to error.")
 	if _, err = stream.Recv(); err == nil {
diff --git a/test/insecure_creds_test.go b/test/insecure_creds_test.go
index 9f6c8b59..52964336 100644
--- a/test/insecure_creds_test.go
+++ b/test/insecure_creds_test.go
@@ -20,7 +20,6 @@ package test
 
 import (
 	"context"
-	"net"
 	"strings"
 	"testing"
 
@@ -29,6 +28,7 @@ import (
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/peer"
 	"google.golang.org/grpc/status"
 
@@ -83,7 +83,13 @@ func (s) TestInsecureCreds(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
+			lis, err := testutils.LocalTCPListener()
+			if err != nil {
+				t.Fatalf("net.Listen(tcp, localhost:0) failed: %v", err)
+			}
+
 			ss := &stubserver.StubServer{
+				Listener: lis,
 				EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
 					if !test.serverInsecureCreds {
 						return &testpb.Empty{}, nil
@@ -104,28 +110,16 @@ func (s) TestInsecureCreds(t *testing.T) {
 					return &testpb.Empty{}, nil
 				},
 			}
-
 			sOpts := []grpc.ServerOption{}
 			if test.serverInsecureCreds {
-				sOpts = append(sOpts, grpc.Creds(insecure.NewCredentials()))
+				ss.S = grpc.NewServer(grpc.Creds(insecure.NewCredentials()))
+			} else {
+				ss.S = grpc.NewServer(sOpts...)
 			}
-			s := grpc.NewServer(sOpts...)
-			defer s.Stop()
-
-			testgrpc.RegisterTestServiceServer(s, ss)
-
-			lis, err := net.Listen("tcp", "localhost:0")
-			if err != nil {
-				t.Fatalf("net.Listen(tcp, localhost:0) failed: %v", err)
-			}
-
-			go s.Serve(lis)
-
+			stubserver.StartTestService(t, ss)
+			defer ss.S.Stop()
 			addr := lis.Addr().String()
 			opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
-			if test.clientInsecureCreds {
-				opts = []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
-			}
 			cc, err := grpc.NewClient(addr, opts...)
 			if err != nil {
 				t.Fatalf("grpc.NewClient(%q) failed: %v", addr, err)
@@ -143,21 +137,20 @@ func (s) TestInsecureCreds(t *testing.T) {
 }
 
 func (s) TestInsecureCreds_WithPerRPCCredentials_AsCallOption(t *testing.T) {
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("net.Listen(tcp, localhost:0) failed: %v", err)
+	}
+
 	ss := &stubserver.StubServer{
+		Listener: lis,
 		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
 			return &testpb.Empty{}, nil
 		},
+		S: grpc.NewServer(grpc.Creds(insecure.NewCredentials())),
 	}
-
-	s := grpc.NewServer(grpc.Creds(insecure.NewCredentials()))
-	defer s.Stop()
-	testgrpc.RegisterTestServiceServer(s, ss)
-
-	lis, err := net.Listen("tcp", "localhost:0")
-	if err != nil {
-		t.Fatalf("net.Listen(tcp, localhost:0) failed: %v", err)
-	}
-	go s.Serve(lis)
+	stubserver.StartTestService(t, ss)
+	defer ss.S.Stop()
 
 	addr := lis.Addr().String()
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
@@ -179,21 +172,19 @@ func (s) TestInsecureCreds_WithPerRPCCredentials_AsCallOption(t *testing.T) {
 }
 
 func (s) TestInsecureCreds_WithPerRPCCredentials_AsDialOption(t *testing.T) {
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("net.Listen(tcp, localhost:0) failed: %v", err)
+	}
 	ss := &stubserver.StubServer{
+		Listener: lis,
 		EmptyCallF: func(_ context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
 			return &testpb.Empty{}, nil
 		},
+		S: grpc.NewServer(grpc.Creds(insecure.NewCredentials())),
 	}
-
-	s := grpc.NewServer(grpc.Creds(insecure.NewCredentials()))
-	defer s.Stop()
-	testgrpc.RegisterTestServiceServer(s, ss)
-
-	lis, err := net.Listen("tcp", "localhost:0")
-	if err != nil {
-		t.Fatalf("net.Listen(tcp, localhost:0) failed: %v", err)
-	}
-	go s.Serve(lis)
+	stubserver.StartTestService(t, ss)
+	defer ss.S.Stop()
 
 	addr := lis.Addr().String()
 	dopts := []grpc.DialOption{
diff --git a/test/local_creds_test.go b/test/local_creds_test.go
index 241f6306..9e64f23e 100644
--- a/test/local_creds_test.go
+++ b/test/local_creds_test.go
@@ -33,6 +33,7 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/credentials/local"
 	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/peer"
 	"google.golang.org/grpc/status"
 
@@ -40,8 +41,13 @@ import (
 	testpb "google.golang.org/grpc/interop/grpc_testing"
 )
 
-func testLocalCredsE2ESucceed(network, address string) error {
+func testLocalCredsE2ESucceed(t *testing.T, network, address string) error {
+	lis, err := net.Listen(network, address)
+	if err != nil {
+		return fmt.Errorf("Failed to create listener: %v", err)
+	}
 	ss := &stubserver.StubServer{
+		Listener: lis,
 		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
 			pr, ok := peer.FromContext(ctx)
 			if !ok {
@@ -69,20 +75,10 @@ func testLocalCredsE2ESucceed(network, address string) error {
 			}
 			return &testpb.Empty{}, nil
 		},
+		S: grpc.NewServer(grpc.Creds(local.NewCredentials())),
 	}
-
-	sopts := []grpc.ServerOption{grpc.Creds(local.NewCredentials())}
-	s := grpc.NewServer(sopts...)
-	defer s.Stop()
-
-	testgrpc.RegisterTestServiceServer(s, ss)
-
-	lis, err := net.Listen(network, address)
-	if err != nil {
-		return fmt.Errorf("Failed to create listener: %v", err)
-	}
-
-	go s.Serve(lis)
+	stubserver.StartTestService(t, ss)
+	defer ss.S.Stop()
 
 	var cc *grpc.ClientConn
 	lisAddr := lis.Addr().String()
@@ -114,14 +110,14 @@ func testLocalCredsE2ESucceed(network, address string) error {
 }
 
 func (s) TestLocalCredsLocalhost(t *testing.T) {
-	if err := testLocalCredsE2ESucceed("tcp", "localhost:0"); err != nil {
+	if err := testLocalCredsE2ESucceed(t, "tcp", "localhost:0"); err != nil {
 		t.Fatalf("Failed e2e test for localhost: %v", err)
 	}
 }
 
 func (s) TestLocalCredsUDS(t *testing.T) {
 	addr := fmt.Sprintf("/tmp/grpc_fullstck_test%d", time.Now().UnixNano())
-	if err := testLocalCredsE2ESucceed("unix", addr); err != nil {
+	if err := testLocalCredsE2ESucceed(t, "unix", addr); err != nil {
 		t.Fatalf("Failed e2e test for UDS: %v", err)
 	}
 }
@@ -162,24 +158,11 @@ func spoofDialer(addr net.Addr) func(target string, t time.Duration) (net.Conn,
 	}
 }
 
-func testLocalCredsE2EFail(dopts []grpc.DialOption) error {
-	ss := &stubserver.StubServer{
-		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
-			return &testpb.Empty{}, nil
-		},
-	}
-
-	sopts := []grpc.ServerOption{grpc.Creds(local.NewCredentials())}
-	s := grpc.NewServer(sopts...)
-	defer s.Stop()
-
-	testgrpc.RegisterTestServiceServer(s, ss)
-
-	lis, err := net.Listen("tcp", "localhost:0")
+func testLocalCredsE2EFail(t *testing.T, dopts []grpc.DialOption) error {
+	lis, err := testutils.LocalTCPListener()
 	if err != nil {
 		return fmt.Errorf("Failed to create listener: %v", err)
 	}
-
 	var fakeClientAddr, fakeServerAddr net.Addr
 	fakeClientAddr = &net.IPAddr{
 		IP:   netip.MustParseAddr("10.8.9.10").AsSlice(),
@@ -189,8 +172,15 @@ func testLocalCredsE2EFail(dopts []grpc.DialOption) error {
 		IP:   netip.MustParseAddr("10.8.9.11").AsSlice(),
 		Zone: "",
 	}
-
-	go s.Serve(spoofListener(lis, fakeClientAddr))
+	ss := &stubserver.StubServer{
+		Listener: spoofListener(lis, fakeClientAddr),
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		S: grpc.NewServer(grpc.Creds(local.NewCredentials())),
+	}
+	stubserver.StartTestService(t, ss)
+	defer ss.S.Stop()
 
 	cc, err := grpc.NewClient(lis.Addr().String(), append(dopts, grpc.WithDialer(spoofDialer(fakeServerAddr)))...)
 	if err != nil {
@@ -214,7 +204,7 @@ func (s) TestLocalCredsClientFail(t *testing.T) {
 	// Use local creds at client-side which should lead to client-side failure.
 	opts := []grpc.DialOption{grpc.WithTransportCredentials(local.NewCredentials())}
 	want := status.Error(codes.Unavailable, "transport: authentication handshake failed: local credentials rejected connection to non-local address")
-	if err := testLocalCredsE2EFail(opts); !isExpected(err, want) {
+	if err := testLocalCredsE2EFail(t, opts); !isExpected(err, want) {
 		t.Fatalf("testLocalCredsE2EFail() = %v; want %v", err, want)
 	}
 }
@@ -222,7 +212,7 @@ func (s) TestLocalCredsClientFail(t *testing.T) {
 func (s) TestLocalCredsServerFail(t *testing.T) {
 	// Use insecure at client-side which should lead to server-side failure.
 	opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
-	if err := testLocalCredsE2EFail(opts); status.Code(err) != codes.Unavailable {
+	if err := testLocalCredsE2EFail(t, opts); status.Code(err) != codes.Unavailable {
 		t.Fatalf("testLocalCredsE2EFail() = %v; want %v", err, codes.Unavailable)
 	}
 }
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 9bf5d956..88b944d4 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -23,6 +23,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -32,7 +33,6 @@ import (
 	"github.com/google/uuid"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/internal"
-	"google.golang.org/grpc/internal/grpcsync"
 	iresolver "google.golang.org/grpc/internal/resolver"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/internal/testutils/xds/e2e"
@@ -271,7 +271,7 @@ func (s) TestResolverCloseClosesXDSClient(t *testing.T) {
 			Name:               t.Name(),
 			WatchExpiryTimeout: defaultTestTimeout,
 		})
-		return c, grpcsync.OnceFunc(func() {
+		return c, sync.OnceFunc(func() {
 			close(closeCh)
 			cancel()
 		}), err
diff --git a/xds/internal/xdsclient/tests/client_custom_dialopts_test.go b/xds/internal/xdsclient/tests/client_custom_dialopts_test.go
new file mode 100644
index 00000000..268bc79e
--- /dev/null
+++ b/xds/internal/xdsclient/tests/client_custom_dialopts_test.go
@@ -0,0 +1,165 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	internalbootstrap "google.golang.org/grpc/internal/xds/bootstrap"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/xds/bootstrap"
+	xci "google.golang.org/grpc/xds/internal/xdsclient/internal"
+)
+
+// nopDialOption is a no-op grpc.DialOption with a name.
+type nopDialOption struct {
+	grpc.EmptyDialOption
+	name string
+}
+
+// testCredsBundle implements `credentials.Bundle` and `extraDialOptions`.
+type testCredsBundle struct {
+	credentials.Bundle
+	testDialOptNames []string
+}
+
+func (t *testCredsBundle) DialOptions() []grpc.DialOption {
+	var opts []grpc.DialOption
+	for _, name := range t.testDialOptNames {
+		opts = append(opts, &nopDialOption{name: name})
+	}
+	return opts
+}
+
+type testCredsBuilder struct {
+	testDialOptNames []string
+}
+
+func (t *testCredsBuilder) Build(config json.RawMessage) (credentials.Bundle, func(), error) {
+	return &testCredsBundle{
+		Bundle:           insecure.NewBundle(),
+		testDialOptNames: t.testDialOptNames,
+	}, func() {}, nil
+}
+
+func (t *testCredsBuilder) Name() string {
+	return "test_dialer_creds"
+}
+
+func (s) TestClientCustomDialOptsFromCredentialsBundle(t *testing.T) {
+	// Create and register the credentials bundle builder.
+	credsBuilder := &testCredsBuilder{
+		testDialOptNames: []string{"opt1", "opt2", "opt3"},
+	}
+	bootstrap.RegisterCredentials(credsBuilder)
+
+	// Start an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc, err := internalbootstrap.NewContentsForTesting(internalbootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{
+				"type": %q,
+				"config": {"mgmt_server_address": %q}
+			}]
+		}]`, mgmtServer.Address, credsBuilder.Name(), mgmtServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Spin up a test backend.
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	// Configure client side xDS resources on the management server.
+	const serviceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Intercept a grpc.NewClient call from the xds client to validate DialOptions.
+	xci.GRPCNewClient = func(target string, opts ...grpc.DialOption) (conn *grpc.ClientConn, err error) {
+		got := map[string]int{}
+		for _, opt := range opts {
+			if mo, ok := opt.(*nopDialOption); ok {
+				got[mo.name]++
+			}
+		}
+		want := map[string]int{}
+		for _, name := range credsBuilder.testDialOptNames {
+			want[name]++
+		}
+		if !cmp.Equal(got, want) {
+			t.Errorf("grpc.NewClient() was called with unexpected DialOptions: got %v, want %v", got, want)
+		}
+		return grpc.NewClient(target, opts...)
+	}
+	defer func() { xci.GRPCNewClient = grpc.NewClient }()
+
+	// Create a ClientConn and make a successful RPC. The insecure transport
+	// credentials passed into the gRPC.NewClient is the credentials for the
+	// data plane communication with the test backend.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("Failed to dial local test server: %v", err)
+	}
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	cc.Close()
+}