Repository: grpc/grpc-go

Issue:
Flaky Test: Test/PickFirstLeaf_StopConnectedServer_SecondServerToFirst 
```
--- FAIL: Test (3.10s)
    --- FAIL: Test/PickFirstLeaf_StopConnectedServer_SecondServerToFirst (0.17s)
        tlogger.go:116: INFO server.go:685 [core] [Server #331]Server created  (t=+1.460984ms)
        stubserver.go:288: Started test service backend at "127.0.0.1:39795"
        tlogger.go:116: INFO server.go:881 [core] [Server #331 ListenSocket #332]ListenSocket created  (t=+2.063269ms)
        tlogger.go:116: INFO server.go:685 [core] [Server #333]Server created  (t=+2.215611ms)
        stubserver.go:288: Started test service backend at "127.0.0.1:38005"
        tlogger.go:116: INFO clientconn.go:1[65](https://github.com/grpc/grpc-go/actions/runs/11989223270/job/33425355899?pr=7865#step:8:66)4 [core] original dial target is: "whatever:///test.server"  (t=+2.453214ms)
        tlogger.go:116: INFO server.go:881 [core] [Server #333 ListenSocket #334]ListenSocket created  (t=+2.608856ms)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #335]Channel created  (t=+2.703702ms)
        tlogger.go:116: INFO clientconn.go:193 [core] [Channel #335]parsed dial target is: resolver.Target{URL:url.URL{Scheme:"whatever", Opaque:"", User:(*url.Userinfo)(nil), Host:"", Path:"/test.server", RawPath:"", OmitHost:false, ForceQuery:false, RawQuery:"", Fragment:"", RawFragment:""}}  (t=+2.833413ms)
        tlogger.go:116: INFO clientconn.go:194 [core] [Channel #335]Channel authority set to "test.server"  (t=+2.909595ms)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #335]Channel exiting idle mode  (t=+3.189119ms)
        tlogger.go:116: INFO server.go:817 [core] [Server #331 ListenSocket #332]ListenSocket deleted  (t=+103.746268ms)
        tlogger.go:116: INFO resolver_wrapper.go:200 [core] [Channel #335]Resolver state updated: {
              "Addresses": [
                {
                  "Addr": "127.0.0.1:39795",
                  "ServerName": "",
                  "Attributes": null,
                  "BalancerAttributes": null,
                  "Metadata": null
                },
                {
                  "Addr": "127.0.0.1:38005",
                  "ServerName": "",
                  "Attributes": null,
                  "BalancerAttributes": null,
                  "Metadata": null
                }
              ],
              "Endpoints": [
                {
                  "Addresses": [
                    {
                      "Addr": "127.0.0.1:39795",
                      "ServerName": "",
                      "Attributes": null,
                      "BalancerAttributes": null,
                      "Metadata": null
                    }
                  ],
                  "Attributes": null
                },
                {
                  "Addresses": [
                    {
                      "Addr": "127.0.0.1:38005",
                      "ServerName": "",
                      "Attributes": null,
                      "BalancerAttributes": null,
                      "Metadata": null
                    }
                  ],
                  "Attributes": null
                }
              ],
              "ServiceConfig": null,
              "Attributes": null
            } (resolver returned new addresses)  (t=+104.399937ms)
        tlogger.go:116: INFO balancer_wrapper.go:109 [core] [Channel #335]Channel switches to new LB policy "state_storing"  (t=+104.79791ms)
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #335]Channel Connectivity change to CONNECTING  (t=+105.222231ms)
        tlogger.go:116: INFO clientconn.go:838 [core] [Channel #335 SubChannel #336]Subchannel created  (t=+105.458091ms)
        tlogger.go:116: INFO clientconn.go:1199 [core] [Channel #335 SubChannel #336]Subchannel Connectivity change to CONNECTING  (t=+105.976808ms)
        tlogger.go:116: INFO clientconn.go:1319 [core] [Channel #335 SubChannel #336]Subchannel picks a new address "127.0.0.1:39795" to connect  (t=+106.240219ms)
        tlogger.go:116: WARNING clientconn.go:1381 [core] [Channel #335 SubChannel #336]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:39795", ServerName: "test.server", }. Err: connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:39795: connect: connection refused"  (t=+106.914386ms)
        tlogger.go:116: INFO clientconn.go:1201 [core] [Channel #335 SubChannel #336]Subchannel Connectivity change to TRANSIENT_FAILURE, last error: connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:39795: connect: connection refused"  (t=+107.201271ms)
        tlogger.go:116: INFO clientconn.go:838 [core] [Channel #335 SubChannel #337]Subchannel created  (t=+107.446539ms)
        tlogger.go:116: INFO clientconn.go:1199 [core] [Channel #335 SubChannel #337]Subchannel Connectivity change to CONNECTING  (t=+107.761787ms)
        tlogger.go:116: INFO clientconn.go:1319 [core] [Channel #335 SubChannel #337]Subchannel picks a new address "127.0.0.1:38005" to connect  (t=+107.9297ms)
        tlogger.go:116: INFO clientconn.go:1199 [core] [Channel #335 SubChannel #337]Subchannel Connectivity change to READY  (t=+108.860055ms)
        tlogger.go:116: INFO clientconn.go:1199 [core] [Channel #335 SubChannel #336]Subchannel Connectivity change to SHUTDOWN  (t=+109.075006ms)
        tlogger.go:116: INFO clientconn.go:1527 [core] [Channel #335 SubChannel #336]Subchannel deleted  (t=+109.244752ms)
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #335]Channel Connectivity change to READY  (t=+109.448222ms)
        tlogger.go:116: INFO server.go:817 [core] [Server #333 ListenSocket #334]ListenSocket deleted  (t=+1[66](https://github.com/grpc/grpc-go/actions/runs/11989223270/job/33425355899?pr=7865#step:8:67).155594ms)
        tlogger.go:116: INFO clientconn.go:1199 [core] [Channel #335 SubChannel #337]Subchannel Connectivity change to IDLE  (t=+166.711573ms)
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #335]Channel Connectivity change to IDLE  (t=+1[67](https://github.com/grpc/grpc-go/actions/runs/11989223270/job/33425355899?pr=7865#step:8:68).187921ms)
        pickfirstleaf_ext_test.go:700: Failed to start server: net.Listen("tcp", "127.0.0.1:39795") = listen tcp 127.0.0.1:39795: bind: address already in use
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #335]Channel Connectivity change to SHUTDOWN  (t=+1[68](https://github.com/grpc/grpc-go/actions/runs/11989223270/job/33425355899?pr=7865#step:8:69).877822ms)
        tlogger.go:116: INFO resolver_wrapper.go:100 [core] [Channel #335]Closing the name resolver  (t=+1[69](https://github.com/grpc/grpc-go/actions/runs/11989223270/job/33425355899?pr=7865#step:8:70).140923ms)
        tlogger.go:116: INFO balancer_wrapper.go:147 [core] [Channel #335]ccBalancerWrapper: closing  (t=+169.365181ms)
        tlogger.go:116: INFO clientconn.go:1199 [core] [Channel #335 SubChannel #337]Subchannel Connectivity change to SHUTDOWN  (t=+169.843233ms)
        tlogger.go:116: INFO clientconn.go:1527 [core] [Channel #335 SubChannel #337]Subchannel deleted  (t=+1[70](https://github.com/grpc/grpc-go/actions/runs/11989223270/job/33425355899?pr=7865#step:8:71).014462ms)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #335]Channel deleted  (t=+170.217891ms)
FAIL
FAIL	google.golang.org/grpc/balancer/pickfirst/pickfirstleaf	7.381s
```

Hints from issue comments:
@arjan-bal: Failure link: 

The error log: https://github.com/grpc/grpc-go/actions/runs/11989223270/job/33425355899?pr=7865
```
pickfirstleaf_ext_test.go:700: Failed to start server: net.Listen("tcp", "127.0.0.1:39795") = listen tcp 127.0.0.1:39795: bind: address already in use
```

The server was shut down, it couldn't bind to the same port when it tried to start again. I'll raise a PR that uses `RestartableListeners` so that the listener is not closed when the server is temporarily stopped.

Solution patch:
diff --git a/balancer/pickfirst/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirst/pickfirstleaf/pickfirstleaf.go
index aaec8749..7a250e6e 100644
--- a/balancer/pickfirst/pickfirstleaf/pickfirstleaf.go
+++ b/balancer/pickfirst/pickfirstleaf/pickfirstleaf.go
@@ -30,12 +30,14 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"net/netip"
 	"sync"
 	"time"
 
 	"google.golang.org/grpc/balancer"
 	"google.golang.org/grpc/balancer/pickfirst/internal"
 	"google.golang.org/grpc/connectivity"
+	expstats "google.golang.org/grpc/experimental/stats"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/envconfig"
 	internalgrpclog "google.golang.org/grpc/internal/grpclog"
@@ -57,7 +59,28 @@ var (
 	// Name is the name of the pick_first_leaf balancer.
 	// It is changed to "pick_first" in init() if this balancer is to be
 	// registered as the default pickfirst.
-	Name = "pick_first_leaf"
+	Name                 = "pick_first_leaf"
+	disconnectionsMetric = expstats.RegisterInt64Count(expstats.MetricDescriptor{
+		Name:        "grpc.lb.pick_first.disconnections",
+		Description: "EXPERIMENTAL. Number of times the selected subchannel becomes disconnected.",
+		Unit:        "disconnection",
+		Labels:      []string{"grpc.target"},
+		Default:     false,
+	})
+	connectionAttemptsSucceededMetric = expstats.RegisterInt64Count(expstats.MetricDescriptor{
+		Name:        "grpc.lb.pick_first.connection_attempts_succeeded",
+		Description: "EXPERIMENTAL. Number of successful connection attempts.",
+		Unit:        "attempt",
+		Labels:      []string{"grpc.target"},
+		Default:     false,
+	})
+	connectionAttemptsFailedMetric = expstats.RegisterInt64Count(expstats.MetricDescriptor{
+		Name:        "grpc.lb.pick_first.connection_attempts_failed",
+		Description: "EXPERIMENTAL. Number of failed connection attempts.",
+		Unit:        "attempt",
+		Labels:      []string{"grpc.target"},
+		Default:     false,
+	})
 )
 
 const (
@@ -80,9 +103,12 @@ const (
 
 type pickfirstBuilder struct{}
 
-func (pickfirstBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer {
+func (pickfirstBuilder) Build(cc balancer.ClientConn, bo balancer.BuildOptions) balancer.Balancer {
 	b := &pickfirstBalancer{
-		cc:                    cc,
+		cc:              cc,
+		target:          bo.Target.String(),
+		metricsRecorder: bo.MetricsRecorder, // ClientConn will always create a Metrics Recorder.
+
 		addressList:           addressList{},
 		subConns:              resolver.NewAddressMap(),
 		state:                 connectivity.Connecting,
@@ -147,8 +173,10 @@ func (b *pickfirstBalancer) newSCData(addr resolver.Address) (*scData, error) {
 type pickfirstBalancer struct {
 	// The following fields are initialized at build time and read-only after
 	// that and therefore do not need to be guarded by a mutex.
-	logger *internalgrpclog.PrefixLogger
-	cc     balancer.ClientConn
+	logger          *internalgrpclog.PrefixLogger
+	cc              balancer.ClientConn
+	target          string
+	metricsRecorder expstats.MetricsRecorder // guaranteed to be non nil
 
 	// The mutex is used to ensure synchronization of updates triggered
 	// from the idle picker and the already serialized resolver,
@@ -390,11 +418,14 @@ func addressFamily(address string) ipAddrFamily {
 	if err != nil {
 		return ipAddrFamilyUnknown
 	}
-	ip := net.ParseIP(host)
+	ip, err := netip.ParseAddr(host)
+	if err != nil {
+		return ipAddrFamilyUnknown
+	}
 	switch {
-	case ip.To4() != nil:
+	case ip.Is4() || ip.Is4In6():
 		return ipAddrFamilyV4
-	case ip.To16() != nil:
+	case ip.Is6():
 		return ipAddrFamilyV6
 	default:
 		return ipAddrFamilyUnknown
@@ -532,10 +563,6 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub
 	b.mu.Lock()
 	defer b.mu.Unlock()
 	oldState := sd.state
-	// Record a connection attempt when exiting CONNECTING.
-	if newState.ConnectivityState == connectivity.TransientFailure {
-		sd.connectionFailedInFirstPass = true
-	}
 	sd.state = newState.ConnectivityState
 	// Previously relevant SubConns can still callback with state updates.
 	// To prevent pickers from returning these obsolete SubConns, this logic
@@ -548,7 +575,14 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub
 		return
 	}
 
+	// Record a connection attempt when exiting CONNECTING.
+	if newState.ConnectivityState == connectivity.TransientFailure {
+		sd.connectionFailedInFirstPass = true
+		connectionAttemptsFailedMetric.Record(b.metricsRecorder, 1, b.target)
+	}
+
 	if newState.ConnectivityState == connectivity.Ready {
+		connectionAttemptsSucceededMetric.Record(b.metricsRecorder, 1, b.target)
 		b.shutdownRemainingLocked(sd)
 		if !b.addressList.seekTo(sd.addr) {
 			// This should not fail as we should have only one SubConn after
@@ -575,6 +609,15 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub
 		// the first address when the picker is used.
 		b.shutdownRemainingLocked(sd)
 		b.state = connectivity.Idle
+		// READY SubConn interspliced in between CONNECTING and IDLE, need to
+		// account for that.
+		if oldState == connectivity.Connecting {
+			// A known issue (https://github.com/grpc/grpc-go/issues/7862)
+			// causes a race that prevents the READY state change notification.
+			// This works around it.
+			connectionAttemptsSucceededMetric.Record(b.metricsRecorder, 1, b.target)
+		}
+		disconnectionsMetric.Record(b.metricsRecorder, 1, b.target)
 		b.addressList.reset()
 		b.cc.UpdateState(balancer.State{
 			ConnectivityState: connectivity.Idle,
diff --git a/credentials/google/google.go b/credentials/google/google.go
index d18e370d..5a9c9461 100644
--- a/credentials/google/google.go
+++ b/credentials/google/google.go
@@ -30,6 +30,8 @@ import (
 	"google.golang.org/grpc/internal"
 )
 
+const defaultCloudPlatformScope = "https://www.googleapis.com/auth/cloud-platform"
+
 var logger = grpclog.Component("credentials")
 
 // DefaultCredentialsOptions constructs options to build DefaultCredentials.
@@ -120,7 +122,7 @@ var (
 		return alts.NewClientCreds(alts.DefaultClientOptions())
 	}
 	newADC = func(ctx context.Context) (credentials.PerRPCCredentials, error) {
-		return oauth.NewApplicationDefault(ctx)
+		return oauth.NewApplicationDefault(ctx, defaultCloudPlatformScope)
 	}
 )
 
diff --git a/examples/features/csm_observability/client/main.go b/examples/features/csm_observability/client/main.go
index 20b357c2..cf33e6cb 100644
--- a/examples/features/csm_observability/client/main.go
+++ b/examples/features/csm_observability/client/main.go
@@ -29,6 +29,7 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
 	"google.golang.org/grpc/examples/features/proto/echo"
 	"google.golang.org/grpc/stats/opentelemetry"
 	"google.golang.org/grpc/stats/opentelemetry/csm"
@@ -56,7 +57,13 @@ func main() {
 	cleanup := csm.EnableObservability(context.Background(), opentelemetry.Options{MetricsOptions: opentelemetry.MetricsOptions{MeterProvider: provider}})
 	defer cleanup()
 
-	cc, err := grpc.NewClient(*target, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	// Set up xds credentials that fall back to insecure as described in:
+	// https://cloud.google.com/service-mesh/docs/service-routing/security-proxyless-setup#workloads_are_unable_to_communicate_in_the_security_setup.
+	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		log.Fatalf("Failed to create xDS credentials: %v", err)
+	}
+	cc, err := grpc.NewClient(*target, grpc.WithTransportCredentials(creds))
 	if err != nil {
 		log.Fatalf("Failed to start NewClient: %v", err)
 	}
diff --git a/examples/features/csm_observability/server/main.go b/examples/features/csm_observability/server/main.go
index b87f859a..3cafe233 100644
--- a/examples/features/csm_observability/server/main.go
+++ b/examples/features/csm_observability/server/main.go
@@ -28,9 +28,12 @@ import (
 	"net/http"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
 	pb "google.golang.org/grpc/examples/features/proto/echo"
 	"google.golang.org/grpc/stats/opentelemetry"
 	"google.golang.org/grpc/stats/opentelemetry/csm"
+	"google.golang.org/grpc/xds"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"go.opentelemetry.io/otel/exporters/prometheus"
@@ -67,7 +70,16 @@ func main() {
 	if err != nil {
 		log.Fatalf("Failed to listen: %v", err)
 	}
-	s := grpc.NewServer()
+	// Set up xds credentials that fall back to insecure as described in:
+	// https://cloud.google.com/service-mesh/docs/service-routing/security-proxyless-setup#workloads_are_unable_to_communicate_in_the_security_setup.
+	creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		log.Fatalf("Failed to create xDS credentials: %v", err)
+	}
+	s, err := xds.NewGRPCServer(grpc.Creds(creds))
+	if err != nil {
+		log.Fatalf("Failed to start xDS Server: %v", err)
+	}
 	pb.RegisterEchoServer(s, &echoServer{addr: ":" + *port})
 
 	log.Printf("Serving on %s\n", *port)
diff --git a/experimental/stats/metricregistry.go b/experimental/stats/metricregistry.go
index 1d827dd5..ad75313a 100644
--- a/experimental/stats/metricregistry.go
+++ b/experimental/stats/metricregistry.go
@@ -23,6 +23,7 @@ import (
 
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/stats"
 )
 
 func init() {
@@ -34,7 +35,7 @@ var logger = grpclog.Component("metrics-registry")
 // DefaultMetrics are the default metrics registered through global metrics
 // registry. This is written to at initialization time only, and is read only
 // after initialization.
-var DefaultMetrics = NewMetrics()
+var DefaultMetrics = stats.NewMetricSet()
 
 // MetricDescriptor is the data for a registered metric.
 type MetricDescriptor struct {
@@ -42,7 +43,7 @@ type MetricDescriptor struct {
 	// (including any per call metrics). See
 	// https://github.com/grpc/proposal/blob/master/A79-non-per-call-metrics-architecture.md#metric-instrument-naming-conventions
 	// for metric naming conventions.
-	Name Metric
+	Name string
 	// The description of this metric.
 	Description string
 	// The unit (e.g. entries, seconds) of this metric.
@@ -154,27 +155,27 @@ func (h *Int64GaugeHandle) Record(recorder MetricsRecorder, incr int64, labels .
 }
 
 // registeredMetrics are the registered metric descriptor names.
-var registeredMetrics = make(map[Metric]bool)
+var registeredMetrics = make(map[string]bool)
 
 // metricsRegistry contains all of the registered metrics.
 //
 // This is written to only at init time, and read only after that.
-var metricsRegistry = make(map[Metric]*MetricDescriptor)
+var metricsRegistry = make(map[string]*MetricDescriptor)
 
 // DescriptorForMetric returns the MetricDescriptor from the global registry.
 //
 // Returns nil if MetricDescriptor not present.
-func DescriptorForMetric(metric Metric) *MetricDescriptor {
-	return metricsRegistry[metric]
+func DescriptorForMetric(metricName string) *MetricDescriptor {
+	return metricsRegistry[metricName]
 }
 
-func registerMetric(name Metric, def bool) {
-	if registeredMetrics[name] {
-		logger.Fatalf("metric %v already registered", name)
+func registerMetric(metricName string, def bool) {
+	if registeredMetrics[metricName] {
+		logger.Fatalf("metric %v already registered", metricName)
 	}
-	registeredMetrics[name] = true
+	registeredMetrics[metricName] = true
 	if def {
-		DefaultMetrics = DefaultMetrics.Add(name)
+		DefaultMetrics = DefaultMetrics.Add(metricName)
 	}
 }
 
@@ -256,8 +257,8 @@ func snapshotMetricsRegistryForTesting() func() {
 	oldRegisteredMetrics := registeredMetrics
 	oldMetricsRegistry := metricsRegistry
 
-	registeredMetrics = make(map[Metric]bool)
-	metricsRegistry = make(map[Metric]*MetricDescriptor)
+	registeredMetrics = make(map[string]bool)
+	metricsRegistry = make(map[string]*MetricDescriptor)
 	maps.Copy(registeredMetrics, registeredMetrics)
 	maps.Copy(metricsRegistry, metricsRegistry)
 
diff --git a/experimental/stats/metrics.go b/experimental/stats/metrics.go
index 3221f7a6..bf9e7f98 100644
--- a/experimental/stats/metrics.go
+++ b/experimental/stats/metrics.go
@@ -19,8 +19,6 @@
 // Package stats contains experimental metrics/stats API's.
 package stats
 
-import "maps"
-
 // MetricsRecorder records on metrics derived from metric registry.
 type MetricsRecorder interface {
 	// RecordInt64Count records the measurement alongside labels on the int
@@ -39,76 +37,3 @@ type MetricsRecorder interface {
 	// gauge associated with the provided handle.
 	RecordInt64Gauge(handle *Int64GaugeHandle, incr int64, labels ...string)
 }
-
-// Metric is an identifier for a metric.
-type Metric string
-
-// Metrics is a set of metrics to record. Once created, Metrics is immutable,
-// however Add and Remove can make copies with specific metrics added or
-// removed, respectively.
-//
-// Do not construct directly; use NewMetrics instead.
-type Metrics struct {
-	// metrics are the set of metrics to initialize.
-	metrics map[Metric]bool
-}
-
-// NewMetrics returns a Metrics containing Metrics.
-func NewMetrics(metrics ...Metric) *Metrics {
-	newMetrics := make(map[Metric]bool)
-	for _, metric := range metrics {
-		newMetrics[metric] = true
-	}
-	return &Metrics{
-		metrics: newMetrics,
-	}
-}
-
-// Metrics returns the metrics set. The returned map is read-only and must not
-// be modified.
-func (m *Metrics) Metrics() map[Metric]bool {
-	return m.metrics
-}
-
-// Add adds the metrics to the metrics set and returns a new copy with the
-// additional metrics.
-func (m *Metrics) Add(metrics ...Metric) *Metrics {
-	newMetrics := make(map[Metric]bool)
-	for metric := range m.metrics {
-		newMetrics[metric] = true
-	}
-
-	for _, metric := range metrics {
-		newMetrics[metric] = true
-	}
-	return &Metrics{
-		metrics: newMetrics,
-	}
-}
-
-// Join joins the metrics passed in with the metrics set, and returns a new copy
-// with the merged metrics.
-func (m *Metrics) Join(metrics *Metrics) *Metrics {
-	newMetrics := make(map[Metric]bool)
-	maps.Copy(newMetrics, m.metrics)
-	maps.Copy(newMetrics, metrics.metrics)
-	return &Metrics{
-		metrics: newMetrics,
-	}
-}
-
-// Remove removes the metrics from the metrics set and returns a new copy with
-// the metrics removed.
-func (m *Metrics) Remove(metrics ...Metric) *Metrics {
-	newMetrics := make(map[Metric]bool)
-	for metric := range m.metrics {
-		newMetrics[metric] = true
-	}
-
-	for _, metric := range metrics {
-		delete(newMetrics, metric)
-	}
-	return &Metrics{
-		metrics: newMetrics,
-	}
-}
diff --git a/gcp/observability/go.sum b/gcp/observability/go.sum
index 30e984fb..472ac41d 100644
--- a/gcp/observability/go.sum
+++ b/gcp/observability/go.sum
@@ -1107,6 +1107,7 @@ go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozR
 go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
 go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=
 go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0=
+go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc=
 go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=
 go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
 go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
diff --git a/grpclog/internal/loggerv2.go b/grpclog/internal/loggerv2.go
index 07df71e9..ed90060c 100644
--- a/grpclog/internal/loggerv2.go
+++ b/grpclog/internal/loggerv2.go
@@ -101,6 +101,22 @@ var severityName = []string{
 	fatalLog:   "FATAL",
 }
 
+// sprintf is fmt.Sprintf.
+// These vars exist to make it possible to test that expensive format calls aren't made unnecessarily.
+var sprintf = fmt.Sprintf
+
+// sprint is fmt.Sprint.
+// These vars exist to make it possible to test that expensive format calls aren't made unnecessarily.
+var sprint = fmt.Sprint
+
+// sprintln is fmt.Sprintln.
+// These vars exist to make it possible to test that expensive format calls aren't made unnecessarily.
+var sprintln = fmt.Sprintln
+
+// exit is os.Exit.
+// This var exists to make it possible to test functions calling os.Exit.
+var exit = os.Exit
+
 // loggerT is the default logger used by grpclog.
 type loggerT struct {
 	m          []*log.Logger
@@ -111,7 +127,7 @@ type loggerT struct {
 func (g *loggerT) output(severity int, s string) {
 	sevStr := severityName[severity]
 	if !g.jsonFormat {
-		g.m[severity].Output(2, fmt.Sprintf("%v: %v", sevStr, s))
+		g.m[severity].Output(2, sevStr+": "+s)
 		return
 	}
 	// TODO: we can also include the logging component, but that needs more
@@ -123,55 +139,79 @@ func (g *loggerT) output(severity int, s string) {
 	g.m[severity].Output(2, string(b))
 }
 
+func (g *loggerT) printf(severity int, format string, args ...any) {
+	// Note the discard check is duplicated in each print func, rather than in
+	// output, to avoid the expensive Sprint calls.
+	// De-duplicating this by moving to output would be a significant performance regression!
+	if lg := g.m[severity]; lg.Writer() == io.Discard {
+		return
+	}
+	g.output(severity, sprintf(format, args...))
+}
+
+func (g *loggerT) print(severity int, v ...any) {
+	if lg := g.m[severity]; lg.Writer() == io.Discard {
+		return
+	}
+	g.output(severity, sprint(v...))
+}
+
+func (g *loggerT) println(severity int, v ...any) {
+	if lg := g.m[severity]; lg.Writer() == io.Discard {
+		return
+	}
+	g.output(severity, sprintln(v...))
+}
+
 func (g *loggerT) Info(args ...any) {
-	g.output(infoLog, fmt.Sprint(args...))
+	g.print(infoLog, args...)
 }
 
 func (g *loggerT) Infoln(args ...any) {
-	g.output(infoLog, fmt.Sprintln(args...))
+	g.println(infoLog, args...)
 }
 
 func (g *loggerT) Infof(format string, args ...any) {
-	g.output(infoLog, fmt.Sprintf(format, args...))
+	g.printf(infoLog, format, args...)
 }
 
 func (g *loggerT) Warning(args ...any) {
-	g.output(warningLog, fmt.Sprint(args...))
+	g.print(warningLog, args...)
 }
 
 func (g *loggerT) Warningln(args ...any) {
-	g.output(warningLog, fmt.Sprintln(args...))
+	g.println(warningLog, args...)
 }
 
 func (g *loggerT) Warningf(format string, args ...any) {
-	g.output(warningLog, fmt.Sprintf(format, args...))
+	g.printf(warningLog, format, args...)
 }
 
 func (g *loggerT) Error(args ...any) {
-	g.output(errorLog, fmt.Sprint(args...))
+	g.print(errorLog, args...)
 }
 
 func (g *loggerT) Errorln(args ...any) {
-	g.output(errorLog, fmt.Sprintln(args...))
+	g.println(errorLog, args...)
 }
 
 func (g *loggerT) Errorf(format string, args ...any) {
-	g.output(errorLog, fmt.Sprintf(format, args...))
+	g.printf(errorLog, format, args...)
 }
 
 func (g *loggerT) Fatal(args ...any) {
-	g.output(fatalLog, fmt.Sprint(args...))
-	os.Exit(1)
+	g.print(fatalLog, args...)
+	exit(1)
 }
 
 func (g *loggerT) Fatalln(args ...any) {
-	g.output(fatalLog, fmt.Sprintln(args...))
-	os.Exit(1)
+	g.println(fatalLog, args...)
+	exit(1)
 }
 
 func (g *loggerT) Fatalf(format string, args ...any) {
-	g.output(fatalLog, fmt.Sprintf(format, args...))
-	os.Exit(1)
+	g.printf(fatalLog, format, args...)
+	exit(1)
 }
 
 func (g *loggerT) V(l int) bool {
@@ -186,19 +226,42 @@ type LoggerV2Config struct {
 	FormatJSON bool
 }
 
+// combineLoggers returns a combined logger for both higher & lower severity logs,
+// or only one if the other is io.Discard.
+//
+// This uses io.Discard instead of io.MultiWriter when all loggers
+// are set to io.Discard. Both this package and the standard log package have
+// significant optimizations for io.Discard, which io.MultiWriter lacks (as of
+// this writing).
+func combineLoggers(lower, higher io.Writer) io.Writer {
+	if lower == io.Discard {
+		return higher
+	}
+	if higher == io.Discard {
+		return lower
+	}
+	return io.MultiWriter(lower, higher)
+}
+
 // NewLoggerV2 creates a new LoggerV2 instance with the provided configuration.
 // The infoW, warningW, and errorW writers are used to write log messages of
 // different severity levels.
 func NewLoggerV2(infoW, warningW, errorW io.Writer, c LoggerV2Config) LoggerV2 {
-	var m []*log.Logger
 	flag := log.LstdFlags
 	if c.FormatJSON {
 		flag = 0
 	}
-	m = append(m, log.New(infoW, "", flag))
-	m = append(m, log.New(io.MultiWriter(infoW, warningW), "", flag))
-	ew := io.MultiWriter(infoW, warningW, errorW) // ew will be used for error and fatal.
-	m = append(m, log.New(ew, "", flag))
-	m = append(m, log.New(ew, "", flag))
+
+	warningW = combineLoggers(infoW, warningW)
+	errorW = combineLoggers(errorW, warningW)
+
+	fatalW := errorW
+
+	m := []*log.Logger{
+		log.New(infoW, "", flag),
+		log.New(warningW, "", flag),
+		log.New(errorW, "", flag),
+		log.New(fatalW, "", flag),
+	}
 	return &loggerT{m: m, v: c.Verbosity, jsonFormat: c.FormatJSON}
 }
diff --git a/internal/resolver/dns/dns_resolver.go b/internal/resolver/dns/dns_resolver.go
index b080ae30..ba5c5a95 100644
--- a/internal/resolver/dns/dns_resolver.go
+++ b/internal/resolver/dns/dns_resolver.go
@@ -26,6 +26,7 @@ import (
 	"fmt"
 	rand "math/rand/v2"
 	"net"
+	"net/netip"
 	"os"
 	"strconv"
 	"strings"
@@ -122,7 +123,7 @@ func (b *dnsBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts
 	}
 
 	// IP address.
-	if ipAddr, ok := formatIP(host); ok {
+	if ipAddr, err := formatIP(host); err == nil {
 		addr := []resolver.Address{{Addr: ipAddr + ":" + port}}
 		cc.UpdateState(resolver.State{Addresses: addr})
 		return deadResolver{}, nil
@@ -260,9 +261,9 @@ func (d *dnsResolver) lookupSRV(ctx context.Context) ([]resolver.Address, error)
 			return nil, err
 		}
 		for _, a := range lbAddrs {
-			ip, ok := formatIP(a)
-			if !ok {
-				return nil, fmt.Errorf("dns: error parsing A record IP address %v", a)
+			ip, err := formatIP(a)
+			if err != nil {
+				return nil, fmt.Errorf("dns: error parsing A record IP address %v: %v", a, err)
 			}
 			addr := ip + ":" + strconv.Itoa(int(s.Port))
 			newAddrs = append(newAddrs, resolver.Address{Addr: addr, ServerName: s.Target})
@@ -322,9 +323,9 @@ func (d *dnsResolver) lookupHost(ctx context.Context) ([]resolver.Address, error
 	}
 	newAddrs := make([]resolver.Address, 0, len(addrs))
 	for _, a := range addrs {
-		ip, ok := formatIP(a)
-		if !ok {
-			return nil, fmt.Errorf("dns: error parsing A record IP address %v", a)
+		ip, err := formatIP(a)
+		if err != nil {
+			return nil, fmt.Errorf("dns: error parsing A record IP address %v: %v", a, err)
 		}
 		addr := ip + ":" + d.port
 		newAddrs = append(newAddrs, resolver.Address{Addr: addr})
@@ -351,19 +352,19 @@ func (d *dnsResolver) lookup() (*resolver.State, error) {
 	return &state, nil
 }
 
-// formatIP returns ok = false if addr is not a valid textual representation of
-// an IP address. If addr is an IPv4 address, return the addr and ok = true.
+// formatIP returns an error if addr is not a valid textual representation of
+// an IP address. If addr is an IPv4 address, return the addr and error = nil.
 // If addr is an IPv6 address, return the addr enclosed in square brackets and
-// ok = true.
-func formatIP(addr string) (addrIP string, ok bool) {
-	ip := net.ParseIP(addr)
-	if ip == nil {
-		return "", false
+// error = nil.
+func formatIP(addr string) (string, error) {
+	ip, err := netip.ParseAddr(addr)
+	if err != nil {
+		return "", err
 	}
-	if ip.To4() != nil {
-		return addr, true
+	if ip.Is4() {
+		return addr, nil
 	}
-	return "[" + addr + "]", true
+	return "[" + addr + "]", nil
 }
 
 // parseTarget takes the user input target string and default port, returns
@@ -379,7 +380,7 @@ func parseTarget(target, defaultPort string) (host, port string, err error) {
 	if target == "" {
 		return "", "", internal.ErrMissingAddr
 	}
-	if ip := net.ParseIP(target); ip != nil {
+	if _, err := netip.ParseAddr(target); err == nil {
 		// target is an IPv4 or IPv6(without brackets) address
 		return target, defaultPort, nil
 	}
diff --git a/internal/xds/rbac/matchers.go b/internal/xds/rbac/matchers.go
index e1c15018..07bb59ce 100644
--- a/internal/xds/rbac/matchers.go
+++ b/internal/xds/rbac/matchers.go
@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"net/netip"
 	"regexp"
 
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
@@ -344,7 +345,8 @@ func newRemoteIPMatcher(cidrRange *v3corepb.CidrRange) (*remoteIPMatcher, error)
 }
 
 func (sim *remoteIPMatcher) match(data *rpcData) bool {
-	return sim.ipNet.Contains(net.IP(net.ParseIP(data.peerInfo.Addr.String())))
+	ip, _ := netip.ParseAddr(data.peerInfo.Addr.String())
+	return sim.ipNet.Contains(net.IP(ip.AsSlice()))
 }
 
 type localIPMatcher struct {
@@ -361,7 +363,8 @@ func newLocalIPMatcher(cidrRange *v3corepb.CidrRange) (*localIPMatcher, error) {
 }
 
 func (dim *localIPMatcher) match(data *rpcData) bool {
-	return dim.ipNet.Contains(net.IP(net.ParseIP(data.localAddr.String())))
+	ip, _ := netip.ParseAddr(data.localAddr.String())
+	return dim.ipNet.Contains(net.IP(ip.AsSlice()))
 }
 
 // portMatcher matches on whether the destination port of the RPC matches the
diff --git a/interop/observability/go.sum b/interop/observability/go.sum
index a749b30f..4cdbd27b 100644
--- a/interop/observability/go.sum
+++ b/interop/observability/go.sum
@@ -1109,6 +1109,7 @@ go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozR
 go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
 go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=
 go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0=
+go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc=
 go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=
 go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
 go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
diff --git a/scripts/common.sh b/scripts/common.sh
index dede0789..3e641bae 100755
--- a/scripts/common.sh
+++ b/scripts/common.sh
@@ -11,6 +11,13 @@ not() {
   ! "$@"
 }
 
+# noret_grep will return 0 if zero or more lines were selected, and >1 if an
+# error occurred. Suppresses grep's return code of 1 when there are no matches
+# (for eg, empty file).
+noret_grep() {
+  grep "$@" || [[ $? == 1 ]]
+}
+
 die() {
   echo "$@" >&2
   exit 1
diff --git a/scripts/gen-deps.sh b/scripts/gen-deps.sh
index bc647c4d..f4d5ec63 100755
--- a/scripts/gen-deps.sh
+++ b/scripts/gen-deps.sh
@@ -3,6 +3,8 @@
 set -e          # Exit on error
 set -o pipefail # Fail a pipe if any sub-command fails.
 
+source "$(dirname $0)/common.sh"
+
 if [[ "$#" -ne 1 || ! -d "$1" ]]; then
     echo "Specify a valid output directory as the first parameter."
     exit 1
@@ -16,6 +18,6 @@ cd "${SCRIPTS_DIR}/.."
 git ls-files -- '*.go' | grep -v '\(^\|/\)\(internal\|examples\|benchmark\|interop\|test\|testdata\)\(/\|$\)' | xargs dirname | sort -u | while read d; do
   pushd "$d" > /dev/null
   pkg="$(echo "$d" | sed 's;\.;grpc;' | sed 's;/;_;g')"
-  go list -deps . | sort >| "${OUTPUT_DIR}/$pkg"
+  go list -deps . | sort | noret_grep -v 'google.golang.org/grpc' >| "${OUTPUT_DIR}/$pkg"
   popd > /dev/null
 done
diff --git a/scripts/vet.sh b/scripts/vet.sh
index 8db7e19c..4867e8c6 100755
--- a/scripts/vet.sh
+++ b/scripts/vet.sh
@@ -8,13 +8,6 @@ source "$(dirname $0)/common.sh"
 # Check to make sure it's safe to modify the user's git repo.
 git status --porcelain | fail_on_output
 
-# noret_grep will return 0 if zero or more lines were selected, and >1 if an
-# error occurred. Suppresses grep's return code of 1 when there are no matches
-# (for eg, empty file).
-noret_grep() {
-  grep "$@" || [[ $? == 1 ]]
-}
-
 # Undo any edits made by this script.
 cleanup() {
   git reset --hard HEAD
@@ -89,6 +82,10 @@ git grep '"github.com/envoyproxy/go-control-plane/envoy' -- '*.go' ':(exclude)*.
 git grep -e 'context.Background()' --or -e 'context.TODO()' -- "*_test.go" | grep -v "benchmark/primitives/context_test.go" | grep -v "credential
 s/google" | grep -v "internal/transport/" | grep -v "xds/internal/" | grep -v "security/advancedtls" | grep -v 'context.WithTimeout(' | not grep -v 'context.WithCancel('
 
+# Disallow usage of net.ParseIP in favour of netip.ParseAddr as the former
+# can't parse link local IPv6 addresses.
+not git grep 'net.ParseIP' -- '*.go'
+
 misspell -error .
 
 # - gofmt, goimports, go vet, go mod tidy.
diff --git a/security/advancedtls/examples/go.sum b/security/advancedtls/examples/go.sum
index 9102af78..2192e859 100644
--- a/security/advancedtls/examples/go.sum
+++ b/security/advancedtls/examples/go.sum
@@ -1,7 +1,23 @@
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=
+go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=
+go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
+go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
+go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=
+go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0=
+go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc=
+go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=
+go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
+go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
 golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
 golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
 golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
diff --git a/security/advancedtls/go.sum b/security/advancedtls/go.sum
index 9102af78..2192e859 100644
--- a/security/advancedtls/go.sum
+++ b/security/advancedtls/go.sum
@@ -1,7 +1,23 @@
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=
+go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=
+go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
+go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
+go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=
+go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0=
+go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc=
+go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=
+go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
+go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
 golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
 golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
 golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
diff --git a/service_config.go b/service_config.go
index 2671c5ef..7e83027d 100644
--- a/service_config.go
+++ b/service_config.go
@@ -168,6 +168,7 @@ func init() {
 		return parseServiceConfig(js, defaultMaxCallAttempts)
 	}
 }
+
 func parseServiceConfig(js string, maxAttempts int) *serviceconfig.ParseResult {
 	if len(js) == 0 {
 		return &serviceconfig.ParseResult{Err: fmt.Errorf("no JSON service config provided")}
@@ -297,7 +298,7 @@ func convertRetryPolicy(jrp *jsonRetryPolicy, maxAttempts int) (p *internalservi
 	return rp, nil
 }
 
-func min(a, b *int) *int {
+func minPointers(a, b *int) *int {
 	if *a < *b {
 		return a
 	}
@@ -309,7 +310,7 @@ func getMaxSize(mcMax, doptMax *int, defaultVal int) *int {
 		return &defaultVal
 	}
 	if mcMax != nil && doptMax != nil {
-		return min(mcMax, doptMax)
+		return minPointers(mcMax, doptMax)
 	}
 	if mcMax != nil {
 		return mcMax
diff --git a/stats/metrics.go b/stats/metrics.go
new file mode 100644
index 00000000..641c8e97
--- /dev/null
+++ b/stats/metrics.go
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package stats
+
+import "maps"
+
+// MetricSet is a set of metrics to record. Once created, MetricSet is immutable,
+// however Add and Remove can make copies with specific metrics added or
+// removed, respectively.
+//
+// Do not construct directly; use NewMetricSet instead.
+type MetricSet struct {
+	// metrics are the set of metrics to initialize.
+	metrics map[string]bool
+}
+
+// NewMetricSet returns a MetricSet containing metricNames.
+func NewMetricSet(metricNames ...string) *MetricSet {
+	newMetrics := make(map[string]bool)
+	for _, metric := range metricNames {
+		newMetrics[metric] = true
+	}
+	return &MetricSet{metrics: newMetrics}
+}
+
+// Metrics returns the metrics set. The returned map is read-only and must not
+// be modified.
+func (m *MetricSet) Metrics() map[string]bool {
+	return m.metrics
+}
+
+// Add adds the metricNames to the metrics set and returns a new copy with the
+// additional metrics.
+func (m *MetricSet) Add(metricNames ...string) *MetricSet {
+	newMetrics := make(map[string]bool)
+	for metric := range m.metrics {
+		newMetrics[metric] = true
+	}
+
+	for _, metric := range metricNames {
+		newMetrics[metric] = true
+	}
+	return &MetricSet{metrics: newMetrics}
+}
+
+// Join joins the metrics passed in with the metrics set, and returns a new copy
+// with the merged metrics.
+func (m *MetricSet) Join(metrics *MetricSet) *MetricSet {
+	newMetrics := make(map[string]bool)
+	maps.Copy(newMetrics, m.metrics)
+	maps.Copy(newMetrics, metrics.metrics)
+	return &MetricSet{metrics: newMetrics}
+}
+
+// Remove removes the metricNames from the metrics set and returns a new copy
+// with the metrics removed.
+func (m *MetricSet) Remove(metricNames ...string) *MetricSet {
+	newMetrics := make(map[string]bool)
+	for metric := range m.metrics {
+		newMetrics[metric] = true
+	}
+
+	for _, metric := range metricNames {
+		delete(newMetrics, metric)
+	}
+	return &MetricSet{metrics: newMetrics}
+}
diff --git a/stats/opencensus/go.sum b/stats/opencensus/go.sum
index 447a12f3..2e88e8bf 100644
--- a/stats/opencensus/go.sum
+++ b/stats/opencensus/go.sum
@@ -821,7 +821,9 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2
 github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U=
 github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M=
 github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M=
@@ -914,6 +916,7 @@ github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkj
 github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
 github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
@@ -1037,10 +1040,15 @@ go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/contrib/detectors/gcp v1.31.0/go.mod h1:tzQL6E1l+iV44YFTkcAeNQqzXUiekSYP9jjJjXwEd00=
+go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=
 go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=
+go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
 go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
+go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=
 go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0=
+go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc=
 go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=
+go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
 go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
diff --git a/stats/opentelemetry/client_metrics.go b/stats/opentelemetry/client_metrics.go
index 4af7f933..265791e5 100644
--- a/stats/opentelemetry/client_metrics.go
+++ b/stats/opentelemetry/client_metrics.go
@@ -260,18 +260,19 @@ func (h *clientStatsHandler) processRPCEnd(ctx context.Context, ai *attemptInfo,
 }
 
 const (
-	// ClientAttemptStarted is the number of client call attempts started.
-	ClientAttemptStarted estats.Metric = "grpc.client.attempt.started"
-	// ClientAttemptDuration is the end-to-end time taken to complete a client
-	// call attempt.
-	ClientAttemptDuration estats.Metric = "grpc.client.attempt.duration"
-	// ClientAttemptSentCompressedTotalMessageSize is the compressed message
-	// bytes sent per client call attempt.
-	ClientAttemptSentCompressedTotalMessageSize estats.Metric = "grpc.client.attempt.sent_total_compressed_message_size"
-	// ClientAttemptRcvdCompressedTotalMessageSize is the compressed message
-	// bytes received per call attempt.
-	ClientAttemptRcvdCompressedTotalMessageSize estats.Metric = "grpc.client.attempt.rcvd_total_compressed_message_size"
-	// ClientCallDuration is the time taken by gRPC to complete an RPC from
-	// application's perspective.
-	ClientCallDuration estats.Metric = "grpc.client.call.duration"
+	// ClientAttemptStartedMetricName is the number of client call attempts
+	// started.
+	ClientAttemptStartedMetricName string = "grpc.client.attempt.started"
+	// ClientAttemptDurationMetricName is the end-to-end time taken to complete
+	// a client call attempt.
+	ClientAttemptDurationMetricName string = "grpc.client.attempt.duration"
+	// ClientAttemptSentCompressedTotalMessageSizeMetricName is the compressed
+	// message bytes sent per client call attempt.
+	ClientAttemptSentCompressedTotalMessageSizeMetricName string = "grpc.client.attempt.sent_total_compressed_message_size"
+	// ClientAttemptRcvdCompressedTotalMessageSizeMetricName is the compressed
+	// message bytes received per call attempt.
+	ClientAttemptRcvdCompressedTotalMessageSizeMetricName string = "grpc.client.attempt.rcvd_total_compressed_message_size"
+	// ClientCallDurationMetricName is the time taken by gRPC to complete an RPC
+	// from application's perspective.
+	ClientCallDurationMetricName string = "grpc.client.call.duration"
 )
diff --git a/stats/opentelemetry/opentelemetry.go b/stats/opentelemetry/opentelemetry.go
index cc5ad387..dcc42477 100644
--- a/stats/opentelemetry/opentelemetry.go
+++ b/stats/opentelemetry/opentelemetry.go
@@ -16,6 +16,10 @@
 
 // Package opentelemetry implements opentelemetry instrumentation code for
 // gRPC-Go clients and servers.
+//
+// For details on configuring opentelemetry and various instruments that this
+// package creates, see
+// [gRPC OpenTelemetry Metrics](https://grpc.io/docs/guides/opentelemetry-metrics/).
 package opentelemetry
 
 import (
@@ -28,6 +32,7 @@ import (
 	estats "google.golang.org/grpc/experimental/stats"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/stats"
 	otelinternal "google.golang.org/grpc/stats/opentelemetry/internal"
 
 	otelattribute "go.opentelemetry.io/otel/attribute"
@@ -57,27 +62,28 @@ type Options struct {
 type MetricsOptions struct {
 	// MeterProvider is the MeterProvider instance that will be used to create
 	// instruments. To enable metrics collection, set a meter provider. If
-	// unset, no metrics will be recorded. Any implementation knobs (i.e. views,
-	// bounds) set in the MeterProvider take precedence over the API calls from
-	// this interface. (i.e. it will create default views for unset views).
+	// unset, no metrics will be recorded.
 	MeterProvider otelmetric.MeterProvider
 
 	// Metrics are the metrics to instrument. Will create instrument and record telemetry
 	// for corresponding metric supported by the client and server
 	// instrumentation components if applicable. If not set, the default metrics
 	// will be recorded.
-	Metrics *estats.Metrics
-
-	// MethodAttributeFilter is to record the method name of RPCs handled by
-	// grpc.UnknownServiceHandler, but take care to limit the values allowed, as
-	// allowing too many will increase cardinality and could cause severe memory
-	// or performance problems. On Client Side, pass a
-	// grpc.StaticMethodCallOption as a call option into Invoke or NewStream.
-	// This only applies for server side metrics.
+	Metrics *stats.MetricSet
+
+	// MethodAttributeFilter is a function that determines whether to record the
+	// method name of RPCs as an attribute, or to bucket into "other". Take care
+	// to limit the values allowed, as allowing too many will increase
+	// cardinality and could cause severe memory or performance problems.
+	//
+	// This only applies for server-side metrics.  For clients, to record the
+	// method name in the attributes, pass grpc.StaticMethodCallOption to Invoke
+	// or NewStream. Note that when using protobuf generated clients, this
+	// CallOption is included automatically.
 	MethodAttributeFilter func(string) bool
 
-	// OptionalLabels are labels received from LB Policies that this component
-	// should add to metrics that record after receiving incoming metadata.
+	// OptionalLabels specifies a list of optional labels to enable on any
+	// metrics that support them.
 	OptionalLabels []string
 
 	// pluginOption is used to get labels to attach to certain metrics, if set.
@@ -207,7 +213,7 @@ type serverMetrics struct {
 	callDuration otelmetric.Float64Histogram
 }
 
-func createInt64Counter(setOfMetrics map[estats.Metric]bool, metricName estats.Metric, meter otelmetric.Meter, options ...otelmetric.Int64CounterOption) otelmetric.Int64Counter {
+func createInt64Counter(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Int64CounterOption) otelmetric.Int64Counter {
 	if _, ok := setOfMetrics[metricName]; !ok {
 		return noop.Int64Counter{}
 	}
@@ -219,7 +225,7 @@ func createInt64Counter(setOfMetrics map[estats.Metric]bool, metricName estats.M
 	return ret
 }
 
-func createFloat64Counter(setOfMetrics map[estats.Metric]bool, metricName estats.Metric, meter otelmetric.Meter, options ...otelmetric.Float64CounterOption) otelmetric.Float64Counter {
+func createFloat64Counter(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Float64CounterOption) otelmetric.Float64Counter {
 	if _, ok := setOfMetrics[metricName]; !ok {
 		return noop.Float64Counter{}
 	}
@@ -231,7 +237,7 @@ func createFloat64Counter(setOfMetrics map[estats.Metric]bool, metricName estats
 	return ret
 }
 
-func createInt64Histogram(setOfMetrics map[estats.Metric]bool, metricName estats.Metric, meter otelmetric.Meter, options ...otelmetric.Int64HistogramOption) otelmetric.Int64Histogram {
+func createInt64Histogram(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Int64HistogramOption) otelmetric.Int64Histogram {
 	if _, ok := setOfMetrics[metricName]; !ok {
 		return noop.Int64Histogram{}
 	}
@@ -243,7 +249,7 @@ func createInt64Histogram(setOfMetrics map[estats.Metric]bool, metricName estats
 	return ret
 }
 
-func createFloat64Histogram(setOfMetrics map[estats.Metric]bool, metricName estats.Metric, meter otelmetric.Meter, options ...otelmetric.Float64HistogramOption) otelmetric.Float64Histogram {
+func createFloat64Histogram(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Float64HistogramOption) otelmetric.Float64Histogram {
 	if _, ok := setOfMetrics[metricName]; !ok {
 		return noop.Float64Histogram{}
 	}
@@ -255,7 +261,7 @@ func createFloat64Histogram(setOfMetrics map[estats.Metric]bool, metricName esta
 	return ret
 }
 
-func createInt64Gauge(setOfMetrics map[estats.Metric]bool, metricName estats.Metric, meter otelmetric.Meter, options ...otelmetric.Int64GaugeOption) otelmetric.Int64Gauge {
+func createInt64Gauge(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Int64GaugeOption) otelmetric.Int64Gauge {
 	if _, ok := setOfMetrics[metricName]; !ok {
 		return noop.Int64Gauge{}
 	}
@@ -298,7 +304,7 @@ type registryMetrics struct {
 	optionalLabels []string
 }
 
-func (rm *registryMetrics) registerMetrics(metrics *estats.Metrics, meter otelmetric.Meter) {
+func (rm *registryMetrics) registerMetrics(metrics *stats.MetricSet, meter otelmetric.Meter) {
 	rm.intCounts = make(map[*estats.MetricDescriptor]otelmetric.Int64Counter)
 	rm.floatCounts = make(map[*estats.MetricDescriptor]otelmetric.Float64Counter)
 	rm.intHistos = make(map[*estats.MetricDescriptor]otelmetric.Int64Histogram)
@@ -379,12 +385,12 @@ var (
 	// DefaultSizeBounds are the default bounds for metrics which record size.
 	DefaultSizeBounds = []float64{0, 1024, 2048, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864, 268435456, 1073741824, 4294967296}
 	// defaultPerCallMetrics are the default metrics provided by this module.
-	defaultPerCallMetrics = estats.NewMetrics(ClientAttemptStarted, ClientAttemptDuration, ClientAttemptSentCompressedTotalMessageSize, ClientAttemptRcvdCompressedTotalMessageSize, ClientCallDuration, ServerCallStarted, ServerCallSentCompressedTotalMessageSize, ServerCallRcvdCompressedTotalMessageSize, ServerCallDuration)
+	defaultPerCallMetrics = stats.NewMetricSet(ClientAttemptStartedMetricName, ClientAttemptDurationMetricName, ClientAttemptSentCompressedTotalMessageSizeMetricName, ClientAttemptRcvdCompressedTotalMessageSizeMetricName, ClientCallDurationMetricName, ServerCallStartedMetricName, ServerCallSentCompressedTotalMessageSizeMetricName, ServerCallRcvdCompressedTotalMessageSizeMetricName, ServerCallDurationMetricName)
 )
 
 // DefaultMetrics returns a set of default OpenTelemetry metrics.
 //
 // This should only be invoked after init time.
-func DefaultMetrics() *estats.Metrics {
+func DefaultMetrics() *stats.MetricSet {
 	return defaultPerCallMetrics.Join(estats.DefaultMetrics)
 }
diff --git a/stats/opentelemetry/server_metrics.go b/stats/opentelemetry/server_metrics.go
index eaea559b..4765afa8 100644
--- a/stats/opentelemetry/server_metrics.go
+++ b/stats/opentelemetry/server_metrics.go
@@ -264,15 +264,15 @@ func (h *serverStatsHandler) processRPCEnd(ctx context.Context, ai *attemptInfo,
 }
 
 const (
-	// ServerCallStarted is the number of server calls started.
-	ServerCallStarted estats.Metric = "grpc.server.call.started"
-	// ServerCallSentCompressedTotalMessageSize is the compressed message bytes
-	// sent per server call.
-	ServerCallSentCompressedTotalMessageSize estats.Metric = "grpc.server.call.sent_total_compressed_message_size"
-	// ServerCallRcvdCompressedTotalMessageSize is the compressed message bytes
-	// received per server call.
-	ServerCallRcvdCompressedTotalMessageSize estats.Metric = "grpc.server.call.rcvd_total_compressed_message_size"
-	// ServerCallDuration is the end-to-end time taken to complete a call from
-	// server transport's perspective.
-	ServerCallDuration estats.Metric = "grpc.server.call.duration"
+	// ServerCallStartedMetricName is the number of server calls started.
+	ServerCallStartedMetricName string = "grpc.server.call.started"
+	// ServerCallSentCompressedTotalMessageSizeMetricName is the compressed
+	// message bytes sent per server call.
+	ServerCallSentCompressedTotalMessageSizeMetricName string = "grpc.server.call.sent_total_compressed_message_size"
+	// ServerCallRcvdCompressedTotalMessageSizeMetricName is the compressed
+	// message bytes received per server call.
+	ServerCallRcvdCompressedTotalMessageSizeMetricName string = "grpc.server.call.rcvd_total_compressed_message_size"
+	// ServerCallDurationMetricName is the end-to-end time taken to complete a
+	// call from server transport's perspective.
+	ServerCallDurationMetricName string = "grpc.server.call.duration"
 )
diff --git a/stream.go b/stream.go
index 6d10d0ac..17e2267b 100644
--- a/stream.go
+++ b/stream.go
@@ -218,7 +218,7 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth
 
 	var mc serviceconfig.MethodConfig
 	var onCommit func()
-	var newStream = func(ctx context.Context, done func()) (iresolver.ClientStream, error) {
+	newStream := func(ctx context.Context, done func()) (iresolver.ClientStream, error) {
 		return newClientStreamWithParams(ctx, desc, cc, method, mc, onCommit, done, opts...)
 	}
 
@@ -708,11 +708,10 @@ func (a *csAttempt) shouldRetry(err error) (bool, error) {
 		cs.numRetriesSincePushback = 0
 	} else {
 		fact := math.Pow(rp.BackoffMultiplier, float64(cs.numRetriesSincePushback))
-		cur := float64(rp.InitialBackoff) * fact
-		if max := float64(rp.MaxBackoff); cur > max {
-			cur = max
-		}
-		dur = time.Duration(rand.Int64N(int64(cur)))
+		cur := min(float64(rp.InitialBackoff)*fact, float64(rp.MaxBackoff))
+		// Apply jitter by multiplying with a random factor between 0.8 and 1.2
+		cur *= 0.8 + 0.4*rand.Float64()
+		dur = time.Duration(int64(cur))
 		cs.numRetriesSincePushback++
 	}
 



Test patch:
diff --git a/balancer/pickfirst/pickfirst_test.go b/balancer/pickfirst/pickfirst_test.go
index 0b360c3b..1da680fb 100644
--- a/balancer/pickfirst/pickfirst_test.go
+++ b/balancer/pickfirst/pickfirst_test.go
@@ -29,6 +29,7 @@ import (
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/stats"
 	"google.golang.org/grpc/resolver"
 )
 
@@ -55,7 +56,7 @@ func (s) TestPickFirst_InitialResolverError(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	cc := testutils.NewBalancerClientConn(t)
-	bal := balancer.Get(Name).Build(cc, balancer.BuildOptions{})
+	bal := balancer.Get(Name).Build(cc, balancer.BuildOptions{MetricsRecorder: &stats.NoopMetricsRecorder{}})
 	defer bal.Close()
 	bal.ResolverError(errors.New("resolution failed: test error"))
 
@@ -88,7 +89,7 @@ func (s) TestPickFirst_ResolverErrorinTF(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	cc := testutils.NewBalancerClientConn(t)
-	bal := balancer.Get(Name).Build(cc, balancer.BuildOptions{})
+	bal := balancer.Get(Name).Build(cc, balancer.BuildOptions{MetricsRecorder: &stats.NoopMetricsRecorder{}})
 	defer bal.Close()
 
 	// After sending a valid update, the LB policy should report CONNECTING.
diff --git a/balancer/pickfirst/pickfirstleaf/metrics_test.go b/balancer/pickfirst/pickfirstleaf/metrics_test.go
new file mode 100644
index 00000000..90beca6a
--- /dev/null
+++ b/balancer/pickfirst/pickfirstleaf/metrics_test.go
@@ -0,0 +1,273 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package pickfirstleaf_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer/pickfirst/pickfirstleaf"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/stats"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/stats/opentelemetry"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest"
+)
+
+var pfConfig string
+
+func init() {
+	pfConfig = fmt.Sprintf(`{
+  		"loadBalancingConfig": [
+    		{
+      			%q: {
+      		}
+    	}
+  	]
+	}`, pickfirstleaf.Name)
+}
+
+// TestPickFirstMetrics tests pick first metrics. It configures a pick first
+// balancer, causes it to connect and then disconnect, and expects the
+// subsequent metrics to emit from that.
+func (s) TestPickFirstMetrics(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+	}
+	ss.StartServer()
+	defer ss.Stop()
+
+	sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(pfConfig)
+
+	r := manual.NewBuilderWithScheme("whatever")
+	r.InitialState(resolver.State{
+		ServiceConfig: sc,
+		Addresses:     []resolver.Address{{Addr: ss.Address}}},
+	)
+
+	tmr := stats.NewTestMetricsRecorder()
+	cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithStatsHandler(tmr), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("NewClient() failed with error: %v", err)
+	}
+	defer cc.Close()
+
+	tsc := testgrpc.NewTestServiceClient(cc)
+	if _, err := tsc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	if got, _ := tmr.Metric("grpc.lb.pick_first.connection_attempts_succeeded"); got != 1 {
+		t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.connection_attempts_succeeded", got, 1)
+	}
+	if got, _ := tmr.Metric("grpc.lb.pick_first.connection_attempts_failed"); got != 0 {
+		t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.connection_attempts_failed", got, 0)
+	}
+	if got, _ := tmr.Metric("grpc.lb.pick_first.disconnections"); got != 0 {
+		t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.disconnections", got, 0)
+	}
+
+	ss.Stop()
+	testutils.AwaitState(ctx, t, cc, connectivity.Idle)
+	if got, _ := tmr.Metric("grpc.lb.pick_first.disconnections"); got != 1 {
+		t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.disconnections", got, 1)
+	}
+}
+
+// TestPickFirstMetricsFailure tests the connection attempts failed metric. It
+// configures a channel and scenario that causes a pick first connection attempt
+// to fail, and then expects that metric to emit.
+func (s) TestPickFirstMetricsFailure(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(pfConfig)
+
+	r := manual.NewBuilderWithScheme("whatever")
+	r.InitialState(resolver.State{
+		ServiceConfig: sc,
+		Addresses:     []resolver.Address{{Addr: "bad address"}}},
+	)
+	grpcTarget := r.Scheme() + ":///"
+	tmr := stats.NewTestMetricsRecorder()
+	cc, err := grpc.NewClient(grpcTarget, grpc.WithStatsHandler(tmr), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("NewClient() failed with error: %v", err)
+	}
+	defer cc.Close()
+
+	tsc := testgrpc.NewTestServiceClient(cc)
+	if _, err := tsc.EmptyCall(ctx, &testpb.Empty{}); err == nil {
+		t.Fatalf("EmptyCall() passed when expected to fail")
+	}
+
+	if got, _ := tmr.Metric("grpc.lb.pick_first.connection_attempts_succeeded"); got != 0 {
+		t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.connection_attempts_succeeded", got, 0)
+	}
+	if got, _ := tmr.Metric("grpc.lb.pick_first.connection_attempts_failed"); got != 1 {
+		t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.connection_attempts_failed", got, 1)
+	}
+	if got, _ := tmr.Metric("grpc.lb.pick_first.disconnections"); got != 0 {
+		t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.disconnections", got, 0)
+	}
+}
+
+// TestPickFirstMetricsE2E tests the pick first metrics end to end. It
+// configures a channel with an OpenTelemetry plugin, induces all 3 pick first
+// metrics to emit, and makes sure the correct OpenTelemetry metrics atoms emit.
+func (s) TestPickFirstMetricsE2E(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+	}
+	ss.StartServer()
+	defer ss.Stop()
+
+	sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(pfConfig)
+	r := manual.NewBuilderWithScheme("whatever")
+	r.InitialState(resolver.State{
+		ServiceConfig: sc,
+		Addresses:     []resolver.Address{{Addr: "bad address"}}},
+	) // Will trigger connection failed.
+
+	grpcTarget := r.Scheme() + ":///"
+	reader := metric.NewManualReader()
+	provider := metric.NewMeterProvider(metric.WithReader(reader))
+	mo := opentelemetry.MetricsOptions{
+		MeterProvider: provider,
+		Metrics:       opentelemetry.DefaultMetrics().Add("grpc.lb.pick_first.disconnections", "grpc.lb.pick_first.connection_attempts_succeeded", "grpc.lb.pick_first.connection_attempts_failed"),
+	}
+
+	cc, err := grpc.NewClient(grpcTarget, opentelemetry.DialOption(opentelemetry.Options{MetricsOptions: mo}), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("NewClient() failed with error: %v", err)
+	}
+	defer cc.Close()
+
+	tsc := testgrpc.NewTestServiceClient(cc)
+	if _, err := tsc.EmptyCall(ctx, &testpb.Empty{}); err == nil {
+		t.Fatalf("EmptyCall() passed when expected to fail")
+	}
+
+	r.UpdateState(resolver.State{
+		ServiceConfig: sc,
+		Addresses:     []resolver.Address{{Addr: ss.Address}},
+	}) // Will trigger successful connection metric.
+	if _, err := tsc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	// Stop the server, that should send signal to disconnect, which will
+	// eventually emit disconnection metric before ClientConn goes IDLE.
+	ss.Stop()
+	testutils.AwaitState(ctx, t, cc, connectivity.Idle)
+	wantMetrics := []metricdata.Metrics{
+		{
+			Name:        "grpc.lb.pick_first.connection_attempts_succeeded",
+			Description: "EXPERIMENTAL. Number of successful connection attempts.",
+			Unit:        "attempt",
+			Data: metricdata.Sum[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget)),
+						Value:      1,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+			},
+		},
+		{
+			Name:        "grpc.lb.pick_first.connection_attempts_failed",
+			Description: "EXPERIMENTAL. Number of failed connection attempts.",
+			Unit:        "attempt",
+			Data: metricdata.Sum[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget)),
+						Value:      1,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+			},
+		},
+		{
+			Name:        "grpc.lb.pick_first.disconnections",
+			Description: "EXPERIMENTAL. Number of times the selected subchannel becomes disconnected.",
+			Unit:        "disconnection",
+			Data: metricdata.Sum[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget)),
+						Value:      1,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+			},
+		},
+	}
+
+	gotMetrics := metricsDataFromReader(ctx, reader)
+	for _, metric := range wantMetrics {
+		val, ok := gotMetrics[metric.Name]
+		if !ok {
+			t.Fatalf("Metric %v not present in recorded metrics", metric.Name)
+		}
+		if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars()) {
+			t.Fatalf("Metrics data type not equal for metric: %v", metric.Name)
+		}
+	}
+}
+
+func metricsDataFromReader(ctx context.Context, reader *metric.ManualReader) map[string]metricdata.Metrics {
+	rm := &metricdata.ResourceMetrics{}
+	reader.Collect(ctx, rm)
+	gotMetrics := map[string]metricdata.Metrics{}
+	for _, sm := range rm.ScopeMetrics {
+		for _, m := range sm.Metrics {
+			gotMetrics[m.Name] = m
+		}
+	}
+	return gotMetrics
+}
diff --git a/balancer/pickfirst/pickfirstleaf/pickfirstleaf_ext_test.go b/balancer/pickfirst/pickfirstleaf/pickfirstleaf_ext_test.go
index bf957f98..9e835a67 100644
--- a/balancer/pickfirst/pickfirstleaf/pickfirstleaf_ext_test.go
+++ b/balancer/pickfirst/pickfirstleaf/pickfirstleaf_ext_test.go
@@ -39,6 +39,7 @@ import (
 	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/internal/testutils/pickfirst"
+	"google.golang.org/grpc/internal/testutils/stats"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/resolver/manual"
 	"google.golang.org/grpc/status"
@@ -66,20 +67,54 @@ func Test(t *testing.T) {
 	grpctest.RunSubTests(t, s{})
 }
 
+// testServer is a server than can be stopped and resumed without closing
+// the listener. This guarantees the same port number (and address) is used
+// after restart. When a server is stopped, it accepts and closes all tcp
+// connections from clients.
+type testServer struct {
+	stubserver.StubServer
+	lis *testutils.RestartableListener
+}
+
+func (s *testServer) stop() {
+	s.lis.Stop()
+}
+
+func (s *testServer) resume() {
+	s.lis.Restart()
+}
+
+func newTestServer(t *testing.T) *testServer {
+	l, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("Failed to create listener: %v", err)
+	}
+	rl := testutils.NewRestartableListener(l)
+	ss := stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
+		Listener:   rl,
+	}
+	return &testServer{
+		StubServer: ss,
+		lis:        rl,
+	}
+}
+
 // setupPickFirstLeaf performs steps required for pick_first tests. It starts a
 // bunch of backends exporting the TestService, and creates a ClientConn to them.
 func setupPickFirstLeaf(t *testing.T, backendCount int, opts ...grpc.DialOption) (*grpc.ClientConn, *manual.Resolver, *backendManager) {
 	t.Helper()
 	r := manual.NewBuilderWithScheme("whatever")
-	backends := make([]*stubserver.StubServer, backendCount)
+	backends := make([]*testServer, backendCount)
 	addrs := make([]resolver.Address, backendCount)
 
 	for i := 0; i < backendCount; i++ {
-		backend := stubserver.StartTestService(t, nil)
+		server := newTestServer(t)
+		backend := stubserver.StartTestService(t, &server.StubServer)
 		t.Cleanup(func() {
 			backend.Stop()
 		})
-		backends[i] = backend
+		backends[i] = server
 		addrs[i] = resolver.Address{Addr: backend.Address}
 	}
 
@@ -263,8 +298,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_DisjointLists(t *testing.T) {
 	stateSubscriber := &ccStateSubscriber{}
 	internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber)
 
-	bm.backends[0].S.Stop()
-	bm.backends[0].S = nil
+	bm.backends[0].stop()
 	r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}})
 	var bal *stateStoringBalancer
 	select {
@@ -286,8 +320,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_DisjointLists(t *testing.T) {
 		t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
 	}
 
-	bm.backends[2].S.Stop()
-	bm.backends[2].S = nil
+	bm.backends[2].stop()
 	r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[2], addrs[3]}})
 
 	if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[3]); err != nil {
@@ -326,8 +359,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_ActiveBackendInUpdatedList(t *testing
 	stateSubscriber := &ccStateSubscriber{}
 	internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber)
 
-	bm.backends[0].S.Stop()
-	bm.backends[0].S = nil
+	bm.backends[0].stop()
 	r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}})
 	var bal *stateStoringBalancer
 	select {
@@ -349,8 +381,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_ActiveBackendInUpdatedList(t *testing
 		t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
 	}
 
-	bm.backends[2].S.Stop()
-	bm.backends[2].S = nil
+	bm.backends[2].stop()
 	r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[2], addrs[1]}})
 
 	// Verify that the ClientConn stays in READY.
@@ -390,8 +421,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_InActiveBackendInUpdatedList(t *testi
 	stateSubscriber := &ccStateSubscriber{}
 	internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber)
 
-	bm.backends[0].S.Stop()
-	bm.backends[0].S = nil
+	bm.backends[0].stop()
 	r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}})
 	var bal *stateStoringBalancer
 	select {
@@ -413,11 +443,9 @@ func (s) TestPickFirstLeaf_ResolverUpdates_InActiveBackendInUpdatedList(t *testi
 		t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
 	}
 
-	bm.backends[2].S.Stop()
-	bm.backends[2].S = nil
-	if err := bm.backends[0].StartServer(); err != nil {
-		t.Fatalf("Failed to re-start test backend: %v", err)
-	}
+	bm.backends[2].stop()
+	bm.backends[0].resume()
+
 	r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[2]}})
 
 	if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil {
@@ -455,8 +483,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_IdenticalLists(t *testing.T) {
 	stateSubscriber := &ccStateSubscriber{}
 	internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber)
 
-	bm.backends[0].S.Stop()
-	bm.backends[0].S = nil
+	bm.backends[0].stop()
 	r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}})
 	var bal *stateStoringBalancer
 	select {
@@ -553,14 +580,11 @@ func (s) TestPickFirstLeaf_StopConnectedServer_FirstServerRestart(t *testing.T)
 	}
 
 	// Shut down the connected server.
-	bm.backends[0].S.Stop()
-	bm.backends[0].S = nil
+	bm.backends[0].stop()
 	testutils.AwaitState(ctx, t, cc, connectivity.Idle)
 
 	// Start the new target server.
-	if err := bm.backends[0].StartServer(); err != nil {
-		t.Fatalf("Failed to start server: %v", err)
-	}
+	bm.backends[0].resume()
 
 	if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil {
 		t.Fatal(err)
@@ -619,14 +643,11 @@ func (s) TestPickFirstLeaf_StopConnectedServer_SecondServerRestart(t *testing.T)
 	}
 
 	// Shut down the connected server.
-	bm.backends[1].S.Stop()
-	bm.backends[1].S = nil
+	bm.backends[1].stop()
 	testutils.AwaitState(ctx, t, cc, connectivity.Idle)
 
 	// Start the new target server.
-	if err := bm.backends[1].StartServer(); err != nil {
-		t.Fatalf("Failed to start server: %v", err)
-	}
+	bm.backends[1].resume()
 
 	if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil {
 		t.Fatal(err)
@@ -691,14 +712,11 @@ func (s) TestPickFirstLeaf_StopConnectedServer_SecondServerToFirst(t *testing.T)
 	}
 
 	// Shut down the connected server.
-	bm.backends[1].S.Stop()
-	bm.backends[1].S = nil
+	bm.backends[1].stop()
 	testutils.AwaitState(ctx, t, cc, connectivity.Idle)
 
 	// Start the new target server.
-	if err := bm.backends[0].StartServer(); err != nil {
-		t.Fatalf("Failed to start server: %v", err)
-	}
+	bm.backends[0].resume()
 
 	if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil {
 		t.Fatal(err)
@@ -762,14 +780,11 @@ func (s) TestPickFirstLeaf_StopConnectedServer_FirstServerToSecond(t *testing.T)
 	}
 
 	// Shut down the connected server.
-	bm.backends[0].S.Stop()
-	bm.backends[0].S = nil
+	bm.backends[0].stop()
 	testutils.AwaitState(ctx, t, cc, connectivity.Idle)
 
 	// Start the new target server.
-	if err := bm.backends[1].StartServer(); err != nil {
-		t.Fatalf("Failed to start server: %v", err)
-	}
+	bm.backends[1].resume()
 
 	if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil {
 		t.Fatal(err)
@@ -863,10 +878,12 @@ func (s) TestPickFirstLeaf_HappyEyeballs_TF_AfterEndOfList(t *testing.T) {
 	triggerTimer, timeAfter := mockTimer()
 	pfinternal.TimeAfterFunc = timeAfter
 
+	tmr := stats.NewTestMetricsRecorder()
 	dialer := testutils.NewBlockingDialer()
 	opts := []grpc.DialOption{
 		grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pickfirstleaf.Name)),
 		grpc.WithContextDialer(dialer.DialContext),
+		grpc.WithStatsHandler(tmr),
 	}
 	cc, rb, bm := setupPickFirstLeaf(t, 3, opts...)
 	addrs := bm.resolverAddrs()
@@ -906,6 +923,7 @@ func (s) TestPickFirstLeaf_HappyEyeballs_TF_AfterEndOfList(t *testing.T) {
 
 	// First SubConn Fails.
 	holds[0].Fail(fmt.Errorf("test error"))
+	tmr.WaitForInt64CountIncr(ctx, 1)
 
 	// No TF should be reported until the first pass is complete.
 	shortCtx, shortCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
@@ -916,11 +934,24 @@ func (s) TestPickFirstLeaf_HappyEyeballs_TF_AfterEndOfList(t *testing.T) {
 	shortCtx, shortCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
 	defer shortCancel()
 	holds[2].Fail(fmt.Errorf("test error"))
+	tmr.WaitForInt64CountIncr(ctx, 1)
 	testutils.AwaitNotState(shortCtx, t, cc, connectivity.TransientFailure)
 
 	// Last SubConn fails, this should result in a TF update.
 	holds[1].Fail(fmt.Errorf("test error"))
+	tmr.WaitForInt64CountIncr(ctx, 1)
 	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	// Only connection attempt fails in this test.
+	if got, _ := tmr.Metric("grpc.lb.pick_first.connection_attempts_succeeded"); got != 0 {
+		t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.connection_attempts_succeeded", got, 0)
+	}
+	if got, _ := tmr.Metric("grpc.lb.pick_first.connection_attempts_failed"); got != 1 {
+		t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.connection_attempts_failed", got, 1)
+	}
+	if got, _ := tmr.Metric("grpc.lb.pick_first.disconnections"); got != 0 {
+		t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.disconnections", got, 0)
+	}
 }
 
 // Test verifies that pickfirst attempts to connect to the second backend once
@@ -936,10 +967,12 @@ func (s) TestPickFirstLeaf_HappyEyeballs_TriggerConnectionDelay(t *testing.T) {
 	triggerTimer, timeAfter := mockTimer()
 	pfinternal.TimeAfterFunc = timeAfter
 
+	tmr := stats.NewTestMetricsRecorder()
 	dialer := testutils.NewBlockingDialer()
 	opts := []grpc.DialOption{
 		grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pickfirstleaf.Name)),
 		grpc.WithContextDialer(dialer.DialContext),
+		grpc.WithStatsHandler(tmr),
 	}
 	cc, rb, bm := setupPickFirstLeaf(t, 2, opts...)
 	addrs := bm.resolverAddrs()
@@ -968,6 +1001,17 @@ func (s) TestPickFirstLeaf_HappyEyeballs_TriggerConnectionDelay(t *testing.T) {
 	// that the channel becomes READY.
 	holds[1].Resume()
 	testutils.AwaitState(ctx, t, cc, connectivity.Ready)
+
+	// Only connection attempt successes in this test.
+	if got, _ := tmr.Metric("grpc.lb.pick_first.connection_attempts_succeeded"); got != 1 {
+		t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.connection_attempts_succeeded", got, 1)
+	}
+	if got, _ := tmr.Metric("grpc.lb.pick_first.connection_attempts_failed"); got != 0 {
+		t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.connection_attempts_failed", got, 0)
+	}
+	if got, _ := tmr.Metric("grpc.lb.pick_first.disconnections"); got != 0 {
+		t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.disconnections", got, 0)
+	}
 }
 
 // Test tests the pickfirst balancer by causing a SubConn to fail and then
@@ -983,10 +1027,12 @@ func (s) TestPickFirstLeaf_HappyEyeballs_TF_ThenTimerFires(t *testing.T) {
 	triggerTimer, timeAfter := mockTimer()
 	pfinternal.TimeAfterFunc = timeAfter
 
+	tmr := stats.NewTestMetricsRecorder()
 	dialer := testutils.NewBlockingDialer()
 	opts := []grpc.DialOption{
 		grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pickfirstleaf.Name)),
 		grpc.WithContextDialer(dialer.DialContext),
+		grpc.WithStatsHandler(tmr),
 	}
 	cc, rb, bm := setupPickFirstLeaf(t, 3, opts...)
 	addrs := bm.resolverAddrs()
@@ -1014,6 +1060,9 @@ func (s) TestPickFirstLeaf_HappyEyeballs_TF_ThenTimerFires(t *testing.T) {
 	if holds[1].Wait(ctx) != true {
 		t.Fatalf("Timeout waiting for server %d with address %q to be contacted", 1, addrs[1])
 	}
+	if got, _ := tmr.Metric("grpc.lb.pick_first.connection_attempts_failed"); got != 1 {
+		t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.connection_attempts_failed", got, 1)
+	}
 	if holds[2].IsStarted() != false {
 		t.Fatalf("Server %d with address %q contacted unexpectedly", 2, addrs[2])
 	}
@@ -1030,13 +1079,20 @@ func (s) TestPickFirstLeaf_HappyEyeballs_TF_ThenTimerFires(t *testing.T) {
 	// that the channel becomes READY.
 	holds[1].Resume()
 	testutils.AwaitState(ctx, t, cc, connectivity.Ready)
+
+	if got, _ := tmr.Metric("grpc.lb.pick_first.connection_attempts_succeeded"); got != 1 {
+		t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.connection_attempts_succeeded", got, 1)
+	}
+	if got, _ := tmr.Metric("grpc.lb.pick_first.disconnections"); got != 0 {
+		t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.disconnections", got, 0)
+	}
 }
 
 func (s) TestPickFirstLeaf_InterleavingIPV4Preffered(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	cc := testutils.NewBalancerClientConn(t)
-	bal := balancer.Get(pickfirstleaf.Name).Build(cc, balancer.BuildOptions{})
+	bal := balancer.Get(pickfirstleaf.Name).Build(cc, balancer.BuildOptions{MetricsRecorder: &stats.NoopMetricsRecorder{}})
 	defer bal.Close()
 	ccState := balancer.ClientConnState{
 		ResolverState: resolver.State{
@@ -1049,7 +1105,7 @@ func (s) TestPickFirstLeaf_InterleavingIPV4Preffered(t *testing.T) {
 				{Addresses: []resolver.Address{{Addr: "[::FFFF:192.168.0.1]:2222"}}},
 				{Addresses: []resolver.Address{{Addr: "[0001:0001:0001:0001:0001:0001:0001:0001]:8080"}}},
 				{Addresses: []resolver.Address{{Addr: "[0002:0002:0002:0002:0002:0002:0002:0002]:8080"}}},
-				{Addresses: []resolver.Address{{Addr: "[0003:0003:0003:0003:0003:0003:0003:0003]:3333"}}},
+				{Addresses: []resolver.Address{{Addr: "[fe80::1%eth0]:3333"}}},
 				{Addresses: []resolver.Address{{Addr: "grpc.io:80"}}}, // not an IP.
 			},
 		},
@@ -1065,7 +1121,7 @@ func (s) TestPickFirstLeaf_InterleavingIPV4Preffered(t *testing.T) {
 		{Addr: "2.2.2.2:2"},
 		{Addr: "[0002:0002:0002:0002:0002:0002:0002:0002]:8080"},
 		{Addr: "3.3.3.3:3"},
-		{Addr: "[0003:0003:0003:0003:0003:0003:0003:0003]:3333"},
+		{Addr: "[fe80::1%eth0]:3333"},
 		{Addr: "[::FFFF:192.168.0.1]:2222"},
 	}
 
@@ -1082,7 +1138,7 @@ func (s) TestPickFirstLeaf_InterleavingIPv6Preffered(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	cc := testutils.NewBalancerClientConn(t)
-	bal := balancer.Get(pickfirstleaf.Name).Build(cc, balancer.BuildOptions{})
+	bal := balancer.Get(pickfirstleaf.Name).Build(cc, balancer.BuildOptions{MetricsRecorder: &stats.NoopMetricsRecorder{}})
 	defer bal.Close()
 	ccState := balancer.ClientConnState{
 		ResolverState: resolver.State{
@@ -1093,7 +1149,7 @@ func (s) TestPickFirstLeaf_InterleavingIPv6Preffered(t *testing.T) {
 				{Addresses: []resolver.Address{{Addr: "3.3.3.3:3"}}},
 				{Addresses: []resolver.Address{{Addr: "[::FFFF:192.168.0.1]:2222"}}},
 				{Addresses: []resolver.Address{{Addr: "[0002:0002:0002:0002:0002:0002:0002:0002]:2222"}}},
-				{Addresses: []resolver.Address{{Addr: "[0003:0003:0003:0003:0003:0003:0003:0003]:3333"}}},
+				{Addresses: []resolver.Address{{Addr: "[fe80::1%eth0]:3333"}}},
 				{Addresses: []resolver.Address{{Addr: "grpc.io:80"}}}, // not an IP.
 			},
 		},
@@ -1108,7 +1164,7 @@ func (s) TestPickFirstLeaf_InterleavingIPv6Preffered(t *testing.T) {
 		{Addr: "grpc.io:80"},
 		{Addr: "[0002:0002:0002:0002:0002:0002:0002:0002]:2222"},
 		{Addr: "2.2.2.2:2"},
-		{Addr: "[0003:0003:0003:0003:0003:0003:0003:0003]:3333"},
+		{Addr: "[fe80::1%eth0]:3333"},
 		{Addr: "3.3.3.3:3"},
 		{Addr: "[::FFFF:192.168.0.1]:2222"},
 	}
@@ -1126,7 +1182,7 @@ func (s) TestPickFirstLeaf_InterleavingUnknownPreffered(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	cc := testutils.NewBalancerClientConn(t)
-	bal := balancer.Get(pickfirstleaf.Name).Build(cc, balancer.BuildOptions{})
+	bal := balancer.Get(pickfirstleaf.Name).Build(cc, balancer.BuildOptions{MetricsRecorder: &stats.NoopMetricsRecorder{}})
 	defer bal.Close()
 	ccState := balancer.ClientConnState{
 		ResolverState: resolver.State{
@@ -1138,7 +1194,7 @@ func (s) TestPickFirstLeaf_InterleavingUnknownPreffered(t *testing.T) {
 				{Addresses: []resolver.Address{{Addr: "[::FFFF:192.168.0.1]:2222"}}},
 				{Addresses: []resolver.Address{{Addr: "[0001:0001:0001:0001:0001:0001:0001:0001]:8080"}}},
 				{Addresses: []resolver.Address{{Addr: "[0002:0002:0002:0002:0002:0002:0002:0002]:8080"}}},
-				{Addresses: []resolver.Address{{Addr: "[0003:0003:0003:0003:0003:0003:0003:0003]:3333"}}},
+				{Addresses: []resolver.Address{{Addr: "[fe80::1%eth0]:3333"}}},
 				{Addresses: []resolver.Address{{Addr: "example.com:80"}}}, // not an IP.
 			},
 		},
@@ -1155,7 +1211,7 @@ func (s) TestPickFirstLeaf_InterleavingUnknownPreffered(t *testing.T) {
 		{Addr: "2.2.2.2:2"},
 		{Addr: "[0002:0002:0002:0002:0002:0002:0002:0002]:8080"},
 		{Addr: "3.3.3.3:3"},
-		{Addr: "[0003:0003:0003:0003:0003:0003:0003:0003]:3333"},
+		{Addr: "[fe80::1%eth0]:3333"},
 		{Addr: "[::FFFF:192.168.0.1]:2222"},
 	}
 
@@ -1266,14 +1322,13 @@ type scState struct {
 }
 
 type backendManager struct {
-	backends []*stubserver.StubServer
+	backends []*testServer
 }
 
 func (b *backendManager) stopAllExcept(index int) {
 	for idx, b := range b.backends {
 		if idx != index {
-			b.S.Stop()
-			b.S = nil
+			b.stop()
 		}
 	}
 }
diff --git a/balancer/pickfirst/pickfirstleaf/pickfirstleaf_test.go b/balancer/pickfirst/pickfirstleaf/pickfirstleaf_test.go
index 71984a23..f269a71a 100644
--- a/balancer/pickfirst/pickfirstleaf/pickfirstleaf_test.go
+++ b/balancer/pickfirst/pickfirstleaf/pickfirstleaf_test.go
@@ -29,6 +29,7 @@ import (
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/stats"
 	"google.golang.org/grpc/resolver"
 )
 
@@ -195,7 +196,7 @@ func (s) TestPickFirstLeaf_TFPickerUpdate(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	cc := testutils.NewBalancerClientConn(t)
-	bal := pickfirstBuilder{}.Build(cc, balancer.BuildOptions{})
+	bal := pickfirstBuilder{}.Build(cc, balancer.BuildOptions{MetricsRecorder: &stats.NoopMetricsRecorder{}})
 	defer bal.Close()
 	ccState := balancer.ClientConnState{
 		ResolverState: resolver.State{
diff --git a/channelz/service/service_test.go b/channelz/service/service_test.go
index 78a5f2f8..111e5d17 100644
--- a/channelz/service/service_test.go
+++ b/channelz/service/service_test.go
@@ -22,6 +22,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"net/netip"
 	"strconv"
 	"strings"
 	"testing"
@@ -605,8 +606,8 @@ func (s) TestGetSocket(t *testing.T) {
 		lastMessageReceivedTimestamp:     time.Unix(3, 0),
 		localFlowControlWindow:           65536,
 		remoteFlowControlWindow:          1024,
-		localAddr:                        &net.TCPAddr{IP: net.ParseIP("1.0.0.1"), Port: 10001},
-		remoteAddr:                       &net.TCPAddr{IP: net.ParseIP("12.0.0.1"), Port: 10002},
+		localAddr:                        &net.TCPAddr{IP: netip.MustParseAddr("1.0.0.1").AsSlice(), Port: 10001},
+		remoteAddr:                       &net.TCPAddr{IP: netip.MustParseAddr("12.0.0.1").AsSlice(), Port: 10002},
 		remoteName:                       "remote.remote",
 	}), newSocket(czSocket{
 		streamsStarted:                   10,
@@ -637,11 +638,11 @@ func (s) TestGetSocket(t *testing.T) {
 		lastMessageReceivedTimestamp:     time.Unix(0, 0),
 		localFlowControlWindow:           65536,
 		remoteFlowControlWindow:          10240,
-		localAddr:                        &net.IPAddr{IP: net.ParseIP("1.0.0.1")},
-		remoteAddr:                       &net.IPAddr{IP: net.ParseIP("9.0.0.1")},
+		localAddr:                        &net.IPAddr{IP: netip.MustParseAddr("1.0.0.1").AsSlice()},
+		remoteAddr:                       &net.IPAddr{IP: netip.MustParseAddr("9.0.0.1").AsSlice()},
 		remoteName:                       "",
 	}), newSocket(czSocket{
-		localAddr: &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 10001},
+		localAddr: &net.TCPAddr{IP: netip.MustParseAddr("127.0.0.1").AsSlice(), Port: 10001},
 	}), newSocket(czSocket{
 		security: &credentials.TLSChannelzSecurityValue{
 			StandardName:      "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
diff --git a/grpclog/internal/loggerv2_test.go b/grpclog/internal/loggerv2_test.go
index b22ecbde..3369448b 100644
--- a/grpclog/internal/loggerv2_test.go
+++ b/grpclog/internal/loggerv2_test.go
@@ -20,11 +20,86 @@ package internal
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
+	"io"
+	"os"
+	"reflect"
 	"regexp"
+	"strings"
 	"testing"
 )
 
+// logFuncStr is a string used via testCheckLogContainsFuncStr to test the
+// logger output.
+const logFuncStr = "called-func"
+
+func makeSprintfErr(t *testing.T) func(format string, a ...any) string {
+	return func(string, ...any) string {
+		t.Errorf("got: sprintf called on io.Discard logger, want: expensive sprintf to not be called for io.Discard")
+		return ""
+	}
+}
+
+func makeSprintErr(t *testing.T) func(a ...any) string {
+	return func(...any) string {
+		t.Errorf("got: sprint called on io.Discard logger, want: expensive sprint to not be called for io.Discard")
+		return ""
+	}
+}
+
+// checkLogContainsFuncStr checks that the logger buffer logBuf contains
+// logFuncStr.
+func checkLogContainsFuncStr(t *testing.T, logBuf []byte) {
+	if !bytes.Contains(logBuf, []byte(logFuncStr)) {
+		t.Errorf("got '%v', want logger func to be called and print '%v'", string(logBuf), logFuncStr)
+	}
+}
+
+// checkBufferWasWrittenAsExpected checks that the log buffer buf was written as expected,
+// per the discard, logTYpe, msg, and isJSON arguments.
+func checkBufferWasWrittenAsExpected(t *testing.T, buf *bytes.Buffer, discard bool, logType string, msg string, isJSON bool) {
+	bts, err := buf.ReadBytes('\n')
+	if discard {
+		if err == nil {
+			t.Fatalf("got '%v', want discard %v to not write", string(bts), logType)
+		} else if err != io.EOF {
+			t.Fatalf("got '%v', want discard %v buffer to be EOF", err, logType)
+		}
+	} else {
+		if err != nil {
+			t.Fatalf("got '%v', want non-discard %v to not error", err, logType)
+		} else if !bytes.Contains(bts, []byte(msg)) {
+			t.Fatalf("got '%v', want non-discard %v buffer contain message '%v'", string(bts), logType, msg)
+		}
+		if isJSON {
+			obj := map[string]string{}
+			if err := json.Unmarshal(bts, &obj); err != nil {
+				t.Fatalf("got '%v', want non-discard json %v to unmarshal", err, logType)
+			} else if _, ok := obj["severity"]; !ok {
+				t.Fatalf("got '%v', want non-discard json %v to have severity field", "missing severity", logType)
+
+			} else if jsonMsg, ok := obj["message"]; !ok {
+				t.Fatalf("got '%v', want non-discard json %v to have message field", "missing message", logType)
+
+			} else if !strings.Contains(jsonMsg, msg) {
+				t.Fatalf("got '%v', want non-discard json %v buffer contain message '%v'", string(bts), logType, msg)
+			}
+		}
+	}
+}
+
+// check if b is in the format of:
+//
+//	2017/04/07 14:55:42 WARNING: WARNING
+func checkLogForSeverity(s int, b []byte) error {
+	expected := regexp.MustCompile(fmt.Sprintf(`^[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} %s: %s\n$`, severityName[s], severityName[s]))
+	if m := expected.Match(b); !m {
+		return fmt.Errorf("got: %v, want string in format of: %v", string(b), severityName[s]+": 2016/10/05 17:09:26 "+severityName[s])
+	}
+	return nil
+}
+
 func TestLoggerV2Severity(t *testing.T) {
 	buffers := []*bytes.Buffer{new(bytes.Buffer), new(bytes.Buffer), new(bytes.Buffer)}
 	l := NewLoggerV2(buffers[infoLog], buffers[warningLog], buffers[errorLog], LoggerV2Config{})
@@ -42,7 +117,7 @@ func TestLoggerV2Severity(t *testing.T) {
 		for j := i; j < fatalLog; j++ {
 			b, err := buf.ReadBytes('\n')
 			if err != nil {
-				t.Fatal(err)
+				t.Fatalf("level %d: %v", j, err)
 			}
 			if err := checkLogForSeverity(j, b); err != nil {
 				t.Fatal(err)
@@ -51,13 +126,225 @@ func TestLoggerV2Severity(t *testing.T) {
 	}
 }
 
-// check if b is in the format of:
-//
-//	2017/04/07 14:55:42 WARNING: WARNING
-func checkLogForSeverity(s int, b []byte) error {
-	expected := regexp.MustCompile(fmt.Sprintf(`^[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} %s: %s\n$`, severityName[s], severityName[s]))
-	if m := expected.Match(b); !m {
-		return fmt.Errorf("got: %v, want string in format of: %v", string(b), severityName[s]+": 2016/10/05 17:09:26 "+severityName[s])
+// TestLoggerV2PrintFuncDiscardOnlyInfo ensures that logs at the INFO level are
+// discarded when set to io.Discard, while logs at other levels (WARN, ERROR)
+// are still printed. It does this by using a custom error function that raises
+// an error if the logger attempts to print at the INFO level, ensuring early
+// return when io.Discard is used.
+func TestLoggerV2PrintFuncDiscardOnlyInfo(t *testing.T) {
+	buffers := []*bytes.Buffer{nil, new(bytes.Buffer), new(bytes.Buffer)}
+	logger := NewLoggerV2(io.Discard, buffers[warningLog], buffers[errorLog], LoggerV2Config{})
+	loggerTp := logger.(*loggerT)
+
+	// test that output doesn't call expensive printf funcs on an io.Discard logger
+	sprintf = makeSprintfErr(t)
+	sprint = makeSprintErr(t)
+	sprintln = makeSprintErr(t)
+
+	loggerTp.output(infoLog, "something")
+
+	sprintf = fmt.Sprintf
+	sprint = fmt.Sprint
+	sprintln = fmt.Sprintln
+
+	loggerTp.output(errorLog, logFuncStr)
+	warnB, err := buffers[warningLog].ReadBytes('\n')
+	if err != nil {
+		t.Fatalf("level %v: %v", warningLog, err)
+	}
+	checkLogContainsFuncStr(t, warnB)
+
+	errB, err := buffers[errorLog].ReadBytes('\n')
+	if err != nil {
+		t.Fatalf("level %v: %v", errorLog, err)
+	}
+	checkLogContainsFuncStr(t, errB)
+}
+
+func TestLoggerV2PrintFuncNoDiscard(t *testing.T) {
+	buffers := []*bytes.Buffer{new(bytes.Buffer), new(bytes.Buffer), new(bytes.Buffer)}
+	logger := NewLoggerV2(buffers[infoLog], buffers[warningLog], buffers[errorLog], LoggerV2Config{})
+	loggerTp := logger.(*loggerT)
+
+	loggerTp.output(errorLog, logFuncStr)
+
+	infoB, err := buffers[infoLog].ReadBytes('\n')
+	if err != nil {
+		t.Fatalf("level %v: %v", infoLog, err)
+	}
+	checkLogContainsFuncStr(t, infoB)
+
+	warnB, err := buffers[warningLog].ReadBytes('\n')
+	if err != nil {
+		t.Fatalf("level %v: %v", warningLog, err)
+	}
+	checkLogContainsFuncStr(t, warnB)
+
+	errB, err := buffers[errorLog].ReadBytes('\n')
+	if err != nil {
+		t.Fatalf("level %v: %v", errorLog, err)
+	}
+	checkLogContainsFuncStr(t, errB)
+}
+
+// TestLoggerV2PrintFuncAllDiscard tests that discard loggers don't log.
+func TestLoggerV2PrintFuncAllDiscard(t *testing.T) {
+	logger := NewLoggerV2(io.Discard, io.Discard, io.Discard, LoggerV2Config{})
+	loggerTp := logger.(*loggerT)
+
+	sprintf = makeSprintfErr(t)
+	sprint = makeSprintErr(t)
+	sprintln = makeSprintErr(t)
+
+	// test that printFunc doesn't call the log func on discard loggers
+	// makeLogFuncErr will fail the test if it's called
+	loggerTp.output(infoLog, logFuncStr)
+	loggerTp.output(warningLog, logFuncStr)
+	loggerTp.output(errorLog, logFuncStr)
+
+	sprintf = fmt.Sprintf
+	sprint = fmt.Sprint
+	sprintln = fmt.Sprintln
+}
+
+func TestLoggerV2PrintFuncAllCombinations(t *testing.T) {
+	const (
+		print int = iota
+		printf
+		println
+	)
+
+	type testDiscard struct {
+		discardInf  bool
+		discardWarn bool
+		discardErr  bool
+
+		printType  int
+		formatJSON bool
+	}
+
+	discardName := func(td testDiscard) string {
+		strs := []string{}
+		if td.discardInf {
+			strs = append(strs, "discardInfo")
+		}
+		if td.discardWarn {
+			strs = append(strs, "discardWarn")
+		}
+		if td.discardErr {
+			strs = append(strs, "discardErr")
+		}
+		if len(strs) == 0 {
+			strs = append(strs, "noDiscard")
+		}
+		return strings.Join(strs, " ")
+	}
+	var printName = []string{
+		print:   "print",
+		printf:  "printf",
+		println: "println",
+	}
+	var jsonName = map[bool]string{
+		true:  "json",
+		false: "noJson",
+	}
+
+	discardTests := []testDiscard{}
+	for _, di := range []bool{true, false} {
+		for _, dw := range []bool{true, false} {
+			for _, de := range []bool{true, false} {
+				for _, pt := range []int{print, printf, println} {
+					for _, fj := range []bool{true, false} {
+						discardTests = append(discardTests, testDiscard{discardInf: di, discardWarn: dw, discardErr: de, printType: pt, formatJSON: fj})
+					}
+				}
+			}
+		}
+	}
+
+	for _, discardTest := range discardTests {
+		testName := fmt.Sprintf("%v %v %v", jsonName[discardTest.formatJSON], printName[discardTest.printType], discardName(discardTest))
+		t.Run(testName, func(t *testing.T) {
+			cfg := LoggerV2Config{FormatJSON: discardTest.formatJSON}
+
+			buffers := []*bytes.Buffer{new(bytes.Buffer), new(bytes.Buffer), new(bytes.Buffer)}
+			writers := []io.Writer{buffers[infoLog], buffers[warningLog], buffers[errorLog]}
+			if discardTest.discardInf {
+				writers[infoLog] = io.Discard
+			}
+			if discardTest.discardWarn {
+				writers[warningLog] = io.Discard
+			}
+			if discardTest.discardErr {
+				writers[errorLog] = io.Discard
+			}
+			logger := NewLoggerV2(writers[infoLog], writers[warningLog], writers[errorLog], cfg)
+
+			msgInf := "someinfo"
+			msgWarn := "somewarn"
+			msgErr := "someerr"
+			if discardTest.printType == print {
+				logger.Info(msgInf)
+				logger.Warning(msgWarn)
+				logger.Error(msgErr)
+			} else if discardTest.printType == printf {
+				logger.Infof("%v", msgInf)
+				logger.Warningf("%v", msgWarn)
+				logger.Errorf("%v", msgErr)
+			} else if discardTest.printType == println {
+				logger.Infoln(msgInf)
+				logger.Warningln(msgWarn)
+				logger.Errorln(msgErr)
+			}
+
+			// verify the test Discard, log type, message, and json arguments were logged as-expected
+
+			checkBufferWasWrittenAsExpected(t, buffers[infoLog], discardTest.discardInf, "info", msgInf, cfg.FormatJSON)
+			checkBufferWasWrittenAsExpected(t, buffers[warningLog], discardTest.discardWarn, "warning", msgWarn, cfg.FormatJSON)
+			checkBufferWasWrittenAsExpected(t, buffers[errorLog], discardTest.discardErr, "error", msgErr, cfg.FormatJSON)
+		})
+	}
+}
+
+func TestLoggerV2Fatal(t *testing.T) {
+	const (
+		print   = "print"
+		println = "println"
+		printf  = "printf"
+	)
+	printFuncs := []string{print, println, printf}
+
+	exitCalls := []int{}
+
+	if reflect.ValueOf(exit).Pointer() != reflect.ValueOf(os.Exit).Pointer() {
+		t.Error("got: exit isn't os.Exit, want exit var to be os.Exit")
+	}
+
+	exit = func(code int) {
+		exitCalls = append(exitCalls, code)
+	}
+	defer func() {
+		exit = os.Exit
+	}()
+
+	for _, printFunc := range printFuncs {
+		buffers := []*bytes.Buffer{new(bytes.Buffer), new(bytes.Buffer), new(bytes.Buffer)}
+		logger := NewLoggerV2(buffers[infoLog], buffers[warningLog], buffers[errorLog], LoggerV2Config{})
+		switch printFunc {
+		case print:
+			logger.Fatal("fatal")
+		case println:
+			logger.Fatalln("fatalln")
+		case printf:
+			logger.Fatalf("fatalf %d", 42)
+		default:
+			t.Errorf("unknown print type '%v'", printFunc)
+		}
+
+		checkBufferWasWrittenAsExpected(t, buffers[errorLog], false, "fatal", "fatal", false)
+		if len(exitCalls) == 0 {
+			t.Error("got: no exit call, want fatal log to call exit")
+		}
+		exitCalls = []int{}
 	}
-	return nil
 }
diff --git a/internal/balancergroup/balancergroup_test.go b/internal/balancergroup/balancergroup_test.go
index c154c029..e49e8135 100644
--- a/internal/balancergroup/balancergroup_test.go
+++ b/internal/balancergroup/balancergroup_test.go
@@ -33,6 +33,7 @@ import (
 	"google.golang.org/grpc/internal/channelz"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/stats"
 	"google.golang.org/grpc/resolver"
 )
 
@@ -603,6 +604,7 @@ func (s) TestBalancerGracefulSwitch(t *testing.T) {
 	childPolicyName := t.Name()
 	stub.Register(childPolicyName, stub.BalancerFuncs{
 		Init: func(bd *stub.BalancerData) {
+			bd.BuildOptions.MetricsRecorder = &stats.NoopMetricsRecorder{}
 			bd.Data = balancer.Get(pickfirst.Name).Build(bd.ClientConn, bd.BuildOptions)
 		},
 		Close: func(bd *stub.BalancerData) {
diff --git a/internal/credentials/xds/handshake_info_test.go b/internal/credentials/xds/handshake_info_test.go
index a6802405..a3e598be 100644
--- a/internal/credentials/xds/handshake_info_test.go
+++ b/internal/credentials/xds/handshake_info_test.go
@@ -21,6 +21,7 @@ package xds
 import (
 	"crypto/x509"
 	"net"
+	"net/netip"
 	"net/url"
 	"regexp"
 	"testing"
@@ -142,8 +143,11 @@ func TestMatchingSANExists_FailureCases(t *testing.T) {
 	inputCert := &x509.Certificate{
 		DNSNames:       []string{"foo.bar.example.com", "bar.baz.test.com", "*.example.com"},
 		EmailAddresses: []string{"foobar@example.com", "barbaz@test.com"},
-		IPAddresses:    []net.IP{net.ParseIP("192.0.0.1"), net.ParseIP("2001:db8::68")},
-		URIs:           []*url.URL{url1, url2},
+		IPAddresses: []net.IP{
+			netip.MustParseAddr("192.0.0.1").AsSlice(),
+			netip.MustParseAddr("2001:db8::68").AsSlice(),
+		},
+		URIs: []*url.URL{url1, url2},
 	}
 
 	tests := []struct {
@@ -214,8 +218,11 @@ func TestMatchingSANExists_Success(t *testing.T) {
 	inputCert := &x509.Certificate{
 		DNSNames:       []string{"baz.test.com", "*.example.com"},
 		EmailAddresses: []string{"foobar@example.com", "barbaz@test.com"},
-		IPAddresses:    []net.IP{net.ParseIP("192.0.0.1"), net.ParseIP("2001:db8::68")},
-		URIs:           []*url.URL{url1, url2},
+		IPAddresses: []net.IP{
+			netip.MustParseAddr("192.0.0.1").AsSlice(),
+			netip.MustParseAddr("2001:db8::68").AsSlice(),
+		},
+		URIs: []*url.URL{url1, url2},
 	}
 
 	tests := []struct {
diff --git a/internal/resolver/dns/dns_resolver_test.go b/internal/resolver/dns/dns_resolver_test.go
index 4e6a4d54..47285052 100644
--- a/internal/resolver/dns/dns_resolver_test.go
+++ b/internal/resolver/dns/dns_resolver_test.go
@@ -755,7 +755,16 @@ func (s) TestIPResolver(t *testing.T) {
 			target:   "[2001:db8::1]:http",
 			wantAddr: []resolver.Address{{Addr: "[2001:db8::1]:http"}},
 		},
-		// TODO(yuxuanli): zone support?
+		{
+			name:     "ipv6 with zone and port",
+			target:   "[fe80::1%25eth0]:1234",
+			wantAddr: []resolver.Address{{Addr: "[fe80::1%eth0]:1234"}},
+		},
+		{
+			name:     "ipv6 with zone and default port",
+			target:   "fe80::1%25eth0",
+			wantAddr: []resolver.Address{{Addr: "[fe80::1%eth0]:443"}},
+		},
 	}
 
 	for _, test := range tests {
diff --git a/internal/testutils/fakegrpclb/server.go b/internal/testutils/fakegrpclb/server.go
index 82be2c1a..b19e863f 100644
--- a/internal/testutils/fakegrpclb/server.go
+++ b/internal/testutils/fakegrpclb/server.go
@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"net/netip"
 	"strconv"
 	"sync"
 	"time"
@@ -85,9 +86,9 @@ func NewServer(params ServerParams) (*Server, error) {
 		if err != nil {
 			return nil, fmt.Errorf("failed to parse list of backend address %q: %v", addr, err)
 		}
-		ip := net.ParseIP(ipStr)
-		if ip == nil {
-			return nil, fmt.Errorf("failed to parse ip: %q", ipStr)
+		ip, err := netip.ParseAddr(ipStr)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse ip %q: %v", ipStr, err)
 		}
 		port, err := strconv.Atoi(portStr)
 		if err != nil {
@@ -95,7 +96,7 @@ func NewServer(params ServerParams) (*Server, error) {
 		}
 		logger.Infof("Adding backend ip: %q, port: %d to server list", ip.String(), port)
 		servers = append(servers, &lbpb.Server{
-			IpAddress: ip,
+			IpAddress: ip.AsSlice(),
 			Port:      int32(port),
 		})
 	}
diff --git a/internal/testutils/stats/test_metrics_recorder.go b/internal/testutils/stats/test_metrics_recorder.go
index 72a20c1c..e1a03b8d 100644
--- a/internal/testutils/stats/test_metrics_recorder.go
+++ b/internal/testutils/stats/test_metrics_recorder.go
@@ -44,7 +44,7 @@ type TestMetricsRecorder struct {
 	// mu protects data.
 	mu sync.Mutex
 	// data is the most recent update for each metric name.
-	data map[estats.Metric]float64
+	data map[string]float64
 }
 
 // NewTestMetricsRecorder returns a new TestMetricsRecorder.
@@ -56,14 +56,16 @@ func NewTestMetricsRecorder() *TestMetricsRecorder {
 		floatHistoCh: testutils.NewChannelWithSize(10),
 		intGaugeCh:   testutils.NewChannelWithSize(10),
 
-		data: make(map[estats.Metric]float64),
+		data: make(map[string]float64),
 	}
 }
 
 // Metric returns the most recent data for a metric, and whether this recorder
 // has received data for a metric.
 func (r *TestMetricsRecorder) Metric(name string) (float64, bool) {
-	data, ok := r.data[estats.Metric(name)]
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	data, ok := r.data[name]
 	return data, ok
 }
 
@@ -71,7 +73,7 @@ func (r *TestMetricsRecorder) Metric(name string) (float64, bool) {
 func (r *TestMetricsRecorder) ClearMetrics() {
 	r.mu.Lock()
 	defer r.mu.Unlock()
-	r.data = make(map[estats.Metric]float64)
+	r.data = make(map[string]float64)
 }
 
 // MetricsData represents data associated with a metric.
@@ -102,6 +104,21 @@ func (r *TestMetricsRecorder) WaitForInt64Count(ctx context.Context, metricsData
 	return nil
 }
 
+// WaitForInt64CountIncr waits for an int64 count metric to be recorded and
+// verifies that the recorded metrics data incr matches the expected incr.
+// Returns an error if failed to wait or received wrong data.
+func (r *TestMetricsRecorder) WaitForInt64CountIncr(ctx context.Context, incrWant int64) error {
+	got, err := r.intCountCh.Receive(ctx)
+	if err != nil {
+		return fmt.Errorf("timeout waiting for int64Count")
+	}
+	metricsDataGot := got.(MetricsData)
+	if diff := cmp.Diff(metricsDataGot.IntIncr, incrWant); diff != "" {
+		return fmt.Errorf("int64count metricsData received unexpected value (-got, +want): %v", diff)
+	}
+	return nil
+}
+
 // RecordInt64Count sends the metrics data to the intCountCh channel and updates
 // the internal data map with the recorded value.
 func (r *TestMetricsRecorder) RecordInt64Count(handle *estats.Int64CountHandle, incr int64, labels ...string) {
diff --git a/security/advancedtls/crl_test.go b/security/advancedtls/crl_test.go
index e24a8f9b..450ab561 100644
--- a/security/advancedtls/crl_test.go
+++ b/security/advancedtls/crl_test.go
@@ -30,6 +30,7 @@ import (
 	"fmt"
 	"math/big"
 	"net"
+	"net/netip"
 	"os"
 	"path"
 	"strings"
@@ -463,7 +464,7 @@ func setupTLSConn(t *testing.T) (net.Listener, *x509.Certificate, *ecdsa.Private
 		Subject:               pkix.Name{CommonName: "test-cert"},
 		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
 		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
-		IPAddresses:           []net.IP{net.ParseIP("::1")},
+		IPAddresses:           []net.IP{netip.MustParseAddr("::1").AsSlice()},
 		CRLDistributionPoints: []string{"http://static.corp.google.com/crl/campus-sln/borg"},
 	}
 
diff --git a/stats/opentelemetry/example_test.go b/stats/opentelemetry/example_test.go
index 66676f3c..e87e4ebb 100644
--- a/stats/opentelemetry/example_test.go
+++ b/stats/opentelemetry/example_test.go
@@ -19,7 +19,7 @@ package opentelemetry_test
 import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
-	estats "google.golang.org/grpc/experimental/stats"
+	"google.golang.org/grpc/stats"
 	"google.golang.org/grpc/stats/opentelemetry"
 
 	"go.opentelemetry.io/otel/sdk/metric"
@@ -88,7 +88,7 @@ func ExampleMetrics_excludeSome() {
 	// To exclude specific metrics, initialize Options as follows:
 	opts := opentelemetry.Options{
 		MetricsOptions: opentelemetry.MetricsOptions{
-			Metrics: opentelemetry.DefaultMetrics().Remove(opentelemetry.ClientAttemptDuration, opentelemetry.ClientAttemptRcvdCompressedTotalMessageSize),
+			Metrics: opentelemetry.DefaultMetrics().Remove(opentelemetry.ClientAttemptDurationMetricName, opentelemetry.ClientAttemptRcvdCompressedTotalMessageSizeMetricName),
 		},
 	}
 	do := opentelemetry.DialOption(opts)
@@ -103,7 +103,7 @@ func ExampleMetrics_disableAll() {
 	// To disable all metrics, initialize Options as follows:
 	opts := opentelemetry.Options{
 		MetricsOptions: opentelemetry.MetricsOptions{
-			Metrics: estats.NewMetrics(), // Distinct to nil, which creates default metrics. This empty set creates no metrics.
+			Metrics: stats.NewMetricSet(), // Distinct to nil, which creates default metrics. This empty set creates no metrics.
 		},
 	}
 	do := opentelemetry.DialOption(opts)
@@ -118,7 +118,7 @@ func ExampleMetrics_enableSome() {
 	// To only create specific metrics, initialize Options as follows:
 	opts := opentelemetry.Options{
 		MetricsOptions: opentelemetry.MetricsOptions{
-			Metrics: estats.NewMetrics(opentelemetry.ClientAttemptDuration, opentelemetry.ClientAttemptRcvdCompressedTotalMessageSize), // only create these metrics
+			Metrics: stats.NewMetricSet(opentelemetry.ClientAttemptDurationMetricName, opentelemetry.ClientAttemptRcvdCompressedTotalMessageSizeMetricName), // only create these metrics
 		},
 	}
 	do := opentelemetry.DialOption(opts)
diff --git a/test/local_creds_test.go b/test/local_creds_test.go
index ac32ab52..241f6306 100644
--- a/test/local_creds_test.go
+++ b/test/local_creds_test.go
@@ -22,6 +22,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"net/netip"
 	"strings"
 	"testing"
 	"time"
@@ -181,11 +182,11 @@ func testLocalCredsE2EFail(dopts []grpc.DialOption) error {
 
 	var fakeClientAddr, fakeServerAddr net.Addr
 	fakeClientAddr = &net.IPAddr{
-		IP:   net.ParseIP("10.8.9.10"),
+		IP:   netip.MustParseAddr("10.8.9.10").AsSlice(),
 		Zone: "",
 	}
 	fakeServerAddr = &net.IPAddr{
-		IP:   net.ParseIP("10.8.9.11"),
+		IP:   netip.MustParseAddr("10.8.9.11").AsSlice(),
 		Zone: "",
 	}
 
diff --git a/test/xds/xds_client_ignore_resource_deletion_test.go b/test/xds/xds_client_ignore_resource_deletion_test.go
index a8078cd2..b85ec16a 100644
--- a/test/xds/xds_client_ignore_resource_deletion_test.go
+++ b/test/xds/xds_client_ignore_resource_deletion_test.go
@@ -310,6 +310,7 @@ func setupGRPCServerWithModeChangeChannelAndServe(t *testing.T, bootstrapContent
 		updateCh <- args.Mode
 	})
 	stub := &stubserver.StubServer{
+		Listener: lis,
 		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 			return &testpb.Empty{}, nil
 		},
@@ -321,17 +322,10 @@ func setupGRPCServerWithModeChangeChannelAndServe(t *testing.T, bootstrapContent
 	if err != nil {
 		t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err)
 	}
-	t.Cleanup(server.Stop)
-
 	stub.S = server
-	stubserver.StartTestService(t, stub)
+	t.Cleanup(stub.S.Stop)
 
-	// Serve.
-	go func() {
-		if err := server.Serve(lis); err != nil {
-			t.Errorf("Serve() failed: %v", err)
-		}
-	}()
+	stubserver.StartTestService(t, stub)
 
 	return updateCh
 }
diff --git a/test/xds/xds_server_certificate_providers_test.go b/test/xds/xds_server_certificate_providers_test.go
index f277db13..9fcf6f49 100644
--- a/test/xds/xds_server_certificate_providers_test.go
+++ b/test/xds/xds_server_certificate_providers_test.go
@@ -158,27 +158,21 @@ func (s) TestServerSideXDS_WithNoCertificateProvidersInBootstrap_Failure(t *test
 			close(servingModeCh)
 		}
 	})
-	server, err := xds.NewGRPCServer(grpc.Creds(creds), modeChangeOpt, xds.BootstrapContentsForTesting(bs))
-	if err != nil {
-		t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err)
-	}
-	defer server.Stop()
 
-	stub := &stubserver.StubServer{}
-	stub.S = server
-	stubserver.StartTestService(t, stub)
-
-	// Create a local listener and pass it to Serve().
+	// Create a local listener and assign it to the stub server.
 	lis, err := testutils.LocalTCPListener()
 	if err != nil {
 		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
 	}
 
-	go func() {
-		if err := server.Serve(lis); err != nil {
-			t.Errorf("Serve() failed: %v", err)
-		}
-	}()
+	stub := &stubserver.StubServer{
+		Listener: lis,
+	}
+	if stub.S, err = xds.NewGRPCServer(grpc.Creds(creds), modeChangeOpt, xds.BootstrapContentsForTesting(bs)); err != nil {
+		t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err)
+	}
+	defer stub.S.Stop()
+	stubserver.StartTestService(t, stub)
 
 	// Create an inbound xDS listener resource for the server side that contains
 	// mTLS security configuration. Since the received certificate provider
@@ -288,30 +282,10 @@ func (s) TestServerSideXDS_WithValidAndInvalidSecurityConfiguration(t *testing.T
 		}
 	})
 
-	stub := &stubserver.StubServer{
-		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
-			return &testpb.Empty{}, nil
-		},
-	}
-	server, err := xds.NewGRPCServer(grpc.Creds(creds), modeChangeOpt, xds.BootstrapContentsForTesting(bootstrapContents))
-	if err != nil {
-		t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err)
-	}
-	defer server.Stop()
-
-	stub.S = server
-	stubserver.StartTestService(t, stub)
-
-	go func() {
-		if err := server.Serve(lis1); err != nil {
-			t.Errorf("Serve() failed: %v", err)
-		}
-	}()
-	go func() {
-		if err := server.Serve(lis2); err != nil {
-			t.Errorf("Serve() failed: %v", err)
-		}
-	}()
+	stub1 := createStubServer(t, lis1, creds, modeChangeOpt, bootstrapContents)
+	defer stub1.S.Stop()
+	stub2 := createStubServer(t, lis2, creds, modeChangeOpt, bootstrapContents)
+	defer stub2.S.Stop()
 
 	// Create inbound xDS listener resources for the server side that contains
 	// mTLS security configuration.
diff --git a/test/xds/xds_server_integration_test.go b/test/xds/xds_server_integration_test.go
index eacc6463..054e1fc7 100644
--- a/test/xds/xds_server_integration_test.go
+++ b/test/xds/xds_server_integration_test.go
@@ -111,12 +111,10 @@ func setupGRPCServer(t *testing.T, bootstrapContents []byte) (net.Listener, func
 		},
 	}
 
-	server, err := xds.NewGRPCServer(grpc.Creds(creds), testModeChangeServerOption(t), xds.BootstrapContentsForTesting(bootstrapContents))
-	if err != nil {
+	if stub.S, err = xds.NewGRPCServer(grpc.Creds(creds), testModeChangeServerOption(t), xds.BootstrapContentsForTesting(bootstrapContents)); err != nil {
 		t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err)
 	}
 
-	stub.S = server
 	stubserver.StartTestService(t, stub)
 
 	// Create a local listener and pass it to Serve().
@@ -131,7 +129,7 @@ func setupGRPCServer(t *testing.T, bootstrapContents []byte) (net.Listener, func
 	}
 
 	go func() {
-		if err := server.Serve(readyLis); err != nil {
+		if err := stub.S.Serve(readyLis); err != nil {
 			t.Errorf("Serve() failed: %v", err)
 		}
 	}()
@@ -144,7 +142,7 @@ func setupGRPCServer(t *testing.T, bootstrapContents []byte) (net.Listener, func
 	}
 
 	return lis, func() {
-		server.Stop()
+		stub.S.Stop()
 	}
 }
 
diff --git a/test/xds/xds_server_serving_mode_test.go b/test/xds/xds_server_serving_mode_test.go
index 3ed6750a..0299d695 100644
--- a/test/xds/xds_server_serving_mode_test.go
+++ b/test/xds/xds_server_serving_mode_test.go
@@ -27,6 +27,7 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
 	xdscreds "google.golang.org/grpc/credentials/xds"
 	"google.golang.org/grpc/internal/stubserver"
@@ -65,19 +66,8 @@ func (s) TestServerSideXDS_RedundantUpdateSuppression(t *testing.T) {
 
 	// Initialize a test gRPC server, assign it to the stub server, and start
 	// the test service.
-	stub := &stubserver.StubServer{
-		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
-			return &testpb.Empty{}, nil
-		},
-	}
-	server, err := xds.NewGRPCServer(grpc.Creds(creds), modeChangeOpt, xds.BootstrapContentsForTesting(bootstrapContents))
-	if err != nil {
-		t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err)
-	}
-	defer server.Stop()
-
-	stub.S = server
-	stubserver.StartTestService(t, stub)
+	stub := createStubServer(t, lis, creds, modeChangeOpt, bootstrapContents)
+	defer stub.S.Stop()
 
 	// Setup the management server to respond with the listener resources.
 	host, port, err := hostPortFromListener(lis)
@@ -95,12 +85,6 @@ func (s) TestServerSideXDS_RedundantUpdateSuppression(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	go func() {
-		if err := server.Serve(lis); err != nil {
-			t.Errorf("Serve() failed: %v", err)
-		}
-	}()
-
 	// Wait for the listener to move to "serving" mode.
 	select {
 	case <-ctx.Done():
@@ -217,19 +201,10 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
 
 	// Initialize a test gRPC server, assign it to the stub server, and start
 	// the test service.
-	stub := &stubserver.StubServer{
-		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
-			return &testpb.Empty{}, nil
-		},
-	}
-	server, err := xds.NewGRPCServer(grpc.Creds(creds), modeChangeOpt, xds.BootstrapContentsForTesting(bootstrapContents))
-	if err != nil {
-		t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err)
-	}
-	defer server.Stop()
-
-	stub.S = server
-	stubserver.StartTestService(t, stub)
+	stub1 := createStubServer(t, lis1, creds, modeChangeOpt, bootstrapContents)
+	defer stub1.S.Stop()
+	stub2 := createStubServer(t, lis2, creds, modeChangeOpt, bootstrapContents)
+	defer stub2.S.Stop()
 
 	// Setup the management server to respond with server-side Listener
 	// resources for both listeners.
@@ -251,17 +226,6 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	go func() {
-		if err := server.Serve(lis1); err != nil {
-			t.Errorf("Serve() failed: %v", err)
-		}
-	}()
-	go func() {
-		if err := server.Serve(lis2); err != nil {
-			t.Errorf("Serve() failed: %v", err)
-		}
-	}()
-
 	// Wait for both listeners to move to "serving" mode.
 	select {
 	case <-ctx.Done():
@@ -384,6 +348,22 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
 	waitForSuccessfulRPC(ctx, t, cc2)
 }
 
+func createStubServer(t *testing.T, lis net.Listener, creds credentials.TransportCredentials, modeChangeOpt grpc.ServerOption, bootstrapContents []byte) *stubserver.StubServer {
+	stub := &stubserver.StubServer{
+		Listener: lis,
+		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+	}
+	server, err := xds.NewGRPCServer(grpc.Creds(creds), modeChangeOpt, xds.BootstrapContentsForTesting(bootstrapContents))
+	if err != nil {
+		t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err)
+	}
+	stub.S = server
+	stubserver.StartTestService(t, stub)
+	return stub
+}
+
 func waitForSuccessfulRPC(ctx context.Context, t *testing.T, cc *grpc.ClientConn) {
 	t.Helper()
 
diff --git a/test/xds/xds_server_test.go b/test/xds/xds_server_test.go
index 6912757e..bee9d401 100644
--- a/test/xds/xds_server_test.go
+++ b/test/xds/xds_server_test.go
@@ -30,6 +30,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/internal/testutils"
@@ -93,26 +94,15 @@ func (s) TestServeLDSRDS(t *testing.T) {
 			serving.Fire()
 		}
 	})
-
-	stub := &stubserver.StubServer{
-		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
-			return &testpb.Empty{}, nil
-		},
-	}
-	server, err := xds.NewGRPCServer(grpc.Creds(insecure.NewCredentials()), modeChangeOpt, xds.BootstrapContentsForTesting(bootstrapContents))
+	// Configure xDS credentials with an insecure fallback to be used on the
+	// server-side.
+	creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{FallbackCreds: insecure.NewCredentials()})
 	if err != nil {
-		t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err)
+		t.Fatalf("failed to create server credentials: %v", err)
 	}
-	defer server.Stop()
-
-	stub.S = server
-	stubserver.StartTestService(t, stub)
+	stub := createStubServer(t, lis, creds, modeChangeOpt, bootstrapContents)
+	defer stub.S.Stop()
 
-	go func() {
-		if err := server.Serve(lis); err != nil {
-			t.Errorf("Serve() failed: %v", err)
-		}
-	}()
 	select {
 	case <-ctx.Done():
 		t.Fatal("timeout waiting for the xDS Server to go Serving")
@@ -210,25 +200,15 @@ func (s) TestRDSNack(t *testing.T) {
 		}
 	})
 
-	stub := &stubserver.StubServer{
-		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
-			return &testpb.Empty{}, nil
-		},
-	}
-	server, err := xds.NewGRPCServer(grpc.Creds(insecure.NewCredentials()), modeChangeOpt, xds.BootstrapContentsForTesting(bootstrapContents))
+	// Configure xDS credentials with an insecure fallback to be used on the
+	// server-side.
+	creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{FallbackCreds: insecure.NewCredentials()})
 	if err != nil {
-		t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err)
+		t.Fatalf("failed to create server credentials: %v", err)
 	}
-	defer server.Stop()
 
-	stub.S = server
-	stubserver.StartTestService(t, stub)
-
-	go func() {
-		if err := server.Serve(lis); err != nil {
-			t.Errorf("Serve() failed: %v", err)
-		}
-	}()
+	stub := createStubServer(t, lis, creds, modeChangeOpt, bootstrapContents)
+	defer stub.S.Stop()
 
 	cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
@@ -278,6 +258,7 @@ func (s) TestMultipleUpdatesImmediatelySwitch(t *testing.T) {
 	}
 
 	stub := &stubserver.StubServer{
+		Listener: lis,
 		EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) {
 			return &testpb.Empty{}, nil
 		},
@@ -291,21 +272,12 @@ func (s) TestMultipleUpdatesImmediatelySwitch(t *testing.T) {
 		},
 	}
 
-	server, err := xds.NewGRPCServer(grpc.Creds(insecure.NewCredentials()), testModeChangeServerOption(t), xds.BootstrapContentsForTesting(bootstrapContents))
-	if err != nil {
+	if stub.S, err = xds.NewGRPCServer(grpc.Creds(insecure.NewCredentials()), testModeChangeServerOption(t), xds.BootstrapContentsForTesting(bootstrapContents)); err != nil {
 		t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err)
 	}
-	defer server.Stop()
-
-	stub.S = server
+	defer stub.S.Stop()
 	stubserver.StartTestService(t, stub)
 
-	go func() {
-		if err := server.Serve(lis); err != nil {
-			t.Errorf("Serve() failed: %v", err)
-		}
-	}()
-
 	cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
 		t.Fatalf("failed to dial local test server: %v", err)
diff --git a/xds/internal/balancer/clustermanager/clustermanager_test.go b/xds/internal/balancer/clustermanager/clustermanager_test.go
index b606cb9e..6ef8738d 100644
--- a/xds/internal/balancer/clustermanager/clustermanager_test.go
+++ b/xds/internal/balancer/clustermanager/clustermanager_test.go
@@ -34,6 +34,7 @@ import (
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/hierarchy"
 	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/stats"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/status"
 )
@@ -643,6 +644,7 @@ func TestClusterGracefulSwitch(t *testing.T) {
 	childPolicyName := t.Name()
 	stub.Register(childPolicyName, stub.BalancerFuncs{
 		Init: func(bd *stub.BalancerData) {
+			bd.BuildOptions.MetricsRecorder = &stats.NoopMetricsRecorder{}
 			bd.Data = balancer.Get(pickfirst.Name).Build(bd.ClientConn, bd.BuildOptions)
 		},
 		Close: func(bd *stub.BalancerData) {
diff --git a/xds/internal/xdsclient/xdsresource/filter_chain_test.go b/xds/internal/xdsclient/xdsresource/filter_chain_test.go
index 1897e387..cdcc8660 100644
--- a/xds/internal/xdsclient/xdsresource/filter_chain_test.go
+++ b/xds/internal/xdsclient/xdsresource/filter_chain_test.go
@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"net/netip"
 	"strings"
 	"testing"
 
@@ -2438,7 +2439,7 @@ func (s) TestLookup_Successes(t *testing.T) {
 			lis:  lisWithoutDefaultChain,
 			params: FilterChainLookupParams{
 				IsUnspecifiedListener: true,
-				DestAddr:              net.ParseIP("2001:68::db8"),
+				DestAddr:              netip.MustParseAddr("2001:68::db8").AsSlice(),
 				SourceAddr:            net.IPv4(10, 1, 1, 1),
 				SourcePort:            1,
 			},
@@ -2468,8 +2469,8 @@ func (s) TestLookup_Successes(t *testing.T) {
 			lis:  lisWithoutDefaultChain,
 			params: FilterChainLookupParams{
 				IsUnspecifiedListener: true,
-				DestAddr:              net.ParseIP("2001:68::1"),
-				SourceAddr:            net.ParseIP("2001:68::2"),
+				DestAddr:              netip.MustParseAddr("2001:68::1").AsSlice(),
+				SourceAddr:            netip.MustParseAddr("2001:68::2").AsSlice(),
 				SourcePort:            1,
 			},
 			wantFC: &FilterChain{