Repository: grpc/grpc-go

Issue:
Deprecate `stats` package trace methods
Due to https://opentelemetry.io/blog/2023/sunsetting-opencensus, we need to migrate the gRPC instrumentation to OpenTelemetry while still supporting OpenCensus until everyone migrates to OpenTelemetry which has the identical trace context format but is injected/extracted directly to/from metadata.

Considering that `grpc-trace-bin` is not a reserved header, both OpenCensus and OpenTelemetry can be allowed to directly set/get traces to/from metadata and we can mark the trace methods of `stats` package deprecated and recommend to use the metadata directly. While marking it deprecated, we can modify them to use the metadata directly for consistency.

Open Questions
- Should we remove all usages of trace methods of `stats` packages? or keep using them internally for now in opencensus and tests
- What about tags methods? Should they be deprecated as well as `grpc-tags-bin` 


Discussion:
@purnesh42H: @dfawley @dfawley: > Should we remove all usages of trace methods of stats packages? or keep using them internally for now in opencensus and tests

Yes, we should stop calling it entirely.

> What about tags methods? Should they be deprecated as well as grpc-tags-bin

Yes, that too.

Solution patch:
diff --git a/codec.go b/codec.go
index e840858b..959c2f99 100644
--- a/codec.go
+++ b/codec.go
@@ -71,7 +71,7 @@ func (c codecV0Bridge) Marshal(v any) (mem.BufferSlice, error) {
 	if err != nil {
 		return nil, err
 	}
-	return mem.BufferSlice{mem.NewBuffer(&data, nil)}, nil
+	return mem.BufferSlice{mem.SliceBuffer(data)}, nil
 }
 
 func (c codecV0Bridge) Unmarshal(data mem.BufferSlice, v any) (err error) {
diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go
index f0c5cbc4..fb5b0c13 100644
--- a/internal/transport/http2_client.go
+++ b/internal/transport/http2_client.go
@@ -599,12 +599,6 @@ func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr)
 	for k, v := range callAuthData {
 		headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)})
 	}
-	if b := stats.OutgoingTags(ctx); b != nil {
-		headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-tags-bin", Value: encodeBinHeader(b)})
-	}
-	if b := stats.OutgoingTrace(ctx); b != nil {
-		headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-trace-bin", Value: encodeBinHeader(b)})
-	}
 
 	if md, added, ok := metadataFromOutgoingContextRaw(ctx); ok {
 		var k string
diff --git a/internal/transport/http2_server.go b/internal/transport/http2_server.go
index f6faa29b..fefd9020 100644
--- a/internal/transport/http2_server.go
+++ b/internal/transport/http2_server.go
@@ -539,12 +539,6 @@ func (t *http2Server) operateHeaders(ctx context.Context, frame *http2.MetaHeade
 	// Attach the received metadata to the context.
 	if len(mdata) > 0 {
 		s.ctx = metadata.NewIncomingContext(s.ctx, mdata)
-		if statsTags := mdata["grpc-tags-bin"]; len(statsTags) > 0 {
-			s.ctx = stats.SetIncomingTags(s.ctx, []byte(statsTags[len(statsTags)-1]))
-		}
-		if statsTrace := mdata["grpc-trace-bin"]; len(statsTrace) > 0 {
-			s.ctx = stats.SetIncomingTrace(s.ctx, []byte(statsTrace[len(statsTrace)-1]))
-		}
 	}
 	t.mu.Lock()
 	if t.state != reachable {
diff --git a/internal/xds/matcher/matcher_header.go b/internal/xds/matcher/matcher_header.go
index 1893526c..780257ec 100644
--- a/internal/xds/matcher/matcher_header.go
+++ b/internal/xds/matcher/matcher_header.go
@@ -36,11 +36,11 @@ type HeaderMatcher interface {
 	String() string
 }
 
-// mdValuesFromOutgoingCtx retrieves metadata from context. If there are
+// valueFromMD retrieves metadata from context. If there are
 // multiple values, the values are concatenated with "," (comma and no space).
 //
 // All header matchers only match against the comma-concatenated string.
-func mdValuesFromOutgoingCtx(md metadata.MD, key string) (string, bool) {
+func valueFromMD(md metadata.MD, key string) (string, bool) {
 	vs, ok := md[key]
 	if !ok {
 		return "", false
@@ -63,7 +63,7 @@ func NewHeaderExactMatcher(key, exact string, invert bool) *HeaderExactMatcher {
 // Match returns whether the passed in HTTP Headers match according to the
 // HeaderExactMatcher.
 func (hem *HeaderExactMatcher) Match(md metadata.MD) bool {
-	v, ok := mdValuesFromOutgoingCtx(md, hem.key)
+	v, ok := valueFromMD(md, hem.key)
 	if !ok {
 		return false
 	}
@@ -90,7 +90,7 @@ func NewHeaderRegexMatcher(key string, re *regexp.Regexp, invert bool) *HeaderRe
 // Match returns whether the passed in HTTP Headers match according to the
 // HeaderRegexMatcher.
 func (hrm *HeaderRegexMatcher) Match(md metadata.MD) bool {
-	v, ok := mdValuesFromOutgoingCtx(md, hrm.key)
+	v, ok := valueFromMD(md, hrm.key)
 	if !ok {
 		return false
 	}
@@ -117,7 +117,7 @@ func NewHeaderRangeMatcher(key string, start, end int64, invert bool) *HeaderRan
 // Match returns whether the passed in HTTP Headers match according to the
 // HeaderRangeMatcher.
 func (hrm *HeaderRangeMatcher) Match(md metadata.MD) bool {
-	v, ok := mdValuesFromOutgoingCtx(md, hrm.key)
+	v, ok := valueFromMD(md, hrm.key)
 	if !ok {
 		return false
 	}
@@ -149,7 +149,7 @@ func NewHeaderPresentMatcher(key string, present bool, invert bool) *HeaderPrese
 // Match returns whether the passed in HTTP Headers match according to the
 // HeaderPresentMatcher.
 func (hpm *HeaderPresentMatcher) Match(md metadata.MD) bool {
-	vs, ok := mdValuesFromOutgoingCtx(md, hpm.key)
+	vs, ok := valueFromMD(md, hpm.key)
 	present := ok && len(vs) > 0 // TODO: Are we sure we need this len(vs) > 0?
 	return present == hpm.present
 }
@@ -174,7 +174,7 @@ func NewHeaderPrefixMatcher(key string, prefix string, invert bool) *HeaderPrefi
 // Match returns whether the passed in HTTP Headers match according to the
 // HeaderPrefixMatcher.
 func (hpm *HeaderPrefixMatcher) Match(md metadata.MD) bool {
-	v, ok := mdValuesFromOutgoingCtx(md, hpm.key)
+	v, ok := valueFromMD(md, hpm.key)
 	if !ok {
 		return false
 	}
@@ -201,7 +201,7 @@ func NewHeaderSuffixMatcher(key string, suffix string, invert bool) *HeaderSuffi
 // Match returns whether the passed in HTTP Headers match according to the
 // HeaderSuffixMatcher.
 func (hsm *HeaderSuffixMatcher) Match(md metadata.MD) bool {
-	v, ok := mdValuesFromOutgoingCtx(md, hsm.key)
+	v, ok := valueFromMD(md, hsm.key)
 	if !ok {
 		return false
 	}
@@ -231,7 +231,7 @@ func NewHeaderContainsMatcher(key string, contains string, invert bool) *HeaderC
 // Match returns whether the passed in HTTP Headers match according to the
 // HeaderContainsMatcher.
 func (hcm *HeaderContainsMatcher) Match(md metadata.MD) bool {
-	v, ok := mdValuesFromOutgoingCtx(md, hcm.key)
+	v, ok := valueFromMD(md, hcm.key)
 	if !ok {
 		return false
 	}
@@ -262,7 +262,7 @@ func NewHeaderStringMatcher(key string, sm StringMatcher, invert bool) *HeaderSt
 // Match returns whether the passed in HTTP Headers match according to the
 // specified StringMatcher.
 func (hsm *HeaderStringMatcher) Match(md metadata.MD) bool {
-	v, ok := mdValuesFromOutgoingCtx(md, hsm.key)
+	v, ok := valueFromMD(md, hsm.key)
 	if !ok {
 		return false
 	}
diff --git a/preloader.go b/preloader.go
index e87a17f3..ee0ff969 100644
--- a/preloader.go
+++ b/preloader.go
@@ -62,7 +62,7 @@ func (p *PreparedMsg) Encode(s Stream, msg any) error {
 
 	materializedData := data.Materialize()
 	data.Free()
-	p.encodedData = mem.BufferSlice{mem.NewBuffer(&materializedData, nil)}
+	p.encodedData = mem.BufferSlice{mem.SliceBuffer(materializedData)}
 
 	// TODO: it should be possible to grab the bufferPool from the underlying
 	//  stream implementation with a type cast to its actual type (such as
@@ -76,7 +76,7 @@ func (p *PreparedMsg) Encode(s Stream, msg any) error {
 	if p.pf.isCompressed() {
 		materializedCompData := compData.Materialize()
 		compData.Free()
-		compData = mem.BufferSlice{mem.NewBuffer(&materializedCompData, nil)}
+		compData = mem.BufferSlice{mem.SliceBuffer(materializedCompData)}
 	}
 
 	p.hdr, p.payload = msgHeader(p.encodedData, compData, p.pf)
diff --git a/rpc_util.go b/rpc_util.go
index 06c1f1b2..7bcc5590 100644
--- a/rpc_util.go
+++ b/rpc_util.go
@@ -841,7 +841,7 @@ func recvAndDecompress(p *parser, s recvCompressor, dc Decompressor, maxReceiveM
 			var uncompressedBuf []byte
 			uncompressedBuf, err = dc.Do(compressed.Reader())
 			if err == nil {
-				out = mem.BufferSlice{mem.NewBuffer(&uncompressedBuf, nil)}
+				out = mem.BufferSlice{mem.SliceBuffer(uncompressedBuf)}
 			}
 			size = len(uncompressedBuf)
 		} else {
diff --git a/stats/opencensus/stats.go b/stats/opencensus/stats.go
index 017f39b3..fecf55a6 100644
--- a/stats/opencensus/stats.go
+++ b/stats/opencensus/stats.go
@@ -29,6 +29,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/stats"
 	"google.golang.org/grpc/status"
 )
@@ -86,7 +87,7 @@ func (csh *clientStatsHandler) statsTagRPC(ctx context.Context, info *stats.RPCT
 
 	// Populate gRPC Metadata with OpenCensus tag map if set by application.
 	if tm := tag.FromContext(ctx); tm != nil {
-		ctx = stats.SetTags(ctx, tag.Encode(tm))
+		ctx = metadata.AppendToOutgoingContext(ctx, "grpc-tags-bin", string(tag.Encode(tm)))
 	}
 	return ctx, mi
 }
@@ -94,18 +95,21 @@ func (csh *clientStatsHandler) statsTagRPC(ctx context.Context, info *stats.RPCT
 // statsTagRPC creates a recording object to derive measurements from in the
 // context, scoping the recordings to per RPC server side (scope of the
 // context). It also deserializes the opencensus tags set in the context's gRPC
-// Metadata, and adds a server method tag to the opencensus tags.
+// Metadata, and adds a server method tag to the opencensus tags. If multiple
+// tags exist, it adds the last one.
 func (ssh *serverStatsHandler) statsTagRPC(ctx context.Context, info *stats.RPCTagInfo) (context.Context, *metricsInfo) {
 	mi := &metricsInfo{
 		startTime: time.Now(),
 		method:    info.FullMethodName,
 	}
 
-	if tagsBin := stats.Tags(ctx); tagsBin != nil {
+	if tgValues := metadata.ValueFromIncomingContext(ctx, "grpc-tags-bin"); len(tgValues) > 0 {
+		tagsBin := []byte(tgValues[len(tgValues)-1])
 		if tags, err := tag.Decode(tagsBin); err == nil {
 			ctx = tag.NewContext(ctx, tags)
 		}
 	}
+
 	// We can ignore the error here because in the error case, the context
 	// passed in is returned. If the call errors, the server side application
 	// layer won't get this key server method information in the tag map, but
diff --git a/stats/opencensus/trace.go b/stats/opencensus/trace.go
index 56f824f7..55a058e0 100644
--- a/stats/opencensus/trace.go
+++ b/stats/opencensus/trace.go
@@ -24,6 +24,7 @@ import (
 	"go.opencensus.io/trace"
 	"go.opencensus.io/trace/propagation"
 
+	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/stats"
 	"google.golang.org/grpc/status"
 )
@@ -46,7 +47,7 @@ func (csh *clientStatsHandler) traceTagRPC(ctx context.Context, rti *stats.RPCTa
 	_, span := trace.StartSpan(ctx, mn, trace.WithSampler(csh.to.TS))
 
 	tcBin := propagation.Binary(span.SpanContext())
-	return stats.SetTrace(ctx, tcBin), &traceInfo{
+	return metadata.AppendToOutgoingContext(ctx, "grpc-trace-bin", string(tcBin)), &traceInfo{
 		span:         span,
 		countSentMsg: 0, // msg events scoped to scope of context, per attempt client side
 		countRecvMsg: 0,
@@ -55,12 +56,17 @@ func (csh *clientStatsHandler) traceTagRPC(ctx context.Context, rti *stats.RPCTa
 
 // traceTagRPC populates context with new span data, with a parent based on the
 // spanContext deserialized from context passed in (wire data in gRPC metadata)
-// if present.
+// if present. If multiple spanContexts exist, it takes the last one.
 func (ssh *serverStatsHandler) traceTagRPC(ctx context.Context, rti *stats.RPCTagInfo) (context.Context, *traceInfo) {
 	mn := strings.ReplaceAll(removeLeadingSlash(rti.FullMethodName), "/", ".")
 
+	var tcBin []byte
+	if tcValues := metadata.ValueFromIncomingContext(ctx, "grpc-trace-bin"); len(tcValues) > 0 {
+		tcBin = []byte(tcValues[len(tcValues)-1])
+	}
+
 	var span *trace.Span
-	if sc, ok := propagation.FromBinary(stats.Trace(ctx)); ok {
+	if sc, ok := propagation.FromBinary(tcBin); ok {
 		// Returned context is ignored because will populate context with data
 		// that wraps the span instead.
 		_, span = trace.StartSpanWithRemoteParent(ctx, mn, sc, trace.WithSpanKind(trace.SpanKindServer), trace.WithSampler(ssh.to.TS))
diff --git a/stats/stats.go b/stats/stats.go
index 71195c49..6f20d2d5 100644
--- a/stats/stats.go
+++ b/stats/stats.go
@@ -260,84 +260,42 @@ func (s *ConnEnd) IsClient() bool { return s.Client }
 
 func (s *ConnEnd) isConnStats() {}
 
-type incomingTagsKey struct{}
-type outgoingTagsKey struct{}
-
 // SetTags attaches stats tagging data to the context, which will be sent in
 // the outgoing RPC with the header grpc-tags-bin.  Subsequent calls to
 // SetTags will overwrite the values from earlier calls.
 //
-// NOTE: this is provided only for backward compatibility with existing clients
-// and will likely be removed in an upcoming release.  New uses should transmit
-// this type of data using metadata with a different, non-reserved (i.e. does
-// not begin with "grpc-") header name.
+// Deprecated: set the `grpc-tags-bin` header in the metadata instead.
 func SetTags(ctx context.Context, b []byte) context.Context {
-	return context.WithValue(ctx, outgoingTagsKey{}, b)
+	return metadata.AppendToOutgoingContext(ctx, "grpc-tags-bin", string(b))
 }
 
 // Tags returns the tags from the context for the inbound RPC.
 //
-// NOTE: this is provided only for backward compatibility with existing clients
-// and will likely be removed in an upcoming release.  New uses should transmit
-// this type of data using metadata with a different, non-reserved (i.e. does
-// not begin with "grpc-") header name.
+// Deprecated: obtain the `grpc-tags-bin` header from metadata instead.
 func Tags(ctx context.Context) []byte {
-	b, _ := ctx.Value(incomingTagsKey{}).([]byte)
-	return b
-}
-
-// SetIncomingTags attaches stats tagging data to the context, to be read by
-// the application (not sent in outgoing RPCs).
-//
-// This is intended for gRPC-internal use ONLY.
-func SetIncomingTags(ctx context.Context, b []byte) context.Context {
-	return context.WithValue(ctx, incomingTagsKey{}, b)
-}
-
-// OutgoingTags returns the tags from the context for the outbound RPC.
-//
-// This is intended for gRPC-internal use ONLY.
-func OutgoingTags(ctx context.Context) []byte {
-	b, _ := ctx.Value(outgoingTagsKey{}).([]byte)
-	return b
+	traceValues := metadata.ValueFromIncomingContext(ctx, "grpc-tags-bin")
+	if len(traceValues) == 0 {
+		return nil
+	}
+	return []byte(traceValues[len(traceValues)-1])
 }
 
-type incomingTraceKey struct{}
-type outgoingTraceKey struct{}
-
 // SetTrace attaches stats tagging data to the context, which will be sent in
 // the outgoing RPC with the header grpc-trace-bin.  Subsequent calls to
 // SetTrace will overwrite the values from earlier calls.
 //
-// NOTE: this is provided only for backward compatibility with existing clients
-// and will likely be removed in an upcoming release.  New uses should transmit
-// this type of data using metadata with a different, non-reserved (i.e. does
-// not begin with "grpc-") header name.
+// Deprecated: set the `grpc-trace-bin` header in the metadata instead.
 func SetTrace(ctx context.Context, b []byte) context.Context {
-	return context.WithValue(ctx, outgoingTraceKey{}, b)
+	return metadata.AppendToOutgoingContext(ctx, "grpc-trace-bin", string(b))
 }
 
 // Trace returns the trace from the context for the inbound RPC.
 //
-// NOTE: this is provided only for backward compatibility with existing clients
-// and will likely be removed in an upcoming release.  New uses should transmit
-// this type of data using metadata with a different, non-reserved (i.e. does
-// not begin with "grpc-") header name.
+// Deprecated: obtain the `grpc-trace-bin` header from metadata instead.
 func Trace(ctx context.Context) []byte {
-	b, _ := ctx.Value(incomingTraceKey{}).([]byte)
-	return b
-}
-
-// SetIncomingTrace attaches stats tagging data to the context, to be read by
-// the application (not sent in outgoing RPCs).  It is intended for
-// gRPC-internal use.
-func SetIncomingTrace(ctx context.Context, b []byte) context.Context {
-	return context.WithValue(ctx, incomingTraceKey{}, b)
-}
-
-// OutgoingTrace returns the trace from the context for the outbound RPC.  It is
-// intended for gRPC-internal use.
-func OutgoingTrace(ctx context.Context) []byte {
-	b, _ := ctx.Value(outgoingTraceKey{}).([]byte)
-	return b
+	traceValues := metadata.ValueFromIncomingContext(ctx, "grpc-trace-bin")
+	if len(traceValues) == 0 {
+		return nil
+	}
+	return []byte(traceValues[len(traceValues)-1])
 }
diff --git a/xds/internal/balancer/clusterimpl/clusterimpl.go b/xds/internal/balancer/clusterimpl/clusterimpl.go
index 7f4218d1..71a4c9c9 100644
--- a/xds/internal/balancer/clusterimpl/clusterimpl.go
+++ b/xds/internal/balancer/clusterimpl/clusterimpl.go
@@ -107,6 +107,7 @@ type clusterImplBalancer struct {
 	mu                    sync.Mutex
 	clusterName           string                            // The cluster name for credentials handshaking.
 	inhibitPickerUpdates  bool                              // Inhibits state updates from child policy when processing an update from the parent.
+	pendingPickerUpdates  bool                              // True if a picker update from the child policy was inhibited when processing an update from the parent.
 	childState            balancer.State                    // Most recent state update from the child policy.
 	drops                 []*dropper                        // Drops implementation.
 	requestCounterCluster string                            // The cluster name for the request counter, from LB config.
@@ -293,7 +294,15 @@ func (b *clusterImplBalancer) UpdateClientConnState(s balancer.ClientConnState)
 
 	b.mu.Lock()
 	b.telemetryLabels = newConfig.TelemetryLabels
-	if b.handleDropAndRequestCountLocked(newConfig) && b.childState.Picker != nil {
+	// We want to send a picker update to the parent if one of the two
+	// conditions are met:
+	// - drop/request config has changed *and* there is already a picker from
+	//   the child, or
+	// - there is a pending picker update from the child (and this covers the
+	//   case where the drop/request config has not changed, but the child sent
+	//   a picker update while we were still processing config from our parent).
+	if (b.handleDropAndRequestCountLocked(newConfig) && b.childState.Picker != nil) || b.pendingPickerUpdates {
+		b.pendingPickerUpdates = false
 		b.ClientConn.UpdateState(balancer.State{
 			ConnectivityState: b.childState.ConnectivityState,
 			Picker:            b.newPickerLocked(),
@@ -356,6 +365,10 @@ func (b *clusterImplBalancer) UpdateState(state balancer.State) {
 	// parent. Update the childState field regardless.
 	b.childState = state
 	if b.inhibitPickerUpdates {
+		b.pendingPickerUpdates = true
+		if b.logger.V(2) {
+			b.logger.Infof("Received a picker update from the child when processing an update from the parent")
+		}
 		return
 	}
 
diff --git a/xds/internal/xdsclient/channel.go b/xds/internal/xdsclient/channel.go
index 383720e6..df6162ad 100644
--- a/xds/internal/xdsclient/channel.go
+++ b/xds/internal/xdsclient/channel.go
@@ -60,7 +60,7 @@ type xdsChannelEventHandler interface {
 
 // xdsChannelOpts holds the options for creating a new xdsChannel.
 type xdsChannelOpts struct {
-	transport          transport.Interface           // Takes ownership of this transport.
+	transport          transport.Transport           // Takes ownership of this transport.
 	serverConfig       *bootstrap.ServerConfig       // Configuration of the server to connect to.
 	bootstrapConfig    *bootstrap.Config             // Complete bootstrap configuration, used to decode resources.
 	resourceTypeGetter func(string) xdsresource.Type // Function to retrieve resource parsing functionality, based on resource type.
@@ -126,7 +126,7 @@ func newXDSChannel(opts xdsChannelOpts) (*xdsChannel, error) {
 type xdsChannel struct {
 	// The following fields are initialized at creation time and are read-only
 	// after that, and hence need not be guarded by a mutex.
-	transport          transport.Interface           // Takes ownership of this transport (used to make streaming calls).
+	transport          transport.Transport           // Takes ownership of this transport (used to make streaming calls).
 	ads                *ads.StreamImpl               // An ADS stream to the management server.
 	lrs                *lrs.StreamImpl               // An LRS stream to the management server.
 	serverConfig       *bootstrap.ServerConfig       // Configuration of the server to connect to.
diff --git a/xds/internal/xdsclient/transport/ads/ads_stream.go b/xds/internal/xdsclient/transport/ads/ads_stream.go
index 27ef4d10..4c4856a0 100644
--- a/xds/internal/xdsclient/transport/ads/ads_stream.go
+++ b/xds/internal/xdsclient/transport/ads/ads_stream.go
@@ -113,7 +113,7 @@ type StreamImpl struct {
 	// The following fields are initialized from arguments passed to the
 	// constructor and are read-only afterwards, and hence can be accessed
 	// without a mutex.
-	transport          transport.Interface     // Transport to use for ADS stream.
+	transport          transport.Transport     // Transport to use for ADS stream.
 	eventHandler       StreamEventHandler      // Callbacks into the xdsChannel.
 	backoff            func(int) time.Duration // Backoff for retries, after stream failures.
 	nodeProto          *v3corepb.Node          // Identifies the gRPC application.
@@ -136,7 +136,7 @@ type StreamImpl struct {
 
 // StreamOpts contains the options for creating a new ADS Stream.
 type StreamOpts struct {
-	Transport          transport.Interface     // xDS transport to create the stream on.
+	Transport          transport.Transport     // xDS transport to create the stream on.
 	EventHandler       StreamEventHandler      // Callbacks for stream events.
 	Backoff            func(int) time.Duration // Backoff for retries, after stream failures.
 	NodeProto          *v3corepb.Node          // Node proto to identify the gRPC application.
diff --git a/xds/internal/xdsclient/transport/grpctransport/grpctransport.go b/xds/internal/xdsclient/transport/grpctransport/grpctransport.go
index 9bb8c737..fb740ade 100644
--- a/xds/internal/xdsclient/transport/grpctransport/grpctransport.go
+++ b/xds/internal/xdsclient/transport/grpctransport/grpctransport.go
@@ -48,7 +48,7 @@ type Builder struct{}
 // Build creates a new gRPC-based transport to an xDS server using the provided
 // options. This involves creating a grpc.ClientConn to the server identified by
 // the server URI in the provided options.
-func (b *Builder) Build(opts transport.BuildOptions) (transport.Interface, error) {
+func (b *Builder) Build(opts transport.BuildOptions) (transport.Transport, error) {
 	if opts.ServerConfig == nil {
 		return nil, fmt.Errorf("ServerConfig field in opts cannot be nil")
 	}
diff --git a/xds/internal/xdsclient/transport/lrs/lrs_stream.go b/xds/internal/xdsclient/transport/lrs/lrs_stream.go
index f9140b2e..bab92bcb 100644
--- a/xds/internal/xdsclient/transport/lrs/lrs_stream.go
+++ b/xds/internal/xdsclient/transport/lrs/lrs_stream.go
@@ -55,7 +55,7 @@ const perRPCVerbosityLevel = 9
 type StreamImpl struct {
 	// The following fields are initialized when a Stream instance is created
 	// and are read-only afterwards, and hence can be accessed without a mutex.
-	transport transport.Interface     // Transport to use for LRS stream.
+	transport transport.Transport     // Transport to use for LRS stream.
 	backoff   func(int) time.Duration // Backoff for retries, after stream failures.
 	nodeProto *v3corepb.Node          // Identifies the gRPC application.
 	doneCh    chan struct{}           // To notify exit of LRS goroutine.
@@ -70,7 +70,7 @@ type StreamImpl struct {
 
 // StreamOpts holds the options for creating an lrsStream.
 type StreamOpts struct {
-	Transport transport.Interface     // xDS transport to create the stream on.
+	Transport transport.Transport     // xDS transport to create the stream on.
 	Backoff   func(int) time.Duration // Backoff for retries, after stream failures.
 	NodeProto *v3corepb.Node          // Node proto to identify the gRPC application.
 	LogPrefix string                  // Prefix to be used for log messages.
diff --git a/xds/internal/xdsclient/transport/transport_interface.go b/xds/internal/xdsclient/transport/transport_interface.go
index 6a33e449..48ce82a0 100644
--- a/xds/internal/xdsclient/transport/transport_interface.go
+++ b/xds/internal/xdsclient/transport/transport_interface.go
@@ -28,7 +28,7 @@ import (
 // Builder is an interface for building a new xDS transport.
 type Builder interface {
 	// Build creates a new xDS transport with the provided options.
-	Build(opts BuildOptions) (Interface, error)
+	Build(opts BuildOptions) (Transport, error)
 }
 
 // BuildOptions contains the options for building a new xDS transport.
@@ -39,12 +39,9 @@ type BuildOptions struct {
 	ServerConfig *bootstrap.ServerConfig
 }
 
-// Interface provides the functionality to communicate with an xDS server using
+// Transport provides the functionality to communicate with an xDS server using
 // streaming calls.
-//
-// TODO(easwars): Rename this to Transport once the existing Transport type is
-// removed.
-type Interface interface {
+type Transport interface {
 	// CreateStreamingCall creates a new streaming call to the xDS server for the
 	// specified method name. The returned StreamingCall interface can be used to
 	// send and receive messages on the stream.
diff --git a/xds/internal/xdsclient/xdsresource/resource_type.go b/xds/internal/xdsclient/xdsresource/resource_type.go
index 3f23378b..e14f56f7 100644
--- a/xds/internal/xdsclient/xdsresource/resource_type.go
+++ b/xds/internal/xdsclient/xdsresource/resource_type.go
@@ -119,8 +119,6 @@ type Type interface {
 // provide an implementation of this interface to represent the configuration
 // received from the xDS management server.
 type ResourceData interface {
-	isResourceData()
-
 	// RawEqual returns true if the passed in resource data is equal to that of
 	// the receiver, based on the underlying raw protobuf message.
 	RawEqual(ResourceData) bool



Test patch:
diff --git a/internal/testutils/xds/e2e/clientresources.go b/internal/testutils/xds/e2e/clientresources.go
index c6f77c60..526f22ae 100644
--- a/internal/testutils/xds/e2e/clientresources.go
+++ b/internal/testutils/xds/e2e/clientresources.go
@@ -667,6 +667,13 @@ func ClusterResourceWithOptions(opts ClusterOptions) *v3clusterpb.Cluster {
 	return cluster
 }
 
+// LocalityID represents a locality identifier.
+type LocalityID struct {
+	Region  string
+	Zone    string
+	SubZone string
+}
+
 // LocalityOptions contains options to configure a Locality.
 type LocalityOptions struct {
 	// Name is the unique locality name.
@@ -677,6 +684,9 @@ type LocalityOptions struct {
 	Backends []BackendOptions
 	// Priority is the priority of the locality. Defaults to 0.
 	Priority uint32
+	// Locality is the locality identifier. If not specified, a random
+	// identifier is generated.
+	Locality LocalityID
 }
 
 // BackendOptions contains options to configure individual backends in a
@@ -752,12 +762,16 @@ func EndpointResourceWithOptions(opts EndpointOptions) *v3endpointpb.ClusterLoad
 			})
 		}
 
-		endpoints = append(endpoints, &v3endpointpb.LocalityLbEndpoints{
-			Locality: &v3corepb.Locality{
+		l := locality.Locality
+		if l == (LocalityID{}) {
+			l = LocalityID{
 				Region:  fmt.Sprintf("region-%d", i+1),
 				Zone:    fmt.Sprintf("zone-%d", i+1),
 				SubZone: fmt.Sprintf("subzone-%d", i+1),
-			},
+			}
+		}
+		endpoints = append(endpoints, &v3endpointpb.LocalityLbEndpoints{
+			Locality:            &v3corepb.Locality{Region: l.Region, Zone: l.Zone, SubZone: l.SubZone},
 			LbEndpoints:         lbEndpoints,
 			LoadBalancingWeight: &wrapperspb.UInt32Value{Value: locality.Weight},
 			Priority:            locality.Priority,
diff --git a/internal/transport/transport_test.go b/internal/transport/transport_test.go
index 832435f6..d315718d 100644
--- a/internal/transport/transport_test.go
+++ b/internal/transport/transport_test.go
@@ -76,7 +76,7 @@ func init() {
 }
 
 func newBufferSlice(b []byte) mem.BufferSlice {
-	return mem.BufferSlice{mem.NewBuffer(&b, nil)}
+	return mem.BufferSlice{mem.SliceBuffer(b)}
 }
 
 func (s *Stream) readTo(p []byte) (int, error) {
@@ -1796,7 +1796,7 @@ func (s) TestReadGivesSameErrorAfterAnyErrorOccurs(t *testing.T) {
 	testData := make([]byte, 1)
 	testData[0] = 5
 	testErr := errors.New("test error")
-	s.write(recvMsg{buffer: mem.NewBuffer(&testData, nil), err: testErr})
+	s.write(recvMsg{buffer: mem.SliceBuffer(testData), err: testErr})
 
 	inBuf := make([]byte, 1)
 	actualCount, actualErr := s.readTo(inBuf)
@@ -1807,8 +1807,8 @@ func (s) TestReadGivesSameErrorAfterAnyErrorOccurs(t *testing.T) {
 		t.Errorf("_ , actualErr := s.Read(_) differs; want actualErr.Error() to be %v; got %v", testErr.Error(), actualErr.Error())
 	}
 
-	s.write(recvMsg{buffer: mem.NewBuffer(&testData, nil), err: nil})
-	s.write(recvMsg{buffer: mem.NewBuffer(&testData, nil), err: errors.New("different error from first")})
+	s.write(recvMsg{buffer: mem.SliceBuffer(testData), err: nil})
+	s.write(recvMsg{buffer: mem.SliceBuffer(testData), err: errors.New("different error from first")})
 
 	for i := 0; i < 2; i++ {
 		inBuf := make([]byte, 1)
diff --git a/rpc_util_test.go b/rpc_util_test.go
index 1daa3a6c..ca7990a0 100644
--- a/rpc_util_test.go
+++ b/rpc_util_test.go
@@ -59,13 +59,13 @@ func (f *fullReader) Read(n int) (mem.BufferSlice, error) {
 	if len(f.data) < n {
 		data := f.data
 		f.data = nil
-		return mem.BufferSlice{mem.NewBuffer(&data, nil)}, io.ErrUnexpectedEOF
+		return mem.BufferSlice{mem.SliceBuffer(data)}, io.ErrUnexpectedEOF
 	}
 
 	buf := f.data[:n]
 	f.data = f.data[n:]
 
-	return mem.BufferSlice{mem.NewBuffer(&buf, nil)}, nil
+	return mem.BufferSlice{mem.SliceBuffer(buf)}, nil
 }
 
 var _ CallOption = EmptyCallOption{} // ensure EmptyCallOption implements the interface
diff --git a/stats/stats_test.go b/stats/stats_test.go
index 13a027f8..ec5ffa04 100644
--- a/stats/stats_test.go
+++ b/stats/stats_test.go
@@ -94,6 +94,27 @@ func payloadToID(p *testpb.Payload) int32 {
 	return int32(p.Body[0]) + int32(p.Body[1])<<8 + int32(p.Body[2])<<16 + int32(p.Body[3])<<24
 }
 
+func setIncomingStats(ctx context.Context, mdKey string, b []byte) context.Context {
+	md, ok := metadata.FromIncomingContext(ctx)
+	if !ok {
+		md = metadata.MD{}
+	}
+	md.Set(mdKey, string(b))
+	return metadata.NewIncomingContext(ctx, md)
+}
+
+func getOutgoingStats(ctx context.Context, mdKey string) []byte {
+	md, ok := metadata.FromOutgoingContext(ctx)
+	if !ok {
+		return nil
+	}
+	tagValues := md.Get(mdKey)
+	if len(tagValues) == 0 {
+		return nil
+	}
+	return []byte(tagValues[len(tagValues)-1])
+}
+
 type testServer struct {
 	testgrpc.UnimplementedTestServiceServer
 }
@@ -1312,19 +1333,19 @@ func (s) TestTags(t *testing.T) {
 	tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	ctx := stats.SetTags(tCtx, b)
-	if tg := stats.OutgoingTags(ctx); !reflect.DeepEqual(tg, b) {
-		t.Errorf("OutgoingTags(%v) = %v; want %v", ctx, tg, b)
+	if tg := getOutgoingStats(ctx, "grpc-tags-bin"); !reflect.DeepEqual(tg, b) {
+		t.Errorf("getOutgoingStats(%v, grpc-tags-bin) = %v; want %v", ctx, tg, b)
 	}
 	if tg := stats.Tags(ctx); tg != nil {
 		t.Errorf("Tags(%v) = %v; want nil", ctx, tg)
 	}
 
-	ctx = stats.SetIncomingTags(tCtx, b)
+	ctx = setIncomingStats(tCtx, "grpc-tags-bin", b)
 	if tg := stats.Tags(ctx); !reflect.DeepEqual(tg, b) {
 		t.Errorf("Tags(%v) = %v; want %v", ctx, tg, b)
 	}
-	if tg := stats.OutgoingTags(ctx); tg != nil {
-		t.Errorf("OutgoingTags(%v) = %v; want nil", ctx, tg)
+	if tg := getOutgoingStats(ctx, "grpc-tags-bin"); tg != nil {
+		t.Errorf("getOutgoingStats(%v, grpc-tags-bin) = %v; want nil", ctx, tg)
 	}
 }
 
@@ -1333,19 +1354,19 @@ func (s) TestTrace(t *testing.T) {
 	tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	ctx := stats.SetTrace(tCtx, b)
-	if tr := stats.OutgoingTrace(ctx); !reflect.DeepEqual(tr, b) {
-		t.Errorf("OutgoingTrace(%v) = %v; want %v", ctx, tr, b)
+	if tr := getOutgoingStats(ctx, "grpc-trace-bin"); !reflect.DeepEqual(tr, b) {
+		t.Errorf("getOutgoingStats(%v, grpc-trace-bin) = %v; want %v", ctx, tr, b)
 	}
 	if tr := stats.Trace(ctx); tr != nil {
 		t.Errorf("Trace(%v) = %v; want nil", ctx, tr)
 	}
 
-	ctx = stats.SetIncomingTrace(tCtx, b)
+	ctx = setIncomingStats(tCtx, "grpc-trace-bin", b)
 	if tr := stats.Trace(ctx); !reflect.DeepEqual(tr, b) {
 		t.Errorf("Trace(%v) = %v; want %v", ctx, tr, b)
 	}
-	if tr := stats.OutgoingTrace(ctx); tr != nil {
-		t.Errorf("OutgoingTrace(%v) = %v; want nil", ctx, tr)
+	if tr := getOutgoingStats(ctx, "grpc-trace-bin"); tr != nil {
+		t.Errorf("getOutgoingStats(%v, grpc-trace-bin) = %v; want nil", ctx, tr)
 	}
 }
 
diff --git a/test/end2end_test.go b/test/end2end_test.go
index fcb686ec..2238f8f1 100644
--- a/test/end2end_test.go
+++ b/test/end2end_test.go
@@ -41,6 +41,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"
 	"google.golang.org/grpc"
@@ -4618,6 +4619,16 @@ func (s) TestStreamingProxyDoesNotForwardMetadata(t *testing.T) {
 }
 
 func (s) TestStatsTagsAndTrace(t *testing.T) {
+	const mdTraceKey = "grpc-trace-bin"
+	const mdTagsKey = "grpc-tags-bin"
+
+	setTrace := func(ctx context.Context, b []byte) context.Context {
+		return metadata.AppendToOutgoingContext(ctx, mdTraceKey, string(b))
+	}
+	setTags := func(ctx context.Context, b []byte) context.Context {
+		return metadata.AppendToOutgoingContext(ctx, mdTagsKey, string(b))
+	}
+
 	// Data added to context by client (typically in a stats handler).
 	tags := []byte{1, 5, 2, 4, 3}
 	trace := []byte{5, 2, 1, 3, 4}
@@ -4627,17 +4638,11 @@ func (s) TestStatsTagsAndTrace(t *testing.T) {
 	endpoint := &stubserver.StubServer{
 		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
 			md, _ := metadata.FromIncomingContext(ctx)
-			if tg := stats.Tags(ctx); !reflect.DeepEqual(tg, tags) {
-				return nil, status.Errorf(codes.Internal, "stats.Tags(%v)=%v; want %v", ctx, tg, tags)
-			}
-			if !reflect.DeepEqual(md["grpc-tags-bin"], []string{string(tags)}) {
-				return nil, status.Errorf(codes.Internal, "md['grpc-tags-bin']=%v; want %v", md["grpc-tags-bin"], tags)
-			}
-			if tr := stats.Trace(ctx); !reflect.DeepEqual(tr, trace) {
-				return nil, status.Errorf(codes.Internal, "stats.Trace(%v)=%v; want %v", ctx, tr, trace)
+			if md[mdTagsKey] == nil || !cmp.Equal(md[mdTagsKey][len(md[mdTagsKey])-1], string(tags)) {
+				return nil, status.Errorf(codes.Internal, "md['grpc-tags-bin']=%v; want %v", md[mdTagsKey], tags)
 			}
-			if !reflect.DeepEqual(md["grpc-trace-bin"], []string{string(trace)}) {
-				return nil, status.Errorf(codes.Internal, "md['grpc-trace-bin']=%v; want %v", md["grpc-trace-bin"], trace)
+			if md[mdTraceKey] == nil || !cmp.Equal(md[mdTraceKey][len(md[mdTraceKey])-1], string(trace)) {
+				return nil, status.Errorf(codes.Internal, "md['grpc-trace-bin']=%v; want %v", md[mdTraceKey], trace)
 			}
 			return &testpb.Empty{}, nil
 		},
@@ -4655,10 +4660,10 @@ func (s) TestStatsTagsAndTrace(t *testing.T) {
 		want codes.Code
 	}{
 		{ctx: ctx, want: codes.Internal},
-		{ctx: stats.SetTags(ctx, tags), want: codes.Internal},
-		{ctx: stats.SetTrace(ctx, trace), want: codes.Internal},
-		{ctx: stats.SetTags(stats.SetTrace(ctx, tags), tags), want: codes.Internal},
-		{ctx: stats.SetTags(stats.SetTrace(ctx, trace), tags), want: codes.OK},
+		{ctx: setTags(ctx, tags), want: codes.Internal},
+		{ctx: setTrace(ctx, trace), want: codes.Internal},
+		{ctx: setTags(setTrace(ctx, tags), tags), want: codes.Internal},
+		{ctx: setTags(setTrace(ctx, trace), tags), want: codes.OK},
 	}
 
 	for _, tc := range testCases {
diff --git a/test/xds/xds_client_priority_locality_test.go b/test/xds/xds_client_priority_locality_test.go
new file mode 100644
index 00000000..0f7c5e21
--- /dev/null
+++ b/test/xds/xds_client_priority_locality_test.go
@@ -0,0 +1,211 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	rrutil "google.golang.org/grpc/internal/testutils/roundrobin"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/resolver"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// backendAddressesAndPorts extracts the address and port of each of the
+// StubServers passed in and returns them. Fails the test if any of the
+// StubServers passed have an invalid address.
+func backendAddressesAndPorts(t *testing.T, servers []*stubserver.StubServer) ([]resolver.Address, []uint32) {
+	addrs := make([]resolver.Address, len(servers))
+	ports := make([]uint32, len(servers))
+	for i := 0; i < len(servers); i++ {
+		addrs[i] = resolver.Address{Addr: servers[i].Address}
+		ports[i] = testutils.ParsePort(t, servers[i].Address)
+	}
+	return addrs, ports
+}
+
+// Tests scenarios involving localities moving between priorities.
+//   - The test starts off with a cluster that contains two priorities, one
+//     locality in each, and one endpoint in each. Verifies that traffic reaches
+//     the endpoint in the higher priority.
+//   - The test then moves the locality in the lower priority over to the higher
+//     priority. At that point, we would have a cluster with a single priority,
+//     but two localities, and one endpoint in each. Verifies that traffic is
+//     split between the endpoints.
+//   - The test then deletes the locality that was originally in the higher
+//     priority.Verifies that all traffic is now reaching the only remaining
+//     endpoint.
+func (s) TestClientSideXDS_LocalityChangesPriority(t *testing.T) {
+	// Spin up a management server and two test service backends.
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+	backend0 := stubserver.StartTestService(t, nil)
+	defer backend0.Stop()
+	backend1 := stubserver.StartTestService(t, nil)
+	defer backend1.Stop()
+	addrs, ports := backendAddressesAndPorts(t, []*stubserver.StubServer{backend0, backend1})
+
+	// Configure resources on the management server. We use default client side
+	// resources for listener, route configuration and cluster. For the
+	// endpoints resource though, we create one with two priorities, and one
+	// locality each, and one endpoint each.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName = "cluster-" + serviceName
+	const endpointsName = "endpoints-" + serviceName
+	locality1 := e2e.LocalityID{Region: "my-region-1", Zone: "my-zone-1", SubZone: "my-subzone-1"}
+	locality2 := e2e.LocalityID{Region: "my-region-2", Zone: "my-zone-2", SubZone: "my-subzone-2"}
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointsName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+			ClusterName: endpointsName,
+			Host:        "localhost",
+			Localities: []e2e.LocalityOptions{
+				{
+					Name:     "my-locality-1",
+					Weight:   1000000,
+					Priority: 0,
+					Backends: []e2e.BackendOptions{{Port: ports[0]}},
+					Locality: locality1,
+				},
+				{
+					Name:     "my-locality-2",
+					Weight:   1000000,
+					Priority: 1,
+					Backends: []e2e.BackendOptions{{Port: ports[1]}},
+					Locality: locality2,
+				},
+			},
+		})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	// // Ensure that RPCs get routed to the backend in the higher priority.
+	client := testgrpc.NewTestServiceClient(cc)
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[:1]); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update the endpoints resource to contain a single priority with two
+	// localities, and one endpoint each. The locality weights are equal at this
+	// point, and we expect RPCs to be round-robined across the two localities.
+	resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+		ClusterName: endpointsName,
+		Host:        "localhost",
+		Localities: []e2e.LocalityOptions{
+			{
+				Name:     "my-locality-1",
+				Weight:   500000,
+				Priority: 0,
+				Backends: []e2e.BackendOptions{{Port: testutils.ParsePort(t, backend0.Address)}},
+				Locality: locality1,
+			},
+			{
+				Name:     "my-locality-2",
+				Weight:   500000,
+				Priority: 0,
+				Backends: []e2e.BackendOptions{{Port: testutils.ParsePort(t, backend1.Address)}},
+				Locality: locality2,
+			},
+		},
+	})}
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update the locality weights ever so slightly. We still expect RPCs to be
+	// round-robined across the two localities.
+	resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+		ClusterName: endpointsName,
+		Host:        "localhost",
+		Localities: []e2e.LocalityOptions{
+			{
+				Name:     "my-locality-1",
+				Weight:   499884,
+				Priority: 0,
+				Backends: []e2e.BackendOptions{{Port: testutils.ParsePort(t, backend0.Address)}},
+				Locality: locality1,
+			},
+			{
+				Name:     "my-locality-2",
+				Weight:   500115,
+				Priority: 0,
+				Backends: []e2e.BackendOptions{{Port: testutils.ParsePort(t, backend1.Address)}},
+				Locality: locality2,
+			},
+		},
+	})}
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update the endpoints resource to contain a single priority with one
+	// locality. The locality which was originally in the higher priority is now
+	// dropped.
+	resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+		ClusterName: endpointsName,
+		Host:        "localhost",
+		Localities: []e2e.LocalityOptions{
+			{
+				Name:     "my-locality-2",
+				Weight:   1000000,
+				Priority: 0,
+				Backends: []e2e.BackendOptions{{Port: testutils.ParsePort(t, backend1.Address)}},
+				Locality: locality2,
+			},
+		},
+	})}
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[1:]); err != nil {
+		t.Fatal(err)
+	}
+}
diff --git a/xds/internal/xdsclient/channel_test.go b/xds/internal/xdsclient/channel_test.go
index 670e76c5..bd8f3c4c 100644
--- a/xds/internal/xdsclient/channel_test.go
+++ b/xds/internal/xdsclient/channel_test.go
@@ -148,7 +148,7 @@ func verifyUpdateAndMetadata(ctx context.Context, t *testing.T, eh *testEventHan
 // serverConfig, bootstrapConfig, or resourceTypeGetter) are missing or nil.
 func (s) TestChannel_New_FailureCases(t *testing.T) {
 	type fakeTransport struct {
-		transport.Interface
+		transport.Transport
 	}
 
 	tests := []struct {