Repository: grpc/grpc-go
Pull request #8169: xds: fix support for circuit breakers in LOGICAL_DNS clusters
Author: login: dfawley
Labels: name: Type: Bug
Created: 2025-03-13T18:28:24Z | Merged: 2025-03-20T16:39:49Z

Previously this setting was only plumbed to EDS, which meant it didn't apply to LOGICAL_DNS, but it should apply to both.

RELEASE NOTES:
* xds: fix support for circuit breakers in LOGICAL_DNS clusters


Diff:
diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
index 99e766abfb33..5106129c4ae0 100644
--- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go
+++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go
@@ -652,9 +652,10 @@ func (b *cdsBalancer) generateDMsForCluster(name string, depth int, dms []cluste
 		}
 	case xdsresource.ClusterTypeLogicalDNS:
 		dm = clusterresolver.DiscoveryMechanism{
-			Type:        clusterresolver.DiscoveryMechanismTypeLogicalDNS,
-			Cluster:     cluster.ClusterName,
-			DNSHostname: cluster.DNSHostName,
+			Type:                  clusterresolver.DiscoveryMechanismTypeLogicalDNS,
+			Cluster:               cluster.ClusterName,
+			DNSHostname:           cluster.DNSHostName,
+			MaxConcurrentRequests: cluster.MaxRequests,
 		}
 	}
 	odJSON := cluster.OutlierDetection
diff --git a/xds/internal/balancer/clusterimpl/tests/balancer_test.go b/xds/internal/balancer/clusterimpl/tests/balancer_test.go
index a8e7ac7a10ad..4287084611ae 100644
--- a/xds/internal/balancer/clusterimpl/tests/balancer_test.go
+++ b/xds/internal/balancer/clusterimpl/tests/balancer_test.go
@@ -21,6 +21,8 @@ package clusterimpl_test
 import (
 	"context"
 	"fmt"
+	"net"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -40,6 +42,7 @@ import (
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
 
 	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
@@ -354,3 +357,218 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+// Tests that circuit breaking limits RPCs E2E.
+func (s) TestCircuitBreaking(t *testing.T) {
+	// Create an xDS management server that serves ADS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+
+	// Configure the xDS management server with default resources.
+	const serviceName = "my-test-xds-service"
+	const maxRequests = 3
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+	})
+	resources.Clusters[0].CircuitBreakers = &v3clusterpb.CircuitBreakers{
+		Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{
+			{
+				Priority:    v3corepb.RoutingPriority_DEFAULT,
+				MaxRequests: wrapperspb.UInt32(maxRequests),
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Start maxRequests streams.
+	for range maxRequests {
+		if _, err := client.FullDuplexCall(ctx); err != nil {
+			t.Fatalf("rpc FullDuplexCall() failed: %v", err)
+		}
+	}
+
+	// Since we are at the max, new streams should fail.  It's possible some are
+	// allowed due to inherent raciness in the tracking, however.
+	for i := 0; i < 100; i++ {
+		stream, err := client.FullDuplexCall(ctx)
+		if status.Code(err) == codes.Unavailable {
+			return
+		}
+		if err == nil {
+			// Terminate the stream (the server immediately exits upon a client
+			// CloseSend) to ensure we never go over the limit.
+			stream.CloseSend()
+			stream.Recv()
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	t.Fatalf("RPCs unexpectedly allowed beyond circuit breaking maximum")
+}
+
+// Tests that circuit breaking limits RPCs in LOGICAL_DNS clusters E2E.
+func (s) TestCircuitBreakingLogicalDNS(t *testing.T) {
+	// Create an xDS management server that serves ADS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+	host, port := hostAndPortFromAddress(t, server.Address)
+
+	// Configure the xDS management server with default resources. Override the
+	// default cluster to include a circuit breaking config.
+	const serviceName = "my-test-xds-service"
+	const maxRequests = 3
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+	})
+	resources.Clusters = []*v3clusterpb.Cluster{
+		e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+			ClusterName: "cluster-" + serviceName,
+			Type:        e2e.ClusterTypeLogicalDNS,
+			DNSHostName: host,
+			DNSPort:     port,
+		}),
+	}
+	resources.Clusters[0].CircuitBreakers = &v3clusterpb.CircuitBreakers{
+		Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{
+			{
+				Priority:    v3corepb.RoutingPriority_DEFAULT,
+				MaxRequests: wrapperspb.UInt32(maxRequests),
+			},
+		},
+	}
+	resources.Endpoints = nil
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Start maxRequests streams.
+	for range maxRequests {
+		if _, err := client.FullDuplexCall(ctx); err != nil {
+			t.Fatalf("rpc FullDuplexCall() failed: %v", err)
+		}
+	}
+
+	// Since we are at the max, new streams should fail.  It's possible some are
+	// allowed due to inherent raciness in the tracking, however.
+	for i := 0; i < 100; i++ {
+		stream, err := client.FullDuplexCall(ctx)
+		if status.Code(err) == codes.Unavailable {
+			return
+		}
+		if err == nil {
+			// Terminate the stream (the server immediately exits upon a client
+			// CloseSend) to ensure we never go over the limit.
+			stream.CloseSend()
+			stream.Recv()
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	t.Fatalf("RPCs unexpectedly allowed beyond circuit breaking maximum")
+}
+
+func hostAndPortFromAddress(t *testing.T, addr string) (string, uint32) {
+	t.Helper()
+
+	host, p, err := net.SplitHostPort(addr)
+	if err != nil {
+		t.Fatalf("Invalid serving address: %v", addr)
+	}
+	port, err := strconv.ParseUint(p, 10, 32)
+	if err != nil {
+		t.Fatalf("Invalid serving port %q: %v", p, err)
+	}
+	return host, uint32(port)
+}
diff --git a/xds/internal/balancer/clusterresolver/configbuilder.go b/xds/internal/balancer/clusterresolver/configbuilder.go
index f7ae4e2799c3..2488f221d4bd 100644
--- a/xds/internal/balancer/clusterresolver/configbuilder.go
+++ b/xds/internal/balancer/clusterresolver/configbuilder.go
@@ -149,9 +149,10 @@ func buildClusterImplConfigForDNS(g *nameGenerator, endpoints []resolver.Endpoin
 		retEndpoints[i].Addresses = append([]resolver.Address{}, e.Addresses...)
 	}
 	return pName, &clusterimpl.LBConfig{
-		Cluster:         mechanism.Cluster,
-		TelemetryLabels: mechanism.TelemetryLabels,
-		ChildPolicy:     &internalserviceconfig.BalancerConfig{Name: childPolicy},
+		Cluster:               mechanism.Cluster,
+		TelemetryLabels:       mechanism.TelemetryLabels,
+		ChildPolicy:           &internalserviceconfig.BalancerConfig{Name: childPolicy},
+		MaxConcurrentRequests: mechanism.MaxConcurrentRequests,
 	}, retEndpoints
 }
 


Reviews:
user: login: purnesh42H
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T09:57:06Z
user: login: dfawley
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T15:49:50Z
user: login: purnesh42H
body: LGTM. Kept some comments unresolved. PTAL
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T14:12:42Z
user: login: dfawley
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-19T20:00:45Z
user: login: dfawley
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-19T20:01:16Z
user: login: dfawley
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-19T20:47:03Z

Review comments:
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: purnesh42H
body: nit: Do we need to server LRS for this test?
created_at: 2025-03-18T07:25:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: purnesh42H
body: Add docstring?
created_at: 2025-03-18T07:25:58Z
author_association: CONTRIBUTOR
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+
+	// Configure the xDS management server with default resources. Override the
+	// default cluster to include an LRS server config pointing to self.
+	const serviceName = "my-test-xds-service"
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: purnesh42H
body: nit: similar to maxRequests, `serviceName` can be declared at the start as well?
created_at: 2025-03-18T07:29:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+
+	// Configure the xDS management server with default resources. Override the
+	// default cluster to include an LRS server config pointing to self.
+	const serviceName = "my-test-xds-service"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+	})
+	resources.Clusters[0].CircuitBreakers = &v3clusterpb.CircuitBreakers{
+		Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{
+			{
+				Priority:    v3corepb.RoutingPriority_DEFAULT,
+				MaxRequests: wrapperspb.UInt32(maxRequests),
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Start maxRequests streams.
+	for range maxRequests {
+		if _, err := client.FullDuplexCall(ctx); err != nil {
+			t.Fatalf("rpc FullDuplexCall() failed: %v", err)
+		}
+	}
+
+	// Since we are at the max, new streams should fail.  It's possible some are
+	// allowed due to inherent raciness in the tracking, however.
+	for i := 0; i < 100; i++ {
+		stream, err := client.FullDuplexCall(ctx)
+		if status.Code(err) == codes.Unavailable {
+			return
+		}
+		if err == nil {
+			stream.CloseSend()
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: purnesh42H
body: nit: may be comment saying we are closing unintended new streams?
created_at: 2025-03-18T09:40:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+
+	// Configure the xDS management server with default resources. Override the
+	// default cluster to include an LRS server config pointing to self.
+	const serviceName = "my-test-xds-service"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+	})
+	resources.Clusters[0].CircuitBreakers = &v3clusterpb.CircuitBreakers{
+		Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{
+			{
+				Priority:    v3corepb.RoutingPriority_DEFAULT,
+				MaxRequests: wrapperspb.UInt32(maxRequests),
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Start maxRequests streams.
+	for range maxRequests {
+		if _, err := client.FullDuplexCall(ctx); err != nil {
+			t.Fatalf("rpc FullDuplexCall() failed: %v", err)
+		}
+	}
+
+	// Since we are at the max, new streams should fail.  It's possible some are
+	// allowed due to inherent raciness in the tracking, however.
+	for i := 0; i < 100; i++ {
+		stream, err := client.FullDuplexCall(ctx)
+		if status.Code(err) == codes.Unavailable {
+			return
+		}
+		if err == nil {
+			stream.CloseSend()
+			stream.Recv()
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	t.Fatalf("RPCs unexpectedly allowed beyond circuit breaking maximum")
+}
+
+func (s) TestCircuitBreakingLogicalDNS(t *testing.T) {
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: purnesh42H
body: Add docstring for this as well explaining what the test is verifying
created_at: 2025-03-18T09:40:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+
+	// Configure the xDS management server with default resources. Override the
+	// default cluster to include an LRS server config pointing to self.
+	const serviceName = "my-test-xds-service"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+	})
+	resources.Clusters[0].CircuitBreakers = &v3clusterpb.CircuitBreakers{
+		Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{
+			{
+				Priority:    v3corepb.RoutingPriority_DEFAULT,
+				MaxRequests: wrapperspb.UInt32(maxRequests),
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Start maxRequests streams.
+	for range maxRequests {
+		if _, err := client.FullDuplexCall(ctx); err != nil {
+			t.Fatalf("rpc FullDuplexCall() failed: %v", err)
+		}
+	}
+
+	// Since we are at the max, new streams should fail.  It's possible some are
+	// allowed due to inherent raciness in the tracking, however.
+	for i := 0; i < 100; i++ {
+		stream, err := client.FullDuplexCall(ctx)
+		if status.Code(err) == codes.Unavailable {
+			return
+		}
+		if err == nil {
+			stream.CloseSend()
+			stream.Recv()
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	t.Fatalf("RPCs unexpectedly allowed beyond circuit breaking maximum")
+}
+
+func (s) TestCircuitBreakingLogicalDNS(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: purnesh42H
body: same. Do we need LRS for this test?
created_at: 2025-03-18T09:49:55Z
author_association: CONTRIBUTOR
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+
+	// Configure the xDS management server with default resources. Override the
+	// default cluster to include an LRS server config pointing to self.
+	const serviceName = "my-test-xds-service"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+	})
+	resources.Clusters[0].CircuitBreakers = &v3clusterpb.CircuitBreakers{
+		Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{
+			{
+				Priority:    v3corepb.RoutingPriority_DEFAULT,
+				MaxRequests: wrapperspb.UInt32(maxRequests),
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Start maxRequests streams.
+	for range maxRequests {
+		if _, err := client.FullDuplexCall(ctx); err != nil {
+			t.Fatalf("rpc FullDuplexCall() failed: %v", err)
+		}
+	}
+
+	// Since we are at the max, new streams should fail.  It's possible some are
+	// allowed due to inherent raciness in the tracking, however.
+	for i := 0; i < 100; i++ {
+		stream, err := client.FullDuplexCall(ctx)
+		if status.Code(err) == codes.Unavailable {
+			return
+		}
+		if err == nil {
+			stream.CloseSend()
+			stream.Recv()
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	t.Fatalf("RPCs unexpectedly allowed beyond circuit breaking maximum")
+}
+
+func (s) TestCircuitBreakingLogicalDNS(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+	host, port := hostAndPortFromAddress(t, server.Address)
+
+	// Configure the xDS management server with default resources. Override the
+	// default cluster to include an LRS server config pointing to self.
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: purnesh42H
body: > default cluster to include an LRS server config pointing to self.

not LRS server. Comment should mention CircuitBreakers ?
created_at: 2025-03-18T09:55:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: dfawley
body: Nope.  This was copy/pasted from another test that probably did.
created_at: 2025-03-18T15:43:06Z
author_association: MEMBER
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: dfawley
body: Done
created_at: 2025-03-18T15:43:38Z
author_association: MEMBER
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+
+	// Configure the xDS management server with default resources. Override the
+	// default cluster to include an LRS server config pointing to self.
+	const serviceName = "my-test-xds-service"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+	})
+	resources.Clusters[0].CircuitBreakers = &v3clusterpb.CircuitBreakers{
+		Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{
+			{
+				Priority:    v3corepb.RoutingPriority_DEFAULT,
+				MaxRequests: wrapperspb.UInt32(maxRequests),
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Start maxRequests streams.
+	for range maxRequests {
+		if _, err := client.FullDuplexCall(ctx); err != nil {
+			t.Fatalf("rpc FullDuplexCall() failed: %v", err)
+		}
+	}
+
+	// Since we are at the max, new streams should fail.  It's possible some are
+	// allowed due to inherent raciness in the tracking, however.
+	for i := 0; i < 100; i++ {
+		stream, err := client.FullDuplexCall(ctx)
+		if status.Code(err) == codes.Unavailable {
+			return
+		}
+		if err == nil {
+			stream.CloseSend()
+			stream.Recv()
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	t.Fatalf("RPCs unexpectedly allowed beyond circuit breaking maximum")
+}
+
+func (s) TestCircuitBreakingLogicalDNS(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+	host, port := hostAndPortFromAddress(t, server.Address)
+
+	// Configure the xDS management server with default resources. Override the
+	// default cluster to include an LRS server config pointing to self.
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: dfawley
body: Done
created_at: 2025-03-18T15:44:22Z
author_association: MEMBER
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+
+	// Configure the xDS management server with default resources. Override the
+	// default cluster to include an LRS server config pointing to self.
+	const serviceName = "my-test-xds-service"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+	})
+	resources.Clusters[0].CircuitBreakers = &v3clusterpb.CircuitBreakers{
+		Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{
+			{
+				Priority:    v3corepb.RoutingPriority_DEFAULT,
+				MaxRequests: wrapperspb.UInt32(maxRequests),
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Start maxRequests streams.
+	for range maxRequests {
+		if _, err := client.FullDuplexCall(ctx); err != nil {
+			t.Fatalf("rpc FullDuplexCall() failed: %v", err)
+		}
+	}
+
+	// Since we are at the max, new streams should fail.  It's possible some are
+	// allowed due to inherent raciness in the tracking, however.
+	for i := 0; i < 100; i++ {
+		stream, err := client.FullDuplexCall(ctx)
+		if status.Code(err) == codes.Unavailable {
+			return
+		}
+		if err == nil {
+			stream.CloseSend()
+			stream.Recv()
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	t.Fatalf("RPCs unexpectedly allowed beyond circuit breaking maximum")
+}
+
+func (s) TestCircuitBreakingLogicalDNS(t *testing.T) {
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: dfawley
body: Done
created_at: 2025-03-18T15:44:50Z
author_association: MEMBER
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+
+	// Configure the xDS management server with default resources. Override the
+	// default cluster to include an LRS server config pointing to self.
+	const serviceName = "my-test-xds-service"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+	})
+	resources.Clusters[0].CircuitBreakers = &v3clusterpb.CircuitBreakers{
+		Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{
+			{
+				Priority:    v3corepb.RoutingPriority_DEFAULT,
+				MaxRequests: wrapperspb.UInt32(maxRequests),
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Start maxRequests streams.
+	for range maxRequests {
+		if _, err := client.FullDuplexCall(ctx); err != nil {
+			t.Fatalf("rpc FullDuplexCall() failed: %v", err)
+		}
+	}
+
+	// Since we are at the max, new streams should fail.  It's possible some are
+	// allowed due to inherent raciness in the tracking, however.
+	for i := 0; i < 100; i++ {
+		stream, err := client.FullDuplexCall(ctx)
+		if status.Code(err) == codes.Unavailable {
+			return
+		}
+		if err == nil {
+			stream.CloseSend()
+			stream.Recv()
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	t.Fatalf("RPCs unexpectedly allowed beyond circuit breaking maximum")
+}
+
+func (s) TestCircuitBreakingLogicalDNS(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: dfawley
body: Done
created_at: 2025-03-18T15:44:58Z
author_association: MEMBER
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+
+	// Configure the xDS management server with default resources. Override the
+	// default cluster to include an LRS server config pointing to self.
+	const serviceName = "my-test-xds-service"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+	})
+	resources.Clusters[0].CircuitBreakers = &v3clusterpb.CircuitBreakers{
+		Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{
+			{
+				Priority:    v3corepb.RoutingPriority_DEFAULT,
+				MaxRequests: wrapperspb.UInt32(maxRequests),
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Start maxRequests streams.
+	for range maxRequests {
+		if _, err := client.FullDuplexCall(ctx); err != nil {
+			t.Fatalf("rpc FullDuplexCall() failed: %v", err)
+		}
+	}
+
+	// Since we are at the max, new streams should fail.  It's possible some are
+	// allowed due to inherent raciness in the tracking, however.
+	for i := 0; i < 100; i++ {
+		stream, err := client.FullDuplexCall(ctx)
+		if status.Code(err) == codes.Unavailable {
+			return
+		}
+		if err == nil {
+			stream.CloseSend()
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: dfawley
body: Done
created_at: 2025-03-18T15:47:53Z
author_association: MEMBER
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+
+	// Configure the xDS management server with default resources. Override the
+	// default cluster to include an LRS server config pointing to self.
+	const serviceName = "my-test-xds-service"
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: dfawley
body: Moved maxRequests here instead.
created_at: 2025-03-18T15:49:33Z
author_association: MEMBER
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: purnesh42H
body: You need not set `SupportLoadReportingService: true` if LRS is not needed
created_at: 2025-03-19T14:09:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -354,3 +357,219 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+// Tests that circuit breaking limits RPCs E2E.
+func (s) TestCircuitBreaking(t *testing.T) {
+	// Create an xDS management server that serves ADS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+
+	// Configure the xDS management server with default resources. Override the
+	// default cluster to include an LRS server config pointing to self.
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: purnesh42H
body: > default cluster to include an LRS server config pointing to self.

need to change this as well
created_at: 2025-03-19T14:10:32Z
author_association: CONTRIBUTOR
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+
+	// Configure the xDS management server with default resources. Override the
+	// default cluster to include an LRS server config pointing to self.
+	const serviceName = "my-test-xds-service"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+	})
+	resources.Clusters[0].CircuitBreakers = &v3clusterpb.CircuitBreakers{
+		Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{
+			{
+				Priority:    v3corepb.RoutingPriority_DEFAULT,
+				MaxRequests: wrapperspb.UInt32(maxRequests),
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Start maxRequests streams.
+	for range maxRequests {
+		if _, err := client.FullDuplexCall(ctx); err != nil {
+			t.Fatalf("rpc FullDuplexCall() failed: %v", err)
+		}
+	}
+
+	// Since we are at the max, new streams should fail.  It's possible some are
+	// allowed due to inherent raciness in the tracking, however.
+	for i := 0; i < 100; i++ {
+		stream, err := client.FullDuplexCall(ctx)
+		if status.Code(err) == codes.Unavailable {
+			return
+		}
+		if err == nil {
+			stream.CloseSend()
+			stream.Recv()
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	t.Fatalf("RPCs unexpectedly allowed beyond circuit breaking maximum")
+}
+
+func (s) TestCircuitBreakingLogicalDNS(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: purnesh42H
body: same. Don't set `SupportLoadReportingService: true`
created_at: 2025-03-19T14:11:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: dfawley
body: Ah sorry I thought you were just talking about the comment.
created_at: 2025-03-19T20:00:45Z
author_association: MEMBER
diff_hunk: @@ -354,3 +357,213 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+func (s) TestCircuitBreaking(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+
+	// Configure the xDS management server with default resources. Override the
+	// default cluster to include an LRS server config pointing to self.
+	const serviceName = "my-test-xds-service"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+	})
+	resources.Clusters[0].CircuitBreakers = &v3clusterpb.CircuitBreakers{
+		Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{
+			{
+				Priority:    v3corepb.RoutingPriority_DEFAULT,
+				MaxRequests: wrapperspb.UInt32(maxRequests),
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Start maxRequests streams.
+	for range maxRequests {
+		if _, err := client.FullDuplexCall(ctx); err != nil {
+			t.Fatalf("rpc FullDuplexCall() failed: %v", err)
+		}
+	}
+
+	// Since we are at the max, new streams should fail.  It's possible some are
+	// allowed due to inherent raciness in the tracking, however.
+	for i := 0; i < 100; i++ {
+		stream, err := client.FullDuplexCall(ctx)
+		if status.Code(err) == codes.Unavailable {
+			return
+		}
+		if err == nil {
+			stream.CloseSend()
+			stream.Recv()
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	t.Fatalf("RPCs unexpectedly allowed beyond circuit breaking maximum")
+}
+
+func (s) TestCircuitBreakingLogicalDNS(t *testing.T) {
+	const maxRequests = 3
+	// Create an xDS management server that serves ADS and LRS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: dfawley
body: Done
created_at: 2025-03-19T20:01:15Z
author_association: MEMBER
diff_hunk: @@ -354,3 +357,219 @@ func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Serv
 		}
 	}
 }
+
+// Tests that circuit breaking limits RPCs E2E.
+func (s) TestCircuitBreaking(t *testing.T) {
+	// Create an xDS management server that serves ADS requests.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Start a server backend exposing the test service.
+	f := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testpb.TestService_FullDuplexCallServer) error {
+			for {
+				if _, err := stream.Recv(); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	server := stubserver.StartTestService(t, f)
+	defer server.Stop()
+
+	// Configure the xDS management server with default resources. Override the
+	// default cluster to include an LRS server config pointing to self.
path: xds/internal/balancer/clusterimpl/tests/balancer_test.go
user: login: dfawley
body: Done
created_at: 2025-03-19T20:47:02Z
author_association: MEMBER

PR comments:
user: login: codecov[bot]
created_at: 2025-03-13T18:33:03Z
body: ## [Codecov](https://app.codecov.io/gh/grpc/grpc-go/pull/8169?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=grpc) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 82.07%. Comparing base [(`775150f`)](https://app.codecov.io/gh/grpc/grpc-go/commit/775150f68c1759ad4b1230d09fe41efb6f3c76ad?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=grpc) to head [(`3bc9d53`)](https://app.codecov.io/gh/grpc/grpc-go/commit/3bc9d53cc7c8455d3dc8080fd462d0312a09b664?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=grpc).
> Report is 7 commits behind head on master.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #8169      +/-   ##
==========================================
- Coverage   82.25%   82.07%   -0.19%     
==========================================
  Files         393      410      +17     
  Lines       39143    40235    +1092     
==========================================
+ Hits        32197    33021     +824     
- Misses       5616     5851     +235     
- Partials     1330     1363      +33     
```

| [Files with missing lines](https://app.codecov.io/gh/grpc/grpc-go/pull/8169?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=grpc) | Coverage Δ | |
|---|---|---|
| [xds/internal/balancer/cdsbalancer/cdsbalancer.go](https://app.codecov.io/gh/grpc/grpc-go/pull/8169?src=pr&el=tree&filepath=xds%2Finternal%2Fbalancer%2Fcdsbalancer%2Fcdsbalancer.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=grpc#diff-eGRzL2ludGVybmFsL2JhbGFuY2VyL2Nkc2JhbGFuY2VyL2Nkc2JhbGFuY2VyLmdv) | `84.59% <100.00%> (+0.03%)` | :arrow_up: |
| [...internal/balancer/clusterresolver/configbuilder.go](https://app.codecov.io/gh/grpc/grpc-go/pull/8169?src=pr&el=tree&filepath=xds%2Finternal%2Fbalancer%2Fclusterresolver%2Fconfigbuilder.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=grpc#diff-eGRzL2ludGVybmFsL2JhbGFuY2VyL2NsdXN0ZXJyZXNvbHZlci9jb25maWdidWlsZGVyLmdv) | `91.12% <100.00%> (+0.05%)` | :arrow_up: |

... and [43 files with indirect coverage changes](https://app.codecov.io/gh/grpc/grpc-go/pull/8169/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=grpc)

</details>
<details><summary>🚀 New features to boost your workflow: </summary>

- ❄ [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: dfawley
created_at: 2025-03-18T20:12:19Z
body: Thanks, PTAL
author_association: MEMBER
user: login: purnesh42H
created_at: 2025-03-19T14:15:22Z
body: Doesn't look like related to change but since its xds test, might be good to confirm https://github.com/grpc/grpc-go/actions/runs/13932590107/job/38993150684?pr=8169
author_association: CONTRIBUTOR
user: login: dfawley
created_at: 2025-03-19T20:55:42Z
body: > Doesn't look like related to change but since its xds test, might be good to confirm grpc/grpc-go/actions/runs/13932590107/job/38993150684?pr=8169

Filed #8185 for the flake.
author_association: MEMBER