Repository: grpc/grpc-go

Issue:
Flaky test: TestServeAndCloseDoNotRace
pkg `xds`

link: https://github.com/grpc/grpc-go/actions/runs/10810537881/job/29988003566?pr=7619
```
--- FAIL: Test (0.59s)
    --- FAIL: Test/ServeAndCloseDoNotRace (0.25s)
    ...
    <removed logs>
    ...

        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc0005a2140] Created xds.GRPCServer  (t=+153.588724ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #95]Server created  (t=+154.986653ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc0005a2480] Created xds.GRPCServer  (t=+155.563565ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #96]Server created  (t=+156.83574ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc0005a2800] Created xds.GRPCServer  (t=+157.410739ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #97]Server created  (t=+158.692692ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc0005a2b40] Created xds.GRPCServer  (t=+159.290173ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #98]Server created  (t=+160.656623ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc0005a2e80] Created xds.GRPCServer  (t=+161.232794ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #99]Server created  (t=+162.553039ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc0005a31c0] Created xds.GRPCServer  (t=+163.130632ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc0000fbd00] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+163.626253ms)
        tlogger.go:116: INFO clientconn.go:1675 [core] original dial target is: "passthrough:///non-existent-management-server"  (t=+163.79105ms)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #100]Channel created  (t=+163.858686ms)
        tlogger.go:116: INFO clientconn.go:193 [core] [Channel #100]parsed dial target is: resolver.Target{URL:url.URL{Scheme:"passthrough", Opaque:"", User:(*url.Userinfo)(nil), Host:"", Path:"/non-existent-management-server", RawPath:"", OmitHost:false, ForceQuery:false, RawQuery:"", Fragment:"", RawFragment:""}}  (t=+163.948372ms)
        tlogger.go:116: INFO clientconn.go:194 [core] [Channel #100]Channel authority set to "non-existent-management-server"  (t=+164.007292ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc0005a2140] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+164.16227ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc0005a2800] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+164.236809ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc0005a2b40] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+164.347354ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc0005a2480] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+164.421442ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc0005a2e80] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+164.495059ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc0005a31c0] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+164.58166ms)
        tlogger.go:116: INFO resolver_wrapper.go:200 [core] [Channel #100]Resolver state updated: {
              "Addresses": [
                {
                  "Addr": "non-existent-management-server",
                  "ServerName": "",
                  "Attributes": null,
                  "BalancerAttributes": null,
                  "Metadata": null
                }
              ],
              "Endpoints": [
                {
                  "Addresses": [
                    {
                      "Addr": "non-existent-management-server",
                      "ServerName": "",
                      "Attributes": null,
                      "BalancerAttributes": null,
                      "Metadata": null
                    }
                  ],
                  "Attributes": null
                }
              ],
              "ServiceConfig": null,
              "Attributes": null
            } (resolver returned new addresses)  (t=+164.785068ms)
        tlogger.go:116: INFO balancer_wrapper.go:107 [core] [Channel #100]Channel switches to new LB policy "pick_first"  (t=+164.88831ms)
        tlogger.go:116: INFO clientconn.go:842 [core] [Channel #100 SubChannel #101]Subchannel created  (t=+164.977095ms)
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #100]Channel Connectivity change to CONNECTING  (t=+165.056853ms)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #100]Channel exiting idle mode  (t=+165.136331ms)
        tlogger.go:116: INFO transport.go:245 [xds] [xds-client 0xc000958000] [passthrough:///non-existent-management-server] Created transport to server "passthrough:///non-existent-management-server"  (t=+165.210659ms)
        tlogger.go:116: INFO clientconn.go:1199 [core] [Channel #100 SubChannel #101]Subchannel Connectivity change to CONNECTING  (t=+165.348806ms)
        tlogger.go:116: INFO clientconn.go:1317 [core] [Channel #100 SubChannel #101]Subchannel picks a new address "non-existent-management-server" to connect  (t=+165.446858ms)
        tlogger.go:116: WARNING clientconn.go:1379 [core] [Channel #100 SubChannel #101]grpc: addrConn.createTransport failed to connect to {Addr: "non-existent-management-server", ServerName: "non-existent-management-server", }. Err: connection error: desc = "transport: Error while dialing: dial tcp: address non-existent-management-server: missing port in address"  (t=+165.605563ms)
        tlogger.go:116: INFO clientconn.go:1201 [core] [Channel #100 SubChannel #101]Subchannel Connectivity change to TRANSIENT_FAILURE, last error: connection error: desc = "transport: Error while dialing: dial tcp: address non-existent-management-server: missing port in address"  (t=+165.686534ms)
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #100]Channel Connectivity change to TRANSIENT_FAILURE  (t=+165.805425ms)
        tlogger.go:116: WARNING clientimpl_watchers.go:56 [xds] Watch registered for name "ListenerResource" of type "grpc/server?xds.resource.listening_address=127.0.0.1:35925", authority "" is not found  (t=+165.865877ms)
        tlogger.go:116: WARNING listener_wrapper.go:428 [xds] [xds-server-listener 0xc000542340] Resource "grpc/server?xds.resource.listening_address=127.0.0.1:35925" received error: the xds-client is closed after listener was closed  (t=+165.957588ms)
        tlogger.go:116: WARNING transport.go:338 [xds] [xds-client 0xc000958000] [passthrough:///non-existent-management-server] Creating new ADS stream failed: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp: address non-existent-management-server: missing port in address"  (t=+166.135138ms)
        tlogger.go:116: WARNING clientimpl_watchers.go:56 [xds] Watch registered for name "ListenerResource" of type "grpc/server?xds.resource.listening_address=127.0.0.1:35925", authority "" is not found  (t=+166.202814ms)
        tlogger.go:116: WARNING listener_wrapper.go:428 [xds] [xds-server-listener 0xc000542410] Resource "grpc/server?xds.resource.listening_address=127.0.0.1:35925" received error: the xds-client is closed after listener was closed  (t=+166.267524ms)
        tlogger.go:116: WARNING clientimpl_watchers.go:56 [xds] Watch registered for name "ListenerResource" of type "grpc/server?xds.resource.listening_address=127.0.0.1:35925", authority "" is not found  (t=+166.320803ms)
        tlogger.go:116: WARNING listener_wrapper.go:428 [xds] [xds-server-listener 0xc000542750] Resource "grpc/server?xds.resource.listening_address=127.0.0.1:35925" received error: the xds-client is closed after listener was closed  (t=+166.398217ms)
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #100]Channel Connectivity change to SHUTDOWN  (t=+166.493464ms)
        tlogger.go:116: INFO resolver_wrapper.go:100 [core] [Channel #100]Closing the name resolver  (t=+166.546583ms)
        tlogger.go:116: INFO balancer_wrapper.go:145 [core] [Channel #100]ccBalancerWrapper: closing  (t=+166.607386ms)
        tlogger.go:116: INFO clientconn.go:1199 [core] [Channel #100 SubChannel #101]Subchannel Connectivity change to SHUTDOWN  (t=+166.777602ms)
        tlogger.go:116: INFO clientconn.go:1548 [core] [Channel #100 SubChannel #101]Subchannel deleted  (t=+166.845288ms)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #100]Channel deleted  (t=+166.893928ms)
        tlogger.go:116: INFO clientimpl.go:100 [xds] [xds-client 0xc000958000] Shutdown  (t=+166.955433ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #102]Server created  (t=+167.959569ms)
        tlogger.go:116: INFO client_refcounted.go:81 [xds] [xds-client 0xc000958f00] Created client with name "#server" and bootstrap configuration:
             {
              "xds_servers": [
               {
                "server_uri": "passthrough:///non-existent-management-server",
                "channel_creds": [
                 {
                  "type": "insecure"
                 }
                ]
               }
              ],
              "certificate_providers": {
               "client-side-certificate-provider-instance": {
                "plugin_name": "file_watcher",
                "config": {
                 "certificate_file": "/tmp/testClientSideXDS4106081044/cert.pem",
                 "private_key_file": "/tmp/testClientSideXDS4106081044/key.pem",
                 "ca_certificate_file": "/tmp/testClientSideXDS4106081044/ca.pem",
                 "refresh_interval": "600s"
                }
               },
               "server-side-certificate-provider-instance": {
                "plugin_name": "file_watcher",
                "config": {
                 "certificate_file": "/tmp/testServerSideXDS3856636603/cert.pem",
                 "private_key_file": "/tmp/testServerSideXDS3856636603/key.pem",
                 "ca_certificate_file": "/tmp/testServerSideXDS3856636603/ca.pem",
                 "refresh_interval": "600s"
                }
               }
              },
              "server_listener_resource_name_template": "grpc/server?xds.resource.listening_address=%s",
              "client_default_listener_resource_name_template": "%s",
              "node": {
               "id": "f692b8eb-2144-4204-9b6e-a0b079[198](https://github.com/grpc/grpc-go/actions/runs/10810537881/job/29988003566?pr=7619#step:8:199)c0f",
               "locality": {}
              }
             }  (t=+168.843673ms)
        tlogger.go:116: INFO client_refcounted.go:86 [xds] xDS node ID: f692b8eb-2144-4204-9b6e-a0b079198c0f  (t=+168.937157ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc0005a3900] Created xds.GRPCServer  (t=+168.986118ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #103]Server created  (t=+170.348851ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc0005a3e80] Created xds.GRPCServer  (t=+171.002346ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #104]Server created  (t=+172.364278ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc00084c3c0] Created xds.GRPCServer  (t=+172.949345ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #105]Server created  (t=+174.305587ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc00084c780] Created xds.GRPCServer  (t=+174.912024ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #106]Server created  (t=+176.248178ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc00084cb80] Created xds.GRPCServer  (t=+176.842713ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #107]Server created  (t=+178.18602ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc00084cb80] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+178.538345ms)
        tlogger.go:116: INFO clientconn.go:1675 [core] original dial target is: "passthrough:///non-existent-management-server"  (t=+178.729711ms)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #108]Channel created  (t=+178.815891ms)
        tlogger.go:116: INFO clientconn.go:193 [core] [Channel #108]parsed dial target is: resolver.Target{URL:url.URL{Scheme:"passthrough", Opaque:"", User:(*url.Userinfo)(nil), Host:"", Path:"/non-existent-management-server", RawPath:"", OmitHost:false, ForceQuery:false, RawQuery:"", Fragment:"", RawFragment:""}}  (t=+178.899617ms)
        tlogger.go:116: INFO clientconn.go:194 [core] [Channel #108]Channel authority set to "non-existent-management-server"  (t=+178.949179ms)
        tlogger.go:116: INFO resolver_wrapper.go:200 [core] [Channel #108]Resolver state updated: {
              "Addresses": [
                {
                  "Addr": "non-existent-management-server",
                  "ServerName": "",
                  "Attributes": null,
                  "BalancerAttributes": null,
                  "Metadata": null
                }
              ],
              "Endpoints": [
                {
                  "Addresses": [
                    {
                      "Addr": "non-existent-management-server",
                      "ServerName": "",
                      "Attributes": null,
                      "BalancerAttributes": null,
                      "Metadata": null
                    }
                  ],
                  "Attributes": null
                }
              ],
              "ServiceConfig": null,
              "Attributes": null
            } (resolver returned new addresses)  (t=+179.203322ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc0005a3900] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+179.311122ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc0005a3e80] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+179.424132ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc00084c3c0] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+179.521463ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc00084c780] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+179.634142ms)
        tlogger.go:116: INFO balancer_wrapper.go:107 [core] [Channel #108]Channel switches to new LB policy "pick_first"  (t=+179.747042ms)
        tlogger.go:116: INFO clientconn.go:842 [core] [Channel #108 SubChannel #109]Subchannel created  (t=+179.830968ms)
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #108]Channel Connectivity change to CONNECTING  (t=+179.897382ms)
        tlogger.go:116: INFO clientconn.go:1[199](https://github.com/grpc/grpc-go/actions/runs/10810537881/job/29988003566?pr=7619#step:8:200) [core] [Channel #108 SubChannel #109]Subchannel Connectivity change to CONNECTING  (t=+179.982059ms)
        tlogger.go:116: INFO clientconn.go:1317 [core] [Channel #108 SubChannel #109]Subchannel picks a new address "non-existent-management-server" to connect  (t=+180.040157ms)
        tlogger.go:116: WARNING clientconn.go:1379 [core] [Channel #108 SubChannel #109]grpc: addrConn.createTransport failed to connect to {Addr: "non-existent-management-server", ServerName: "non-existent-management-server", }. Err: connection error: desc = "transport: Error while dialing: dial tcp: address non-existent-management-server: missing port in address"  (t=+180.190967ms)
        tlogger.go:116: INFO clientconn.go:1201 [core] [Channel #108 SubChannel #109]Subchannel Connectivity change to TRANSIENT_FAILURE, last error: connection error: desc = "transport: Error while dialing: dial tcp: address non-existent-management-server: missing port in address"  (t=+180.2731ms)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #108]Channel exiting idle mode  (t=+180.333663ms)
        tlogger.go:116: INFO transport.go:245 [xds] [xds-client 0xc000958f00] [passthrough:///non-existent-management-server] Created transport to server "passthrough:///non-existent-management-server"  (t=+180.408622ms)
        tlogger.go:116: WARNING clientimpl_watchers.go:56 [xds] Watch registered for name "ListenerResource" of type "grpc/server?xds.resource.listening_address=127.0.0.1:35925", authority "" is not found  (t=+180.507997ms)
        tlogger.go:116: WARNING listener_wrapper.go:428 [xds] [xds-server-listener 0xc0005c12b0] Resource "grpc/server?xds.resource.listening_address=127.0.0.1:35925" received error: the xds-client is closed after listener was closed  (t=+180.586693ms)
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #108]Channel Connectivity change to TRANSIENT_FAILURE  (t=+180.672132ms)
        tlogger.go:116: WARNING transport.go:338 [xds] [xds-client 0xc000958f00] [passthrough:///non-existent-management-server] Creating new ADS stream failed: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp: address non-existent-management-server: missing port in address"  (t=+180.845464ms)
        tlogger.go:116: WARNING clientimpl_watchers.go:56 [xds] Watch registered for name "ListenerResource" of type "grpc/server?xds.resource.listening_address=127.0.0.1:35925", authority "" is not found  (t=+180.900867ms)
        tlogger.go:116: WARNING listener_wrapper.go:428 [xds] [xds-server-listener 0xc0005c1450] Resource "grpc/server?xds.resource.listening_address=127.0.0.1:35925" received error: the xds-client is closed after listener was closed  (t=+180.970897ms)
        tlogger.go:116: WARNING clientimpl_watchers.go:56 [xds] Watch registered for name "ListenerResource" of type "grpc/server?xds.resource.listening_address=127.0.0.1:35925", authority "" is not found  (t=+181.040146ms)
        tlogger.go:116: WARNING listener_wrapper.go:428 [xds] [xds-server-listener 0xc0005c1860] Resource "grpc/server?xds.resource.listening_address=127.0.0.1:35925" received error: the xds-client is closed after listener was closed  (t=+181.110527ms)
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #108]Channel Connectivity change to SHUTDOWN  (t=+181.188853ms)
        tlogger.go:116: INFO resolver_wrapper.go:100 [core] [Channel #108]Closing the name resolver  (t=+181.242372ms)
        tlogger.go:116: INFO balancer_wrapper.go:145 [core] [Channel #108]ccBalancerWrapper: closing  (t=+181.30061ms)
        tlogger.go:116: INFO clientconn.go:1199 [core] [Channel #108 SubChannel #109]Subchannel Connectivity change to SHUTDOWN  (t=+181.389045ms)
        tlogger.go:116: INFO clientconn.go:1548 [core] [Channel #108 SubChannel #109]Subchannel deleted  (t=+181.450208ms)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #108]Channel deleted  (t=+181.497296ms)
        tlogger.go:116: INFO clientimpl.go:100 [xds] [xds-client 0xc000958f00] Shutdown  (t=+181.556676ms)
        tlogger.go:116: INFO client_refcounted.go:81 [xds] [xds-client 0xc000959c20] Created client with name "#server" and bootstrap configuration:
             {
              "xds_servers": [
               {
                "server_uri": "passthrough:///non-existent-management-server",
                "channel_creds": [
                 {
                  "type": "insecure"
                 }
                ]
               }
              ],
              "certificate_providers": {
               "client-side-certificate-provider-instance": {
                "plugin_name": "file_watcher",
                "config": {
                 "certificate_file": "/tmp/testClientSideXDS808658363/cert.pem",
                 "private_key_file": "/tmp/testClientSideXDS808658363/key.pem",
                 "ca_certificate_file": "/tmp/testClientSideXDS808658363/ca.pem",
                 "refresh_interval": "600s"
                }
               },
               "server-side-certificate-provider-instance": {
                "plugin_name": "file_watcher",
                "config": {
                 "certificate_file": "/tmp/testServerSideXDS1611495745/cert.pem",
                 "private_key_file": "/tmp/testServerSideXDS1611495745/key.pem",
                 "ca_certificate_file": "/tmp/testServerSideXDS1611495745/ca.pem",
                 "refresh_interval": "600s"
                }
               }
              },
              "server_listener_resource_name_template": "grpc/server?xds.resource.listening_address=%s",
              "client_default_listener_resource_name_template": "%s",
              "node": {
               "id": "21640e10-c9a5-4c71-bf99-18be00c73a02",
               "locality": {}
              }
             }  (t=+182.092522ms)
        tlogger.go:116: INFO client_refcounted.go:86 [xds] xDS node ID: 21640e10-c9a5-4c71-bf99-18be00c73a02  (t=+182.184403ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc00084cfc0] Created xds.GRPCServer  (t=+182.231ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #110]Server created  (t=+184.224175ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc00084d740] Created xds.GRPCServer  (t=+184.866199ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #111]Server created  (t=+186.307268ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc00084db00] Created xds.GRPCServer  (t=+187.049297ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #112]Server created  (t=+188.630367ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc00063a000] Created xds.GRPCServer  (t=+189.339425ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #113]Server created  (t=+191.184725ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc00063a380] Created xds.GRPCServer  (t=+191.834824ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #114]Server created  (t=+193.439156ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc00063a780] Created xds.GRPCServer  (t=+194.14588ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #115]Server created  (t=+195.76[200](https://github.com/grpc/grpc-go/actions/runs/10810537881/job/29988003566?pr=7619#step:8:201)4ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc00063ab00] Created xds.GRPCServer  (t=+196.451546ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #116]Server created  (t=+197.996438ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc00063ae00] Created xds.GRPCServer  (t=+198.627171ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #117]Server created  (t=+203.682278ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc00063b140] Created xds.GRPCServer  (t=+204.306198ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #118]Server created  (t=+205.675795ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc00063b480] Created xds.GRPCServer  (t=+206.286069ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #119]Server created  (t=+207.731576ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc00063b840] Created xds.GRPCServer  (t=+208.322855ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #120]Server created  (t=+209.698232ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc00063bc00] Created xds.GRPCServer  (t=+210.296013ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #121]Server created  (t=+211.756408ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc0008c4040] Created xds.GRPCServer  (t=+212.320757ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #122]Server created  (t=+213.720269ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc0008c43c0] Created xds.GRPCServer  (t=+214.481204ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #123]Server created  (t=+215.977976ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc0008c4700] Created xds.GRPCServer  (t=+216.614299ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #124]Server created  (t=+217.931728ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc00084cfc0] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+226.548868ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc00063ae00] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+226.638074ms)
        tlogger.go:116: INFO clientconn.go:1675 [core] original dial target is: "passthrough:///non-existent-management-server"  (t=+226.847954ms)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #125]Channel created  (t=+226.913886ms)
        tlogger.go:116: INFO clientconn.go:193 [core] [Channel #125]parsed dial target is: resolver.Target{URL:url.URL{Scheme:"passthrough", Opaque:"", User:(*url.Userinfo)(nil), Host:"", Path:"/non-existent-management-server", RawPath:"", OmitHost:false, ForceQuery:false, RawQuery:"", Fragment:"", RawFragment:""}}  (t=+227.006839ms)
        tlogger.go:116: INFO clientconn.go:194 [core] [Channel #125]Channel authority set to "non-existent-management-server"  (t=+227.076769ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc0008c4a40] Created xds.GRPCServer  (t=+227.41587ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #126]Server created  (t=+228.817094ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc0008c4dc0] Created xds.GRPCServer  (t=+229.414144ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #127]Server created  (t=+230.860673ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc0008c5180] Created xds.GRPCServer  (t=+231.453896ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #128]Server created  (t=+232.817932ms)
        tlogger.go:116: INFO server.go:123 [xds] [xds-server 0xc0008c5500] Created xds.GRPCServer  (t=+233.4032ms)
        tlogger.go:116: INFO server.go:684 [core] [Server #129]Server created  (t=+240.445061ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc00063b140] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+242.31556ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc00063b480] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+242.829926ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc00063b840] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+242.972712ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc0008c4040] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+243.102112ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc00063bc00] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+243.216846ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc0008c43c0] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+243.475967ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc0008c4700] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+243.543633ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc00084d740] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+243.590981ms)
        tlogger.go:116: INFO resolver_wrapper.go:200 [core] [Channel #125]Resolver state updated: {
              "Addresses": [
                {
                  "Addr": "non-existent-management-server",
                  "ServerName": "",
                  "Attributes": null,
                  "BalancerAttributes": null,
                  "Metadata": null
                }
              ],
              "Endpoints": [
                {
                  "Addresses": [
                    {
                      "Addr": "non-existent-management-server",
                      "ServerName": "",
                      "Attributes": null,
                      "BalancerAttributes": null,
                      "Metadata": null
                    }
                  ],
                  "Attributes": null
                }
              ],
              "ServiceConfig": null,
              "Attributes": null
            } (resolver returned new addresses)  (t=+243.865722ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc0008c4dc0] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+243.969625ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc0008c4a40] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+244.051377ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc0008c5180] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+244.135654ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc0008c5500] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+244.226132ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc00084db00] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+244.268801ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc00063a000] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+244.314116ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc00063a780] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+244.360241ms)
        tlogger.go:116: INFO balancer_wrapper.go:107 [core] [Channel #125]Channel switches to new LB policy "pick_first"  (t=+244.444278ms)
        tlogger.go:116: INFO clientconn.go:842 [core] [Channel #125 SubChannel #130]Subchannel created  (t=+244.53698ms)
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #125]Channel Connectivity change to CONNECTING  (t=+244.611508ms)
        tlogger.go:116: INFO clientconn.go:1199 [core] [Channel #125 SubChannel #130]Subchannel Connectivity change to CONNECTING  (t=+244.701496ms)
        tlogger.go:116: INFO clientconn.go:1317 [core] [Channel #125 SubChannel #130]Subchannel picks a new address "non-existent-management-server" to connect  (t=+244.781825ms)
        tlogger.go:116: WARNING clientconn.go:1379 [core] [Channel #125 SubChannel #130]grpc: addrConn.createTransport failed to connect to {Addr: "non-existent-management-server", ServerName: "non-existent-management-server", }. Err: connection error: desc = "transport: Error while dialing: dial tcp: address non-existent-management-server: missing port in address"  (t=+244.963403ms)
        tlogger.go:116: INFO clientconn.go:1[201](https://github.com/grpc/grpc-go/actions/runs/10810537881/job/29988003566?pr=7619#step:8:202) [core] [Channel #125 SubChannel #130]Subchannel Connectivity change to TRANSIENT_FAILURE, last error: connection error: desc = "transport: Error while dialing: dial tcp: address non-existent-management-server: missing port in address"  (t=+245.060553ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc00063a380] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+245.105537ms)
        tlogger.go:116: INFO server.go:182 [xds] [xds-server 0xc00063ab00] Serve() passed a net.Listener on 127.0.0.1:35925  (t=+245.154818ms)
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #125]Channel Connectivity change to TRANSIENT_FAILURE  (t=+245.255596ms)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #125]Channel exiting idle mode  (t=+245.310217ms)
        tlogger.go:116: INFO transport.go:245 [xds] [xds-client 0xc000959c20] [passthrough:///non-existent-management-server] Created transport to server "passthrough:///non-existent-management-server"  (t=+245.390506ms)
        tlogger.go:116: WARNING clientimpl_watchers.go:56 [xds] Watch registered for name "ListenerResource" of type "grpc/server?xds.resource.listening_address=127.0.0.1:35925", authority "" is not found  (t=+245.50021ms)
        tlogger.go:116: WARNING transport.go:338 [xds] [xds-client 0xc000959c20] [passthrough:///non-existent-management-server] Creating new ADS stream failed: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp: address non-existent-management-server: missing port in address"  (t=+245.706634ms)
        tlogger.go:116: WARNING clientimpl_watchers.go:56 [xds] Watch registered for name "ListenerResource" of type "grpc/server?xds.resource.listening_address=127.0.0.1:35925", authority "" is not found  (t=+245.763941ms)
        tlogger.go:116: WARNING clientimpl_watchers.go:56 [xds] Watch registered for name "ListenerResource" of type "grpc/server?xds.resource.listening_address=127.0.0.1:35925", authority "" is not found  (t=+245.834883ms)
        tlogger.go:116: WARNING clientimpl_watchers.go:56 [xds] Watch registered for name "ListenerResource" of type "grpc/server?xds.resource.listening_address=127.0.0.1:35925", authority "" is not found  (t=+245.897118ms)
        tlogger.go:116: WARNING clientimpl_watchers.go:56 [xds] Watch registered for name "ListenerResource" of type "grpc/server?xds.resource.listening_address=127.0.0.1:35925", authority "" is not found  (t=+245.972398ms)
        tlogger.go:116: WARNING listener_wrapper.go:428 [xds] [xds-server-listener 0xc0001f7ba0] Resource "grpc/server?xds.resource.listening_address=127.0.0.1:35925" received error: the xds-client is closed after listener was closed  (t=+246.060612ms)
        tlogger.go:116: WARNING listener_wrapper.go:428 [xds] [xds-server-listener 0xc0001f7d40] Resource "grpc/server?xds.resource.listening_address=127.0.0.1:35925" received error: the xds-client is closed after listener was closed  (t=+246.1[213](https://github.com/grpc/grpc-go/actions/runs/10810537881/job/29988003566?pr=7619#step:8:214)55ms)
        tlogger.go:116: WARNING listener_wrapper.go:428 [xds] [xds-server-listener 0xc000198270] Resource "grpc/server?xds.resource.listening_address=127.0.0.1:35925" received error: the xds-client is closed after listener was closed  (t=+246.166619ms)
        tlogger.go:116: WARNING listener_wrapper.go:428 [xds] [xds-server-listener 0xc000198a90] Resource "grpc/server?xds.resource.listening_address=127.0.0.1:35925" received error: the xds-client is closed after listener was closed  (t=+246.213406ms)
        tlogger.go:116: WARNING listener_wrapper.go:428 [xds] [xds-server-listener 0xc000198f70] Resource "grpc/server?xds.resource.listening_address=127.0.0.1:35925" received error: the xds-client is closed after listener was closed  (t=+246.254753ms)
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #125]Channel Connectivity change to SHUTDOWN  (t=+246.346894ms)
        tlogger.go:116: INFO resolver_wrapper.go:100 [core] [Channel #125]Closing the name resolver  (t=+246.394603ms)
        tlogger.go:116: INFO balancer_wrapper.go:145 [core] [Channel #125]ccBalancerWrapper: closing  (t=+246.451689ms)
        tlogger.go:116: INFO clientconn.go:1199 [core] [Channel #125 SubChannel #130]Subchannel Connectivity change to SHUTDOWN  (t=+246.540635ms)
        tlogger.go:116: INFO clientconn.go:1548 [core] [Channel #125 SubChannel #130]Subchannel deleted  (t=+246.606096ms)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #125]Channel deleted  (t=+246.7[217](https://github.com/grpc/grpc-go/actions/runs/10810537881/job/29988003566?pr=7619#step:8:218)81ms)
        tlogger.go:116: INFO clientimpl.go:100 [xds] [xds-client 0xc000959c20] Shutdown  (t=+246.785069ms)
        server_test.go:703: Failed to create an xDS enabled gRPC server: xDS client creation failed: xds: failed to get xDS bootstrap config: bootstrap environment variables ("GRPC_XDS_BOOTSTRAP" or "GRPC_XDS_BOOTSTRAP_CONFIG") not defined, and no fallback config set
FAIL
```

Discussion:
@purnesh42H: @arvindbr8 looks like no report from 2 weeks. Should we downgrade to P2?@dfawley: We should measure the flakiness w/10k runs and assign priority accordingly.  Also consider new-ness -- if the test is new we should treat with much higher priority since we don't generally want to allow flakiness.@purnesh42H: no failures in 10k forge runs@arjan-bal: https://github.com/grpc/grpc-go/actions/runs/11397970463/job/31714272864?pr=7754@aranjans: FAILED in 106 out of 100000 
http://fusion2/49630097-747f-425d-b94e-3d6519c01e67@zasweq: Taking a look now, but I see that this test has flaked in G/A twice over a year and a third of it being merged, so I don't think it's super urgent. The fact it only showed up this year Sept makes me think some bootstrap parsing code changed, as that's how it specifies to create a new server. Weird that that would induce flakiness though, because the waitgroup doesn't end until after that processes. Maybe there's a race between setting bootstrap and server actually starting.@dfawley: 100/100k = 1/1000 which is pretty low, but taking a look to see if we can find the root cause would be worth it.@dfawley: Flaked 2x on my PR in a single snapshot:

https://github.com/grpc/grpc-go/actions/runs/11634120790/job/32400709681?pr=7798
https://github.com/grpc/grpc-go/actions/runs/11634120790/job/32400708629?pr=7798@zasweq: https://github.com/grpc/grpc-go/actions/runs/11636054765/job/32406682904?pr=7797@easwars: https://github.com/grpc/grpc-go/actions/runs/11674461177/job/32507200078?pr=7804@zasweq: Will take a look at this this week once I get some time looks like it's flaking a lot more now.@zasweq: The fact the first occurence of this occurred sept of this year, over a year after it was merged makes me think it's racing with some of the xDS Client changes for fallback with respect to testing plumbing of xDS Bootstrap.@easwars: I'll take a look at this tomorrow.@arjan-bal: Failure: https://github.com/grpc/grpc-go/actions/runs/11689787872/job/32553307641

Stack traces:
```
panic: test timed out after 7m0s
	running tests:
		Test (7m0s)
		Test/ServeAndCloseDoNotRace (7m0s)

goroutine 369 [running]:
testing.(*M).startAlarm.func1()
	/opt/hostedtoolcache/go/1.23.2/x64/src/testing/testing.go:2373 +0x265
created by time.goFunc
	/opt/hostedtoolcache/go/1.23.2/x64/src/time/sleep.go:215 +0x45

goroutine 1 [chan receive, 7 minutes]:
testing.(*T).Run(0xc00015a4e0, {0x17b7e62, 0x4}, 0x181c398)
	/opt/hostedtoolcache/go/1.23.2/x64/src/testing/testing.go:1751 +0x851
testing.runTests.func1(0xc00015a4e0)
	/opt/hostedtoolcache/go/1.23.2/x64/src/testing/testing.go:[216](https://github.com/grpc/grpc-go/actions/runs/11689787872/job/32553307641#step:8:217)8 +0x86
testing.tRunner(0xc00015a4e0, 0xc00006bae0)
	/opt/hostedtoolcache/go/1.23.2/x64/src/testing/testing.go:1690 +0x227
testing.runTests(0xc00012c2d0, {0x235b860, 0x2, 0x2}, {0x7fe8fec5c108?, 0x40?, 0x237b1c0?})
	/opt/hostedtoolcache/go/1.23.2/x64/src/testing/testing.go:2166 +0x8bf
testing.(*M).Run(0xc000320000)
	/opt/hostedtoolcache/go/1.23.2/x64/src/testing/testing.go:2034 +0xf18
main.main()
	_testmain.go:49 +0x165

goroutine 4 [chan receive, 6 minutes]:
testing.(*T).Run(0xc00015a680, {0x14d6b6f, 0x16}, 0xc00019e440)
	/opt/hostedtoolcache/go/1.23.2/x64/src/testing/testing.go:1751 +0x851
google.golang.org/grpc/internal/grpctest.RunSubTests(0xc00015a680, {0x195c4e0, 0x239c600})
	/home/runner/work/grpc-go/grpc-go/internal/grpctest/grpctest.go:114 +0x352
google.golang.org/grpc/xds.Test(0xc00015a680)
	/home/runner/work/grpc-go/grpc-go/xds/server_test.go:65 +0x35
testing.tRunner(0xc00015a680, 0x181c398)
	/opt/hostedtoolcache/go/1.23.2/x64/src/testing/testing.go:1690 +0x[227](https://github.com/grpc/grpc-go/actions/runs/11689787872/job/32553307641#step:8:228)
created by testing.(*T).Run in goroutine 1
	/opt/hostedtoolcache/go/1.23.2/x64/src/testing/testing.go:1743 +0x826

goroutine 119 [semacquire, 6 minutes]:
sync.runtime_Semacquire(0xc00034a2a8?)
	/opt/hostedtoolcache/go/1.23.2/x64/src/runtime/sema.go:71 +0x25
sync.(*WaitGroup).Wait(0xc00034a2a0)
	/opt/hostedtoolcache/go/1.23.2/x64/src/sync/waitgroup.go:118 +0xa5
google.golang.org/grpc/xds.s.TestServeAndCloseDoNotRace({{}}, 0xc00036a000)
	/home/runner/work/grpc-go/grpc-go/xds/server_test.go:715 +0x54b
google.golang.org/grpc/internal/grpctest.RunSubTests.func1(0xc00036a000)
	/home/runner/work/grpc-go/grpc-go/internal/grpctest/grpctest.go:122 +0x10e

goroutine 183 [chan receive, 6 minutes]:
google.golang.org/grpc/xds/internal/xdsclient.(*authority).watchResource(0xc0005d42a0, {0x19617a0, 0xc000827c50}, {0xc000059e80, 0x3a}, {0x195fa00, 0xc000862780})
	/home/runner/work/grpc-go/grpc-go/xds/internal/xdsclient/authority.go:483 +0x2ab
google.golang.org/grpc/xds/internal/xdsclient.(*clientImpl).WatchResource(0xc000536680, {0x19617a0, 0xc000827c50}, {0xc000059e80, 0x3a}, {0x195fa00, 0xc000862780})
	/home/runner/work/grpc-go/grpc-go/xds/internal/xdsclient/clientimpl_watchers.go:66 +0xae5
google.golang.org/grpc/xds/internal/xdsclient/xdsresource.WatchListener(...)
	/home/runner/work/grpc-go/grpc-go/xds/internal/xdsclient/xdsresource/listener_resource_type.go:185
google.golang.org/grpc/xds/internal/server.NewListenerWrapper({{0x195f6b8, 0xc00036c100}, {0xc000059e80, 0x3a}, {0x7fe8b4dacfa0, 0xc00047dc70}, 0xc000862760})
	/home/runner/work/grpc-go/grpc-go/xds/internal/server/listener_wrapper.go:102 +0xc3d
google.golang.org/grpc/xds.(*GRPCServer).Serve(0xc000802b00, {0x195f6b8, 0xc00036c100})
	/home/runner/work/grpc-go/grpc-go/xds/server.go:201 +0x457
google.golang.org/grpc/xds.s.TestServeAndCloseDoNotRace.func1()
	/home/runner/work/grpc-go/grpc-go/xds/server_test.go:707 +0x50
created by google.golang.org/grpc/xds.s.TestServeAndCloseDoNotRace in goroutine 119
	/home/runner/work/grpc-go/grpc-go/xds/server_test.go:706 +0x47f

goroutine 181 [chan receive, 6 minutes]:
google.golang.org/grpc/xds/internal/xdsclient.(*authority).watchResource(0xc0005d42a0, {0x19617a0, 0xc000827b60}, {0xc000059e00, 0x3a}, {0x195fa00, 0xc0008626c0})
	/home/runner/work/grpc-go/grpc-go/xds/internal/xdsclient/authority.go:483 +0x2ab
google.golang.org/grpc/xds/internal/xdsclient.(*clientImpl).WatchResource(0xc000536680, {0x19617a0, 0xc000827b60}, {0xc000059e00, 0x3a}, {0x195fa00, 0xc0008626c0})
	/home/runner/work/grpc-go/grpc-go/xds/internal/xdsclient/clientimpl_watchers.go:66 +0xae5
google.golang.org/grpc/xds/internal/xdsclient/xdsresource.WatchListener(...)
	/home/runner/work/grpc-go/grpc-go/xds/internal/xdsclient/xdsresource/listener_resource_type.go:185
google.golang.org/grpc/xds/internal/server.NewListenerWrapper({{0x195f6b8, 0xc00036c100}, {0xc000059e00, 0x3a}, {0x7fe8b4dacfa0, 0xc00047dc70}, 0xc0008626a0})
	/home/runner/work/grpc-go/grpc-go/xds/internal/server/listener_wrapper.go:102 +0xc3d
google.golang.org/grpc/xds.(*GRPCServer).Serve(0xc000802780, {0x195f6b8, 0xc00036c100})
	/home/runner/work/grpc-go/grpc-go/xds/server.go:201 +0x457
google.golang.org/grpc/xds.s.TestServeAndCloseDoNotRace.func1()
	/home/runner/work/grpc-go/grpc-go/xds/server_test.go:707 +0x50
created by google.golang.org/grpc/xds.s.TestServeAndCloseDoNotRace in goroutine 119
	/home/runner/work/grpc-go/grpc-go/xds/server_test.go:706 +0x47f

goroutine 169 [chan receive, 6 minutes]:
google.golang.org/grpc/xds/internal/xdsclient.(*authority).watchResource(0xc0005d42a0, {0x19617a0, 0xc000827740}, {0xc000059b80, 0x3a}, {0x195fa00, 0xc0008622a0})
	/home/runner/work/grpc-go/grpc-go/xds/internal/xdsclient/authority.go:483 +0x2ab
google.golang.org/grpc/xds/internal/xdsclient.(*clientImpl).WatchResource(0xc000536680, {0x19617a0, 0xc000827740}, {0xc000059b80, 0x3a}, {0x195fa00, 0xc0008622a0})
	/home/runner/work/grpc-go/grpc-go/xds/internal/xdsclient/clientimpl_watchers.go:66 +0xae5
google.golang.org/grpc/xds/internal/xdsclient/xdsresource.WatchListener(...)
	/home/runner/work/grpc-go/grpc-go/xds/internal/xdsclient/xdsresource/listener_resource_type.go:185
google.golang.org/grpc/xds/internal/server.NewListenerWrapper({{0x195f6b8, 0xc00036c100}, {0xc000059b80, 0x3a}, {0x7fe8b4dacfa0, 0xc00047dc70}, 0xc00086[228](https://github.com/grpc/grpc-go/actions/runs/11689787872/job/32553307641#step:8:229)0})
	/home/runner/work/grpc-go/grpc-go/xds/internal/server/listener_wrapper.go:102 +0xc3d
google.golang.org/grpc/xds.(*GRPCServer).Serve(0xc000595200, {0x195f6b8, 0xc00036c100})
	/home/runner/work/grpc-go/grpc-go/xds/server.go:201 +0x457
google.golang.org/grpc/xds.s.TestServeAndCloseDoNotRace.func1()
	/home/runner/work/grpc-go/grpc-go/xds/server_test.go:707 +0x50
created by google.golang.org/grpc/xds.s.TestServeAndCloseDoNotRace in goroutine 119
	/home/runner/work/grpc-go/grpc-go/xds/server_test.go:706 +0x47f
```@arjan-bal: A bunch of goroutines are stuck on authority.go:483 which reads from a channel: https://github.com/grpc/grpc-go/blob/0ec8fd84fdfb54f1b7f9c2d3d22aa20cd7a8cf09/xds/internal/xdsclient/authority.go#L483-L484

The change was introduced in https://github.com/grpc/grpc-go/pull/7773.@arjan-bal: @easwars maybe we need to close the channel if the callback can't be scheduled because the serializer is closed?
https://github.com/grpc/grpc-go/blob/0ec8fd84fdfb54f1b7f9c2d3d22aa20cd7a8cf09/xds/internal/xdsclient/authority.go#L430-L436@easwars: I think you are right. Let me send a PR. I also see another place which looks very similar to this.@zasweq: Oh yeah makes sense good find Arjan.@easwars: Closing the `done` channel when unable to schedule the serializer callback fixes the failures where it takes `7m` (the configured test timeout) to fail.

I'm now seeing another flake though:
```
grpc/xds/server_test.go:704: Failed to create an xDS enabled gRPC server: xDS client creation failed: xds: failed to get xDS bootstrap config: bootstrap environment variables ("GRPC_XDS_BOOTSTRAP" or "GRPC_XDS_BOOTSTRAP_CONFIG") not defined, and no fallback config set
```

This is weird and I'm trying to get to the bottom of this now.@easwars: Ok, I see what is going on:

- The test creates a new xDS enabled gRPC server in a loop:
```
server, err := NewGRPCServer(BootstrapContentsForTesting(...))
```
- The loop also starts goroutines that calls `Serve` and `Stop`

When the server is created with a server option for bootstrap config, the server calls `xdsclient.NewForTesting()` to create an xDS client instead of calling `xds.New()`, which is what it would do in production code.

The `xdsclient.NewForTesting()`, calls `bootstrap.SetFallbackBootstrapConfig` to set the fallback bootstrap config to the one passed by the test. And it returns a cancel func which calls `bootstrap.UnsetFallbackBootstrapConfigForTesting`.

The above two operations basically race against each other, and a server creation could see that the boostrap config is unset because an previous iteration erased it.

I'm still exploring options for the fix. 

Solution patch:
diff --git a/xds/internal/xdsclient/authority.go b/xds/internal/xdsclient/authority.go
index bd1662e8..27abb64e 100644
--- a/xds/internal/xdsclient/authority.go
+++ b/xds/internal/xdsclient/authority.go
@@ -40,7 +40,7 @@ type resourceState struct {
 	cache             xdsresource.ResourceData             // Most recent ACKed update for this resource.
 	md                xdsresource.UpdateMetadata           // Metadata for the most recent update.
 	deletionIgnored   bool                                 // True, if resource deletion was ignored for a prior update.
-	xdsChannelConfigs []*xdsChannelWithConfig              // List of xdsChannels where this resource is subscribed.
+	xdsChannelConfigs map[*xdsChannelWithConfig]bool       // Set of xdsChannels where this resource is subscribed.
 }
 
 // xdsChannelForADS is used to acquire a reference to an xdsChannel. This
@@ -59,9 +59,9 @@ type xdsChannelForADS func(*bootstrap.ServerConfig, *authority) (*xdsChannel, fu
 // xdsChannelWithConfig is a struct that holds an xdsChannel and its associated
 // ServerConfig, along with a cleanup function to release the xdsChannel.
 type xdsChannelWithConfig struct {
-	xc      *xdsChannel
-	sc      *bootstrap.ServerConfig
-	cleanup func()
+	channel      *xdsChannel
+	serverConfig *bootstrap.ServerConfig
+	cleanup      func()
 }
 
 // authority provides the functionality required to communicate with a
@@ -149,7 +149,7 @@ func newAuthority(args authorityBuildOptions) *authority {
 	// first watch is registered, and channels to other server configurations
 	// are created as needed to support fallback.
 	for _, sc := range args.serverConfigs {
-		ret.xdsChannelConfigs = append(ret.xdsChannelConfigs, &xdsChannelWithConfig{sc: sc})
+		ret.xdsChannelConfigs = append(ret.xdsChannelConfigs, &xdsChannelWithConfig{serverConfig: sc})
 	}
 	return ret
 }
@@ -200,8 +200,101 @@ func (a *authority) handleADSStreamFailure(serverConfig *bootstrap.ServerConfig,
 		}
 	}
 
-	// TODO(easwars-fallback): Trigger fallback here if conditions for fallback
-	// are met.
+	// Two conditions need to be met for fallback to be triggered:
+	// 1. There is a connectivity failure on the ADS stream, as described in
+	//    gRFC A57. For us, this means that the ADS stream was closed before the
+	//    first server response was received. We already checked that condition
+	//    earlier in this method.
+	// 2. There is at least one watcher for a resource that is not cached.
+	//    Cached resources include ones that
+	//    - have been successfully received and can be used.
+	//    - are considered non-existent according to xDS Protocol Specification.
+	if !a.watcherExistsForUncachedResource() {
+		if a.logger.V(2) {
+			a.logger.Infof("No watchers for uncached resources. Not triggering fallback")
+		}
+		return
+	}
+	a.fallbackToNextServerIfPossible(serverConfig)
+}
+
+// serverIndexForConfig returns the index of the xdsChannelConfig that matches
+// the provided ServerConfig. If no match is found, it returns the length of the
+// xdsChannelConfigs slice, which represents the index of a non-existent config.
+func (a *authority) serverIndexForConfig(sc *bootstrap.ServerConfig) int {
+	for i, cfg := range a.xdsChannelConfigs {
+		if cfg.serverConfig.Equal(sc) {
+			return i
+		}
+	}
+	return len(a.xdsChannelConfigs)
+}
+
+// Determines the server to fallback to and triggers fallback to the same. If
+// required, creates an xdsChannel to that server, and re-subscribes to all
+// existing resources.
+//
+// Only executed in the context of a serializer callback.
+func (a *authority) fallbackToNextServerIfPossible(failingServerConfig *bootstrap.ServerConfig) {
+	if a.logger.V(2) {
+		a.logger.Infof("Attempting to initiate fallback after failure from server %q", failingServerConfig)
+	}
+
+	// The server to fallback to is the next server on the list. If the current
+	// server is the last server, then there is nothing that can be done.
+	currentServerIdx := a.serverIndexForConfig(failingServerConfig)
+	if currentServerIdx == len(a.xdsChannelConfigs) {
+		// This can never happen.
+		a.logger.Errorf("Received error from an unknown server: %s", failingServerConfig)
+		return
+	}
+	if currentServerIdx == len(a.xdsChannelConfigs)-1 {
+		if a.logger.V(2) {
+			a.logger.Infof("No more servers to fallback to")
+		}
+		return
+	}
+	fallbackServerIdx := currentServerIdx + 1
+	fallbackChannel := a.xdsChannelConfigs[fallbackServerIdx]
+
+	// If the server to fallback to already has an xdsChannel, it means that
+	// this connectivity error is from a server with a higher priority. There
+	// is not much we can do here.
+	if fallbackChannel.channel != nil {
+		if a.logger.V(2) {
+			a.logger.Infof("Channel to the next server in the list %q already exists", fallbackChannel.serverConfig)
+		}
+		return
+	}
+
+	// Create an xdsChannel for the fallback server.
+	if a.logger.V(2) {
+		a.logger.Infof("Initiating fallback to server %s", fallbackChannel.serverConfig)
+	}
+	xc, cleanup, err := a.getChannelForADS(fallbackChannel.serverConfig, a)
+	if err != nil {
+		a.logger.Errorf("Failed to create XDS channel: %v", err)
+		return
+	}
+	fallbackChannel.channel = xc
+	fallbackChannel.cleanup = cleanup
+	a.activeXDSChannel = fallbackChannel
+
+	// Subscribe to all existing resources from the new management server.
+	for typ, resources := range a.resources {
+		for name, state := range resources {
+			if a.logger.V(2) {
+				a.logger.Infof("Resubscribing to resource of type %q and name %q", typ.TypeName(), name)
+			}
+			xc.subscribe(typ, name)
+
+			// Add the fallback channel to the list of xdsChannels from which
+			// this resource has been requested from. Retain the cached resource
+			// and the set of existing watchers (and other metadata fields) in
+			// the resource state.
+			state.xdsChannelConfigs[fallbackChannel] = true
+		}
+	}
 }
 
 // adsResourceUpdate is called to notify the authority about a resource update
@@ -218,13 +311,15 @@ func (a *authority) adsResourceUpdate(serverConfig *bootstrap.ServerConfig, rTyp
 // handleADSResourceUpdate processes an update from the xDS client, updating the
 // resource cache and notifying any registered watchers of the update.
 //
+// If the update is received from a higher priority xdsChannel that was
+// previously down, we revert to it and close all lower priority xdsChannels.
+//
 // Once the update has been processed by all watchers, the authority is expected
 // to invoke the onDone callback.
 //
 // Only executed in the context of a serializer callback.
 func (a *authority) handleADSResourceUpdate(serverConfig *bootstrap.ServerConfig, rType xdsresource.Type, updates map[string]ads.DataAndErrTuple, md xdsresource.UpdateMetadata, onDone func()) {
-	// TODO(easwars-fallback): Trigger reverting to a higher priority server if
-	// the update is from one.
+	a.handleRevertingToPrimaryOnUpdate(serverConfig)
 
 	// We build a list of callback funcs to invoke, and invoke them at the end
 	// of this method instead of inline (when handling the update for a
@@ -416,6 +511,74 @@ func (a *authority) handleADSResourceDoesNotExist(rType xdsresource.Type, resour
 	}
 }
 
+// handleRevertingToPrimaryOnUpdate is called when a resource update is received
+// from the xDS client.
+//
+// If the update is from the currently active server, nothing is done. Else, all
+// lower priority servers are closed and the active server is reverted to the
+// highest priority server that sent the update.
+//
+// This method is only executed in the context of a serializer callback.
+func (a *authority) handleRevertingToPrimaryOnUpdate(serverConfig *bootstrap.ServerConfig) {
+	if a.activeXDSChannel != nil && a.activeXDSChannel.serverConfig.Equal(serverConfig) {
+		// If the resource update is from the current active server, nothing
+		// needs to be done from fallback point of view.
+		return
+	}
+
+	if a.logger.V(2) {
+		a.logger.Infof("Received update from non-active server %q", serverConfig)
+	}
+
+	// If the resource update is not from the current active server, it means
+	// that we have received an update from a higher priority server and we need
+	// to revert back to it. This method guarantees that when an update is
+	// received from a server, all lower priority servers are closed.
+	serverIdx := a.serverIndexForConfig(serverConfig)
+	if serverIdx == len(a.xdsChannelConfigs) {
+		// This can never happen.
+		a.logger.Errorf("Received update from an unknown server: %s", serverConfig)
+		return
+	}
+	a.activeXDSChannel = a.xdsChannelConfigs[serverIdx]
+
+	// Close all lower priority channels.
+	//
+	// But before closing any channel, we need to unsubscribe from any resources
+	// that were subscribed to on this channel. Resources could be subscribed to
+	// from multiple channels as we fallback to lower priority servers. But when
+	// a higher priority one comes back up, we need to unsubscribe from all
+	// lower priority ones before releasing the reference to them.
+	for i := serverIdx + 1; i < len(a.xdsChannelConfigs); i++ {
+		cfg := a.xdsChannelConfigs[i]
+
+		for rType, rState := range a.resources {
+			for resourceName, state := range rState {
+				for xcc := range state.xdsChannelConfigs {
+					if xcc != cfg {
+						continue
+					}
+					// If the current resource is subscribed to on this channel,
+					// unsubscribe, and remove the channel from the list of
+					// channels that this resource is subscribed to.
+					xcc.channel.unsubscribe(rType, resourceName)
+					delete(state.xdsChannelConfigs, xcc)
+				}
+			}
+		}
+
+		// Release the reference to the channel.
+		if cfg.cleanup != nil {
+			if a.logger.V(2) {
+				a.logger.Infof("Closing lower priority server %q", cfg.serverConfig)
+			}
+			cfg.cleanup()
+			cfg.cleanup = nil
+		}
+		cfg.channel = nil
+	}
+}
+
 // watchResource registers a new watcher for the specified resource type and
 // name. It returns a function that can be called to cancel the watch.
 //
@@ -431,7 +594,7 @@ func (a *authority) watchResource(rType xdsresource.Type, resourceName string, w
 	cleanup := func() {}
 	done := make(chan struct{})
 
-	a.xdsClientSerializer.TrySchedule(func(context.Context) {
+	a.xdsClientSerializer.ScheduleOr(func(context.Context) {
 		defer close(done)
 
 		if a.logger.V(2) {
@@ -462,10 +625,10 @@ func (a *authority) watchResource(rType xdsresource.Type, resourceName string, w
 			state = &resourceState{
 				watchers:          make(map[xdsresource.ResourceWatcher]bool),
 				md:                xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusRequested},
-				xdsChannelConfigs: []*xdsChannelWithConfig{xdsChannel},
+				xdsChannelConfigs: map[*xdsChannelWithConfig]bool{xdsChannel: true},
 			}
 			resources[resourceName] = state
-			xdsChannel.xc.subscribe(rType, resourceName)
+			xdsChannel.channel.subscribe(rType, resourceName)
 		}
 		// Always add the new watcher to the set of watchers.
 		state.watchers[watcher] = true
@@ -479,6 +642,11 @@ func (a *authority) watchResource(rType xdsresource.Type, resourceName string, w
 			a.watcherCallbackSerializer.TrySchedule(func(context.Context) { watcher.OnUpdate(resource, func() {}) })
 		}
 		cleanup = a.unwatchResource(rType, resourceName, watcher)
+	}, func() {
+		if a.logger.V(2) {
+			a.logger.Infof("Failed to schedule a watch for type %q, resource name %q, because the xDS client is closed", rType.TypeName(), resourceName)
+		}
+		close(done)
 	})
 	<-done
 	return cleanup
@@ -516,8 +684,8 @@ func (a *authority) unwatchResource(rType xdsresource.Type, resourceName string,
 			if a.logger.V(2) {
 				a.logger.Infof("Removing last watch for resource name %q", resourceName)
 			}
-			for _, xc := range state.xdsChannelConfigs {
-				xc.xc.unsubscribe(rType, resourceName)
+			for xcc := range state.xdsChannelConfigs {
+				xcc.channel.unsubscribe(rType, resourceName)
 			}
 			delete(resources, resourceName)
 
@@ -553,13 +721,13 @@ func (a *authority) xdsChannelToUse() *xdsChannelWithConfig {
 		return a.activeXDSChannel
 	}
 
-	sc := a.xdsChannelConfigs[0].sc
+	sc := a.xdsChannelConfigs[0].serverConfig
 	xc, cleanup, err := a.getChannelForADS(sc, a)
 	if err != nil {
 		a.logger.Warningf("Failed to create xDS channel: %v", err)
 		return nil
 	}
-	a.xdsChannelConfigs[0].xc = xc
+	a.xdsChannelConfigs[0].channel = xc
 	a.xdsChannelConfigs[0].cleanup = cleanup
 	a.activeXDSChannel = a.xdsChannelConfigs[0]
 	return a.activeXDSChannel
@@ -570,26 +738,41 @@ func (a *authority) xdsChannelToUse() *xdsChannelWithConfig {
 //
 // Only executed in the context of a serializer callback.
 func (a *authority) closeXDSChannels() {
-	for _, xc := range a.xdsChannelConfigs {
-		if xc.cleanup != nil {
-			xc.cleanup()
-			xc.cleanup = nil
+	for _, xcc := range a.xdsChannelConfigs {
+		if xcc.cleanup != nil {
+			xcc.cleanup()
+			xcc.cleanup = nil
 		}
-		xc.xc = nil
+		xcc.channel = nil
 	}
 	a.activeXDSChannel = nil
 }
 
+// watcherExistsForUncachedResource returns true if there is at least one
+// watcher for a resource that has not yet been cached.
+//
+// Only executed in the context of a serializer callback.
+func (a *authority) watcherExistsForUncachedResource() bool {
+	for _, resourceStates := range a.resources {
+		for _, state := range resourceStates {
+			if state.md.Status == xdsresource.ServiceStatusRequested {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // dumpResources returns a dump of the resource configuration cached by this
 // authority, for CSDS purposes.
 func (a *authority) dumpResources() []*v3statuspb.ClientConfig_GenericXdsConfig {
 	var ret []*v3statuspb.ClientConfig_GenericXdsConfig
 	done := make(chan struct{})
 
-	a.xdsClientSerializer.TrySchedule(func(context.Context) {
+	a.xdsClientSerializer.ScheduleOr(func(context.Context) {
 		defer close(done)
 		ret = a.resourceConfig()
-	})
+	}, func() { close(done) })
 	<-done
 	return ret
 }
diff --git a/xds/internal/xdsclient/client_new.go b/xds/internal/xdsclient/client_new.go
index 839cf238..fd5a8d57 100644
--- a/xds/internal/xdsclient/client_new.go
+++ b/xds/internal/xdsclient/client_new.go
@@ -108,6 +108,7 @@ func newClientImpl(config *bootstrap.Config, watchExpiryTimeout, idleChannelExpi
 type OptionsForTesting struct {
 	// Name is a unique name for this xDS client.
 	Name string
+
 	// Contents contain a JSON representation of the bootstrap configuration to
 	// be used when creating the xDS client.
 	Contents []byte
@@ -128,6 +129,11 @@ type OptionsForTesting struct {
 
 // NewForTesting returns an xDS client configured with the provided options.
 //
+// Sets the fallback bootstrap configuration to the contents in the
+// opts.Contents field. This value persists for the life of the test binary. So,
+// tests that want this value to be empty should call
+// bootstrap.UnsetFallbackBootstrapConfigForTesting to ensure the same.
+//
 // The second return value represents a close function which the caller is
 // expected to invoke once they are done using the client.  It is safe for the
 // caller to invoke this close function multiple times.
@@ -152,8 +158,7 @@ func NewForTesting(opts OptionsForTesting) (XDSClient, func(), error) {
 	if err := bootstrap.SetFallbackBootstrapConfig(opts.Contents); err != nil {
 		return nil, nil, err
 	}
-	client, cancel, err := newRefCounted(opts.Name, opts.WatchExpiryTimeout, opts.IdleChannelExpiryTimeout, opts.StreamBackoffAfterFailure)
-	return client, func() { bootstrap.UnsetFallbackBootstrapConfigForTesting(); cancel() }, err
+	return newRefCounted(opts.Name, opts.WatchExpiryTimeout, opts.IdleChannelExpiryTimeout, opts.StreamBackoffAfterFailure)
 }
 
 // GetForTesting returns an xDS client created earlier using the given name.
@@ -180,7 +185,6 @@ func GetForTesting(name string) (XDSClient, func(), error) {
 func init() {
 	internal.TriggerXDSResourceNotFoundForTesting = triggerXDSResourceNotFoundForTesting
 	xdsclientinternal.ResourceWatchStateForTesting = resourceWatchStateForTesting
-
 }
 
 func triggerXDSResourceNotFoundForTesting(client XDSClient, typ xdsresource.Type, name string) error {



Test patch:
diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go
index 77e8c47e..8a1a7427 100644
--- a/xds/internal/resolver/xds_resolver_test.go
+++ b/xds/internal/resolver/xds_resolver_test.go
@@ -71,6 +71,7 @@ import (
 // build fails as well.
 func (s) TestResolverBuilder_ClientCreationFails_NoBootstrap(t *testing.T) {
 	// Build an xDS resolver without specifying bootstrap env vars.
+	bootstrap.UnsetFallbackBootstrapConfigForTesting()
 	builder := resolver.Get(xdsresolver.Scheme)
 	if builder == nil {
 		t.Fatalf("Scheme %q is not registered", xdsresolver.Scheme)
diff --git a/xds/internal/xdsclient/tests/fallback_test.go b/xds/internal/xdsclient/tests/fallback_test.go
new file mode 100644
index 00000000..514945f8
--- /dev/null
+++ b/xds/internal/xdsclient/tests/fallback_test.go
@@ -0,0 +1,628 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/xds/internal/xdsclient"
+	"google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// Give the fallback tests additional time to complete because they need to
+// first identify failed connections before establishing new ones.
+const defaultFallbackTestTimeout = 2 * defaultTestTimeout
+
+func waitForRPCsToReachBackend(ctx context.Context, client testgrpc.TestServiceClient, backend string) error {
+	var lastErr error
+	for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
+		var peer peer.Peer
+		if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil {
+			lastErr = err
+			continue
+		}
+		// Veirfy the peer when the RPC succeeds.
+		if peer.Addr.String() == backend {
+			break
+		}
+	}
+	if ctx.Err() != nil {
+		return fmt.Errorf("timeout when waiting for RPCs to reach expected backend. Last error: %v", lastErr)
+	}
+	return nil
+}
+
+// Tests fallback on startup where the xDS client is unable to establish a
+// connection to the primary server. The test verifies that the xDS client falls
+// back to the secondary server, and when the primary comes back up, it reverts
+// to it. The test also verifies that when all requested resources are cached
+// from the primary, fallback is not triggered when the connection goes down.
+func (s) TestFallback_OnStartup(t *testing.T) {
+	// Enable fallback env var.
+	origFallbackEnv := envconfig.XDSFallbackSupport
+	envconfig.XDSFallbackSupport = true
+	defer func() { envconfig.XDSFallbackSupport = origFallbackEnv }()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout)
+	defer cancel()
+
+	// Create two listeners for the two management servers. The test can
+	// start/stop these listeners and can also get notified when the listener
+	// receives a connection request.
+	primaryWrappedLis := testutils.NewListenerWrapper(t, nil)
+	primaryLis := testutils.NewRestartableListener(primaryWrappedLis)
+	fallbackWrappedLis := testutils.NewListenerWrapper(t, nil)
+	fallbackLis := testutils.NewRestartableListener(fallbackWrappedLis)
+
+	// Start two management servers, primary and fallback, with the above
+	// listeners.
+	primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: primaryLis})
+	fallbackManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: fallbackLis})
+
+	// Start two test service backends.
+	backend1 := stubserver.StartTestService(t, nil)
+	defer backend1.Stop()
+	backend2 := stubserver.StartTestService(t, nil)
+	defer backend2.Stop()
+
+	// Configure xDS resource on the primary management server, with a cluster
+	// resource that contains an endpoint for backend1.
+	nodeID := uuid.New().String()
+	const serviceName = "my-service-fallback-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, backend1.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	if err := primaryManagementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure xDS resource on the secondary management server, with a cluster
+	// resource that contains an endpoint for backend2. Only the listener
+	// resource has the same name on both servers.
+	fallbackRouteConfigName := "fallback-route-" + serviceName
+	fallbackClusterName := "fallback-cluster-" + serviceName
+	fallbackEndpointsName := "fallback-endpoints-" + serviceName
+	resources = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, fallbackRouteConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(fallbackRouteConfigName, serviceName, fallbackClusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(fallbackClusterName, fallbackEndpointsName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(fallbackEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend2.Address)})},
+	}
+	if err := fallbackManagementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Shut both management servers down before starting the gRPC client to
+	// trigger fallback on startup.
+	primaryLis.Stop()
+	fallbackLis.Stop()
+
+	// Generate bootstrap configuration with the above two servers.
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[
+		{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		},
+		{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, primaryManagementServer.Address, fallbackManagementServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap file: %v", err)
+	}
+
+	// Create an xDS client with the above bootstrap configuration and a short
+	// idle channel expiry timeout. This ensures that connections to lower
+	// priority servers get closed quickly, for the test to verify.
+	xdsC, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{
+		Name:                     t.Name(),
+		Contents:                 bootstrapContents,
+		IdleChannelExpiryTimeout: defaultTestIdleChannelExpiryTimeout,
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer close()
+
+	// Get the xDS resolver to use the above xDS client.
+	resolverBuilder := internal.NewXDSResolverWithClientForTesting.(func(xdsclient.XDSClient) (resolver.Builder, error))
+	resolver, err := resolverBuilder(xdsC)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Start a gRPC client that uses the above xDS resolver.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
+	if err != nil {
+		t.Fatalf("Failed to create gRPC client: %v", err)
+	}
+	defer cc.Close()
+	cc.Connect()
+
+	// Ensure that a connection is attempted to the primary.
+	if _, err := primaryWrappedLis.NewConnCh.Receive(ctx); err != nil {
+		t.Fatalf("Failure when waiting for a connection to be opened to the primary management server: %v", err)
+	}
+
+	// Ensure that a connection is attempted to the fallback.
+	if _, err := fallbackWrappedLis.NewConnCh.Receive(ctx); err != nil {
+		t.Fatalf("Failure when waiting for a connection to be opened to the primary management server: %v", err)
+	}
+
+	// Make an RPC with a shortish deadline and expect it to fail.
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(sCtx, &testpb.Empty{}, grpc.WaitForReady(true)); err == nil || status.Code(err) != codes.DeadlineExceeded {
+		t.Fatalf("EmptyCall() = %v, want DeadlineExceeded", err)
+	}
+
+	// Start the fallback server. Ensure that an RPC can succeed, and that it
+	// reaches backend2.
+	fallbackLis.Restart()
+	if err := waitForRPCsToReachBackend(ctx, client, backend2.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Start the primary server. It can take a while before the xDS client
+	// notices this, since the ADS stream implementation uses a backoff before
+	// retrying the stream.
+	primaryLis.Restart()
+
+	// Wait for the connection to the secondary to be closed and ensure that an
+	// RPC can succeed, and that it reaches backend1.
+	c, err := fallbackWrappedLis.NewConnCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Failure when retrieving the most recent connection to the fallback management server: %v", err)
+	}
+	conn := c.(*testutils.ConnWrapper)
+	if _, err := conn.CloseCh.Receive(ctx); err != nil {
+		t.Fatalf("Connection to fallback server not closed once primary becomes ready: %v", err)
+	}
+	if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Stop the primary servers. Since all xDS resources were received from the
+	// primary (and RPCs were succeeding to the clusters returned by the
+	// primary), we will not trigger fallback.
+	primaryLis.Stop()
+	sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := fallbackWrappedLis.NewConnCh.Receive(sCtx); err == nil {
+		t.Fatalf("Fallback attempted when not expected to. There are no uncached resources from the primary server at this point.")
+	}
+
+	// Ensure that RPCs still succeed, and that they use the configuration
+	// received from the primary.
+	if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests fallback when the primary management server fails during an update.
+func (s) TestFallback_MidUpdate(t *testing.T) {
+	// Enable fallback env var.
+	origFallbackEnv := envconfig.XDSFallbackSupport
+	envconfig.XDSFallbackSupport = true
+	defer func() { envconfig.XDSFallbackSupport = origFallbackEnv }()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout)
+	defer cancel()
+
+	// Create two listeners for the two management servers. The test can
+	// start/stop these listeners and can also get notified when the listener
+	// receives a connection request.
+	primaryWrappedLis := testutils.NewListenerWrapper(t, nil)
+	primaryLis := testutils.NewRestartableListener(primaryWrappedLis)
+	fallbackWrappedLis := testutils.NewListenerWrapper(t, nil)
+	fallbackLis := testutils.NewRestartableListener(fallbackWrappedLis)
+
+	// This boolean helps with triggering fallback mid update. When this boolean
+	// is set and the below defined cluster resource is requested, the primary
+	// management server shuts down the connection, forcing the client to
+	// fallback to the secondary server.
+	var closeConnOnMidUpdateClusterResource atomic.Bool
+	const (
+		serviceName              = "my-service-fallback-xds"
+		routeConfigName          = "route-" + serviceName
+		clusterName              = "cluster-" + serviceName
+		endpointsName            = "endpoints-" + serviceName
+		midUpdateRouteConfigName = "mid-update-route-" + serviceName
+		midUpdateClusterName     = "mid-update-cluster-" + serviceName
+		midUpdateEndpointsName   = "mid-update-endpoints-" + serviceName
+	)
+
+	// Start two management servers, primary and fallback, with the above
+	// listeners.
+	primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener: primaryLis,
+		OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error {
+			if closeConnOnMidUpdateClusterResource.Load() == false {
+				return nil
+			}
+			if req.GetTypeUrl() != version.V3ClusterURL {
+				return nil
+			}
+			for _, name := range req.GetResourceNames() {
+				if name == midUpdateClusterName {
+					primaryLis.Stop()
+					return fmt.Errorf("closing ADS stream because %q resource was requested", midUpdateClusterName)
+				}
+			}
+			return nil
+		},
+		AllowResourceSubset: true,
+	})
+	fallbackManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: fallbackLis})
+
+	// Start three test service backends.
+	backend1 := stubserver.StartTestService(t, nil)
+	defer backend1.Stop()
+	backend2 := stubserver.StartTestService(t, nil)
+	defer backend2.Stop()
+	backend3 := stubserver.StartTestService(t, nil)
+	defer backend3.Stop()
+
+	// Configure xDS resource on the primary management server, with a cluster
+	// resource that contains an endpoint for backend1.
+	nodeID := uuid.New().String()
+	primaryResources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointsName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointsName, "localhost", []uint32{testutils.ParsePort(t, backend1.Address)})},
+	}
+	if err := primaryManagementServer.Update(ctx, primaryResources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure xDS resource on the secondary management server, with a cluster
+	// resource that contains an endpoint for backend2. Only the listener
+	// resource has the same name on both servers.
+	const (
+		fallbackRouteConfigName = "fallback-route-" + serviceName
+		fallbackClusterName     = "fallback-cluster-" + serviceName
+		fallbackEndpointsName   = "fallback-endpoints-" + serviceName
+	)
+	fallbackResources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, fallbackRouteConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(fallbackRouteConfigName, serviceName, fallbackClusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(fallbackClusterName, fallbackEndpointsName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(fallbackEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend2.Address)})},
+	}
+	if err := fallbackManagementServer.Update(ctx, fallbackResources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Generate bootstrap configuration with the above two servers.
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[
+		{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		},
+		{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, primaryManagementServer.Address, fallbackManagementServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap file: %v", err)
+	}
+
+	// Create an xDS client with the above bootstrap configuration and a short
+	// idle channel expiry timeout. This ensures that connections to lower
+	// priority servers get closed quickly, for the test to verify.
+	xdsC, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{
+		Name:                     t.Name(),
+		Contents:                 bootstrapContents,
+		IdleChannelExpiryTimeout: defaultTestIdleChannelExpiryTimeout,
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer close()
+
+	// Get the xDS resolver to use the above xDS client.
+	resolverBuilder := internal.NewXDSResolverWithClientForTesting.(func(xdsclient.XDSClient) (resolver.Builder, error))
+	resolver, err := resolverBuilder(xdsC)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Start a gRPC client that uses the above xDS resolver.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
+	if err != nil {
+		t.Fatalf("Failed to create gRPC client: %v", err)
+	}
+	defer cc.Close()
+	cc.Connect()
+
+	// Ensure that RPCs reach the cluster specified by the primary server and
+	// that no connection is attempted to the fallback server.
+	client := testgrpc.NewTestServiceClient(cc)
+	if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil {
+		t.Fatal(err)
+	}
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := fallbackWrappedLis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatalf("Connection attempt made to fallback server when none expected: %v", err)
+	}
+
+	// Instruct the primary server to close the connection if below defined
+	// cluster resource is requested.
+	closeConnOnMidUpdateClusterResource.Store(true)
+
+	// Update the listener resource on the primary server to point to a new
+	// route configuration that points to a new cluster that points to a new
+	// endpoints resource that contains backend3.
+	primaryResources = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, midUpdateRouteConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(midUpdateRouteConfigName, serviceName, midUpdateClusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(midUpdateClusterName, midUpdateEndpointsName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(midUpdateEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend3.Address)})},
+	}
+	if err := primaryManagementServer.Update(ctx, primaryResources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Ensure that a connection is attempted to the fallback (because both
+	// conditions mentioned for fallback in A71 are satisfied: connectivity
+	// failure and a watcher for an uncached resource), and that RPCs are
+	// routed to the cluster returned by the fallback server.
+	c, err := fallbackWrappedLis.NewConnCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Failure when waiting for a connection to be opened to the fallback management server: %v", err)
+	}
+	fallbackConn := c.(*testutils.ConnWrapper)
+	if err := waitForRPCsToReachBackend(ctx, client, backend2.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Set the primary management server to not close the connection anymore if
+	// the mid-update cluster resource is requested, and get it to start serving
+	// again.
+	closeConnOnMidUpdateClusterResource.Store(false)
+	primaryLis.Restart()
+
+	// A new snapshot, with the same resources, is pushed to the management
+	// server to get it to respond for already requested resource names.
+	if err := primaryManagementServer.Update(ctx, primaryResources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Ensure that RPCs reach the backend pointed to by the new cluster.
+	if err := waitForRPCsToReachBackend(ctx, client, backend3.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the connection to the secondary to be closed since we have
+	// reverted back to the primary.
+	if _, err := fallbackConn.CloseCh.Receive(ctx); err != nil {
+		t.Fatalf("Connection to fallback server not closed once primary becomes ready: %v", err)
+	}
+}
+
+// Tests fallback when the primary management server fails during startup.
+func (s) TestFallback_MidStartup(t *testing.T) {
+	// Enable fallback env var.
+	origFallbackEnv := envconfig.XDSFallbackSupport
+	envconfig.XDSFallbackSupport = true
+	defer func() { envconfig.XDSFallbackSupport = origFallbackEnv }()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout)
+	defer cancel()
+
+	// Create two listeners for the two management servers. The test can
+	// start/stop these listeners and can also get notified when the listener
+	// receives a connection request.
+	primaryWrappedLis := testutils.NewListenerWrapper(t, nil)
+	primaryLis := testutils.NewRestartableListener(primaryWrappedLis)
+	fallbackWrappedLis := testutils.NewListenerWrapper(t, nil)
+	fallbackLis := testutils.NewRestartableListener(fallbackWrappedLis)
+
+	// This boolean helps with triggering fallback during startup. When this
+	// boolean is set and a cluster resource is requested, the primary
+	// management server shuts down the connection, forcing the client to
+	// fallback to the secondary server.
+	var closeConnOnClusterResource atomic.Bool
+	closeConnOnClusterResource.Store(true)
+
+	// Start two management servers, primary and fallback, with the above
+	// listeners.
+	primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener: primaryLis,
+		OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error {
+			if closeConnOnClusterResource.Load() == false {
+				return nil
+			}
+			if req.GetTypeUrl() != version.V3ClusterURL {
+				return nil
+			}
+			primaryLis.Stop()
+			return fmt.Errorf("closing ADS stream because cluster resource was requested")
+		},
+		AllowResourceSubset: true,
+	})
+	fallbackManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: fallbackLis})
+
+	// Start two test service backends.
+	backend1 := stubserver.StartTestService(t, nil)
+	defer backend1.Stop()
+	backend2 := stubserver.StartTestService(t, nil)
+	defer backend2.Stop()
+
+	// Configure xDS resource on the primary management server, with a cluster
+	// resource that contains an endpoint for backend1.
+	nodeID := uuid.New().String()
+	const serviceName = "my-service-fallback-xds"
+	primaryResources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, backend1.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	if err := primaryManagementServer.Update(ctx, primaryResources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure xDS resource on the secondary management server, with a cluster
+	// resource that contains an endpoint for backend2. Only the listener
+	// resource has the same name on both servers.
+	fallbackRouteConfigName := "fallback-route-" + serviceName
+	fallbackClusterName := "fallback-cluster-" + serviceName
+	fallbackEndpointsName := "fallback-endpoints-" + serviceName
+	fallbackResources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, fallbackRouteConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(fallbackRouteConfigName, serviceName, fallbackClusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(fallbackClusterName, fallbackEndpointsName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(fallbackEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend2.Address)})},
+	}
+	if err := fallbackManagementServer.Update(ctx, fallbackResources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Generate bootstrap configuration with the above two servers.
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[
+		{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		},
+		{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, primaryManagementServer.Address, fallbackManagementServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap file: %v", err)
+	}
+
+	// Create an xDS client with the above bootstrap configuration and a short
+	// idle channel expiry timeout. This ensures that connections to lower
+	// priority servers get closed quickly, for the test to verify.
+	xdsC, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{
+		Name:                     t.Name(),
+		Contents:                 bootstrapContents,
+		IdleChannelExpiryTimeout: defaultTestIdleChannelExpiryTimeout,
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer close()
+
+	// Get the xDS resolver to use the above xDS client.
+	resolverBuilder := internal.NewXDSResolverWithClientForTesting.(func(xdsclient.XDSClient) (resolver.Builder, error))
+	resolver, err := resolverBuilder(xdsC)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Start a gRPC client that uses the above xDS resolver.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
+	if err != nil {
+		t.Fatalf("Failed to create gRPC client: %v", err)
+	}
+	defer cc.Close()
+	cc.Connect()
+
+	// Ensure that a connection is attempted to the primary.
+	if _, err := primaryWrappedLis.NewConnCh.Receive(ctx); err != nil {
+		t.Fatalf("Failure when waiting for a connection to be opened to the primary management server: %v", err)
+	}
+
+	// Ensure that a connection is attempted to the fallback.
+	c, err := fallbackWrappedLis.NewConnCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Failure when waiting for a connection to be opened to the secondary management server: %v", err)
+	}
+	fallbackConn := c.(*testutils.ConnWrapper)
+
+	// Ensure that RPCs are routed to the cluster returned by the fallback
+	// management server.
+	client := testgrpc.NewTestServiceClient(cc)
+	if err := waitForRPCsToReachBackend(ctx, client, backend2.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Get the primary management server to no longer close the connection when
+	// the cluster resource is requested.
+	closeConnOnClusterResource.Store(false)
+	primaryLis.Restart()
+
+	// A new snapshot, with the same resources, is pushed to the management
+	// server to get it to respond for already requested resource names.
+	if err := primaryManagementServer.Update(ctx, primaryResources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Ensure that RPCs are routed to the cluster returned by the primary
+	// management server.
+	if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the connection to the secondary to be closed since we have
+	// reverted back to the primary.
+	if _, err := fallbackConn.CloseCh.Receive(ctx); err != nil {
+		t.Fatalf("Connection to fallback server not closed once primary becomes ready: %v", err)
+	}
+}
diff --git a/xds/server_test.go b/xds/server_test.go
index 7a8dcbcc..405ded0a 100644
--- a/xds/server_test.go
+++ b/xds/server_test.go
@@ -176,9 +176,14 @@ func (s) TestNewServer_Failure(t *testing.T) {
 		wantErr    string
 	}{
 		{
-			desc:       "bootstrap env var not set",
-			serverOpts: []grpc.ServerOption{grpc.Creds(xdsCreds)},
-			wantErr:    "failed to get xDS bootstrap config",
+			desc: "bootstrap env var not set",
+			serverOpts: func() []grpc.ServerOption {
+				// Ensure that any fallback bootstrap configuration setup by
+				// previous tests is cleared.
+				bootstrap.UnsetFallbackBootstrapConfigForTesting()
+				return []grpc.ServerOption{grpc.Creds(xdsCreds)}
+			}(),
+			wantErr: "failed to get xDS bootstrap config",
 		},
 		{
 			desc: "empty bootstrap config",
@@ -696,10 +701,16 @@ func (s) TestServeAndCloseDoNotRace(t *testing.T) {
 		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
 	}
 
+	// Generate bootstrap contents up front for all servers, and clear the
+	// fallback bootstrap configuration that gets set when a server is created
+	// with the BootstrapContentsForTesting() server option.
+	bootstrapContents := generateBootstrapContents(t, uuid.NewString(), nonExistentManagementServer)
+	defer bootstrap.UnsetFallbackBootstrapConfigForTesting()
+
 	wg := sync.WaitGroup{}
 	wg.Add(200)
 	for i := 0; i < 100; i++ {
-		server, err := NewGRPCServer(BootstrapContentsForTesting(generateBootstrapContents(t, uuid.NewString(), nonExistentManagementServer)))
+		server, err := NewGRPCServer(BootstrapContentsForTesting(bootstrapContents))
 		if err != nil {
 			t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err)
 		}