Repository: grpc/grpc-go

Issue:
Flaky test: Test/RingHash_AggregateClusterFallBackFromRingHashToLogicalDnsAtStartup
```
--- FAIL: Test (23.36s)
    --- FAIL: Test/RingHash_AggregateClusterFallBackFromRingHashToLogicalDnsAtStartup (10.01s)
        tlogger.go:116: INFO server.go:685 [core] [Server #409]Server created  (t=+276.144µs)
        stubserver.go:288: Started test service backend at "127.0.0.1:44079"
        tlogger.go:116: INFO server.go:881 [core] [Server #409 ListenSocket #410]ListenSocket created  (t=+423.177µs)
        tlogger.go:116: INFO restartable_listener.go:84 [testutils] Stopping restartable listener "127.0.0.1:34327"  (t=+762.212µs)
        ringhash_balancer_test.go:438: Created new snapshot cache...
        tlogger.go:116: INFO server.go:685 [core] [Server #411]Server created  (t=+1.204154ms)
        ringhash_balancer_test.go:438: Registered Aggregated Discovery Service (ADS)...
        ringhash_balancer_test.go:438: xDS management server serving at: 127.0.0.1:35659...
        tlogger.go:116: INFO server.go:881 [core] [Server #411 ListenSocket #412]ListenSocket created  (t=+1.870166ms)
        server.go:[229](https://github.com/grpc/grpc-go/actions/runs/12684369727/job/35352925778?pr=7992#step:8:230): Created new resource snapshot...
        server.go:235: Updated snapshot cache with resource snapshot...
        tlogger.go:116: INFO clientconn.go:1674 [core] original dial target is: "xds:///test.server"  (t=+3.552713ms)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #413]Channel created  (t=+3.622452ms)
        tlogger.go:116: INFO clientconn.go:193 [core] [Channel #413]parsed dial target is: resolver.Target{URL:url.URL{Scheme:"xds", Opaque:"", User:(*url.Userinfo)(nil), Host:"", Path:"/test.server", RawPath:"", OmitHost:false, ForceQuery:false, RawQuery:"", Fragment:"", RawFragment:""}}  (t=+3.73948ms)
        tlogger.go:116: INFO clientconn.go:194 [core] [Channel #413]Channel authority set to "test.server"  (t=+3.798049ms)
        tlogger.go:116: INFO xds_resolver.go:100 [xds] [xds-resolver 0xc000785320] Creating resolver for target: xds:///test.server  (t=+4.049948ms)
        tlogger.go:116: INFO client_refcounted.go:76 [xds] [xds-client 0xc000389a40] Created client with name "xds:///test.server" and bootstrap configuration:
             {
              "xds_servers": [
               {
                "server_uri": "passthrough:///127.0.0.1:35659",
                "channel_creds": [
                 {
                  "type": "insecure"
                 }
                ]
               }
              ],
              "certificate_providers": {
               "client-side-certificate-provider-instance": {
                "plugin_name": "file_watcher",
                "config": {
                 "certificate_file": "/tmp/testClientSideXDS35[230](https://github.com/grpc/grpc-go/actions/runs/12684369727/job/35352925778?pr=7992#step:8:231)94675/cert.pem",
                 "private_key_file": "/tmp/testClientSideXDS3523094675/key.pem",
                 "ca_certificate_file": "/tmp/testClientSideXDS3523094675/ca.pem",
                 "refresh_interval": "600s"
                }
               },
               "server-side-certificate-provider-instance": {
                "plugin_name": "file_watcher",
                "config": {
                 "certificate_file": "/tmp/testServerSideXDS420696777/cert.pem",
                 "private_key_file": "/tmp/testServerSideXDS420696777/key.pem",
                 "ca_certificate_file": "/tmp/testServerSideXDS420696777/ca.pem",
                 "refresh_interval": "600s"
                }
               }
              },
              "server_listener_resource_name_template": "grpc/server?xds.resource.listening_address=%s",
              "client_default_listener_resource_name_template": "%s",
              "node": {
               "id": "9deb4ef4-9391-4307-b8ba-e560cb9cc3ed",
               "locality": {}
              }
             }  (t=+5.461273ms)
        tlogger.go:116: INFO client_refcounted.go:81 [xds] xDS node ID: 9deb4ef4-9391-4307-b8ba-e560cb9cc3ed  (t=+5.576247ms)
        tlogger.go:116: INFO clientconn.go:1674 [core] original dial target is: "passthrough:///127.0.0.1:35659"  (t=+5.729802ms)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #414]Channel created  (t=+5.829618ms)
        tlogger.go:116: INFO clientconn.go:193 [core] [Channel #414]parsed dial target is: resolver.Target{URL:url.URL{Scheme:"passthrough", Opaque:"", User:(*url.Userinfo)(nil), Host:"", Path:"/127.0.0.1:35659", RawPath:"", OmitHost:false, ForceQuery:false, RawQuery:"", Fragment:"", RawFragment:""}}  (t=+5.969428ms)
        tlogger.go:116: INFO clientconn.go:194 [core] [Channel #414]Channel authority set to "127.0.0.1:35659"  (t=+6.044247ms)
        tlogger.go:116: INFO resolver_wrapper.go:200 [core] [Channel #414]Resolver state updated: {
              "Addresses": [
                {
                  "Addr": "127.0.0.1:35659",
                  "ServerName": "",
                  "Attributes": null,
                  "BalancerAttributes": null,
                  "Metadata": null
                }
              ],
              "Endpoints": [
                {
                  "Addresses": [
                    {
                      "Addr": "127.0.0.1:35659",
                      "ServerName": "",
                      "Attributes": null,
                      "BalancerAttributes": null,
                      "Metadata": null
                    }
                  ],
                  "Attributes": null
                }
              ],
              "ServiceConfig": null,
              "Attributes": null
            } (resolver returned new addresses)  (t=+6.435836ms)
        tlogger.go:116: INFO balancer_wrapper.go:117 [core] [Channel #414]Channel switches to new LB policy "pick_first"  (t=+6.641057ms)
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #414]Channel Connectivity change to CONNECTING  (t=+6.738749ms)
        tlogger.go:116: INFO clientconn.go:858 [core] [Channel #414 SubChannel #415]Subchannel created  (t=+6.848183ms)
        tlogger.go:116: INFO clientconn.go:1219 [core] [Channel #414 SubChannel #415]Subchannel Connectivity change to CONNECTING  (t=+6.952096ms)
        tlogger.go:116: INFO clientconn.go:1339 [core] [Channel #414 SubChannel #415]Subchannel picks a new address "127.0.0.1:35659" to connect  (t=+7.016496ms)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #414]Channel exiting idle mode  (t=+7.827194ms)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #413]Channel exiting idle mode  (t=+8.216047ms)
        tlogger.go:116: INFO clientconn.go:1219 [core] [Channel #414 SubChannel #415]Subchannel Connectivity change to READY  (t=+10.783082ms)
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #414]Channel Connectivity change to READY  (t=+11.036874ms)
        logging.go:30: nodeID "9deb4ef4-9391-4307-b8ba-e560cb9cc3ed" requested type.googleapis.com/envoy.config.listener.v3.Listener[test.server] and known map[]. Diff [test.server]
        logging.go:30: respond type.googleapis.com/envoy.config.listener.v3.Listener[test.server] version "" with version "1"
        logging.go:30: nodeID "9deb4ef4-9391-4307-b8ba-e560cb9cc3ed" requested type.googleapis.com/envoy.config.listener.v3.Listener[test.server] and known map[test.server:{}]. Diff []
        logging.go:30: open watch 1 for type.googleapis.com/envoy.config.listener.v3.Listener[test.server] from nodeID "9deb4ef4-9391-4307-b8ba-e560cb9cc3ed", version "1"
        logging.go:30: nodeID "9deb4ef4-9391-4307-b8ba-e560cb9cc3ed" requested type.googleapis.com/envoy.config.route.v3.RouteConfiguration[new_route] and known map[]. Diff [new_route]
        logging.go:30: respond type.googleapis.com/envoy.config.route.v3.RouteConfiguration[new_route] version "" with version "1"
        tlogger.go:116: INFO xds_resolver.go:284 [xds] [xds-resolver 0xc000785320] For Listener resource "test.server" and RouteConfiguration resource "new_route", generated service config: {
              "loadBalancingConfig": [
                {
                  "xds_cluster_manager_experimental": {
                    "children": {
                      "cluster:aggregate_cluster": {
                        "childPolicy": [
                          {
                            "cds_experimental": {
                              "cluster": "aggregate_cluster"
                            }
                          }
                        ]
                      }
                    }
                  }
                }
              ]
            }  (t=+14.317477ms)
        tlogger.go:116: INFO resolver_wrapper.go:200 [core] [Channel #413]Resolver state updated: {
              "Addresses": null,
              "Endpoints": [],
              "ServiceConfig": {
                "Config": {
                  "Config": null,
                  "Methods": {}
                },
                "Err": null
              },
              "Attributes": {
                "\u003c%!p(resolver.csKeyType=grpc.internal.resolver.configSelector)\u003e": "\u003c0xc0006f60a0\u003e",
                "\u003c%!p(xdsclient.clientKeyType=grpc.xds.internal.client.Client)\u003e": "\u003c0xc000739c40\u003e"
              }
            } (service config updated)  (t=+14.696121ms)
        tlogger.go:116: INFO balancer_wrapper.go:117 [core] [Channel #413]Channel switches to new LB policy "xds_cluster_manager_experimental"  (t=+14.779026ms)
        tlogger.go:116: INFO clustermanager.go:59 [xds] [xds-cluster-manager-lb 0xc000160cc0] Created  (t=+14.828808ms)
        tlogger.go:116: INFO balancergroup.go:299 [xds] [xds-cluster-manager-lb 0xc000160cc0] Adding child policy of type "cds_experimental" for child "cluster:aggregate_cluster"  (t=+14.879082ms)
        tlogger.go:116: INFO balancergroup.go:102 [xds] [xds-cluster-manager-lb 0xc000160cc0] Creating child policy of type "cds_experimental" for child "cluster:aggregate_cluster"  (t=+14.918615ms)
        tlogger.go:116: INFO cdsbalancer.go:109 [xds] [cds-lb 0xc000247520] Created  (t=+14.966885ms)
        logging.go:30: nodeID "9deb4ef4-9391-4307-b8ba-e560cb9cc3ed" requested type.googleapis.com/envoy.config.route.v3.RouteConfiguration[new_route] and known map[new_route:{}]. Diff []
        logging.go:30: open watch 2 for type.googleapis.com/envoy.config.route.v3.RouteConfiguration[new_route] from nodeID "9deb4ef4-9391-4307-b8ba-e560cb9cc3ed", version "1"
        tlogger.go:116: INFO cdsbalancer.go:121 [xds] [cds-lb 0xc000247520] xDS credentials in use: false  (t=+15.364675ms)
        tlogger.go:116: INFO cdsbalancer.go:290 [xds] [cds-lb 0xc000247520] Received balancer config update: {
              "LoadBalancingConfig": null,
              "Cluster": "aggregate_cluster"
            }  (t=+15.460233ms)
        tlogger.go:116: INFO balancerstateaggregator.go:191 [xds] [xds-cluster-manager-lb 0xc000160cc0] Child pickers: map[cluster:aggregate_cluster:picker:0xc00077a670,state:CONNECTING,stateToAggregate:CONNECTING]  (t=+15.634617ms)
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #413]Channel Connectivity change to CONNECTING  (t=+15.713023ms)
        logging.go:30: nodeID "9deb4ef4-9391-4307-b8ba-e560cb9cc3ed" requested type.googleapis.com/envoy.config.cluster.v3.Cluster[aggregate_cluster] and known map[]. Diff [aggregate_cluster]
        logging.go:30: respond type.googleapis.com/envoy.config.cluster.v3.Cluster[aggregate_cluster] version "" with version "1"
        logging.go:30: nodeID "9deb4ef4-9391-4307-b8ba-e560cb9cc3ed" requested type.googleapis.com/envoy.config.cluster.v3.Cluster[aggregate_cluster] and known map[aggregate_cluster:{}]. Diff []
        logging.go:30: open watch 3 for type.googleapis.com/envoy.config.cluster.v3.Cluster[aggregate_cluster] from nodeID "9deb4ef4-9391-4307-b8ba-e560cb9cc3ed", version "1"
        tlogger.go:116: INFO cdsbalancer.go:414 [xds] [cds-lb 0xc000247520] Received Cluster resource: {
              "ClusterType": 2,
              "ClusterName": "aggregate_cluster",
              "EDSServiceName": "",
              "LRSServerConfig": null,
              "SecurityCfg": null,
              "MaxRequests": null,
              "DNSHostName": "",
              "PrioritizedClusterNames": [
                "eds_cluster",
                "logical_dns_cluster"
              ],
              "LBPolicy": [
                {
                  "ring_hash_experimental": {
                    "minRingSize": 1024,
                    "maxRingSize": 8388608
                  }
                }
              ],
              "OutlierDetection": null,
              "Raw": {
                "type_url": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
                "value": "ChFhZ2dyZWdhdGVfY2x1c3RlcjACsgKKAQoYZW52b3kuY2x1c3RlcnMuYWdncmVnYXRlEm4KSHR5cGUuZ29vZ2xlYXBpcy5jb20vZW52b3kuZXh0ZW5zaW9ucy5jbHVzdGVycy5hZ2dyZWdhdGUudjMuQ2x1c3RlckNvbmZpZxIiCgtlZHNfY2x1c3RlcgoTbG9naWNhbF9kbnNfY2x1c3Rlcg=="
              },
              "TelemetryLabels": {
                "csm.service_name": "unknown",
                "csm.service_namespace_name": "unknown"
              }
            }  (t=+16.746595ms)
        logging.go:30: nodeID "9deb4ef4-9391-4307-b8ba-e560cb9cc3ed" requested type.googleapis.com/envoy.config.cluster.v3.Cluster[aggregate_cluster eds_cluster] and known map[aggregate_cluster:{}]. Diff [eds_cluster]
        logging.go:30: respond type.googleapis.com/envoy.config.cluster.v3.Cluster[aggregate_cluster eds_cluster] version "1" with version "1"
        tlogger.go:116: INFO cdsbalancer.go:414 [xds] [cds-lb 0xc000247520] Received Cluster resource: {
              "ClusterType": 0,
              "ClusterName": "eds_cluster",
              "EDSServiceName": "eds_cluster",
              "LRSServerConfig": null,
              "SecurityCfg": null,
              "MaxRequests": null,
              "DNSHostName": "",
              "PrioritizedClusterNames": null,
              "LBPolicy": [
                {
                  "xds_wrr_locality_experimental": {
                    "childPolicy": [
                      {
                        "round_robin": {}
                      }
                    ]
                  }
                }
              ],
              "OutlierDetection": null,
              "Raw": {
                "type_url": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
                "value": "CgtlZHNfY2x1c3RlchoRCgIaABILZWRzX2NsdXN0ZXIQAw=="
              },
              "TelemetryLabels": {
                "csm.service_name": "unknown",
                "csm.service_namespace_name": "unknown"
              }
            }  (t=+18.182486ms)
        logging.go:30: nodeID "9deb4ef4-9391-4307-b8ba-e560cb9cc3ed" requested type.googleapis.com/envoy.config.cluster.v3.Cluster[aggregate_cluster eds_cluster logical_dns_cluster] and known map[aggregate_cluster:{} eds_cluster:{}]. Diff [logical_dns_cluster]
        logging.go:30: respond type.googleapis.com/envoy.config.cluster.v3.Cluster[aggregate_cluster eds_cluster logical_dns_cluster] version "1" with version "1"
        tlogger.go:116: INFO cdsbalancer.go:414 [xds] [cds-lb 0xc000247520] Received Cluster resource: {
              "ClusterType": 1,
              "ClusterName": "logical_dns_cluster",
              "EDSServiceName": "",
              "LRSServerConfig": null,
              "SecurityCfg": null,
              "MaxRequests": null,
              "DNSHostName": "server.example.com:443",
              "PrioritizedClusterNames": null,
              "LBPolicy": [
                {
                  "xds_wrr_locality_experimental": {
                    "childPolicy": [
                      {
                        "round_robin": {}
                      }
                    ]
                  }
                }
              ],
              "OutlierDetection": null,
              "Raw": {
                "type_url": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
                "value": "ChNsb2dpY2FsX2Ruc19jbHVzdGVyigIhEh8SHQobChkKFxISc2VydmVyLmV4YW1wbGUuY29tGLsDEAI="
              },
              "TelemetryLabels": {
                "csm.service_name": "unknown",
                "csm.service_namespace_name": "unknown"
              }
            }  (t=+19.145708ms)
        logging.go:30: nodeID "9deb4ef4-9391-4307-b8ba-e560cb9cc3ed" requested type.googleapis.com/envoy.config.cluster.v3.Cluster[aggregate_cluster eds_cluster logical_dns_cluster] and known map[aggregate_cluster:{} eds_cluster:{} logical_dns_cluster:{}]. Diff []
        logging.go:30: open watch 4 for type.googleapis.com/envoy.config.cluster.v3.Cluster[aggregate_cluster eds_cluster logical_dns_cluster] from nodeID "9deb4ef4-9391-4307-b8ba-e560cb9cc3ed", version "1"
        tlogger.go:116: INFO clusterresolver.go:85 [xds] [xds-cluster-resolver-lb 0xc0008cea80] Created  (t=+19.61966ms)
        tlogger.go:116: INFO cdsbalancer.go:455 [xds] [cds-lb 0xc000247520] Created child policy 0xc0008cea80 of type cluster_resolver_experimental  (t=+19.708395ms)
        ringhash_balancer_test.go:460: rpc EmptyCall() failed: rpc error: code = DeadlineExceeded desc = received context error while waiting for new LB policy update: context deadline exceeded
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #413]Channel Connectivity change to SHUTDOWN  (t=+10.008614258s)
        tlogger.go:116: INFO resolver_wrapper.go:100 [core] [Channel #413]Closing the name resolver  (t=+10.008656096s)
        tlogger.go:116: INFO balancer_wrapper.go:155 [core] [Channel #413]ccBalancerWrapper: closing  (t=+10.008718722s)
        tlogger.go:116: INFO watch_service.go:56 [xds] [xds-resolver 0xc000785320] Canceling watch on Listener resource "test.server"  (t=+10.008880952s)
        tlogger.go:116: INFO watch_service.go:91 [xds] [xds-resolver 0xc000785320] Canceling watch on RouteConfiguration resource "new_route"  (t=+10.00908305s)
        tlogger.go:116: INFO clusterresolver.go:355 [xds] [xds-cluster-resolver-lb 0xc0008cea80] Shutdown  (t=+10.00918091s)
        tlogger.go:116: WARNING ads_stream.go:514 [xds] [xds-client 0xc000389a40] [xds-channel 0xc00064c120] [ads-stream 0xc0006f0800] ADS stream closed: rpc error: code = Canceled desc = context canceled  (t=+10.009[232](https://github.com/grpc/grpc-go/actions/runs/12684369727/job/35352925778?pr=7992#step:8:233)788s)
        tlogger.go:116: INFO cdsbalancer.go:380 [xds] [cds-lb 0xc000247520] Shutdown  (t=+10.009283252s)
        tlogger.go:116: INFO ads_stream.go:178 [xds] [xds-client 0xc000389a40] [xds-channel 0xc00064c120] [ads-stream 0xc0006f0800] Stopping ADS stream  (t=+10.009327324s)
        logging.go:30: nodeID "9deb4ef4-9391-4307-b8ba-e560cb9cc3ed" requested type.googleapis.com/envoy.config.cluster.v3.Cluster[eds_cluster logical_dns_cluster] and known map[aggregate_cluster:{} eds_cluster:{} logical_dns_cluster:{}]. Diff []
        tlogger.go:116: INFO clientconn.go:544 [core] [Channel #414]Channel Connectivity change to SHUTDOWN  (t=+10.009384731s)
        logging.go:30: open watch 5 for type.googleapis.com/envoy.config.cluster.v3.Cluster[eds_cluster logical_dns_cluster] from nodeID "9deb4ef4-9391-4307-b8ba-e560cb9cc3ed", version "1"
        tlogger.go:116: INFO resolver_wrapper.go:100 [core] [Channel #414]Closing the name resolver  (t=+10.009489346s)
        tlogger.go:116: INFO clustermanager.go:197 [xds] [xds-cluster-manager-lb 0xc000160cc0] Shutdown  (t=+10.009563402s)
        tlogger.go:116: INFO balancer_wrapper.go:155 [core] [Channel #414]ccBalancerWrapper: closing  (t=+10.009610319s)
        tlogger.go:116: INFO clientconn.go:1219 [core] [Channel #414 SubChannel #415]Subchannel Connectivity change to SHUTDOWN  (t=+10.009743041s)
        tlogger.go:116: INFO clientconn.go:1547 [core] [Channel #414 SubChannel #415]Subchannel deleted  (t=+10.009789337s)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #414]Channel deleted  (t=+10.010027204s)
        tlogger.go:116: INFO channel.go:145 [xds] [xds-client 0xc000389a40] [xds-channel 0xc00064c120] Shutdown  (t=+10.010089721s)
        tlogger.go:116: INFO clientimpl.go:185 [xds] [xds-client 0xc000389a40] Shutdown  (t=+10.010159192s)
        tlogger.go:116: INFO xds_resolver.go:253 [xds] [xds-resolver 0xc000785320] Shutdown  (t=+10.01019071s)
        tlogger.go:116: INFO clientconn.go:314 [core] [Channel #413]Channel deleted  (t=+10.010[238](https://github.com/grpc/grpc-go/actions/runs/12684369727/job/35352925778?pr=7992#step:8:239)539s)
        tlogger.go:116: INFO server.go:817 [core] [Server #411 ListenSocket #412]ListenSocket deleted  (t=+10.010355049s)
        tlogger.go:116: INFO server.go:817 [core] [Server #409 ListenSocket #410]ListenSocket deleted  (t=+10.010458542s)
FAIL
```

Hints from issue comments:
@arjan-bal: In the success case, the next few log lines after `Created child policy 0x14000530820 of type cluster_resolver_experimental` are similar to the following:
> logging.go:30: nodeID "267a4abd-bf2b-4af8-9119-b88eeb7e1d98" requested type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment[cluster] and known map[]. Diff [cluster]
    logging.go:30: respond type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment[cluster] version "" with version "1"
    tlogger.go:116: INFO balancer.go:76 [xds] [priority-lb 0xc000378230] Created  (t=+9.91661ms)

It seems like that clusterresolver didn't start a watch for EDS resources.

Able to repro the failure on forge in 7600 runs.

Solution patch:
diff --git a/clientconn.go b/clientconn.go
index a824c262..a319ef97 100644
--- a/clientconn.go
+++ b/clientconn.go
@@ -118,12 +118,26 @@ func (dcs *defaultConfigSelector) SelectConfig(rpcInfo iresolver.RPCInfo) (*ires
 
 // NewClient creates a new gRPC "channel" for the target URI provided.  No I/O
 // is performed.  Use of the ClientConn for RPCs will automatically cause it to
-// connect.  Connect may be used to manually create a connection, but for most
-// users this is unnecessary.
+// connect.  The Connect method may be called to manually create a connection,
+// but for most users this should be unnecessary.
 //
 // The target name syntax is defined in
-// https://github.com/grpc/grpc/blob/master/doc/naming.md.  e.g. to use dns
-// resolver, a "dns:///" prefix should be applied to the target.
+// https://github.com/grpc/grpc/blob/master/doc/naming.md.  E.g. to use the dns
+// name resolver, a "dns:///" prefix may be applied to the target.  The default
+// name resolver will be used if no scheme is detected, or if the parsed scheme
+// is not a registered name resolver.  The default resolver is "dns" but can be
+// overridden using the resolver package's SetDefaultScheme.
+//
+// Examples:
+//
+//   - "foo.googleapis.com:8080"
+//   - "dns:///foo.googleapis.com:8080"
+//   - "dns:///foo.googleapis.com"
+//   - "dns:///10.0.0.213:8080"
+//   - "dns:///%5B2001:db8:85a3:8d3:1319:8a2e:370:7348%5D:443"
+//   - "dns://8.8.8.8/foo.googleapis.com:8080"
+//   - "dns://8.8.8.8/foo.googleapis.com"
+//   - "zookeeper://zk.example.com:9900/example_service"
 //
 // The DialOptions returned by WithBlock, WithTimeout,
 // WithReturnConnectionError, and FailOnNonTempDialError are ignored by this
@@ -872,7 +886,13 @@ func (cc *ClientConn) Target() string {
 	return cc.target
 }
 
-// CanonicalTarget returns the canonical target string of the ClientConn.
+// CanonicalTarget returns the canonical target string used when creating cc.
+//
+// This always has the form "<scheme>://[authority]/<endpoint>".  For example:
+//
+//   - "dns:///example.com:42"
+//   - "dns://8.8.8.8/example.com:42"
+//   - "unix:///path/to/socket"
 func (cc *ClientConn) CanonicalTarget() string {
 	return cc.parsedTarget.String()
 }
diff --git a/xds/internal/xdsclient/authority.go b/xds/internal/xdsclient/authority.go
index d9269821..05c3e49b 100644
--- a/xds/internal/xdsclient/authority.go
+++ b/xds/internal/xdsclient/authority.go
@@ -604,8 +604,9 @@ func (a *authority) watchResource(rType xdsresource.Type, resourceName string, w
 			a.logger.Infof("New watch for type %q, resource name %q", rType.TypeName(), resourceName)
 		}
 
-		xdsChannel := a.xdsChannelToUse()
-		if xdsChannel == nil {
+		xdsChannel, err := a.xdsChannelToUse()
+		if err != nil {
+			a.watcherCallbackSerializer.TrySchedule(func(context.Context) { watcher.OnError(err, func() {}) })
 			return
 		}
 
@@ -739,22 +740,23 @@ func (a *authority) unwatchResource(rType xdsresource.Type, resourceName string,
 // Otherwise, it creates a new channel using the first server configuration in
 // the list of configurations, and returns that.
 //
+// A non-nil error is returned if the channel creation fails.
+//
 // Only executed in the context of a serializer callback.
-func (a *authority) xdsChannelToUse() *xdsChannelWithConfig {
+func (a *authority) xdsChannelToUse() (*xdsChannelWithConfig, error) {
 	if a.activeXDSChannel != nil {
-		return a.activeXDSChannel
+		return a.activeXDSChannel, nil
 	}
 
 	sc := a.xdsChannelConfigs[0].serverConfig
 	xc, cleanup, err := a.getChannelForADS(sc, a)
 	if err != nil {
-		a.logger.Warningf("Failed to create xDS channel: %v", err)
-		return nil
+		return nil, err
 	}
 	a.xdsChannelConfigs[0].channel = xc
 	a.xdsChannelConfigs[0].cleanup = cleanup
 	a.activeXDSChannel = a.xdsChannelConfigs[0]
-	return a.activeXDSChannel
+	return a.activeXDSChannel, nil
 }
 
 // closeXDSChannels closes all the xDS channels associated with this authority,
diff --git a/xds/internal/xdsclient/clientimpl_watchers.go b/xds/internal/xdsclient/clientimpl_watchers.go
index ed4ee360..cc8e0849 100644
--- a/xds/internal/xdsclient/clientimpl_watchers.go
+++ b/xds/internal/xdsclient/clientimpl_watchers.go
@@ -26,6 +26,17 @@ import (
 	"google.golang.org/grpc/xds/internal/xdsclient/xdsresource"
 )
 
+// wrappingWatcher is a wrapper around an xdsresource.ResourceWatcher that adds
+// the node ID to the error messages reported to the watcher.
+type wrappingWatcher struct {
+	xdsresource.ResourceWatcher
+	nodeID string
+}
+
+func (w *wrappingWatcher) OnError(err error, done xdsresource.OnDoneFunc) {
+	w.ResourceWatcher.OnError(fmt.Errorf("[xDS node id: %v]: %v", w.nodeID, err), done)
+}
+
 // WatchResource uses xDS to discover the resource associated with the provided
 // resource name. The resource type implementation determines how xDS responses
 // are are deserialized and validated, as received from the xDS management
@@ -43,6 +54,11 @@ func (c *clientImpl) WatchResource(rType xdsresource.Type, resourceName string,
 		return func() {}
 	}
 
+	watcher = &wrappingWatcher{
+		ResourceWatcher: watcher,
+		nodeID:          c.config.Node().GetId(),
+	}
+
 	if err := c.resourceTypes.maybeRegister(rType); err != nil {
 		logger.Warningf("Watch registered for name %q of type %q which is already registered", rType.TypeName(), resourceName)
 		c.serializer.TrySchedule(func(context.Context) { watcher.OnError(err, func() {}) })
@@ -53,9 +69,7 @@ func (c *clientImpl) WatchResource(rType xdsresource.Type, resourceName string,
 	a := c.getAuthorityForResource(n)
 	if a == nil {
 		logger.Warningf("Watch registered for name %q of type %q, authority %q is not found", rType.TypeName(), resourceName, n.Authority)
-		c.serializer.TrySchedule(func(context.Context) {
-			watcher.OnError(fmt.Errorf("authority %q not found in bootstrap config for resource %q", n.Authority, resourceName), func() {})
-		})
+		watcher.OnError(fmt.Errorf("authority %q not found in bootstrap config for resource %q", n.Authority, resourceName), func() {})
 		return func() {}
 	}
 	// The watchResource method on the authority is invoked with n.String()



Test patch:
diff --git a/balancer/lazy/lazy_ext_test.go b/balancer/lazy/lazy_ext_test.go
index bff01489..c2470ef7 100644
--- a/balancer/lazy/lazy_ext_test.go
+++ b/balancer/lazy/lazy_ext_test.go
@@ -33,6 +33,7 @@ import (
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/internal/testutils"
@@ -196,14 +197,14 @@ func (s) TestGoodUpdateThenResolverError(t *testing.T) {
 	backend := stubserver.StartTestService(t, nil)
 	defer backend.Stop()
 	resolverStateReceived := false
-	resolverErrorReceived := false
+	resolverErrorReceived := grpcsync.NewEvent()
 
 	childBF := stub.BalancerFuncs{
 		Init: func(bd *stub.BalancerData) {
 			bd.Data = balancer.Get(pickfirstleaf.Name).Build(bd.ClientConn, bd.BuildOptions)
 		},
 		UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error {
-			if resolverErrorReceived {
+			if resolverErrorReceived.HasFired() {
 				t.Error("Received resolver error before resolver state.")
 			}
 			resolverStateReceived = true
@@ -213,7 +214,7 @@ func (s) TestGoodUpdateThenResolverError(t *testing.T) {
 			if !resolverStateReceived {
 				t.Error("Received resolver error before resolver state.")
 			}
-			resolverErrorReceived = true
+			resolverErrorReceived.Fire()
 			bd.Data.(balancer.Balancer).ResolverError(err)
 		},
 		Close: func(bd *stub.BalancerData) {
@@ -286,8 +287,10 @@ func (s) TestGoodUpdateThenResolverError(t *testing.T) {
 		t.Fatalf("Child balancer did not receive resolver state.")
 	}
 
-	if !resolverErrorReceived {
-		t.Fatalf("Child balancer did not receive error.")
+	select {
+	case <-resolverErrorReceived.Done():
+	case <-ctx.Done():
+		t.Fatal("Context timed out waiting for resolver error to be delivered to child balancer.")
 	}
 }
 
diff --git a/xds/internal/balancer/ringhash/e2e/ringhash_balancer_test.go b/xds/internal/balancer/ringhash/e2e/ringhash_balancer_test.go
index a98c0c68..7641f917 100644
--- a/xds/internal/balancer/ringhash/e2e/ringhash_balancer_test.go
+++ b/xds/internal/balancer/ringhash/e2e/ringhash_balancer_test.go
@@ -467,13 +467,14 @@ func (s) TestRingHash_AggregateClusterFallBackFromRingHashToLogicalDnsAtStartup(
 		Routes:    []*v3routepb.RouteConfiguration{route},
 		Listeners: []*v3listenerpb.Listener{listener},
 	}
-	if err := xdsServer.Update(ctx, updateOpts); err != nil {
-		t.Fatalf("Failed to update xDS resources: %v", err)
-	}
 
 	dnsR := replaceDNSResolver(t)
 	dnsR.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: backends[0]}}})
 
+	if err := xdsServer.Update(ctx, updateOpts); err != nil {
+		t.Fatalf("Failed to update xDS resources: %v", err)
+	}
+
 	conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
 		t.Fatalf("Failed to create client: %s", err)
@@ -544,13 +545,14 @@ func (s) TestRingHash_AggregateClusterFallBackFromRingHashToLogicalDnsAtStartupN
 		Routes:    []*v3routepb.RouteConfiguration{route},
 		Listeners: []*v3listenerpb.Listener{listener},
 	}
-	if err := xdsServer.Update(ctx, updateOpts); err != nil {
-		t.Fatalf("Failed to update xDS resources: %v", err)
-	}
 
 	dnsR := replaceDNSResolver(t)
 	dnsR.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: backends[0]}}})
 
+	if err := xdsServer.Update(ctx, updateOpts); err != nil {
+		t.Fatalf("Failed to update xDS resources: %v", err)
+	}
+
 	dialer := testutils.NewBlockingDialer()
 	cp := grpc.ConnectParams{
 		// Increase backoff time, so that subconns stay in TRANSIENT_FAILURE
diff --git a/xds/internal/xdsclient/tests/misc_watchers_test.go b/xds/internal/xdsclient/tests/misc_watchers_test.go
index a11661f2..3fcb20bd 100644
--- a/xds/internal/xdsclient/tests/misc_watchers_test.go
+++ b/xds/internal/xdsclient/tests/misc_watchers_test.go
@@ -22,9 +22,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/google/uuid"
+	"google.golang.org/grpc"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/internal/testutils/xds/e2e"
 	"google.golang.org/grpc/internal/testutils/xds/fakeserver"
@@ -32,6 +34,7 @@ import (
 	"google.golang.org/grpc/xds/internal"
 	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
 	"google.golang.org/grpc/xds/internal/xdsclient"
+	xdsclientinternal "google.golang.org/grpc/xds/internal/xdsclient/internal"
 	"google.golang.org/grpc/xds/internal/xdsclient/xdsresource"
 	"google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version"
 	"google.golang.org/protobuf/types/known/anypb"
@@ -371,3 +374,135 @@ func readDiscoveryResponseAndCheckForNonEmptyNodeProto(ctx context.Context, reqC
 	}
 	return nil
 }
+
+type testRouteConfigResourceType struct{}
+
+func (testRouteConfigResourceType) TypeURL() string                  { return version.V3RouteConfigURL }
+func (testRouteConfigResourceType) TypeName() string                 { return "RouteConfigResource" }
+func (testRouteConfigResourceType) AllResourcesRequiredInSotW() bool { return false }
+func (testRouteConfigResourceType) Decode(*xdsresource.DecodeOptions, *anypb.Any) (*xdsresource.DecodeResult, error) {
+	return nil, nil
+}
+
+// Tests that the errors returned by the xDS client when watching a resource
+// contain the node ID that was used to create the client. This test covers two
+// scenarios:
+//
+//  1. When a watch is registered for an already registered resource type, but
+//     this time with a different implementation,
+//  2. When a watch is registered for a resource name whose authority is not
+//     found in the bootstrap configuration.
+func (s) TestWatchErrorsContainNodeID(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create an xDS client with the above bootstrap contents.
+	config, err := bootstrap.NewConfigFromContents(bc)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
+	}
+	pool := xdsclient.NewPool(config)
+	client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+		Name: t.Name(),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	t.Run("Multiple_ResourceType_Implementations", func(t *testing.T) {
+		const routeConfigName = "route-config-name"
+		watcher := xdstestutils.NewTestResourceWatcher()
+		client.WatchResource(routeConfigResourceType, routeConfigName, watcher)
+
+		sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+		defer sCancel()
+		select {
+		case <-sCtx.Done():
+		case <-watcher.UpdateCh:
+			t.Fatal("Unexpected resource update")
+		case <-watcher.ErrorCh:
+			t.Fatal("Unexpected resource error")
+		case <-watcher.ResourceDoesNotExistCh:
+			t.Fatal("Unexpected resource does not exist")
+		}
+
+		client.WatchResource(testRouteConfigResourceType{}, routeConfigName, watcher)
+		select {
+		case <-ctx.Done():
+			t.Fatal("Timeout when waiting for error callback to be invoked")
+		case err := <-watcher.ErrorCh:
+			if err == nil || !strings.Contains(err.Error(), nodeID) {
+				t.Fatalf("Unexpected error: %v, want error with node ID: %q", err, nodeID)
+			}
+		}
+	})
+
+	t.Run("Missing_Authority", func(t *testing.T) {
+		const routeConfigName = "xdstp://nonexistant-authority/envoy.config.route.v3.RouteConfiguration/route-config-name"
+		watcher := xdstestutils.NewTestResourceWatcher()
+		client.WatchResource(routeConfigResourceType, routeConfigName, watcher)
+
+		select {
+		case <-ctx.Done():
+			t.Fatal("Timeout when waiting for error callback to be invoked")
+		case err := <-watcher.ErrorCh:
+			if err == nil || !strings.Contains(err.Error(), nodeID) {
+				t.Fatalf("Unexpected error: %v, want error with node ID: %q", err, nodeID)
+			}
+		}
+	})
+}
+
+// Tests that the errors returned by the xDS client when watching a resource
+// contain the node ID when channel creation to the management server fails.
+func (s) TestWatchErrorsContainNodeID_ChannelCreationFailure(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create an xDS client with the above bootstrap contents.
+	config, err := bootstrap.NewConfigFromContents(bc)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
+	}
+	pool := xdsclient.NewPool(config)
+	client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+		Name: t.Name(),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Override the xDS channel dialer with one that always fails.
+	origDialer := xdsclientinternal.GRPCNewClient
+	xdsclientinternal.GRPCNewClient = func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
+		return nil, fmt.Errorf("failed to create channel")
+	}
+	defer func() { xdsclientinternal.GRPCNewClient = origDialer }()
+
+	const routeConfigName = "route-config-name"
+	watcher := xdstestutils.NewTestResourceWatcher()
+	client.WatchResource(routeConfigResourceType, routeConfigName, watcher)
+
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for error callback to be invoked")
+	case err := <-watcher.ErrorCh:
+		if err == nil || !strings.Contains(err.Error(), nodeID) {
+			t.Fatalf("Unexpected error: %v, want error with node ID: %q", err, nodeID)
+		}
+	}
+}