Repository: linkerd/linkerd2
Pull request #11795 (2023-12-20 00:43:38)

destination: Add optional experimental endpoint weighting
This change adds a runtime flag to the destination controller, --experimental-endpoint-zone-weights=true, that causes endpoints in the local zone to receive higher weights. This feature is disabled by default, since the weight value is not honored by proxies. No helm configuration is exposed yet, either.

This weighting is instrumented in the endpoint translator. Tests are added to confirm that the behavior is feature-gated.

Additionally, this PR adds the "zone" metric label to endpoint metadata responses.

<!--  Thanks for sending a pull request!

If you already have a well-structured git commit message, chances are GitHub
set the title and description of this PR to the git commit message subject and
body, respectively. If so, you may delete these instructions and submit your PR.

If this is your first time, please read our contributor guide:
https://github.com/linkerd/linkerd2/blob/main/CONTRIBUTING.md

The title and description of your Pull Request should match the git commit
subject and body, respectively. Git commit messages are structured as follows:

```
Subject

Problem

Solution

Validation

Fixes #[GitHub issue ID]

DCO Sign off
```

Example git commit message:

```
Introduce Pull Request Template

GitHub's community guidelines recommend a pull request template, the repo was
lacking one.

Introduce a `PULL_REQUEST_TEMPLATE.md` file.

Once merged, the
[Community profile checklist](https://github.com/linkerd/linkerd2/community)
should indicate the repo now provides a pull request template.

Fixes #3321

Signed-off-by: Jane Smith <jane.smith@example.com>
```

Note the git commit message subject becomes the pull request title.

For more details around git commits, see the section on Committing in our
contributor guide:
https://github.com/linkerd/linkerd2/blob/main/CONTRIBUTING.md#committing
-->



Patch:
diff --git a/.proxy-version b/.proxy-version
index 51795d461..8549fdd43 100644
--- a/.proxy-version
+++ b/.proxy-version
@@ -1,1 +1,1 @@
-v2.215.0
+v2.216.0
diff --git a/Cargo.lock b/Cargo.lock
index 06dd26e0e..895413419 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -38,9 +38,9 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "0.7.20"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
+checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
 dependencies = [
  "memchr",
 ]
@@ -298,9 +298,9 @@ dependencies = [
 
 [[package]]
 name = "clang-sys"
-version = "1.6.0"
+version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77ed9a53e5d4d9c573ae844bfac6872b159cb1d1585a83b29e7a64b7eef7332a"
+checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f"
 dependencies = [
  "glob",
  "libc",
@@ -363,9 +363,9 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.6"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181"
+checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1"
 dependencies = [
  "libc",
 ]
@@ -394,9 +394,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.15"
+version = "0.8.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
+checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f"
 dependencies = [
  "cfg-if",
 ]
@@ -672,9 +672,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.8"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
+checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f"
 dependencies = [
  "cfg-if",
  "libc",
@@ -1230,9 +1230,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "libc"
-version = "0.2.147"
+version = "0.2.151"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
+checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"
 
 [[package]]
 name = "libloading"
@@ -1442,9 +1442,9 @@ checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
 
 [[package]]
 name = "lock_api"
-version = "0.4.9"
+version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df"
+checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45"
 dependencies = [
  "autocfg",
  "scopeguard",
@@ -1480,7 +1480,7 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
 dependencies = [
- "regex-automata",
+ "regex-automata 0.1.10",
 ]
 
 [[package]]
@@ -1901,9 +1901,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.32"
+version = "1.0.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965"
+checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
 dependencies = [
  "proc-macro2",
 ]
@@ -1958,13 +1958,14 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.7.3"
+version = "1.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d"
+checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax",
+ "regex-automata 0.4.3",
+ "regex-syntax 0.8.2",
 ]
 
 [[package]]
@@ -1973,7 +1974,18 @@ version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
 dependencies = [
- "regex-syntax",
+ "regex-syntax 0.6.29",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax 0.8.2",
 ]
 
 [[package]]
@@ -1982,6 +1994,12 @@ version = "0.6.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
 
+[[package]]
+name = "regex-syntax"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
+
 [[package]]
 name = "ring"
 version = "0.17.3"
diff --git a/controller/api/destination/endpoint_translator.go b/controller/api/destination/endpoint_translator.go
index 1ba0908a0..1b70db7aa 100644
--- a/controller/api/destination/endpoint_translator.go
+++ b/controller/api/destination/endpoint_translator.go
@@ -20,6 +20,7 @@ import (
 
 const (
 	defaultWeight uint32 = 10000
+
 	// inboundListenAddr is the environment variable holding the inbound
 	// listening address for the proxy container.
 	envInboundListenAddr = "LINKERD2_PROXY_INBOUND_LISTEN_ADDR"
@@ -31,13 +32,15 @@ const (
 // into Destination.Get messages.
 type (
 	endpointTranslator struct {
-		controllerNS            string
-		identityTrustDomain     string
-		enableH2Upgrade         bool
-		nodeTopologyZone        string
-		nodeName                string
-		defaultOpaquePorts      map[uint32]struct{}
-		enableEndpointFiltering bool
+		controllerNS        string
+		identityTrustDomain string
+		nodeTopologyZone    string
+		nodeName            string
+		defaultOpaquePorts  map[uint32]struct{}
+
+		enableH2Upgrade,
+		enableEndpointFiltering,
+		experimentalEndpointZoneWeights bool
 
 		availableEndpoints watcher.AddressSet
 		filteredSnapshot   watcher.AddressSet
@@ -76,11 +79,12 @@ var updatesQueueOverflowCounter = promauto.NewCounterVec(
 func newEndpointTranslator(
 	controllerNS string,
 	identityTrustDomain string,
-	enableH2Upgrade bool,
+	enableH2Upgrade,
+	enableEndpointFiltering,
+	experimentalEndpointZoneWeights bool,
 	service string,
 	srcNodeName string,
 	defaultOpaquePorts map[uint32]struct{},
-	enableEndpointFiltering bool,
 	k8sAPI *k8s.MetadataAPI,
 	stream pb.Destination_GetServer,
 	endStream chan struct{},
@@ -102,11 +106,12 @@ func newEndpointTranslator(
 	return &endpointTranslator{
 		controllerNS,
 		identityTrustDomain,
-		enableH2Upgrade,
 		nodeTopologyZone,
 		srcNodeName,
 		defaultOpaquePorts,
+		enableH2Upgrade,
 		enableEndpointFiltering,
+		experimentalEndpointZoneWeights,
 		availableEndpoints,
 		filteredSnapshot,
 		stream,
@@ -373,17 +378,26 @@ func (et *endpointTranslator) sendClientAdd(set watcher.AddressSet) {
 		if address.Pod != nil {
 			opaquePorts = watcher.GetAnnotatedOpaquePorts(address.Pod, et.defaultOpaquePorts)
 			wa, err = createWeightedAddr(address, opaquePorts, et.enableH2Upgrade, et.identityTrustDomain, et.controllerNS, et.log)
+			if err != nil {
+				et.log.Errorf("Failed to translate endpoints to weighted addr: %s", err)
+				continue
+			}
 		} else {
+			// When there's no associated pod, we may still need to set metadata
+			// (especially for remote multi-cluster services).
+			var addr *net.TcpAddress
+			addr, err = toAddr(address)
+			if err != nil {
+				et.log.Errorf("Failed to translate endpoints to weighted addr: %s", err)
+				continue
+			}
+
 			var authOverride *pb.AuthorityOverride
 			if address.AuthorityOverride != "" {
 				authOverride = &pb.AuthorityOverride{
 					AuthorityOverride: address.AuthorityOverride,
 				}
 			}
-
-			// handling address with no associated pod
-			var addr *net.TcpAddress
-			addr, err = toAddr(address)
 			wa = &pb.WeightedAddr{
 				Addr:              addr,
 				Weight:            defaultWeight,
@@ -398,7 +412,6 @@ func (et *endpointTranslator) sendClientAdd(set watcher.AddressSet) {
 						},
 					},
 				}
-				// in this case we most likely have a proxy on the other side, so set protocol hint as well.
 				if et.enableH2Upgrade {
 					wa.ProtocolHint = &pb.ProtocolHint{
 						Protocol: &pb.ProtocolHint_H2_{
@@ -408,10 +421,15 @@ func (et *endpointTranslator) sendClientAdd(set watcher.AddressSet) {
 				}
 			}
 		}
-		if err != nil {
-			et.log.Errorf("Failed to translate endpoints to weighted addr: %s", err)
-			continue
+
+		if et.experimentalEndpointZoneWeights {
+			// EXPERIMENTAL: Use the endpoint weight field to indicate zonal
+			// preference so that local endoints are more heavily weighted.
+			if et.nodeTopologyZone != "" && address.Zone != nil && *address.Zone == et.nodeTopologyZone {
+				wa.Weight *= 10
+			}
 		}
+
 		addrs = append(addrs, wa)
 	}
 
@@ -462,8 +480,14 @@ func toAddr(address watcher.Address) (*net.TcpAddress, error) {
 	}, nil
 }
 
-func createWeightedAddr(address watcher.Address, opaquePorts map[uint32]struct{}, enableH2Upgrade bool, identityTrustDomain string, controllerNS string, log *logging.Entry) (*pb.WeightedAddr, error) {
-
+func createWeightedAddr(
+	address watcher.Address,
+	opaquePorts map[uint32]struct{},
+	enableH2Upgrade bool,
+	identityTrustDomain string,
+	controllerNS string,
+	log *logging.Entry,
+) (*pb.WeightedAddr, error) {
 	tcpAddr, err := toAddr(address)
 	if err != nil {
 		return nil, err
@@ -489,6 +513,14 @@ func createWeightedAddr(address watcher.Address, opaquePorts map[uint32]struct{}
 	controllerNSLabel := address.Pod.Labels[pkgK8s.ControllerNSLabel]
 	sa, ns := pkgK8s.GetServiceAccountAndNS(address.Pod)
 	weightedAddr.MetricLabels = pkgK8s.GetPodLabels(address.OwnerKind, address.OwnerName, address.Pod)
+
+	// Set a zone label, even if it is empty (for consistency).
+	z := ""
+	if address.Zone != nil {
+		z = *address.Zone
+	}
+	weightedAddr.MetricLabels["zone"] = z
+
 	_, isSkippedInboundPort := skippedInboundPorts[address.Port]
 
 	// If the pod is controlled by any Linkerd control plane, then it can be
diff --git a/controller/api/destination/server.go b/controller/api/destination/server.go
index 613ebbaa5..75a07d8fe 100644
--- a/controller/api/destination/server.go
+++ b/controller/api/destination/server.go
@@ -25,22 +25,29 @@ import (
 )
 
 type (
+	Config struct {
+		ControllerNS,
+		IdentityTrustDomain,
+		ClusterDomain string
+
+		EnableH2Upgrade,
+		EnableEndpointSlices,
+		ExperimentalEndpointZoneWeights bool
+
+		DefaultOpaquePorts map[uint32]struct{}
+	}
+
 	server struct {
 		pb.UnimplementedDestinationServer
 
-		pods        *watcher.PodWatcher
-		endpoints   *watcher.EndpointsWatcher
-		opaquePorts *watcher.OpaquePortsWatcher
-		profiles    *watcher.ProfileWatcher
+		config Config
 
+		pods         *watcher.PodWatcher
+		endpoints    *watcher.EndpointsWatcher
+		opaquePorts  *watcher.OpaquePortsWatcher
+		profiles     *watcher.ProfileWatcher
 		clusterStore *watcher.ClusterStore
 
-		enableH2Upgrade     bool
-		controllerNS        string
-		identityTrustDomain string
-		clusterDomain       string
-		defaultOpaquePorts  map[uint32]struct{}
-
 		k8sAPI      *k8s.API
 		metadataAPI *k8s.MetadataAPI
 		log         *logging.Entry
@@ -62,15 +69,10 @@ type (
 // API.
 func NewServer(
 	addr string,
-	controllerNS string,
-	identityTrustDomain string,
-	enableH2Upgrade bool,
-	enableEndpointSlices bool,
+	config Config,
 	k8sAPI *k8s.API,
 	metadataAPI *k8s.MetadataAPI,
 	clusterStore *watcher.ClusterStore,
-	clusterDomain string,
-	defaultOpaquePorts map[uint32]struct{},
 	shutdown <-chan struct{},
 ) (*grpc.Server, error) {
 	log := logging.WithFields(logging.Fields{
@@ -84,15 +86,15 @@ func NewServer(
 		return nil, err
 	}
 
-	pods, err := watcher.NewPodWatcher(k8sAPI, metadataAPI, log, defaultOpaquePorts)
+	pods, err := watcher.NewPodWatcher(k8sAPI, metadataAPI, log, config.DefaultOpaquePorts)
 	if err != nil {
 		return nil, err
 	}
-	endpoints, err := watcher.NewEndpointsWatcher(k8sAPI, metadataAPI, log, enableEndpointSlices, "local")
+	endpoints, err := watcher.NewEndpointsWatcher(k8sAPI, metadataAPI, log, config.EnableEndpointSlices, "local")
 	if err != nil {
 		return nil, err
 	}
-	opaquePorts, err := watcher.NewOpaquePortsWatcher(k8sAPI, log, defaultOpaquePorts)
+	opaquePorts, err := watcher.NewOpaquePortsWatcher(k8sAPI, log, config.DefaultOpaquePorts)
 	if err != nil {
 		return nil, err
 	}
@@ -103,16 +105,12 @@ func NewServer(
 
 	srv := server{
 		pb.UnimplementedDestinationServer{},
+		config,
 		pods,
 		endpoints,
 		opaquePorts,
 		profiles,
 		clusterStore,
-		enableH2Upgrade,
-		controllerNS,
-		identityTrustDomain,
-		clusterDomain,
-		defaultOpaquePorts,
 		k8sAPI,
 		metadataAPI,
 		log,
@@ -155,7 +153,7 @@ func (s *server) Get(dest *pb.GetDestination, stream pb.Destination_GetServer) e
 		return status.Errorf(codes.InvalidArgument, "IP queries not supported by Get API: host=%s", host)
 	}
 
-	service, instanceID, err := parseK8sServiceName(host, s.clusterDomain)
+	service, instanceID, err := parseK8sServiceName(host, s.config.ClusterDomain)
 	if err != nil {
 		log.Debugf("Invalid service %s", dest.GetPath())
 		return status.Errorf(codes.InvalidArgument, "Invalid authority: %s", dest.GetPath())
@@ -184,13 +182,14 @@ func (s *server) Get(dest *pb.GetDestination, stream pb.Destination_GetServer) e
 			return status.Errorf(codes.NotFound, "Remote cluster not found: %s", cluster)
 		}
 		translator := newEndpointTranslator(
-			s.controllerNS,
+			s.config.ControllerNS,
 			remoteConfig.TrustDomain,
-			s.enableH2Upgrade,
+			s.config.EnableH2Upgrade,
+			false, // Disable endpoint filtering for remote discovery.
+			s.config.ExperimentalEndpointZoneWeights,
 			fmt.Sprintf("%s.%s.svc.%s:%d", remoteSvc, service.Namespace, remoteConfig.ClusterDomain, port),
 			token.NodeName,
-			s.defaultOpaquePorts,
-			false, // Disable endpoint filtering for remote discovery.
+			s.config.DefaultOpaquePorts,
 			s.metadataAPI,
 			stream,
 			streamEnd,
@@ -214,13 +213,14 @@ func (s *server) Get(dest *pb.GetDestination, stream pb.Destination_GetServer) e
 	} else {
 		// Local discovery
 		translator := newEndpointTranslator(
-			s.controllerNS,
-			s.identityTrustDomain,
-			s.enableH2Upgrade,
+			s.config.ControllerNS,
+			s.config.IdentityTrustDomain,
+			s.config.EnableH2Upgrade,
+			true,
+			s.config.ExperimentalEndpointZoneWeights,
 			dest.GetPath(),
 			token.NodeName,
-			s.defaultOpaquePorts,
-			true,
+			s.config.DefaultOpaquePorts,
 			s.metadataAPI,
 			stream,
 			streamEnd,
@@ -301,7 +301,7 @@ func (s *server) getProfileByIP(
 		return s.subscribeToEndpointProfile(nil, "", ip.String(), port, log, stream)
 	}
 
-	fqn := fmt.Sprintf("%s.%s.svc.%s", svcID.Name, svcID.Namespace, s.clusterDomain)
+	fqn := fmt.Sprintf("%s.%s.svc.%s", svcID.Name, svcID.Namespace, s.config.ClusterDomain)
 	return s.subscribeToServiceProfile(*svcID, token, fqn, port, log, stream)
 }
 
@@ -312,7 +312,7 @@ func (s *server) getProfileByName(
 	log *logging.Entry,
 	stream pb.Destination_GetProfileServer,
 ) error {
-	service, hostname, err := parseK8sServiceName(host, s.clusterDomain)
+	service, hostname, err := parseK8sServiceName(host, s.config.ClusterDomain)
 	if err != nil {
 		s.log.Debugf("Invalid service %s", host)
 		return status.Errorf(codes.InvalidArgument, "invalid service %q: %q", host, err)
@@ -407,7 +407,7 @@ func (s *server) subscribeToServicesWithContext(
 
 	// The backup lookup ignores the context token to lookup any
 	// server-namespace-hosted profiles.
-	backupID, err := profileID(fqn, contextToken{}, s.clusterDomain)
+	backupID, err := profileID(fqn, contextToken{}, s.config.ClusterDomain)
 	if err != nil {
 		log.Debug("Invalid service")
 		return status.Errorf(codes.InvalidArgument, "invalid profile ID: %s", err)
@@ -419,7 +419,7 @@ func (s *server) subscribeToServicesWithContext(
 	}
 	defer s.profiles.Unsubscribe(backupID, backup)
 
-	primaryID, err := profileID(fqn, token, s.clusterDomain)
+	primaryID, err := profileID(fqn, token, s.config.ClusterDomain)
 	if err != nil {
 		log.Debug("Invalid service")
 		return status.Errorf(codes.InvalidArgument, "invalid profile ID: %s", err)
@@ -450,7 +450,7 @@ func (s *server) subscribeToServiceWithoutContext(
 	log *logging.Entry,
 	streamEnd <-chan struct{},
 ) error {
-	id, err := profileID(fqn, contextToken{}, s.clusterDomain)
+	id, err := profileID(fqn, contextToken{}, s.config.ClusterDomain)
 	if err != nil {
 		log.Debug("Invalid service")
 		return status.Errorf(codes.InvalidArgument, "invalid profile ID: %s", err)
@@ -485,10 +485,10 @@ func (s *server) subscribeToEndpointProfile(
 	stream pb.Destination_GetProfileServer,
 ) error {
 	translator := newEndpointProfileTranslator(
-		s.enableH2Upgrade,
-		s.controllerNS,
-		s.identityTrustDomain,
-		s.defaultOpaquePorts,
+		s.config.EnableH2Upgrade,
+		s.config.ControllerNS,
+		s.config.IdentityTrustDomain,
+		s.config.DefaultOpaquePorts,
 		log,
 		stream,
 	)
diff --git a/controller/api/destination/watcher/endpoints_watcher.go b/controller/api/destination/watcher/endpoints_watcher.go
index 2fffb46a9..63ad57b42 100644
--- a/controller/api/destination/watcher/endpoints_watcher.go
+++ b/controller/api/destination/watcher/endpoints_watcher.go
@@ -51,6 +51,7 @@ type (
 		OwnerKind         string
 		Identity          string
 		AuthorityOverride string
+		Zone              *string
 		ForZones          []discovery.ForZone
 		OpaqueProtocol    bool
 	}
@@ -898,6 +899,7 @@ func (pp *portPublisher) endpointSliceToAddresses(es *discovery.EndpointSlice) A
 					pp.log.Errorf("failed to set address OpaqueProtocol: %s", err)
 					continue
 				}
+				address.Zone = endpoint.Zone
 				if endpoint.Hints != nil {
 					zones := make([]discovery.ForZone, len(endpoint.Hints.ForZones))
 					copy(zones, endpoint.Hints.ForZones)
diff --git a/controller/cmd/destination/main.go b/controller/cmd/destination/main.go
index b36befc25..2f4abb207 100644
--- a/controller/cmd/destination/main.go
+++ b/controller/cmd/destination/main.go
@@ -26,9 +26,11 @@ func Main(args []string) {
 	addr := cmd.String("addr", ":8086", "address to serve on")
 	metricsAddr := cmd.String("metrics-addr", ":9996", "address to serve scrapable metrics on")
 	kubeConfigPath := cmd.String("kubeconfig", "", "path to kube config")
-	enableH2Upgrade := cmd.Bool("enable-h2-upgrade", true, "Enable transparently upgraded HTTP2 connections among pods in the service mesh")
 	controllerNamespace := cmd.String("controller-namespace", "linkerd", "namespace in which Linkerd is installed")
-	enableEndpointSlices := cmd.Bool("enable-endpoint-slices", true, "Enable the usage of EndpointSlice informers and resources")
+	enableH2Upgrade := cmd.Bool("enable-h2-upgrade", true,
+		"Enable transparently upgraded HTTP2 connections among pods in the service mesh")
+	enableEndpointSlices := cmd.Bool("enable-endpoint-slices", true,
+		"Enable the usage of EndpointSlice informers and resources")
 	trustDomain := cmd.String("identity-trust-domain", "", "configures the name suffix used for identities")
 	clusterDomain := cmd.String("cluster-domain", "", "kubernetes cluster domain")
 	defaultOpaquePorts := cmd.String("default-opaque-ports", "", "configures the default opaque ports")
@@ -36,6 +38,12 @@ func Main(args []string) {
 
 	traceCollector := flags.AddTraceFlags(cmd)
 
+	// Zone weighting is disabled by default because it is not consumed by
+	// proxies. This feature exists to support experimentation on top of the
+	// Linkerd control plane API.
+	experimentalEndpointZoneWeights := cmd.Bool("experimental-endpoint-zone-weights", false,
+		"Enable setting endpoint weighting based on zone locality")
+
 	flags.ConfigureAndParse(cmd, args)
 
 	ready := false
@@ -127,17 +135,21 @@ func Main(args []string) {
 		log.Fatalf("Failed to initialize Cluster Store: %s", err)
 	}
 
+	config := destination.Config{
+		ControllerNS:                    *controllerNamespace,
+		IdentityTrustDomain:             *trustDomain,
+		ClusterDomain:                   *clusterDomain,
+		DefaultOpaquePorts:              opaquePorts,
+		EnableH2Upgrade:                 *enableH2Upgrade,
+		EnableEndpointSlices:            *enableEndpointSlices,
+		ExperimentalEndpointZoneWeights: *experimentalEndpointZoneWeights,
+	}
 	server, err := destination.NewServer(
 		*addr,
-		*controllerNamespace,
-		*trustDomain,
-		*enableH2Upgrade,
-		*enableEndpointSlices,
+		config,
 		k8sAPI,
 		metadataAPI,
 		clusterStore,
-		*clusterDomain,
-		opaquePorts,
 		done,
 	)
 
diff --git a/deny.toml b/deny.toml
index c3c71e097..c5eae35e3 100644
--- a/deny.toml
+++ b/deny.toml
@@ -64,6 +64,9 @@ skip = [
     # `tower-http` (a transitive dep via `kubert`) depends on v2.x of `bitflags`,
     # while pretty much the entire rest of the world is still on v1.x
     { name = "bitflags", version = "1.0" },
+    # https://github.com/hawkw/matchers/pull/4
+    { name = "regex-automata", version = "0.1" },
+    { name = "regex-syntax", version = "0.6" },
 ]
 skip-tree = [
     # `serde_json` and `h2` depend on diverged versions of `indexmap` (2.0.x and
diff --git a/go.mod b/go.mod
index e74c63af9..db9d9e435 100644
--- a/go.mod
+++ b/go.mod
@@ -10,7 +10,7 @@ require (
 	github.com/evanphx/json-patch v5.7.0+incompatible
 	github.com/fatih/color v1.16.0
 	github.com/fsnotify/fsnotify v1.7.0
-	github.com/go-openapi/spec v0.20.11
+	github.com/go-openapi/spec v0.20.12
 	github.com/go-test/deep v1.1.0
 	github.com/golang/protobuf v1.5.3
 	github.com/gorilla/websocket v1.5.1
@@ -35,7 +35,7 @@ require (
 	go.opencensus.io v0.24.0
 	golang.org/x/net v0.19.0
 	golang.org/x/tools v0.16.1
-	google.golang.org/grpc v1.60.0
+	google.golang.org/grpc v1.60.1
 	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0
 	google.golang.org/protobuf v1.31.0
 	helm.sh/helm/v3 v3.13.3
@@ -85,9 +85,9 @@ require (
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-logr/logr v1.3.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/go-openapi/jsonpointer v0.19.6 // indirect
-	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-openapi/jsonpointer v0.20.1 // indirect
+	github.com/go-openapi/jsonreference v0.20.3 // indirect
+	github.com/go-openapi/swag v0.22.5 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -126,7 +126,6 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
-	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/spf13/cast v1.5.1 // indirect
diff --git a/go.sum b/go.sum
index 2a8d1d2de..add7f638a 100644
--- a/go.sum
+++ b/go.sum
@@ -87,7 +87,6 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
@@ -151,19 +150,14 @@ github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
 github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
-github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
-github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo=
-github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
-github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
-github.com/go-openapi/spec v0.20.11 h1:J/TzFDLTt4Rcl/l1PmyErvkqlJDncGvPTMnCI39I4gY=
-github.com/go-openapi/spec v0.20.11/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA=
-github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
-github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
-github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-openapi/jsonpointer v0.20.1 h1:MkK4VEIEZMj4wT9PmjaUmGflVBr9nvud4Q4UVFbDoBE=
+github.com/go-openapi/jsonpointer v0.20.1/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs=
+github.com/go-openapi/jsonreference v0.20.3 h1:EjGcjTW8pD1mRis6+w/gmoBdqv5+RbE9B85D1NgDOVQ=
+github.com/go-openapi/jsonreference v0.20.3/go.mod h1:FviDZ46i9ivh810gqzFLl5NttD5q3tSlMLqLr6okedM=
+github.com/go-openapi/spec v0.20.12 h1:cgSLbrsmziAP2iais+Vz7kSazwZ8rsUZd6TUzdDgkVI=
+github.com/go-openapi/spec v0.20.12/go.mod h1:iSCgnBcwbMW9SfzJb8iYynXvcY6C/QFrI7otzF7xGM4=
+github.com/go-openapi/swag v0.22.5 h1:fVS63IE3M0lsuWRzuom3RLwUMVI2peDH01s6M70ugys=
+github.com/go-openapi/swag v0.22.5/go.mod h1:Gl91UqO+btAM0plGGxHqJcQZ1ZTy6jbmridBTsDy8A0=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
@@ -294,7 +288,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -305,9 +298,6 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhn
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
 github.com/linkerd/linkerd2-proxy-api v0.12.0 h1:ZB9a2OF7JAVk3tjtoukPz7zfBRZ3U+jhkdFCpBzJAR4=
 github.com/linkerd/linkerd2-proxy-api v0.12.0/go.mod h1:n9g59OaIKK7y0VF1lIwRkIhuvFtE1KHRtA8iz6Wenh8=
-github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -353,7 +343,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nsf/termbox-go v1.1.1 h1:nksUPLCb73Q++DwbYUBEglYBRPZyoXJdrj5L+TkjyZY=
 github.com/nsf/termbox-go v1.1.1/go.mod h1:T0cTdVuOwf7pHQNtfhnEbzHbcNyCEcVU4YPpouCbVxo=
 github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
@@ -435,7 +424,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
@@ -721,8 +709,8 @@ google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
 google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.60.0 h1:6FQAR0kM31P6MRdeluor2w2gPaS4SVNrD/DNTxrQ15k=
-google.golang.org/grpc v1.60.0/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM=
+google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU=
+google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 h1:rNBFJjBCOgVr9pWD7rs/knKL4FRTKgpZmsRfV214zcA=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
@@ -743,7 +731,6 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
@@ -757,7 +744,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
diff --git a/pkg/prometheus/prometheus.go b/pkg/prometheus/prometheus.go
index 27c6fe4db..8b7f9c82d 100644
--- a/pkg/prometheus/prometheus.go
+++ b/pkg/prometheus/prometheus.go
@@ -62,6 +62,14 @@ var (
 		[]string{"client", "code", "method"},
 	)
 
+	clientErrorCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "http_client_errors_total",
+			Help: "A counter for errors from the wrapped client.",
+		},
+		[]string{"client", "method"},
+	)
+
 	clientLatency = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Name:    "http_client_request_latency_seconds",
@@ -97,7 +105,7 @@ var (
 func init() {
 	prometheus.MustRegister(
 		serverCounter, serverLatency, serverResponseSize, clientCounter,
-		clientLatency, clientInFlight, clientQPS, clientBurst,
+		clientLatency, clientInFlight, clientQPS, clientBurst, clientErrorCounter,
 	)
 }
 
@@ -128,20 +136,33 @@ func ClientWithTelemetry(name string, wt func(http.RoundTripper) http.RoundTripp
 	latency := clientLatency.MustCurryWith(prometheus.Labels{"client": name})
 	counter := clientCounter.MustCurryWith(prometheus.Labels{"client": name})
 	inFlight := clientInFlight.With(prometheus.Labels{"client": name})
+	errors := clientErrorCounter.MustCurryWith(prometheus.Labels{"client": name})
 
 	return func(rt http.RoundTripper) http.RoundTripper {
 		if wt != nil {
 			rt = wt(rt)
 		}
 
-		return promhttp.InstrumentRoundTripperInFlight(inFlight,
-			promhttp.InstrumentRoundTripperCounter(counter,
-				promhttp.InstrumentRoundTripperDuration(latency, rt),
+		return InstrumentErrorCounter(errors,
+			promhttp.InstrumentRoundTripperInFlight(inFlight,
+				promhttp.InstrumentRoundTripperCounter(counter,
+					promhttp.InstrumentRoundTripperDuration(latency, rt),
+				),
 			),
 		)
 	}
 }
 
+func InstrumentErrorCounter(counter *prometheus.CounterVec, next http.RoundTripper) promhttp.RoundTripperFunc {
+	return func(r *http.Request) (*http.Response, error) {
+		resp, err := next.RoundTrip(r)
+		if err != nil {
+			counter.With(prometheus.Labels{"method": r.Method}).Inc()
+		}
+		return resp, err
+	}
+}
+
 func SetClientQPS(name string, qps float32) {
 	clientQPS.With(prometheus.Labels{"client": name}).Set(float64(qps))
 }


Test patch:
diff --git a/controller/api/destination/endpoint_translator_test.go b/controller/api/destination/endpoint_translator_test.go
index f2bdfb6d3..6f3bf80b5 100644
--- a/controller/api/destination/endpoint_translator_test.go
+++ b/controller/api/destination/endpoint_translator_test.go
@@ -346,8 +346,8 @@ func TestEndpointTranslatorForPods(t *testing.T) {
 		sort.Slice(addressesAdded, func(i, j int) bool {
 			return addressesAdded[i].GetAddr().Port < addressesAdded[j].GetAddr().Port
 		})
-		checkAddressAndWeight(t, addressesAdded[0], pod1)
-		checkAddressAndWeight(t, addressesAdded[1], pod2)
+		checkAddressAndWeight(t, addressesAdded[0], pod1, defaultWeight)
+		checkAddressAndWeight(t, addressesAdded[1], pod2, defaultWeight)
 		checkAddress(t, addressesRemoved[0], pod3)
 	})
 
@@ -372,6 +372,7 @@ func TestEndpointTranslatorForPods(t *testing.T) {
 			"replicationcontroller": "rc-name",
 			"serviceaccount":        "serviceaccount-name",
 			"control_plane_ns":      "linkerd",
+			"zone":                  "",
 		}
 		if diff := deep.Equal(actualAddedAddress1MetricLabels, expectedAddedAddress1MetricLabels); diff != nil {
 			t.Fatalf("Expected global metric labels sent to be [%v] but was [%v]", expectedAddedAddress1MetricLabels, actualAddedAddress1MetricLabels)
@@ -428,7 +429,7 @@ func TestEndpointTranslatorForPods(t *testing.T) {
 	})
 }
 
-func TestEndpointTranslatorForZonedAddresses(t *testing.T) {
+func TestEndpointTranslatorTopologyAwareFilter(t *testing.T) {
 	t.Run("Sends one update for add and none for remove", func(t *testing.T) {
 		mockGetServer, translator := makeEndpointTranslator(t)
 		translator.Start()
@@ -449,6 +450,59 @@ func TestEndpointTranslatorForZonedAddresses(t *testing.T) {
 	})
 }
 
+func TestEndpointTranslatorExperimentalZoneWeights(t *testing.T) {
+	zoneA := "west-1a"
+	zoneB := "west-1b"
+	addrA := watcher.Address{
+		IP:   "7.9.7.9",
+		Port: 7979,
+		Zone: &zoneA,
+	}
+	addrB := watcher.Address{
+		IP:   "9.7.9.7",
+		Port: 9797,
+		Zone: &zoneB,
+	}
+
+	t.Run("Disabled", func(t *testing.T) {
+		mockGetServer, translator := makeEndpointTranslator(t)
+		translator.experimentalEndpointZoneWeights = false
+		translator.Start()
+		defer translator.Stop()
+
+		translator.Add(mkAddressSetForServices(addrA, addrB))
+
+		addrs := (<-mockGetServer.updatesReceived).GetAdd().GetAddrs()
+		if len(addrs) != 2 {
+			t.Fatalf("Expected [2] addresses returned, got %v", addrs)
+		}
+		sort.Slice(addrs, func(i, j int) bool {
+			return addrs[i].GetAddr().Port < addrs[j].GetAddr().Port
+		})
+		checkAddressAndWeight(t, addrs[0], addrA, defaultWeight)
+		checkAddressAndWeight(t, addrs[1], addrB, defaultWeight)
+	})
+
+	t.Run("Applies weights", func(t *testing.T) {
+		mockGetServer, translator := makeEndpointTranslator(t)
+		translator.experimentalEndpointZoneWeights = true
+		translator.Start()
+		defer translator.Stop()
+
+		translator.Add(mkAddressSetForServices(addrA, addrB))
+
+		addrs := (<-mockGetServer.updatesReceived).GetAdd().GetAddrs()
+		if len(addrs) != 2 {
+			t.Fatalf("Expected [2] addresses returned, got %v", addrs)
+		}
+		sort.Slice(addrs, func(i, j int) bool {
+			return addrs[i].GetAddr().Port < addrs[j].GetAddr().Port
+		})
+		checkAddressAndWeight(t, addrs[0], addrA, defaultWeight*10)
+		checkAddressAndWeight(t, addrs[1], addrB, defaultWeight)
+	})
+}
+
 func TestEndpointTranslatorForLocalTrafficPolicy(t *testing.T) {
 	t.Run("Sends one update for add and none for remove", func(t *testing.T) {
 		mockGetServer, translator := makeEndpointTranslator(t)
@@ -521,14 +575,18 @@ func mkAddressSetForPods(podAddresses ...watcher.Address) watcher.AddressSet {
 	return set
 }
 
-func checkAddressAndWeight(t *testing.T, actual *pb.WeightedAddr, expected watcher.Address) {
+func checkAddressAndWeight(t *testing.T, actual *pb.WeightedAddr, expected watcher.Address, weight uint32) {
+	t.Helper()
+
 	checkAddress(t, actual.GetAddr(), expected)
-	if actual.GetWeight() != defaultWeight {
-		t.Fatalf("Expected weight [%+v] but got [%+v]", defaultWeight, actual.GetWeight())
+	if actual.GetWeight() != weight {
+		t.Fatalf("Expected weight [%+v] but got [%+v]", weight, actual.GetWeight())
 	}
 }
 
 func checkAddress(t *testing.T, actual *net.TcpAddress, expected watcher.Address) {
+	t.Helper()
+
 	expectedAddr, err := addr.ParseProxyIPV4(expected.IP)
 	expectedTCP := net.TcpAddress{
 		Ip:   expectedAddr,
diff --git a/controller/api/destination/test_util.go b/controller/api/destination/test_util.go
index 96e6bc77c..3b9f8366c 100644
--- a/controller/api/destination/test_util.go
+++ b/controller/api/destination/test_util.go
@@ -509,16 +509,18 @@ spec:
 
 	return &server{
 		pb.UnimplementedDestinationServer{},
+		Config{
+			EnableH2Upgrade:     true,
+			ControllerNS:        "linkerd",
+			ClusterDomain:       "mycluster.local",
+			IdentityTrustDomain: "trust.domain",
+			DefaultOpaquePorts:  defaultOpaquePorts,
+		},
 		pods,
 		endpoints,
 		opaquePorts,
 		profiles,
 		clusterStore,
-		true,
-		"linkerd",
-		"trust.domain",
-		"mycluster.local",
-		defaultOpaquePorts,
 		k8sAPI,
 		metadataAPI,
 		log,
@@ -576,6 +578,7 @@ func (m *mockDestinationGetProfileServer) Send(profile *pb.DestinationProfile) e
 }
 
 func makeEndpointTranslator(t *testing.T) (*mockDestinationGetServer, *endpointTranslator) {
+	t.Helper()
 	node := `apiVersion: v1
 kind: Node
 metadata:
@@ -592,12 +595,6 @@ metadata:
     topology.kubernetes.io/zone: west-1a
   name: test-123
 `
-	k8sAPI, err := k8s.NewFakeAPI(node)
-	if err != nil {
-		t.Fatalf("NewFakeAPI returned an error: %s", err)
-	}
-	k8sAPI.Sync(nil)
-
 	metadataAPI, err := k8s.NewFakeMetadataAPI([]string{node})
 	if err != nil {
 		t.Fatalf("NewFakeMetadataAPI returned an error: %s", err)
@@ -609,10 +606,11 @@ metadata:
 		"linkerd",
 		"trust.domain",
 		true,
+		true,  // enableEndpointFiltering
+		false, // experimentalEndpointZoneWeights
 		"service-name.service-ns",
 		"test-123",
 		map[uint32]struct{}{},
-		true,
 		metadataAPI,
 		mockGetServer,
 		nil,