Repository: linkerd/linkerd2
Pull request #4279 (2020-04-21 12:43:04)

Multicluster checks
Adds multicluster checks that: 

- check that a service mirror controller exists (1 only)
- check that the expected ClusterRoles, ClusterRoleBindings and ServiceAccounts for the controller exist locally
- for each secret of type `remote-kubeconfig` we check that we can parse the contents into a k8s RestConfig object, can instantiate API connection to the remote cluster AND can LIST, WATCH and GET services in all namespace on this cluster. This is done for all clusters and if there is anything wrong a list of clusters violating these constraints is printed
- checks that each of the gateways (for which we have metrics in Prometheus) is alive and if not it prints a list of problematic cluster gateways


Patch:
diff --git a/charts/linkerd2/README.md b/charts/linkerd2/README.md
index 24d158d3b..8ffa8a658 100644
--- a/charts/linkerd2/README.md
+++ b/charts/linkerd2/README.md
@@ -82,80 +82,83 @@ helm install \
 
 The following table lists the configurable parameters of the Linkerd2 chart and their default values.
 
-| Parameter                             | Description                                                                                                                                                                           | Default                              |
-|:--------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------|
-| `controllerImage`                     | Docker image for the controller, tap and identity components                                                                                                                          | `gcr.io/linkerd-io/controller`       |
-| `controllerImageVersion`              | Tag for the controller container docker image                                                                                                                                         | latest version                       |
-| `controllerLogLevel`                  | Log level for the control plane components                                                                                                                                            | `info`                               |
-| `controllerReplicas`                  | Number of replicas for each control plane pod                                                                                                                                         | `1`                                  |
-| `controllerUID`                       | User ID for the control plane components                                                                                                                                              | `2103`                               |
-| `dashboard.replicas`                  | Number of replicas of dashboard                                                                                                                                                       | `1`                                  |
-| `debugContainer.image.name`           | Docker image for the debug container                                                                                                                                                  | `gcr.io/linkerd-io/debug`            |
-| `debugContainer.image.pullPolicy`     | Pull policy for the debug container Docker image                                                                                                                                      | `IfNotPresent`                       |
-| `debugContainer.image.version`        | Tag for the debug container Docker image                                                                                                                                              | latest version                       |
-| `disableHeartBeat`                    | Set to true to not start the heartbeat cronjob                                                                                                                                        | `false`                              |
-| `enableH2Upgrade`                     | Allow proxies to perform transparent HTTP/2 upgrading                                                                                                                                 | `true`                               |
-| `global.clusterDomain`                       | Kubernetes DNS Domain name to use                                                                                                                                                     | `cluster.local`                      |
-| `global.cniEnabled`                          | Omit the NET_ADMIN capability in the PSP and the proxy-init container when injecting the proxy; requires the linkerd-cni plugin to already be installed                               | `false`                      |
-| `global.controllerComponentLabel`            | Control plane label. Do not edit                                                                                                                                                      | `linkerd.io/control-plane-component` |
-| `global.controllerNamespaceLabel`            | Control plane label. Do not edit                                                                                                                                                      | `linkerd.io/control-plane-component` |
-| `global.createdByAnnotation`                 | Annotation label for the proxy create. Do not edit.                                                                                                                                   | `linkerd.io/created-by`              |
-| `global.identityTrustAnchorsPEM`            | Trust root certificate (ECDSA). It must be provided during install.                                                                                                                                                         ||
+| Parameter                                   | Description                                                                                                                                                                           | Default                              |
+|:--------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------|
+| `controllerImage`                           | Docker image for the controller, tap and identity components                                                                                                                          | `gcr.io/linkerd-io/controller`       |
+| `controllerImageVersion`                    | Tag for the controller container docker image                                                                                                                                         | latest version                       |
+| `controllerLogLevel`                        | Log level for the control plane components                                                                                                                                            | `info`                               |
+| `controllerReplicas`                        | Number of replicas for each control plane pod                                                                                                                                         | `1`                                  |
+| `controllerUID`                             | User ID for the control plane components                                                                                                                                              | `2103`                               |
+| `dashboard.replicas`                        | Number of replicas of dashboard                                                                                                                                                       | `1`                                  |
+| `debugContainer.image.name`                 | Docker image for the debug container                                                                                                                                                  | `gcr.io/linkerd-io/debug`            |
+| `debugContainer.image.pullPolicy`           | Pull policy for the debug container Docker image                                                                                                                                      | `IfNotPresent`                       |
+| `debugContainer.image.version`              | Tag for the debug container Docker image                                                                                                                                              | latest version                       |
+| `disableHeartBeat`                          | Set to true to not start the heartbeat cronjob                                                                                                                                        | `false`                              |
+| `enableH2Upgrade`                           | Allow proxies to perform transparent HTTP/2 upgrading                                                                                                                                 | `true`                               |
+| `global.clusterDomain`                      | Kubernetes DNS Domain name to use                                                                                                                                                     | `cluster.local`                      |
+| `global.cniEnabled`                         | Omit the NET_ADMIN capability in the PSP and the proxy-init container when injecting the proxy; requires the linkerd-cni plugin to already be installed                               | `false`                              |
+| `global.controllerComponentLabel`           | Control plane label. Do not edit                                                                                                                                                      | `linkerd.io/control-plane-component` |
+| `global.controllerNamespaceLabel`           | Control plane label. Do not edit                                                                                                                                                      | `linkerd.io/control-plane-component` |
+| `global.createdByAnnotation`                | Annotation label for the proxy create. Do not edit.                                                                                                                                   | `linkerd.io/created-by`              |
+| `global.identityTrustAnchorsPEM`            | Trust root certificate (ECDSA). It must be provided during install.                                                                                                                   |                                      |
 | `global.identityTrustDomain`                | Trust domain used for identity                                                                                                                                                        | `cluster.local`                      |
-| `global.imagePullPolicy`                     | Docker image pull policy                                                                                                                                                              | `IfNotPresent`                       |
-| `global.linkerdNamespaceLabel`               | Control plane label. Do not edit                                                                                                                                                      | `linkerd.io/control-plane-component` |
-| `global.linkerdVersion`                      | Control plane version                                                                                                                                                                 | latest version                       |
-| `global.namespace`                           | Control plane namespace                                                                                                                                                               | `linkerd`                            |
-| `global.proxy.enableExternalProfiles`        | Enable service profiles for non-Kubernetes services                                                                                                                                   | `false`                              |
-| `global.proxy.image.name`                    | Docker image for the proxy                                                                                                                                                            | `gcr.io/linkerd-io/proxy`            |
-| `global.proxy.image.pullPolicy`              | Pull policy for the proxy container Docker image                                                                                                                                      | `IfNotPresent`                       |
-| `global.proxy.image.version`                 | Tag for the proxy container Docker image                                                                                                                                              | latest version                       |
-| `global.proxy.logLevel`                      | Log level for the proxy                                                                                                                                                               | `warn,linkerd=info`           |
-| `global.proxy.ports.admin`                   | Admin port for the proxy container                                                                                                                                                    | `4191`                               |
-| `global.proxy.ports.control`                 | Control port for the proxy container                                                                                                                                                  | `4190`                               |
-| `global.proxy.ports.inbound`                 | Inbound port for the proxy container                                                                                                                                                  | `4143`                               |
-| `global.proxy.ports.outbound`                | Outbound port for the proxy container                                                                                                                                                 | `4140`                               |
-| `global.proxy.resources.cpu.limit`           | Maximum amount of CPU units that the proxy can use                                                                                                                                                                          ||
-| `global.proxy.resources.cpu.request`         | Amount of CPU units that the proxy requests                                                                                                                                                                                 ||
-| `global.proxy.resources.memory.limit`        | Maximum amount of memory that the proxy can use                                                                                                                                                                             ||
-| `global.proxy.resources.memory.request`      | Amount of memory that the proxy requests                                                                                                                                                                                    ||
-| `global.proxy.trace.collectorSvcAccount`     | Service account associated with the Trace collector instance                                                                                                                                                                ||
-| `global.proxy.trace.collectorSvcAddr`        | Collector Service address for the proxies to send Trace Data                                                                                                                                                                ||
-| `global.proxy.uid`                           | User id under which the proxy runs                                                                                                                                                    | `2102`                               |
-| `global.proxy.waitBeforeExitSeconds`         | The proxy sidecar will stay alive for at least the given period before receiving SIGTERM signal from Kubernetes but no longer than pod's `terminationGracePeriodSeconds`.             | `0`                                  |
-| `global.proxyInit.ignoreInboundPorts`        | Inbound ports the proxy should ignore                                                                                                                                                                                       ||
-| `global.proxyInit.ignoreOutboundPorts`       | Outbound ports the proxy should ignore                                                                                                                                                                                      ||
-| `global.proxyInit.image.name`                | Docker image for the proxy-init container                                                                                                                                             | `gcr.io/linkerd-io/proxy-init`       |
-| `global.proxyInit.image.pullPolicy`          | Pull policy for the proxy-init container Docker image                                                                                                                                 | `IfNotPresent`                       |
-| `global.proxyInit.image.version`             | Tag for the proxy-init container Docker image                                                                                                                                         | latest version                       |
-| `global.proxyInit.resources.cpu.limit`       | Maximum amount of CPU units that the proxy-init container can use                                                                                                                     | `100m`                               |
-| `global.proxyInit.resources.cpu.request`     | Amount of CPU units that the proxy-init container requests                                                                                                                            | `10m`                                |
-| `global.ProxyInit.resources.memory.limit`    | Maximum amount of memory that the proxy-init container can use                                                                                                                        | `50Mi`                               |
-| `global.proxyInit.resources.memory.request`  | Amount of memory that the proxy-init container requests                                                                                                                               | `10Mi`                               |
-| `global.proxyInjectAnnotation`               | Annotation label to signal injection. Do not edit.                                                                                                                                                                          ||
-| `global.proxyInjectDisabled`                 | Annotation value to disable injection. Do not edit.                                                                                                                                   | `disabled`                           |
-| `grafanaImage`                        | Docker image for the Grafana container                                                                                                                                                | `gcr.io/linkerd-io/grafana`          |
-| `heartbeatSchedule`                   | Config for the heartbeat cronjob                                                                                                                                                      | `0 0 * * *`                          |
-| `identity.issuer.clockSkewAllowance`  | Amount of time to allow for clock skew within a Linkerd cluster                                                                                                                       | `20s`                                |
-| `identity.issuer.crtExpiry`           | Expiration timestamp for the issuer certificate. It must be provided during install                                                                                                                                         ||
-| `identity.issuer.crtExpiryAnnotation` | Annotation used to identity the issuer certificate expiration timestamp. Do not edit.                                                                                                 | `linkerd.io/identity-issuer-expiry`  |
-| `identity.issuer.issuanceLifetime`    | Amount of time for which the Identity issuer should certify identity                                                                                                                  | `86400s`                             |
-| `identity.issuer.scheme`              | Which scheme is used for the identity issuer secret format                                                                                                                            | `linkerd.io/tls`                     |
-| `identity.issuer.tls.crtPEM`          | Issuer certificate (ECDSA). It must be provided during install.                                                                                                                                                             ||
-| `identity.issuer.tls.keyPEM`          | Key for the issuer certificate (ECDSA). It must be provided during install.                                                                                                                                                 ||
-| `installNamespace`                    | Set to false when installing Linkerd in a custom namespace. See the [Linkerd documentation](https://linkerd.io/2/tasks/install-helm/#customizing-the-namespace) for more information. | `true`                               |
-| `omitWebhookSideEffects`              | Omit the `sideEffects` flag in the webhook manifests                                                                                                                                  | `false`                              |
-| `prometheusImage`                     | Docker image for the Prometheus container                                                                                                                                             | `prom/prometheus:v2.15.2`            |
-| `prometheusLogLevel`                  | Log level for Prometheus                                                                                                                                                              | `info`                               |
-| `proxyInjector.crtPEM`                | Certificate for the proxy injector. If not provided then Helm will generate one.                                                                                                                                            ||
-| `proxyInjector.keyPEM`                | Certificate key for the proxy injector. If not provided then Helm will generate one.                                                                                                                                        ||
-| `profileValidator.crtPEM`             | Certificate for the service profile validator. If not provided then Helm will generate one.                                                                                                                                 ||
-| `profileValidator.keyPEM`             | Certificate key for the service profile validator. If not provided then Helm will generate one.                                                                                                                             ||
-| `tap.crtPEM`                          | Certificate for the Tap component. If not provided then Helm will generate one.                                                                                                                                             ||
-| `tap.keyPEM`                          | Certificate key for Tap component. If not provided then Helm will generate one.                                                                                                                                             ||
-| `webhookFailurePolicy`                | Failure policy for the proxy injector                                                                                                                                                 | `Ignore`                             |
-| `webImage`                            | Docker image for the web container                                                                                                                                                    | `gcr.io/linkerd-io/web`              |
-| `enforcedHostRegexp`                  | Host header validation regex for the dashboard. See the [Linkerd documentation](https://linkerd.io/2/tasks/exposing-dashboard) for more information                                   | `""`              |
+| `global.imagePullPolicy`                    | Docker image pull policy                                                                                                                                                              | `IfNotPresent`                       |
+| `global.linkerdNamespaceLabel`              | Control plane label. Do not edit                                                                                                                                                      | `linkerd.io/control-plane-component` |
+| `global.linkerdVersion`                     | Control plane version                                                                                                                                                                 | latest version                       |
+| `global.namespace`                          | Control plane namespace                                                                                                                                                               | `linkerd`                            |
+| `global.proxy.enableExternalProfiles`       | Enable service profiles for non-Kubernetes services                                                                                                                                   | `false`                              |
+| `global.proxy.image.name`                   | Docker image for the proxy                                                                                                                                                            | `gcr.io/linkerd-io/proxy`            |
+| `global.proxy.image.pullPolicy`             | Pull policy for the proxy container Docker image                                                                                                                                      | `IfNotPresent`                       |
+| `global.proxy.image.version`                | Tag for the proxy container Docker image                                                                                                                                              | latest version                       |
+| `global.proxy.logLevel`                     | Log level for the proxy                                                                                                                                                               | `warn,linkerd=info`                  |
+| `global.proxy.ports.admin`                  | Admin port for the proxy container                                                                                                                                                    | `4191`                               |
+| `global.proxy.ports.control`                | Control port for the proxy container                                                                                                                                                  | `4190`                               |
+| `global.proxy.ports.inbound`                | Inbound port for the proxy container                                                                                                                                                  | `4143`                               |
+| `global.proxy.ports.outbound`               | Outbound port for the proxy container                                                                                                                                                 | `4140`                               |
+| `global.proxy.resources.cpu.limit`          | Maximum amount of CPU units that the proxy can use                                                                                                                                    |                                      |
+| `global.proxy.resources.cpu.request`        | Amount of CPU units that the proxy requests                                                                                                                                           |                                      |
+| `global.proxy.resources.memory.limit`       | Maximum amount of memory that the proxy can use                                                                                                                                       |                                      |
+| `global.proxy.resources.memory.request`     | Amount of memory that the proxy requests                                                                                                                                              |                                      |
+| `global.proxy.trace.collectorSvcAccount`    | Service account associated with the Trace collector instance                                                                                                                          |                                      |
+| `global.proxy.trace.collectorSvcAddr`       | Collector Service address for the proxies to send Trace Data                                                                                                                          |                                      |
+| `global.proxy.uid`                          | User id under which the proxy runs                                                                                                                                                    | `2102`                               |
+| `global.proxy.waitBeforeExitSeconds`        | The proxy sidecar will stay alive for at least the given period before receiving SIGTERM signal from Kubernetes but no longer than pod's `terminationGracePeriodSeconds`.             | `0`                                  |
+| `global.proxyInit.ignoreInboundPorts`       | Inbound ports the proxy should ignore                                                                                                                                                 |                                      |
+| `global.proxyInit.ignoreOutboundPorts`      | Outbound ports the proxy should ignore                                                                                                                                                |                                      |
+| `global.proxyInit.image.name`               | Docker image for the proxy-init container                                                                                                                                             | `gcr.io/linkerd-io/proxy-init`       |
+| `global.proxyInit.image.pullPolicy`         | Pull policy for the proxy-init container Docker image                                                                                                                                 | `IfNotPresent`                       |
+| `global.proxyInit.image.version`            | Tag for the proxy-init container Docker image                                                                                                                                         | latest version                       |
+| `global.proxyInit.resources.cpu.limit`      | Maximum amount of CPU units that the proxy-init container can use                                                                                                                     | `100m`                               |
+| `global.proxyInit.resources.cpu.request`    | Amount of CPU units that the proxy-init container requests                                                                                                                            | `10m`                                |
+| `global.ProxyInit.resources.memory.limit`   | Maximum amount of memory that the proxy-init container can use                                                                                                                        | `50Mi`                               |
+| `global.proxyInit.resources.memory.request` | Amount of memory that the proxy-init container requests                                                                                                                               | `10Mi`                               |
+| `global.proxyInjectAnnotation`              | Annotation label to signal injection. Do not edit.                                                                                                                                    |                                      |
+| `global.proxyInjectDisabled`                | Annotation value to disable injection. Do not edit.                                                                                                                                   | `disabled`                           |
+| `grafanaImage`                              | Docker image for the Grafana container                                                                                                                                                | `gcr.io/linkerd-io/grafana`          |
+| `heartbeatSchedule`                         | Config for the heartbeat cronjob                                                                                                                                                      | `0 0 * * *`                          |
+| `identity.issuer.clockSkewAllowance`        | Amount of time to allow for clock skew within a Linkerd cluster                                                                                                                       | `20s`                                |
+| `identity.issuer.crtExpiry`                 | Expiration timestamp for the issuer certificate. It must be provided during install                                                                                                   |                                      |
+| `identity.issuer.crtExpiryAnnotation`       | Annotation used to identity the issuer certificate expiration timestamp. Do not edit.                                                                                                 | `linkerd.io/identity-issuer-expiry`  |
+| `identity.issuer.issuanceLifetime`          | Amount of time for which the Identity issuer should certify identity                                                                                                                  | `86400s`                             |
+| `identity.issuer.scheme`                    | Which scheme is used for the identity issuer secret format                                                                                                                            | `linkerd.io/tls`                     |
+| `identity.issuer.tls.crtPEM`                | Issuer certificate (ECDSA). It must be provided during install.                                                                                                                       |                                      |
+| `identity.issuer.tls.keyPEM`                | Key for the issuer certificate (ECDSA). It must be provided during install.                                                                                                           |                                      |
+| `installNamespace`                          | Set to false when installing Linkerd in a custom namespace. See the [Linkerd documentation](https://linkerd.io/2/tasks/install-helm/#customizing-the-namespace) for more information. | `true`                               |
+| `omitWebhookSideEffects`                    | Omit the `sideEffects` flag in the webhook manifests                                                                                                                                  | `false`                              |
+| `prometheusAlertmanagers`                   | Alertmanager instances the Prometheus server sends alerts to configured via the static_configs parameter.                                                                             | `[]`                                 |
+| `prometheusExtraArgs`                       | Extra command line options for Prometheus                                                                                                                                             | `{}`                                 |
+| `prometheusImage`                           | Docker image for the Prometheus container                                                                                                                                             | `prom/prometheus:v2.15.2`            |
+| `prometheusLogLevel`                        | Log level for Prometheus                                                                                                                                                              | `info`                               |
+| `prometheusRuleConfigMapMounts`             | Alerting/recording rule ConfigMap mounts (sub-path names must end in `_rules.yml` or `_rules.yaml`)                                                                                   | `[]`                                 |
+| `proxyInjector.crtPEM`                      | Certificate for the proxy injector. If not provided then Helm will generate one.                                                                                                      |                                      |
+| `proxyInjector.keyPEM`                      | Certificate key for the proxy injector. If not provided then Helm will generate one.                                                                                                  |                                      |
+| `profileValidator.crtPEM`                   | Certificate for the service profile validator. If not provided then Helm will generate one.                                                                                           |                                      |
+| `profileValidator.keyPEM`                   | Certificate key for the service profile validator. If not provided then Helm will generate one.                                                                                       |                                      |
+| `tap.crtPEM`                                | Certificate for the Tap component. If not provided then Helm will generate one.                                                                                                       |                                      |
+| `tap.keyPEM`                                | Certificate key for Tap component. If not provided then Helm will generate one.                                                                                                       |                                      |
+| `webhookFailurePolicy`                      | Failure policy for the proxy injector                                                                                                                                                 | `Ignore`                             |
+| `webImage`                                  | Docker image for the web container                                                                                                                                                    | `gcr.io/linkerd-io/web`              |
+| `enforcedHostRegexp`                        | Host header validation regex for the dashboard. See the [Linkerd documentation](https://linkerd.io/2/tasks/exposing-dashboard) for more information                                   | `""`                                 |
 
 ## Add-Ons Configuration
 
@@ -163,21 +166,21 @@ The following table lists the configurable parameters of the Linkerd2 chart and
 
 The following table lists the configurable parameters for the Tracing Add-On.
 
-| Parameter                             | Description                                                                                                                                                                           | Default                              |
-|:--------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------|
-| `tracing.enabled`                     | Flag to enable tracing components to be installed                                                                                                                                                | `false`
-| `tracing.collector.name`                | Name of the trace collector Service                                                                                                                                                 | `linkerd-collector`                             |
-| `tracing.collector.image`                | Docker image for the trace collector                                                                                                                                                 | `omnition/opencensus-collector:0.1.10`                             |
-| `tracing.collector.resources.cpu.limit`       | Maximum amount of CPU units that the trace collector container can use                                                                                                                     | `1`                               |
-| `tracing.collector.resources.cpu.request`     | Amount of CPU units that the trace collector container requests                                                                                                                            | `200m`                                |
-| `tracing.collector.resources.memory.limit`    | Maximum amount of memory that the trace collector container can use                                                                                                                        | `2Gi`                               |
-| `tracing.collector.resources.memory.request`  | Amount of memory that the trace collector container requests                                                                                                                               | `400Mi`                               |
-| `tracing.jaeger.name`                | Name of the jaeger instance                                                                                                                                                 | `linkerd-jaeger`                             |
-| `tracing.jaeger.image`                | Docker image for the jaeger instance                                                                                                                                                 | `jaegertracing/all-in-one:1.8`                             |
-| `tracing.jaeger.resources.cpu.limit`       | Maximum amount of CPU units that the jaeger container can use                                                                                                                     ||
-| `tracing.jaeger.resources.cpu.request`     | Amount of CPU units that the jaeger container requests                                                                                                                            ||
-| `tracing.jaeger.resources.memory.limit`    | Maximum amount of memory that the jaeger container can use                                                                                                                        ||
-| `tracing.jaeger.resources.memory.request`  | Amount of memory that the jaeger container requests                                                                                                                               ||
+| Parameter                                    | Description                                                            | Default                                |
+|:---------------------------------------------|:-----------------------------------------------------------------------|:---------------------------------------|
+| `tracing.enabled`                            | Flag to enable tracing components to be installed                      | `false`                                |
+| `tracing.collector.name`                     | Name of the trace collector Service                                    | `linkerd-collector`                    |
+| `tracing.collector.image`                    | Docker image for the trace collector                                   | `omnition/opencensus-collector:0.1.10` |
+| `tracing.collector.resources.cpu.limit`      | Maximum amount of CPU units that the trace collector container can use | `1`                                    |
+| `tracing.collector.resources.cpu.request`    | Amount of CPU units that the trace collector container requests        | `200m`                                 |
+| `tracing.collector.resources.memory.limit`   | Maximum amount of memory that the trace collector container can use    | `2Gi`                                  |
+| `tracing.collector.resources.memory.request` | Amount of memory that the trace collector container requests           | `400Mi`                                |
+| `tracing.jaeger.name`                        | Name of the jaeger instance                                            | `linkerd-jaeger`                       |
+| `tracing.jaeger.image`                       | Docker image for the jaeger instance                                   | `jaegertracing/all-in-one:1.8`         |
+| `tracing.jaeger.resources.cpu.limit`         | Maximum amount of CPU units that the jaeger container can use          |                                        |
+| `tracing.jaeger.resources.cpu.request`       | Amount of CPU units that the jaeger container requests                 |                                        |
+| `tracing.jaeger.resources.memory.limit`      | Maximum amount of memory that the jaeger container can use             |                                        |
+| `tracing.jaeger.resources.memory.request`    | Amount of memory that the jaeger container requests                    |                                        |
 
 ## Get involved
 
diff --git a/charts/linkerd2/templates/prometheus.yaml b/charts/linkerd2/templates/prometheus.yaml
index 59176fe67..d190d62e1 100644
--- a/charts/linkerd2/templates/prometheus.yaml
+++ b/charts/linkerd2/templates/prometheus.yaml
@@ -15,6 +15,11 @@ metadata:
     {{.Values.global.createdByAnnotation}}: {{default (printf "linkerd/helm %s" .Values.global.linkerdVersion) .Values.global.cliVersion}}
 data:
   prometheus.yml: |-
+    {{- if .Values.prometheusAlertmanagers }}
+    alerting:
+      alertmanagers:
+        {{- toYaml .Values.prometheusAlertmanagers | trim | nindent 8 }}
+    {{- end }}
     global:
       scrape_interval: 10s
       scrape_timeout: 10s
@@ -22,6 +27,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -209,6 +215,9 @@ spec:
         - --storage.tsdb.retention.time=6h
         - --config.file=/etc/prometheus/prometheus.yml
         - --log.level={{lower .Values.prometheusLogLevel}}
+        {{- range $key, $value := .Values.prometheusExtraArgs}}
+        - --{{ $key }}{{ if $value }}={{ $value }}{{ end }}
+        {{- end }}
         image: {{.Values.prometheusImage}}
         imagePullPolicy: {{.Values.global.imagePullPolicy}}
         livenessProbe:
@@ -233,10 +242,17 @@ spec:
         securityContext:
           runAsUser: 65534
         volumeMounts:
+      {{- range .Values.prometheusRuleConfigMapMounts }}
+        - name: {{ .name }}
+          mountPath: /etc/prometheus/{{ .subPath }}
+          subPath: {{ .subPath }}
+          readOnly: true
+      {{- end }}
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - {{- include "partials.proxy" . | indent 8 | trimPrefix (repeat 7 " ") }}
       {{ if not .Values.global.cniEnabled -}}
@@ -245,6 +261,11 @@ spec:
       {{ end -}}
       serviceAccountName: linkerd-prometheus
       volumes:
+    {{- range .Values.prometheusRuleConfigMapMounts }}
+      - name: {{ .name }}
+        configMap:
+          name: {{ .configMap }}
+    {{- end }}
       - emptyDir: {}
         name: data
       - configMap:
diff --git a/charts/linkerd2/values.yaml b/charts/linkerd2/values.yaml
index ad6cb6c69..92cc7ffce 100644
--- a/charts/linkerd2/values.yaml
+++ b/charts/linkerd2/values.yaml
@@ -7,7 +7,7 @@ global:
   clusterDomain: &cluster_domain cluster.local
   imagePullPolicy: &image_pull_policy IfNotPresent
 
- # control plane trace configuration
+  # control plane trace configuration
   controlPlaneTracing: false
 
   # control plane version. See Proxy section for proxy version
@@ -133,6 +133,20 @@ heartbeatSchedule: "0 0 * * *"
 # prometheus configuration
 prometheusImage: prom/prometheus:v2.15.2
 prometheusLogLevel: *controller_log_level
+prometheusExtraArgs: {}
+  # log.format: json
+prometheusAlertmanagers: []
+  # - scheme: http
+  #   static_configs:
+  #   - targets:
+  #     - "alertmanager.linkerd.svc:9093"
+prometheusRuleConfigMapMounts: []
+  # - name: alerting-rules
+  #   subPath: alerting_rules.yml
+  #   configMap: linkerd-prometheus-rules
+  # - name: recording-rules
+  #   subPath: recording_rules.yml
+  #   configMap: linkerd-prometheus-rules
 
 # proxy injector configuration
 proxyInjector:
diff --git a/cli/cmd/check.go b/cli/cmd/check.go
index 8ecc883e0..198664aba 100644
--- a/cli/cmd/check.go
+++ b/cli/cmd/check.go
@@ -21,6 +21,7 @@ import (
 type checkOptions struct {
 	versionOverride    string
 	preInstallOnly     bool
+	multicluster       bool
 	dataPlaneOnly      bool
 	wait               time.Duration
 	namespace          string
@@ -31,6 +32,7 @@ type checkOptions struct {
 
 func newCheckOptions() *checkOptions {
 	return &checkOptions{
+		multicluster:       false,
 		versionOverride:    "",
 		preInstallOnly:     false,
 		dataPlaneOnly:      false,
@@ -50,6 +52,7 @@ func (options *checkOptions) nonConfigFlagSet() *pflag.FlagSet {
 	flags.StringVarP(&options.namespace, "namespace", "n", options.namespace, "Namespace to use for --proxy checks (default: all namespaces)")
 	flags.BoolVar(&options.preInstallOnly, "pre", options.preInstallOnly, "Only run pre-installation checks, to determine if the control plane can be installed")
 	flags.BoolVar(&options.dataPlaneOnly, "proxy", options.dataPlaneOnly, "Only run data-plane checks, to determine if the data plane is healthy")
+	flags.BoolVar(&options.multicluster, "multicluster", options.multicluster, "Run multicluster checks")
 
 	return flags
 }
@@ -185,22 +188,25 @@ func configureAndRunChecks(wout io.Writer, werr io.Writer, stage string, options
 			}
 			checks = append(checks, healthcheck.LinkerdCNIPluginChecks)
 			checks = append(checks, healthcheck.LinkerdHAChecks)
+			checks = append(checks, healthcheck.LinkerdMulticlusterChecks)
+
 		}
 	}
 
 	hc := healthcheck.NewHealthChecker(checks, &healthcheck.Options{
-		ControlPlaneNamespace: controlPlaneNamespace,
-		CNINamespace:          cniNamespace,
-		DataPlaneNamespace:    options.namespace,
-		KubeConfig:            kubeconfigPath,
-		KubeContext:           kubeContext,
-		Impersonate:           impersonate,
-		ImpersonateGroup:      impersonateGroup,
-		APIAddr:               apiAddr,
-		VersionOverride:       options.versionOverride,
-		RetryDeadline:         time.Now().Add(options.wait),
-		CNIEnabled:            options.cniEnabled,
-		InstallManifest:       installManifest,
+		ControlPlaneNamespace:   controlPlaneNamespace,
+		CNINamespace:            cniNamespace,
+		DataPlaneNamespace:      options.namespace,
+		KubeConfig:              kubeconfigPath,
+		KubeContext:             kubeContext,
+		Impersonate:             impersonate,
+		ImpersonateGroup:        impersonateGroup,
+		APIAddr:                 apiAddr,
+		VersionOverride:         options.versionOverride,
+		RetryDeadline:           time.Now().Add(options.wait),
+		CNIEnabled:              options.cniEnabled,
+		InstallManifest:         installManifest,
+		ShouldCheckMulticluster: options.multicluster,
 	})
 
 	success := runChecks(wout, werr, hc, options.output)
diff --git a/cli/cmd/edges.go b/cli/cmd/edges.go
index 2c0d7a6e4..922addeff 100644
--- a/cli/cmd/edges.go
+++ b/cli/cmd/edges.go
@@ -23,7 +23,7 @@ type edgesOptions struct {
 
 func newEdgesOptions() *edgesOptions {
 	return &edgesOptions{
-		namespace:     "",
+		namespace:     getDefaultNamespace(),
 		outputFormat:  tableOutput,
 		allNamespaces: false,
 	}
diff --git a/cli/cmd/get.go b/cli/cmd/get.go
index 9ec844da5..bf13c3283 100644
--- a/cli/cmd/get.go
+++ b/cli/cmd/get.go
@@ -18,7 +18,7 @@ type getOptions struct {
 
 func newGetOptions() *getOptions {
 	return &getOptions{
-		namespace:     "default",
+		namespace:     getDefaultNamespace(),
 		allNamespaces: false,
 	}
 }
diff --git a/cli/cmd/metrics.go b/cli/cmd/metrics.go
index b2236589d..95d1c5f12 100644
--- a/cli/cmd/metrics.go
+++ b/cli/cmd/metrics.go
@@ -22,7 +22,7 @@ type metricsOptions struct {
 
 func newMetricsOptions() *metricsOptions {
 	return &metricsOptions{
-		namespace: "default",
+		namespace: getDefaultNamespace(),
 		pod:       "",
 	}
 }
diff --git a/cli/cmd/profile.go b/cli/cmd/profile.go
index 098b7e6c5..a89ef87a2 100644
--- a/cli/cmd/profile.go
+++ b/cli/cmd/profile.go
@@ -27,7 +27,7 @@ type profileOptions struct {
 func newProfileOptions() *profileOptions {
 	return &profileOptions{
 		name:          "",
-		namespace:     "default",
+		namespace:     getDefaultNamespace(),
 		template:      false,
 		openAPI:       "",
 		proto:         "",
diff --git a/cli/cmd/root.go b/cli/cmd/root.go
index 3cdb33ce4..4f98d3997 100644
--- a/cli/cmd/root.go
+++ b/cli/cmd/root.go
@@ -9,6 +9,8 @@ import (
 	"time"
 
 	"github.com/spf13/pflag"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/tools/clientcmd"
 
 	"github.com/fatih/color"
 	log "github.com/sirupsen/logrus"
@@ -139,7 +141,7 @@ type statOptionsBase struct {
 
 func newStatOptionsBase() *statOptionsBase {
 	return &statOptionsBase{
-		namespace:    "default",
+		namespace:    getDefaultNamespace(),
 		timeWindow:   "1m",
 		outputFormat: tableOutput,
 	}
@@ -189,6 +191,27 @@ func getSuccessRate(success, failure uint64) float64 {
 	return float64(success) / float64(success+failure)
 }
 
+// getDefaultNamespace fetches the default namespace
+// used in the current KubeConfig context
+func getDefaultNamespace() string {
+	rules := clientcmd.NewDefaultClientConfigLoadingRules()
+
+	if kubeconfigPath != "" {
+		rules.ExplicitPath = kubeconfigPath
+	}
+
+	overrides := &clientcmd.ConfigOverrides{CurrentContext: kubeContext}
+	kubeCfg := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides)
+	ns, _, err := kubeCfg.Namespace()
+
+	if err != nil {
+		log.Errorf("failed to set namespace from config context:%v", err)
+		return corev1.NamespaceDefault
+	}
+
+	return ns
+}
+
 // proxyConfigOptions holds values for command line flags that apply to both the
 // install and inject commands. All fields in this struct should have
 // corresponding flags added in the addProxyConfigFlags func later in this file.
diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go
index 003c19d9a..8cff99dd4 100644
--- a/cli/cmd/tap.go
+++ b/cli/cmd/tap.go
@@ -103,7 +103,7 @@ type tapEvent struct {
 
 func newTapOptions() *tapOptions {
 	return &tapOptions{
-		namespace:     "default",
+		namespace:     getDefaultNamespace(),
 		toResource:    "",
 		toNamespace:   "",
 		maxRps:        maxRps,
diff --git a/cli/cmd/top.go b/cli/cmd/top.go
index f1532a7e8..2a35d4d49 100644
--- a/cli/cmd/top.go
+++ b/cli/cmd/top.go
@@ -259,7 +259,7 @@ const (
 
 func newTopOptions() *topOptions {
 	return &topOptions{
-		namespace:     "default",
+		namespace:     getDefaultNamespace(),
 		toResource:    "",
 		toNamespace:   "",
 		maxRps:        maxRps,
diff --git a/controller/cmd/service-mirror/cluster_watcher.go b/controller/cmd/service-mirror/cluster_watcher.go
index f4fc0be70..392f64425 100644
--- a/controller/cmd/service-mirror/cluster_watcher.go
+++ b/controller/cmd/service-mirror/cluster_watcher.go
@@ -433,21 +433,22 @@ func (rcsw *RemoteClusterServiceWatcher) handleRemoteServiceUpdated(ev *RemoteSe
 	}
 
 	gatewaySpec, err := rcsw.resolveGateway(&ev.gatewayData)
+	copiedEndpoints := ev.localEndpoints.DeepCopy()
 	if err == nil {
-		ev.localEndpoints.Subsets = []corev1.EndpointSubset{
+		copiedEndpoints.Subsets = []corev1.EndpointSubset{
 			{
 				Addresses: gatewaySpec.addresses,
 				Ports:     rcsw.getEndpointsPorts(ev.remoteUpdate, int32(gatewaySpec.incomingPort)),
 			},
 		}
 
-		ev.localEndpoints.Labels[consts.RemoteGatewayNameLabel] = ev.gatewayData.Name
-		ev.localEndpoints.Labels[consts.RemoteGatewayNsLabel] = ev.gatewayData.Namespace
+		copiedEndpoints.Labels[consts.RemoteGatewayNameLabel] = ev.gatewayData.Name
+		copiedEndpoints.Labels[consts.RemoteGatewayNsLabel] = ev.gatewayData.Namespace
 
 		if gatewaySpec.identity != "" {
-			ev.localEndpoints.Annotations[consts.RemoteGatewayIdentity] = gatewaySpec.identity
+			copiedEndpoints.Annotations[consts.RemoteGatewayIdentity] = gatewaySpec.identity
 		} else {
-			delete(ev.localEndpoints.Annotations, consts.RemoteGatewayIdentity)
+			delete(copiedEndpoints.Annotations, consts.RemoteGatewayIdentity)
 		}
 
 		if gatewayChanged {
@@ -463,7 +464,7 @@ func (rcsw *RemoteClusterServiceWatcher) handleRemoteServiceUpdated(ev *RemoteSe
 		ev.localEndpoints.Subsets = nil
 	}
 
-	if _, err := rcsw.localAPIClient.Client.CoreV1().Endpoints(ev.localEndpoints.Namespace).Update(ev.localEndpoints); err != nil {
+	if _, err := rcsw.localAPIClient.Client.CoreV1().Endpoints(copiedEndpoints.Namespace).Update(copiedEndpoints); err != nil {
 		return RetryableError{[]error{err}}
 	}
 
@@ -576,6 +577,11 @@ func (rcsw *RemoteClusterServiceWatcher) handleRemoteServiceCreated(ev *RemoteSe
 }
 
 func (rcsw *RemoteClusterServiceWatcher) handleRemoteGatewayDeleted(ev *RemoteGatewayDeleted) error {
+	rcsw.probeEventsSink.send(&GatewayDeleted{
+		gatewayName: ev.gatewayData.Name,
+		gatewayNs:   ev.gatewayData.Namespace,
+		clusterName: rcsw.clusterName,
+	})
 	affectedEndpoints, err := rcsw.endpointsForGateway(&ev.gatewayData)
 	if err != nil {
 		// if we cannot find the endpoints, we can give up
diff --git a/controller/cmd/service-mirror/config_watcher.go b/controller/cmd/service-mirror/config_watcher.go
index a92df5f59..18573de21 100644
--- a/controller/cmd/service-mirror/config_watcher.go
+++ b/controller/cmd/service-mirror/config_watcher.go
@@ -11,6 +11,7 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 
 	consts "github.com/linkerd/linkerd2/pkg/k8s"
+	sm "github.com/linkerd/linkerd2/pkg/servicemirror"
 )
 
 // ProbeEventSink is an interface for a type that can send events to the probe manager
@@ -112,12 +113,13 @@ func (rcw *RemoteClusterConfigWatcher) Stop() {
 }
 
 func (rcw *RemoteClusterConfigWatcher) registerRemoteCluster(secret *corev1.Secret) error {
-	config, err := parseRemoteClusterSecret(secret)
+	config, err := sm.ParseRemoteClusterSecret(secret)
+
 	if err != nil {
 		return err
 	}
 
-	clientConfig, err := clientcmd.RESTConfigFromKubeConfig(config.apiConfig)
+	clientConfig, err := clientcmd.RESTConfigFromKubeConfig(config.APIConfig)
 	if err != nil {
 		return fmt.Errorf("unable to parse kube config: %s", err)
 	}
@@ -125,61 +127,35 @@ func (rcw *RemoteClusterConfigWatcher) registerRemoteCluster(secret *corev1.Secr
 	rcw.Lock()
 	defer rcw.Unlock()
 
-	if _, ok := rcw.clusterWatchers[config.clusterName]; ok {
-		return fmt.Errorf("there is already a cluster with name %s being watcher. Please delete its config before attempting to register a new one", config.clusterName)
+	if _, ok := rcw.clusterWatchers[config.ClusterName]; ok {
+		return fmt.Errorf("there is already a cluster with name %s being watcher. Please delete its config before attempting to register a new one", config.ClusterName)
 	}
 
-	watcher, err := NewRemoteClusterServiceWatcher(rcw.k8sAPI, clientConfig, config.clusterName, rcw.requeueLimit, config.clusterDomain, rcw.probeEventsSink)
+	watcher, err := NewRemoteClusterServiceWatcher(rcw.k8sAPI, clientConfig, config.ClusterName, rcw.requeueLimit, config.ClusterDomain, rcw.probeEventsSink)
 	if err != nil {
 		return err
 	}
 
-	rcw.clusterWatchers[config.clusterName] = watcher
+	rcw.clusterWatchers[config.ClusterName] = watcher
 	watcher.Start()
 	return nil
 
 }
 
 func (rcw *RemoteClusterConfigWatcher) unregisterRemoteCluster(secret *corev1.Secret, cleanState bool) error {
-	config, err := parseRemoteClusterSecret(secret)
+	config, err := sm.ParseRemoteClusterSecret(secret)
+
 	if err != nil {
 		return err
 	}
 	rcw.Lock()
 	defer rcw.Unlock()
-	if watcher, ok := rcw.clusterWatchers[config.clusterName]; ok {
+	if watcher, ok := rcw.clusterWatchers[config.ClusterName]; ok {
 		watcher.Stop(cleanState)
 	} else {
-		return fmt.Errorf("cannot find watcher for cluser: %s", config.clusterName)
+		return fmt.Errorf("cannot find watcher for cluser: %s", config.ClusterName)
 	}
-	delete(rcw.clusterWatchers, config.clusterName)
+	delete(rcw.clusterWatchers, config.ClusterName)
 
 	return nil
 }
-
-type watchedClusterConfig struct {
-	apiConfig     []byte
-	clusterName   string
-	clusterDomain string
-}
-
-func parseRemoteClusterSecret(secret *corev1.Secret) (*watchedClusterConfig, error) {
-	clusterName, hasClusterName := secret.Annotations[consts.RemoteClusterNameLabel]
-	config, hasConfig := secret.Data[consts.ConfigKeyName]
-	domain, hasDomain := secret.Annotations[consts.RemoteClusterDomainAnnotation]
-	if !hasClusterName {
-		return nil, fmt.Errorf("secret of type %s should contain key %s", consts.MirrorSecretType, consts.ConfigKeyName)
-	}
-	if !hasConfig {
-		return nil, fmt.Errorf("secret should contain remote cluster name as annotation %s", consts.RemoteClusterNameLabel)
-	}
-	if !hasDomain {
-		return nil, fmt.Errorf("secret should contain remote cluster domain as annotation %s", consts.RemoteClusterDomainAnnotation)
-	}
-
-	return &watchedClusterConfig{
-		apiConfig:     config,
-		clusterName:   clusterName,
-		clusterDomain: domain,
-	}, nil
-}
diff --git a/controller/cmd/service-mirror/metrics.go b/controller/cmd/service-mirror/metrics.go
index ec2fdf31f..bb7612603 100644
--- a/controller/cmd/service-mirror/metrics.go
+++ b/controller/cmd/service-mirror/metrics.go
@@ -3,6 +3,7 @@ package servicemirror
 import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	logging "github.com/sirupsen/logrus"
 )
 
 const (
@@ -23,10 +24,11 @@ type probeMetricVecs struct {
 }
 
 type probeMetrics struct {
-	services  prometheus.Gauge
-	alive     prometheus.Gauge
-	latencies prometheus.Observer
-	probes    *prometheus.CounterVec
+	services   prometheus.Gauge
+	alive      prometheus.Gauge
+	latencies  prometheus.Observer
+	probes     *prometheus.CounterVec
+	unregister func()
 }
 
 func newProbeMetricVecs() probeMetricVecs {
@@ -112,5 +114,26 @@ func (mv probeMetricVecs) newWorkerMetrics(gatewayNamespace, gatewayName, remote
 		alive:     mv.alive.With(labels),
 		latencies: mv.latencies.With(labels),
 		probes:    curriedProbes,
+		unregister: func() {
+			mv.unregister(gatewayNamespace, gatewayName, remoteClusterName)
+		},
 	}, nil
 }
+
+func (mv probeMetricVecs) unregister(gatewayNamespace, gatewayName, remoteClusterName string) {
+	labels := prometheus.Labels{
+		gatewayNameLabel:      gatewayName,
+		gatewayNamespaceLabel: gatewayNamespace,
+		gatewayClusterName:    remoteClusterName,
+	}
+
+	if !mv.services.Delete(labels) {
+		logging.Warnf("unable to delete num_mirrored_services metric with labels %s", labels)
+	}
+	if !mv.alive.Delete(labels) {
+		logging.Warnf("unable to delete gateway_alive metric with labels %s", labels)
+	}
+	if !mv.latencies.Delete(labels) {
+		logging.Warnf("unable to delete gateway_probe_latency_ms metric with labels %s", labels)
+	}
+}
diff --git a/controller/cmd/service-mirror/probe_manager.go b/controller/cmd/service-mirror/probe_manager.go
index 0febad05d..19d054dd9 100644
--- a/controller/cmd/service-mirror/probe_manager.go
+++ b/controller/cmd/service-mirror/probe_manager.go
@@ -40,6 +40,13 @@ type MirroredServicePaired struct {
 	GatewaySpec
 }
 
+// GatewayDeleted is emitted when a gateway is deleted
+type GatewayDeleted struct {
+	gatewayName string
+	gatewayNs   string
+	clusterName string
+}
+
 // ClusterNotRegistered is is emitted when the cluster is not monitored anymore
 type ClusterNotRegistered struct {
 	clusterName string
@@ -94,6 +101,8 @@ func (m *ProbeManager) handleEvent(ev interface{}) {
 		m.handleMirroredServiceUnpaired(ev)
 	case *GatewayUpdated:
 		m.handleGatewayUpdated(ev)
+	case *GatewayDeleted:
+		m.handleGatewayDeleted(ev)
 	case *ClusterNotRegistered:
 		m.handleClusterNotRegistered(ev)
 	default:
@@ -189,6 +198,11 @@ func (m *ProbeManager) stopProbe(key string) {
 	}
 }
 
+func (m *ProbeManager) handleGatewayDeleted(event *GatewayDeleted) {
+	probeKey := probeKey(event.gatewayNs, event.gatewayName, event.clusterName)
+	m.stopProbe(probeKey)
+}
+
 func (m *ProbeManager) handleClusterNotRegistered(event *ClusterNotRegistered) {
 	matchLabels := map[string]string{
 		consts.MirroredResourceLabel:  "true",
@@ -202,7 +216,7 @@ func (m *ProbeManager) handleClusterNotRegistered(event *ClusterNotRegistered) {
 
 	stopped := make(map[string]bool)
 	for _, svc := range services {
-		probeKey := probeKey(svc.Labels[consts.RemoteGatewayNameLabel], svc.Labels[consts.RemoteGatewayNameLabel], event.clusterName)
+		probeKey := probeKey(svc.Labels[consts.RemoteGatewayNsLabel], svc.Labels[consts.RemoteGatewayNameLabel], event.clusterName)
 
 		if _, ok := stopped[probeKey]; !ok {
 			m.stopProbe(probeKey)
diff --git a/controller/cmd/service-mirror/probe_worker.go b/controller/cmd/service-mirror/probe_worker.go
index b5ffb0bd3..c48f75113 100644
--- a/controller/cmd/service-mirror/probe_worker.go
+++ b/controller/cmd/service-mirror/probe_worker.go
@@ -77,6 +77,7 @@ func (pw *ProbeWorker) UpdateProbeSpec(spec *probeSpec) {
 
 // Stop this probe worker
 func (pw *ProbeWorker) Stop() {
+	pw.metrics.unregister()
 	pw.log.Debug("Stopping probe worker")
 	close(pw.stopCh)
 }
diff --git a/pkg/charts/linkerd2/values.go b/pkg/charts/linkerd2/values.go
index 01e9b119a..7ef3ed525 100644
--- a/pkg/charts/linkerd2/values.go
+++ b/pkg/charts/linkerd2/values.go
@@ -19,34 +19,37 @@ const (
 type (
 	// Values contains the top-level elements in the Helm charts
 	Values struct {
-		Stage                       string            `json:"stage"`
-		ControllerImage             string            `json:"controllerImage"`
-		ControllerImageVersion      string            `json:"controllerImageVersion"`
-		WebImage                    string            `json:"webImage"`
-		PrometheusImage             string            `json:"prometheusImage"`
-		GrafanaImage                string            `json:"grafanaImage"`
-		ControllerReplicas          uint              `json:"controllerReplicas"`
-		ControllerLogLevel          string            `json:"controllerLogLevel"`
-		PrometheusLogLevel          string            `json:"prometheusLogLevel"`
-		ControllerUID               int64             `json:"controllerUID"`
-		EnableH2Upgrade             bool              `json:"enableH2Upgrade"`
-		EnablePodAntiAffinity       bool              `json:"enablePodAntiAffinity"`
-		WebhookFailurePolicy        string            `json:"webhookFailurePolicy"`
-		OmitWebhookSideEffects      bool              `json:"omitWebhookSideEffects"`
-		RestrictDashboardPrivileges bool              `json:"restrictDashboardPrivileges"`
-		DisableHeartBeat            bool              `json:"disableHeartBeat"`
-		HeartbeatSchedule           string            `json:"heartbeatSchedule"`
-		InstallNamespace            bool              `json:"installNamespace"`
-		Configs                     ConfigJSONs       `json:"configs"`
-		Global                      *Global           `json:"global"`
-		Identity                    *Identity         `json:"identity"`
-		Dashboard                   *Dashboard        `json:"dashboard"`
-		DebugContainer              *DebugContainer   `json:"debugContainer"`
-		ProxyInjector               *ProxyInjector    `json:"proxyInjector"`
-		ProfileValidator            *ProfileValidator `json:"profileValidator"`
-		Tap                         *Tap              `json:"tap"`
-		NodeSelector                map[string]string `json:"nodeSelector"`
-		SMIMetrics                  *SMIMetrics       `json:"smiMetrics"`
+		Stage                         string                         `json:"stage"`
+		ControllerImage               string                         `json:"controllerImage"`
+		ControllerImageVersion        string                         `json:"controllerImageVersion"`
+		WebImage                      string                         `json:"webImage"`
+		PrometheusImage               string                         `json:"prometheusImage"`
+		GrafanaImage                  string                         `json:"grafanaImage"`
+		ControllerReplicas            uint                           `json:"controllerReplicas"`
+		ControllerLogLevel            string                         `json:"controllerLogLevel"`
+		PrometheusLogLevel            string                         `json:"prometheusLogLevel"`
+		PrometheusExtraArgs           map[string]string              `json:"prometheusExtraArgs"`
+		PrometheusAlertmanagers       []interface{}                  `json:"prometheusAlertmanagers"`
+		PrometheusRuleConfigMapMounts []PrometheusRuleConfigMapMount `json:"prometheusRuleConfigMapMounts"`
+		ControllerUID                 int64                          `json:"controllerUID"`
+		EnableH2Upgrade               bool                           `json:"enableH2Upgrade"`
+		EnablePodAntiAffinity         bool                           `json:"enablePodAntiAffinity"`
+		WebhookFailurePolicy          string                         `json:"webhookFailurePolicy"`
+		OmitWebhookSideEffects        bool                           `json:"omitWebhookSideEffects"`
+		RestrictDashboardPrivileges   bool                           `json:"restrictDashboardPrivileges"`
+		DisableHeartBeat              bool                           `json:"disableHeartBeat"`
+		HeartbeatSchedule             string                         `json:"heartbeatSchedule"`
+		InstallNamespace              bool                           `json:"installNamespace"`
+		Configs                       ConfigJSONs                    `json:"configs"`
+		Global                        *Global                        `json:"global"`
+		Identity                      *Identity                      `json:"identity"`
+		Dashboard                     *Dashboard                     `json:"dashboard"`
+		DebugContainer                *DebugContainer                `json:"debugContainer"`
+		ProxyInjector                 *ProxyInjector                 `json:"proxyInjector"`
+		ProfileValidator              *ProfileValidator              `json:"profileValidator"`
+		Tap                           *Tap                           `json:"tap"`
+		NodeSelector                  map[string]string              `json:"nodeSelector"`
+		SMIMetrics                    *SMIMetrics                    `json:"smiMetrics"`
 
 		DestinationResources   *Resources `json:"destinationResources"`
 		GrafanaResources       *Resources `json:"grafanaResources"`
@@ -188,6 +191,13 @@ type (
 		TLS                 *TLS      `json:"tls"`
 	}
 
+	// PrometheusRuleConfigMapMount is a user supplied prometheus rule config maps.
+	PrometheusRuleConfigMapMount struct {
+		Name      string `json:"name"`
+		SubPath   string `json:"subPath"`
+		ConfigMap string `json:"configMap"`
+	}
+
 	// ProxyInjector has all the proxy injector's Helm variables
 	ProxyInjector struct {
 		*TLS
diff --git a/pkg/healthcheck/healthcheck.go b/pkg/healthcheck/healthcheck.go
index c53882073..1caee99c3 100644
--- a/pkg/healthcheck/healthcheck.go
+++ b/pkg/healthcheck/healthcheck.go
@@ -11,6 +11,9 @@ import (
 	"strings"
 	"time"
 
+	v1 "k8s.io/api/rbac/v1"
+	"k8s.io/client-go/tools/clientcmd"
+
 	"github.com/linkerd/linkerd2/pkg/issuercerts"
 
 	"github.com/linkerd/linkerd2/controller/api/public"
@@ -20,6 +23,7 @@ import (
 	"github.com/linkerd/linkerd2/pkg/config"
 	"github.com/linkerd/linkerd2/pkg/identity"
 	"github.com/linkerd/linkerd2/pkg/k8s"
+	sm "github.com/linkerd/linkerd2/pkg/servicemirror"
 	"github.com/linkerd/linkerd2/pkg/tls"
 	"github.com/linkerd/linkerd2/pkg/version"
 	log "github.com/sirupsen/logrus"
@@ -137,15 +141,20 @@ const (
 	/// plugin is installed and ready
 	LinkerdCNIPluginChecks CategoryID = "linkerd-cni-plugin"
 
+	// LinkerdMulticlusterChecks adds a series of checks to validate
+	// that the multicluster setup is working as expected
+	LinkerdMulticlusterChecks CategoryID = "linkerd-multicluster"
+
 	// LinkerdCNIResourceLabel is the label key that is used to identify
 	// whether a Kubernetes resource is related to the install-cni command
 	// The value is expected to be "true", "false" or "", where "false" and
 	// "" are equal, making "false" the default
 	LinkerdCNIResourceLabel = "linkerd.io/cni-resource"
 
-	linkerdCNIDisabledSkipReason = "skipping check because CNI is not enabled"
-	linkerdCNIResourceName       = "linkerd-cni"
-	linkerdCNIConfigMapName      = "linkerd-cni-config"
+	linkerdCNIDisabledSkipReason      = "skipping check because CNI is not enabled"
+	linkerdCNIResourceName            = "linkerd-cni"
+	linkerdCNIConfigMapName           = "linkerd-cni-config"
+	linkerdServiceMirrorComponentName = "linkerd-service-mirror"
 
 	// linkerdTapAPIServiceName is the name of the tap api service
 	// This key is passed to checkApiSercice method to check whether
@@ -190,6 +199,20 @@ var ExpectedServiceAccountNames = []string{
 	"linkerd-tap",
 }
 
+var expectedServiceMirrorPolicyVerbs = []string{
+	"list", "get", "watch", "create", "delete", "update",
+}
+
+var expectedServiceMirrorPolicyResources = []string{
+	"endpoints", "services", "secrets", "namespaces",
+}
+
+var expectedServiceMirrorRemoteClusterPolicyVerbs = []string{
+	"get",
+	"list",
+	"watch",
+}
+
 var (
 	retryWindow    = 5 * time.Second
 	requestTimeout = 30 * time.Second
@@ -317,18 +340,19 @@ type category struct {
 
 // Options specifies configuration for a HealthChecker.
 type Options struct {
-	ControlPlaneNamespace string
-	CNINamespace          string
-	DataPlaneNamespace    string
-	KubeConfig            string
-	KubeContext           string
-	Impersonate           string
-	ImpersonateGroup      []string
-	APIAddr               string
-	VersionOverride       string
-	RetryDeadline         time.Time
-	CNIEnabled            bool
-	InstallManifest       string
+	ControlPlaneNamespace   string
+	CNINamespace            string
+	DataPlaneNamespace      string
+	KubeConfig              string
+	KubeContext             string
+	Impersonate             string
+	ImpersonateGroup        []string
+	APIAddr                 string
+	VersionOverride         string
+	RetryDeadline           time.Time
+	CNIEnabled              bool
+	InstallManifest         string
+	ShouldCheckMulticluster bool
 }
 
 // HealthChecker encapsulates all health check checkers, and clients required to
@@ -349,6 +373,7 @@ type HealthChecker struct {
 	issuerCert       *tls.Cred
 	trustAnchors     []*x509.Certificate
 	cniDaemonSet     *appsv1.DaemonSet
+	serviceMirrorNs  string
 }
 
 // NewHealthChecker returns an initialized HealthChecker
@@ -538,14 +563,14 @@ func (hc *HealthChecker) allCategories() []category {
 					description: "no ClusterRoles exist",
 					hintAnchor:  "pre-l5d-existence",
 					check: func(context.Context) error {
-						return hc.checkClusterRoles(false)
+						return hc.checkClusterRoles(false, hc.expectedRBACNames(), hc.controlPlaneComponentsSelector())
 					},
 				},
 				{
 					description: "no ClusterRoleBindings exist",
 					hintAnchor:  "pre-l5d-existence",
 					check: func(context.Context) error {
-						return hc.checkClusterRoleBindings(false)
+						return hc.checkClusterRoleBindings(false, hc.expectedRBACNames(), hc.controlPlaneComponentsSelector())
 					},
 				},
 				{
@@ -601,7 +626,7 @@ func (hc *HealthChecker) allCategories() []category {
 						if hc.isHeartbeatDisabled() {
 							return nil
 						}
-						return hc.checkServiceAccounts([]string{"linkerd-heartbeat"})
+						return hc.checkServiceAccounts([]string{"linkerd-heartbeat"}, hc.ControlPlaneNamespace, hc.controlPlaneComponentsSelector())
 					},
 				},
 				{
@@ -691,7 +716,7 @@ func (hc *HealthChecker) allCategories() []category {
 					hintAnchor:  "l5d-existence-cr",
 					fatal:       true,
 					check: func(context.Context) error {
-						return hc.checkClusterRoles(true)
+						return hc.checkClusterRoles(true, hc.expectedRBACNames(), hc.controlPlaneComponentsSelector())
 					},
 				},
 				{
@@ -699,7 +724,7 @@ func (hc *HealthChecker) allCategories() []category {
 					hintAnchor:  "l5d-existence-crb",
 					fatal:       true,
 					check: func(context.Context) error {
-						return hc.checkClusterRoleBindings(true)
+						return hc.checkClusterRoleBindings(true, hc.expectedRBACNames(), hc.controlPlaneComponentsSelector())
 					},
 				},
 				{
@@ -707,7 +732,7 @@ func (hc *HealthChecker) allCategories() []category {
 					hintAnchor:  "l5d-existence-sa",
 					fatal:       true,
 					check: func(context.Context) error {
-						return hc.checkServiceAccounts(ExpectedServiceAccountNames)
+						return hc.checkServiceAccounts(ExpectedServiceAccountNames, hc.ControlPlaneNamespace, hc.controlPlaneComponentsSelector())
 					},
 				},
 				{
@@ -1209,6 +1234,76 @@ func (hc *HealthChecker) allCategories() []category {
 				},
 			},
 		},
+		{
+			id: LinkerdMulticlusterChecks,
+			checkers: []checker{
+				{
+					description: "service mirror controller exists",
+					hintAnchor:  "l5d-smc-existence",
+					fatal:       true,
+					check: func(context.Context) error {
+						return hc.checkServiceMirrorController()
+					},
+				},
+				{
+					description: "service mirror controller ClusterRoles exist",
+					hintAnchor:  "l5d-smc-existence-cr",
+					fatal:       true,
+					check: func(context.Context) error {
+						if hc.Options.ShouldCheckMulticluster {
+							return hc.checkClusterRoles(true, []string{linkerdServiceMirrorComponentName}, hc.serviceMirrorComponentsSelector())
+						}
+						return &SkipError{Reason: "not checking muticluster"}
+					},
+				},
+				{
+					description: "service mirror controller ClusterRoleBindings exist",
+					hintAnchor:  "l5d-smc-existence-crb",
+					fatal:       true,
+					check: func(context.Context) error {
+						if hc.Options.ShouldCheckMulticluster {
+							return hc.checkClusterRoleBindings(true, []string{linkerdServiceMirrorComponentName}, hc.serviceMirrorComponentsSelector())
+						}
+						return &SkipError{Reason: "not checking muticluster"}
+					},
+				},
+				{
+					description: "service mirror controller ServiceAccounts exist",
+					hintAnchor:  "l5d-smc-existence-sa",
+					fatal:       true,
+					check: func(context.Context) error {
+						if hc.Options.ShouldCheckMulticluster {
+							return hc.checkServiceAccounts([]string{linkerdServiceMirrorComponentName}, hc.serviceMirrorNs, hc.serviceMirrorComponentsSelector())
+						}
+						return &SkipError{Reason: "not checking muticluster"}
+					},
+				},
+				{
+					description: "service mirror controller has required permissions",
+					hintAnchor:  "l5d-smc-local-rbac-existence",
+					fatal:       true,
+					check: func(context.Context) error {
+						return hc.checkServiceMirrorLocalRBAC()
+					},
+				},
+				{
+					description: "service mirror controller can access remote clusters",
+					hintAnchor:  "l5d-smc-remote-remote-clusters-access",
+					fatal:       true,
+					check: func(context.Context) error {
+						return hc.checkRemoteClusterConnectivity()
+					},
+				},
+				{
+					description: "all remote cluster gateways are alive",
+					hintAnchor:  "l5d-remote-gateways-alive",
+					fatal:       true,
+					check: func(ctx context.Context) error {
+						return hc.checkRemoteClusterGatewaysHealth(ctx)
+					},
+				},
+			},
+		},
 	}
 }
 
@@ -1385,6 +1480,10 @@ func (hc *HealthChecker) controlPlaneComponentsSelector() string {
 	return fmt.Sprintf("%s,!%s", k8s.ControllerNSLabel, LinkerdCNIResourceLabel)
 }
 
+func (hc *HealthChecker) serviceMirrorComponentsSelector() string {
+	return fmt.Sprintf("%s=%s", k8s.ControllerComponentLabel, linkerdServiceMirrorComponentName)
+}
+
 // PublicAPIClient returns a fully configured public API client. This client is
 // only configured if the KubernetesAPIChecks and LinkerdAPIChecks are
 // configured and run first.
@@ -1463,6 +1562,188 @@ func FetchLinkerdConfigMap(k kubernetes.Interface, controlPlaneNamespace string)
 	return cm, configPB, nil
 }
 
+func (hc *HealthChecker) checkServiceMirrorController() error {
+	options := metav1.ListOptions{
+		LabelSelector: hc.serviceMirrorComponentsSelector(),
+	}
+	result, err := hc.kubeAPI.AppsV1().Deployments(corev1.NamespaceAll).List(options)
+	if err != nil {
+		return err
+	}
+
+	// if we have explicitly requested for multicluster to be checked, error out
+	if len(result.Items) == 0 && hc.Options.ShouldCheckMulticluster {
+		return errors.New("Service mirror controller is not present")
+	}
+
+	if len(result.Items) > 0 {
+		hc.Options.ShouldCheckMulticluster = true
+
+		if len(result.Items) > 1 {
+			var errors []string
+			for _, smc := range result.Items {
+				errors = append(errors, fmt.Sprintf("%s/%s", smc.Namespace, smc.Name))
+			}
+			return fmt.Errorf("There are more than one service mirror controllers:\n\t%s", strings.Join(errors, "\n\t"))
+		}
+
+		controller := result.Items[0]
+		if controller.Status.AvailableReplicas < 1 {
+			return fmt.Errorf("Service mirror controller is not available: %s/%s", controller.Namespace, controller.Name)
+		}
+		hc.serviceMirrorNs = controller.Namespace
+		return nil
+	}
+
+	return &SkipError{Reason: "not checking muticluster"}
+}
+
+func comparePermissions(expected, actual []string) error {
+	sort.Strings(expected)
+	sort.Strings(actual)
+
+	expectedStr := strings.Join(expected, ",")
+	actualStr := strings.Join(actual, ",")
+
+	if expectedStr != actualStr {
+		return fmt.Errorf("expected %s, got %s", expectedStr, actualStr)
+	}
+
+	return nil
+}
+
+func (hc *HealthChecker) checkServiceMirrorLocalRBAC() error {
+	if hc.Options.ShouldCheckMulticluster {
+		var errors []string
+
+		cr, err := hc.kubeAPI.RbacV1().ClusterRoles().Get(linkerdServiceMirrorComponentName, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("Could not obtain service mirror ClusterRole: %s", err)
+		}
+
+		var policyRule *v1.PolicyRule
+
+		for _, r := range cr.Rules {
+			if len(r.APIGroups) == 1 && r.APIGroups[0] == "" {
+				rule := r
+				policyRule = &rule
+			}
+		}
+
+		if policyRule == nil {
+			return fmt.Errorf("Service mirror ClusterRole is missing expected policy rule")
+		}
+
+		if err := comparePermissions(expectedServiceMirrorPolicyVerbs, policyRule.Verbs); err != nil {
+			errors = append(errors, fmt.Sprintf("Service mirror ClusterRole is missing verbs: %s", err))
+		}
+
+		if err := comparePermissions(expectedServiceMirrorPolicyResources, policyRule.Resources); err != nil {
+			errors = append(errors, fmt.Sprintf("Service mirror ClusterRole is missing required resources: %s", err))
+		}
+
+		if len(errors) > 0 {
+			return fmt.Errorf(strings.Join(errors, "\n\t"))
+		}
+
+		return nil
+	}
+	return &SkipError{Reason: "not checking muticluster"}
+}
+
+func (hc *HealthChecker) checkRemoteClusterConnectivity() error {
+	if hc.Options.ShouldCheckMulticluster {
+		options := metav1.ListOptions{
+			FieldSelector: fmt.Sprintf("%s=%s", "type", k8s.MirrorSecretType),
+		}
+		secrets, err := hc.kubeAPI.CoreV1().Secrets(corev1.NamespaceAll).List(options)
+		if err != nil {
+			return err
+		}
+
+		if len(secrets.Items) == 0 {
+			return &SkipError{Reason: "no remote cluster configs"}
+		}
+
+		var errors []string
+		for _, s := range secrets.Items {
+			secret := s
+			config, err := sm.ParseRemoteClusterSecret(&secret)
+			if err != nil {
+				errors = append(errors, fmt.Sprintf("*  secret: [%s/%s]: could not parse config secret: %s", secret.Namespace, secret.Name, err))
+				continue
+			}
+
+			clientConfig, err := clientcmd.RESTConfigFromKubeConfig(config.APIConfig)
+			if err != nil {
+				errors = append(errors, fmt.Sprintf("* secret: [%s/%s] cluster: [%s]: unable to parse api config: %s", secret.Namespace, secret.Name, config.ClusterName, err))
+				continue
+			}
+
+			remoteAPI, err := k8s.NewAPIForConfig(clientConfig, "", []string{}, requestTimeout)
+			if err != nil {
+				errors = append(errors, fmt.Sprintf("* secret: [%s/%s] cluster: [%s]: could not instantiate remote api: %s", secret.Namespace, secret.Name, config.ClusterName, err))
+				continue
+			}
+
+			var verbs []string
+			if err := hc.checkCanPerformAction(remoteAPI, "get", corev1.NamespaceAll, "", "v1", "services"); err == nil {
+				verbs = append(verbs, "get")
+			}
+
+			if err := hc.checkCanPerformAction(remoteAPI, "list", corev1.NamespaceAll, "", "v1", "services"); err == nil {
+				verbs = append(verbs, "list")
+			}
+
+			if err := hc.checkCanPerformAction(remoteAPI, "watch", corev1.NamespaceAll, "", "v1", "services"); err == nil {
+				verbs = append(verbs, "watch")
+			}
+
+			if err := comparePermissions(expectedServiceMirrorRemoteClusterPolicyVerbs, verbs); err != nil {
+				errors = append(errors, fmt.Sprintf("* cluster: [%s]: Insufficient Service permissions: %s", config.ClusterName, err))
+			}
+
+		}
+
+		if len(errors) > 0 {
+			return fmt.Errorf("Problematic clusters:\n\t%s", strings.Join(errors, "\n\t"))
+		}
+		return nil
+	}
+	return &SkipError{Reason: "not checking muticluster"}
+}
+
+func (hc *HealthChecker) checkRemoteClusterGatewaysHealth(ctx context.Context) error {
+	if hc.Options.ShouldCheckMulticluster {
+		if hc.apiClient == nil {
+			return errors.New("public api client uninitialized")
+		}
+		req := &pb.GatewaysRequest{
+			TimeWindow: "1m",
+		}
+		rsp, err := hc.apiClient.Gateways(ctx, req)
+		if err != nil {
+			return err
+		}
+
+		var deadGateways []string
+		if len(rsp.GetOk().GatewaysTable.Rows) == 0 {
+			return &SkipError{Reason: "no remote gateways"}
+		}
+		for _, gtw := range rsp.GetOk().GatewaysTable.Rows {
+			if !gtw.Alive {
+				deadGateways = append(deadGateways, fmt.Sprintf("* cluster: [%s], gateway: [%s/%s]", gtw.ClusterName, gtw.Namespace, gtw.Name))
+			}
+		}
+
+		if len(deadGateways) > 0 {
+			return fmt.Errorf("Some gateways are not alive:\n\t%s", strings.Join(deadGateways, "\n\t"))
+		}
+		return nil
+	}
+	return &SkipError{Reason: "not checking muticluster"}
+}
+
 // checkNamespace checks whether the given namespace exists, and returns an
 // error if it does not match `shouldExist`.
 func (hc *HealthChecker) checkNamespace(namespace string, shouldExist bool) error {
@@ -1490,9 +1771,9 @@ func (hc *HealthChecker) expectedRBACNames() []string {
 	}
 }
 
-func (hc *HealthChecker) checkClusterRoles(shouldExist bool) error {
+func (hc *HealthChecker) checkClusterRoles(shouldExist bool, expectedNames []string, labelSelector string) error {
 	options := metav1.ListOptions{
-		LabelSelector: hc.controlPlaneComponentsSelector(),
+		LabelSelector: labelSelector,
 	}
 	crList, err := hc.kubeAPI.RbacV1().ClusterRoles().List(options)
 	if err != nil {
@@ -1506,12 +1787,12 @@ func (hc *HealthChecker) checkClusterRoles(shouldExist bool) error {
 		objects = append(objects, &item)
 	}
 
-	return checkResources("ClusterRoles", objects, hc.expectedRBACNames(), shouldExist)
+	return checkResources("ClusterRoles", objects, expectedNames, shouldExist)
 }
 
-func (hc *HealthChecker) checkClusterRoleBindings(shouldExist bool) error {
+func (hc *HealthChecker) checkClusterRoleBindings(shouldExist bool, expectedNames []string, labelSelector string) error {
 	options := metav1.ListOptions{
-		LabelSelector: hc.controlPlaneComponentsSelector(),
+		LabelSelector: labelSelector,
 	}
 	crbList, err := hc.kubeAPI.RbacV1().ClusterRoleBindings().List(options)
 	if err != nil {
@@ -1525,7 +1806,7 @@ func (hc *HealthChecker) checkClusterRoleBindings(shouldExist bool) error {
 		objects = append(objects, &item)
 	}
 
-	return checkResources("ClusterRoleBindings", objects, hc.expectedRBACNames(), shouldExist)
+	return checkResources("ClusterRoleBindings", objects, expectedNames, shouldExist)
 }
 
 func (hc *HealthChecker) isHA() bool {
@@ -1546,11 +1827,11 @@ func (hc *HealthChecker) isHeartbeatDisabled() bool {
 	return false
 }
 
-func (hc *HealthChecker) checkServiceAccounts(saNames []string) error {
+func (hc *HealthChecker) checkServiceAccounts(saNames []string, ns, labelSelector string) error {
 	options := metav1.ListOptions{
-		LabelSelector: hc.controlPlaneComponentsSelector(),
+		LabelSelector: labelSelector,
 	}
-	saList, err := hc.kubeAPI.CoreV1().ServiceAccounts(hc.ControlPlaneNamespace).List(options)
+	saList, err := hc.kubeAPI.CoreV1().ServiceAccounts(ns).List(options)
 	if err != nil {
 		return err
 	}
@@ -1776,14 +2057,14 @@ func (hc *HealthChecker) getDataPlanePods(ctx context.Context) ([]*pb.Pod, error
 	return pods, nil
 }
 
-func (hc *HealthChecker) checkCanPerformAction(verb, namespace, group, version, resource string) error {
-	if hc.kubeAPI == nil {
+func (hc *HealthChecker) checkCanPerformAction(api *k8s.KubernetesAPI, verb, namespace, group, version, resource string) error {
+	if api == nil {
 		// we should never get here
 		return fmt.Errorf("unexpected error: Kubernetes ClientSet not initialized")
 	}
 
 	return k8s.ResourceAuthz(
-		hc.kubeAPI,
+		api,
 		namespace,
 		verb,
 		group,
@@ -1808,7 +2089,7 @@ func (hc *HealthChecker) checkHAMetadataPresentOnKubeSystemNamespace() error {
 }
 
 func (hc *HealthChecker) checkCanCreate(namespace, group, version, resource string) error {
-	return hc.checkCanPerformAction("create", namespace, group, version, resource)
+	return hc.checkCanPerformAction(hc.kubeAPI, "create", namespace, group, version, resource)
 }
 
 func (hc *HealthChecker) checkCanCreateNonNamespacedResources() error {
@@ -1859,7 +2140,7 @@ func (hc *HealthChecker) checkCanCreateNonNamespacedResources() error {
 }
 
 func (hc *HealthChecker) checkCanGet(namespace, group, version, resource string) error {
-	return hc.checkCanPerformAction("get", namespace, group, version, resource)
+	return hc.checkCanPerformAction(hc.kubeAPI, "get", namespace, group, version, resource)
 }
 
 func (hc *HealthChecker) checkAPIService(serviceName string) error {
diff --git a/pkg/servicemirror/util.go b/pkg/servicemirror/util.go
new file mode 100644
index 000000000..b2928a9d6
--- /dev/null
+++ b/pkg/servicemirror/util.go
@@ -0,0 +1,37 @@
+package servicemirror
+
+import (
+	"fmt"
+
+	consts "github.com/linkerd/linkerd2/pkg/k8s"
+	corev1 "k8s.io/api/core/v1"
+)
+
+// WatchedClusterConfig contains the needed data to identify a remote cluster
+type WatchedClusterConfig struct {
+	APIConfig     []byte
+	ClusterName   string
+	ClusterDomain string
+}
+
+// ParseRemoteClusterSecret extracts the credentials used to access the remote cluster
+func ParseRemoteClusterSecret(secret *corev1.Secret) (*WatchedClusterConfig, error) {
+	clusterName, hasClusterName := secret.Annotations[consts.RemoteClusterNameLabel]
+	config, hasConfig := secret.Data[consts.ConfigKeyName]
+	domain, hasDomain := secret.Annotations[consts.RemoteClusterDomainAnnotation]
+	if !hasClusterName {
+		return nil, fmt.Errorf("secret of type %s should contain key %s", consts.MirrorSecretType, consts.ConfigKeyName)
+	}
+	if !hasConfig {
+		return nil, fmt.Errorf("secret should contain remote cluster name as annotation %s", consts.RemoteClusterNameLabel)
+	}
+	if !hasDomain {
+		return nil, fmt.Errorf("secret should contain remote cluster domain as annotation %s", consts.RemoteClusterDomainAnnotation)
+	}
+
+	return &WatchedClusterConfig{
+		APIConfig:     config,
+		ClusterName:   clusterName,
+		ClusterDomain: domain,
+	}, nil
+}


Test patch:
diff --git a/bin/_test-run.sh b/bin/_test-run.sh
index 8420cbe99..deee2ea09 100755
--- a/bin/_test-run.sh
+++ b/bin/_test-run.sh
@@ -228,7 +228,7 @@ helm_cleanup() {
         # We wait for the namespace to be gone so the following call to `cleanup` doesn't fail when it attempts to delete
         # the same namespace that is already being deleted here (error thrown by the NamespaceLifecycle controller).
         # We don't have that problem with global resources, so no need to wait for them to be gone.
-        kubectl wait --for=delete ns/$linkerd_namespace-helm --timeout=40s
+        kubectl wait --for=delete ns/$linkerd_namespace-helm --timeout=120s
         # `helm reset` deletes the tiller pod in $tiller_namespace
         "$helm_path" --kube-context=$k8s_context --tiller-namespace=$tiller_namespace reset
         kubectl --context=$k8s_context delete clusterrolebinding ${tiller_namespace}:tiller-cluster-admin
diff --git a/cli/cmd/testdata/install_addon_control-plane.golden b/cli/cmd/testdata/install_addon_control-plane.golden
index 8e3792739..fd3f58a58 100644
--- a/cli/cmd/testdata/install_addon_control-plane.golden
+++ b/cli/cmd/testdata/install_addon_control-plane.golden
@@ -982,6 +982,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -1188,8 +1189,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/install_control-plane.golden b/cli/cmd/testdata/install_control-plane.golden
index 6bae28cb2..44930c42c 100644
--- a/cli/cmd/testdata/install_control-plane.golden
+++ b/cli/cmd/testdata/install_control-plane.golden
@@ -982,6 +982,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -1188,8 +1189,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/install_controlplane_tracing_output.golden b/cli/cmd/testdata/install_controlplane_tracing_output.golden
index db075eab6..98dc53854 100644
--- a/cli/cmd/testdata/install_controlplane_tracing_output.golden
+++ b/cli/cmd/testdata/install_controlplane_tracing_output.golden
@@ -1882,6 +1882,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2088,8 +2089,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/install_custom_registry.golden b/cli/cmd/testdata/install_custom_registry.golden
index c4ea1464d..8e1c1ea67 100644
--- a/cli/cmd/testdata/install_custom_registry.golden
+++ b/cli/cmd/testdata/install_custom_registry.golden
@@ -1822,6 +1822,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2028,8 +2029,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/install_default.golden b/cli/cmd/testdata/install_default.golden
index e84b34e0f..c5c19db03 100644
--- a/cli/cmd/testdata/install_default.golden
+++ b/cli/cmd/testdata/install_default.golden
@@ -1822,6 +1822,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2028,8 +2029,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/install_ha_output.golden b/cli/cmd/testdata/install_ha_output.golden
index f49bd513c..876d7f58b 100644
--- a/cli/cmd/testdata/install_ha_output.golden
+++ b/cli/cmd/testdata/install_ha_output.golden
@@ -1950,6 +1950,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2163,8 +2164,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/install_ha_with_overrides_output.golden b/cli/cmd/testdata/install_ha_with_overrides_output.golden
index f6cc45d27..01b85ad6c 100644
--- a/cli/cmd/testdata/install_ha_with_overrides_output.golden
+++ b/cli/cmd/testdata/install_ha_with_overrides_output.golden
@@ -1950,6 +1950,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2163,8 +2164,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/install_heartbeat_disabled_output.golden b/cli/cmd/testdata/install_heartbeat_disabled_output.golden
index 3f56776a4..87c82ac26 100644
--- a/cli/cmd/testdata/install_heartbeat_disabled_output.golden
+++ b/cli/cmd/testdata/install_heartbeat_disabled_output.golden
@@ -1733,6 +1733,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -1939,8 +1940,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/install_helm_output.golden b/cli/cmd/testdata/install_helm_output.golden
index 50d5b36cf..f7fd527ba 100644
--- a/cli/cmd/testdata/install_helm_output.golden
+++ b/cli/cmd/testdata/install_helm_output.golden
@@ -1888,6 +1888,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2094,8 +2095,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/install_helm_output_addons.golden b/cli/cmd/testdata/install_helm_output_addons.golden
index 42ee47dfb..7703a7b19 100644
--- a/cli/cmd/testdata/install_helm_output_addons.golden
+++ b/cli/cmd/testdata/install_helm_output_addons.golden
@@ -1888,6 +1888,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2094,8 +2095,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/install_helm_output_ha.golden b/cli/cmd/testdata/install_helm_output_ha.golden
index bef79fc42..0179cc6ec 100644
--- a/cli/cmd/testdata/install_helm_output_ha.golden
+++ b/cli/cmd/testdata/install_helm_output_ha.golden
@@ -2016,6 +2016,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2229,8 +2230,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/install_no_init_container.golden b/cli/cmd/testdata/install_no_init_container.golden
index 1c70b2af7..87db5d455 100644
--- a/cli/cmd/testdata/install_no_init_container.golden
+++ b/cli/cmd/testdata/install_no_init_container.golden
@@ -1687,6 +1687,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -1893,8 +1894,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/install_output.golden b/cli/cmd/testdata/install_output.golden
index 797c59fae..f9286fa2e 100644
--- a/cli/cmd/testdata/install_output.golden
+++ b/cli/cmd/testdata/install_output.golden
@@ -1818,6 +1818,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2024,8 +2025,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/install_proxy_ignores.golden b/cli/cmd/testdata/install_proxy_ignores.golden
index 23a2b5c0f..9442de79f 100644
--- a/cli/cmd/testdata/install_proxy_ignores.golden
+++ b/cli/cmd/testdata/install_proxy_ignores.golden
@@ -1822,6 +1822,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2028,8 +2029,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/install_restricted_dashboard.golden b/cli/cmd/testdata/install_restricted_dashboard.golden
index b387a4057..2c98a783d 100644
--- a/cli/cmd/testdata/install_restricted_dashboard.golden
+++ b/cli/cmd/testdata/install_restricted_dashboard.golden
@@ -1757,6 +1757,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -1963,8 +1964,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/install_tracing.golden b/cli/cmd/testdata/install_tracing.golden
index 04373e024..579532484 100644
--- a/cli/cmd/testdata/install_tracing.golden
+++ b/cli/cmd/testdata/install_tracing.golden
@@ -1822,6 +1822,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2028,8 +2029,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/upgrade_add-on_controlplane.golden b/cli/cmd/testdata/upgrade_add-on_controlplane.golden
index 690543ad0..1a927e0ba 100644
--- a/cli/cmd/testdata/upgrade_add-on_controlplane.golden
+++ b/cli/cmd/testdata/upgrade_add-on_controlplane.golden
@@ -990,6 +990,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -1196,8 +1197,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/upgrade_add-on_overwrite.golden b/cli/cmd/testdata/upgrade_add-on_overwrite.golden
index af4e14047..495bc419e 100644
--- a/cli/cmd/testdata/upgrade_add-on_overwrite.golden
+++ b/cli/cmd/testdata/upgrade_add-on_overwrite.golden
@@ -1830,6 +1830,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2036,8 +2037,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/upgrade_add_add-on.golden b/cli/cmd/testdata/upgrade_add_add-on.golden
index 86351d233..95bf1e3bd 100644
--- a/cli/cmd/testdata/upgrade_add_add-on.golden
+++ b/cli/cmd/testdata/upgrade_add_add-on.golden
@@ -1830,6 +1830,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2036,8 +2037,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/upgrade_default.golden b/cli/cmd/testdata/upgrade_default.golden
index 1ceba9206..9a349a398 100644
--- a/cli/cmd/testdata/upgrade_default.golden
+++ b/cli/cmd/testdata/upgrade_default.golden
@@ -1830,6 +1830,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2036,8 +2037,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/upgrade_external_issuer.golden b/cli/cmd/testdata/upgrade_external_issuer.golden
index b06b5a8e2..ff6e58684 100644
--- a/cli/cmd/testdata/upgrade_external_issuer.golden
+++ b/cli/cmd/testdata/upgrade_external_issuer.golden
@@ -1816,6 +1816,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2022,8 +2023,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/upgrade_ha.golden b/cli/cmd/testdata/upgrade_ha.golden
index f844525a5..23aa6e441 100644
--- a/cli/cmd/testdata/upgrade_ha.golden
+++ b/cli/cmd/testdata/upgrade_ha.golden
@@ -1958,6 +1958,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2171,8 +2172,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/upgrade_overwrite_issuer.golden b/cli/cmd/testdata/upgrade_overwrite_issuer.golden
index 3ec11fce8..d99c2cd70 100644
--- a/cli/cmd/testdata/upgrade_overwrite_issuer.golden
+++ b/cli/cmd/testdata/upgrade_overwrite_issuer.golden
@@ -1822,6 +1822,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2028,8 +2029,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/upgrade_overwrite_trust_anchors-external-issuer.golden b/cli/cmd/testdata/upgrade_overwrite_trust_anchors-external-issuer.golden
index 138cb8485..f2eda01b1 100644
--- a/cli/cmd/testdata/upgrade_overwrite_trust_anchors-external-issuer.golden
+++ b/cli/cmd/testdata/upgrade_overwrite_trust_anchors-external-issuer.golden
@@ -1808,6 +1808,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2014,8 +2015,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/cli/cmd/testdata/upgrade_overwrite_trust_anchors.golden b/cli/cmd/testdata/upgrade_overwrite_trust_anchors.golden
index 3ec11fce8..d99c2cd70 100644
--- a/cli/cmd/testdata/upgrade_overwrite_trust_anchors.golden
+++ b/cli/cmd/testdata/upgrade_overwrite_trust_anchors.golden
@@ -1822,6 +1822,7 @@ data:
 
     rule_files:
     - /etc/prometheus/*_rules.yml
+    - /etc/prometheus/*_rules.yaml
 
     scrape_configs:
     - job_name: 'prometheus'
@@ -2028,8 +2029,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
-        - mountPath: /etc/prometheus
+        - mountPath: /etc/prometheus/prometheus.yml
           name: prometheus-config
+          subPath: prometheus.yml
           readOnly: true
       - env:
         - name: LINKERD2_PROXY_LOG
diff --git a/controller/cmd/service-mirror/cluster_watcher_mirroring_test.go b/controller/cmd/service-mirror/cluster_watcher_mirroring_test.go
new file mode 100644
index 000000000..9762c5d1d
--- /dev/null
+++ b/controller/cmd/service-mirror/cluster_watcher_mirroring_test.go
@@ -0,0 +1,573 @@
+package servicemirror
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/workqueue"
+)
+
+type NoOpProbeEventSink struct{}
+
+func (s *NoOpProbeEventSink) send(event interface{}) {}
+
+type mirroringTestCase struct {
+	description            string
+	environment            *testEnvironment
+	expectedLocalServices  []*corev1.Service
+	expectedLocalEndpoints []*corev1.Endpoints
+	expectedEventsInQueue  []interface{}
+}
+
+func (tc *mirroringTestCase) run(t *testing.T) {
+	t.Run(tc.description, func(t *testing.T) {
+
+		q := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
+		localAPI, err := tc.environment.runEnvironment(&NoOpProbeEventSink{}, q)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if tc.expectedLocalServices == nil {
+			// ensure the are no local services
+			services, err := localAPI.Client.CoreV1().Services(corev1.NamespaceAll).List(metav1.ListOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(services.Items) > 0 {
+				t.Fatalf("Was expecting no local services but instead found %v", services.Items)
+
+			}
+		} else {
+			for _, expected := range tc.expectedLocalServices {
+				actual, err := localAPI.Client.CoreV1().Services(expected.Namespace).Get(expected.Name, metav1.GetOptions{})
+				if err != nil {
+					t.Fatalf("Could not find mirrored service with name %s", expected.Name)
+				}
+
+				if err := diffServices(expected, actual); err != nil {
+					t.Fatal(err)
+				}
+			}
+		}
+
+		if tc.expectedLocalEndpoints == nil {
+			// ensure the are no local endpoints
+			endpoints, err := localAPI.Client.CoreV1().Endpoints(corev1.NamespaceAll).List(metav1.ListOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(endpoints.Items) > 0 {
+				t.Fatalf("Was expecting no local endpoints but instead found %d", len(endpoints.Items))
+
+			}
+		} else {
+			for _, expected := range tc.expectedLocalEndpoints {
+				actual, err := localAPI.Client.CoreV1().Endpoints(expected.Namespace).Get(expected.Name, metav1.GetOptions{})
+				if err != nil {
+					t.Fatalf("Could not find endpoints with name %s", expected.Name)
+				}
+
+				if err := diffEndpoints(expected, actual); err != nil {
+					t.Fatal(err)
+				}
+			}
+		}
+
+		expectedNumEvents := len(tc.expectedEventsInQueue)
+		actualNumEvents := q.Len()
+
+		if expectedNumEvents != actualNumEvents {
+			t.Fatalf("Was expecting %d events but got %d", expectedNumEvents, actualNumEvents)
+		}
+
+		for _, ev := range tc.expectedEventsInQueue {
+			evInQueue, _ := q.Get()
+			if !reflect.DeepEqual(ev, evInQueue) {
+				t.Fatalf("was expecting to see event %T but got %T", ev, evInQueue)
+			}
+		}
+	})
+}
+
+func TestRemoteServiceCreatedMirroring(t *testing.T) {
+	for _, tt := range []mirroringTestCase{
+		{
+			description: "create service and endpoints when gateway cannot be resolved",
+			environment: serviceCreateWithMissingGateway,
+			expectedLocalServices: []*corev1.Service{
+				mirroredService("service-one-remote", "ns1", "missing-gateway", "missing-namespace", "111", "", nil),
+			},
+			expectedLocalEndpoints: []*corev1.Endpoints{
+				endpoints("service-one-remote", "ns1", "missing-gateway", "missing-namespace", "", "", nil),
+			},
+		},
+		{
+			description: "create service and endpoints without subsets when gateway spec is wrong",
+			environment: createServiceWrongGatewaySpec,
+			expectedLocalServices: []*corev1.Service{
+				mirroredService("service-one-remote", "ns1", "existing-gateway", "existing-namespace", "111", "",
+					[]corev1.ServicePort{
+						{
+							Name:     "port1",
+							Protocol: "TCP",
+							Port:     555,
+						},
+						{
+							Name:     "port2",
+							Protocol: "TCP",
+							Port:     666,
+						},
+					}),
+			},
+			expectedLocalEndpoints: []*corev1.Endpoints{
+				endpoints("service-one-remote", "ns1", "existing-gateway", "existing-namespace", "", "", nil),
+			},
+		},
+		{
+			description: "create service and endpoints when gateway can be resolved",
+			environment: createServiceOkeGatewaySpec,
+			expectedLocalServices: []*corev1.Service{
+				mirroredService(
+					"service-one-remote",
+					"ns1",
+					"existing-gateway",
+					"existing-namespace",
+					"111",
+					"222",
+					[]corev1.ServicePort{
+						{
+							Name:     "port1",
+							Protocol: "TCP",
+							Port:     555,
+						},
+						{
+							Name:     "port2",
+							Protocol: "TCP",
+							Port:     666,
+						},
+					}),
+			},
+			expectedLocalEndpoints: []*corev1.Endpoints{
+				endpoints("service-one-remote", "ns1", "existing-gateway", "existing-namespace", "192.0.2.127", "gateway-identity", []corev1.EndpointPort{
+					{
+						Name:     "port1",
+						Port:     888,
+						Protocol: "TCP",
+					},
+					{
+						Name:     "port2",
+						Port:     888,
+						Protocol: "TCP",
+					},
+				}),
+			},
+		},
+	} {
+		tc := tt // pin
+		tc.run(t)
+	}
+}
+
+func TestRemoteServiceDeletedMirroring(t *testing.T) {
+	for _, tt := range []mirroringTestCase{
+		{
+			description: "deletes locally mirrored service",
+			environment: deleteMirroredService,
+		},
+	} {
+		tc := tt // pin
+		tc.run(t)
+	}
+}
+
+func TestRemoteServiceUpdatedMirroring(t *testing.T) {
+	for _, tt := range []mirroringTestCase{
+		{
+			description: "update to new gateway",
+			environment: updateServiceToNewGateway,
+			expectedLocalServices: []*corev1.Service{
+				mirroredService(
+					"test-service-remote",
+					"test-namespace",
+					"gateway-new",
+					"gateway-ns",
+					"currentServiceResVersion",
+					"currentGatewayResVersion",
+					[]corev1.ServicePort{
+						{
+							Name:     "port1",
+							Protocol: "TCP",
+							Port:     111,
+						},
+						{
+							Name:     "port3",
+							Protocol: "TCP",
+							Port:     333,
+						},
+					}),
+			},
+			expectedLocalEndpoints: []*corev1.Endpoints{
+				endpoints("test-service-remote", "test-namespace", "gateway-new", "gateway-ns", "0.0.0.0", "", []corev1.EndpointPort{
+					{
+						Name:     "port1",
+						Port:     999,
+						Protocol: "TCP",
+					},
+					{
+						Name:     "port2",
+						Port:     999,
+						Protocol: "TCP",
+					},
+				}),
+			},
+		},
+		{
+			description: "updates service ports on both service and endpoints",
+			environment: updateServiceWithChangedPorts,
+			expectedLocalServices: []*corev1.Service{
+				mirroredService("test-service-remote", "test-namespace", "gateway", "gateway-ns", "currentServiceResVersion", "currentGatewayResVersion",
+					[]corev1.ServicePort{
+						{
+							Name:     "port1",
+							Protocol: "TCP",
+							Port:     111,
+						},
+						{
+							Name:     "port3",
+							Protocol: "TCP",
+							Port:     333,
+						},
+					}),
+			},
+
+			expectedLocalEndpoints: []*corev1.Endpoints{
+				endpoints("test-service-remote", "test-namespace", "gateway", "gateway-ns", "192.0.2.127", "", []corev1.EndpointPort{
+					{
+						Name:     "port1",
+						Port:     888,
+						Protocol: "TCP",
+					},
+					{
+						Name:     "port3",
+						Port:     888,
+						Protocol: "TCP",
+					},
+				}),
+			},
+		},
+	} {
+		tc := tt // pin
+		tc.run(t)
+	}
+}
+
+func TestRemoteGatewayUpdatedMirroring(t *testing.T) {
+	for _, tt := range []mirroringTestCase{
+		{
+			description: "endpoints ports are updated on gateway change",
+			environment: remoteGatewayUpdated,
+			expectedLocalServices: []*corev1.Service{
+				mirroredService("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "currentGatewayResVersion",
+					[]corev1.ServicePort{
+						{
+							Name:     "svc-1-port",
+							Protocol: "TCP",
+							Port:     8081,
+						},
+					}),
+
+				mirroredService("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "currentGatewayResVersion", []corev1.ServicePort{
+					{
+						Name:     "svc-2-port",
+						Protocol: "TCP",
+						Port:     8082,
+					},
+				}),
+			},
+			expectedLocalEndpoints: []*corev1.Endpoints{
+				endpoints("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
+					[]corev1.EndpointPort{
+						{
+							Name:     "svc-1-port",
+							Port:     999,
+							Protocol: "TCP",
+						}}),
+				endpoints("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
+					[]corev1.EndpointPort{
+						{
+							Name:     "svc-2-port",
+							Port:     999,
+							Protocol: "TCP",
+						}}),
+			},
+		},
+
+		{
+			description: "endpoints addresses are updated on gateway change",
+			environment: gatewayAddressChanged,
+			expectedLocalServices: []*corev1.Service{
+				mirroredService("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "currentGatewayResVersion",
+					[]corev1.ServicePort{
+						{
+							Name:     "svc-1-port",
+							Protocol: "TCP",
+							Port:     8081,
+						},
+					}),
+				mirroredService("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "currentGatewayResVersion", []corev1.ServicePort{
+					{
+						Name:     "svc-2-port",
+						Protocol: "TCP",
+						Port:     8082,
+					},
+				}),
+			},
+			expectedLocalEndpoints: []*corev1.Endpoints{
+				endpoints("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.1", "",
+					[]corev1.EndpointPort{
+						{
+							Name:     "svc-1-port",
+							Port:     888,
+							Protocol: "TCP",
+						}}),
+				endpoints("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.1", "",
+					[]corev1.EndpointPort{
+						{
+							Name:     "svc-2-port",
+							Port:     888,
+							Protocol: "TCP",
+						}}),
+			},
+		},
+
+		{
+			description: "identity is updated on gateway change",
+			environment: gatewayIdentityChanged,
+			expectedLocalServices: []*corev1.Service{
+				mirroredService("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "currentGatewayResVersion",
+					[]corev1.ServicePort{
+						{
+							Name:     "svc-1-port",
+							Protocol: "TCP",
+							Port:     8081,
+						},
+					}),
+				mirroredService("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "currentGatewayResVersion", []corev1.ServicePort{
+					{
+						Name:     "svc-2-port",
+						Protocol: "TCP",
+						Port:     8082,
+					},
+				}),
+			},
+			expectedLocalEndpoints: []*corev1.Endpoints{
+				endpoints("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "new-identity",
+					[]corev1.EndpointPort{
+						{
+							Name:     "svc-1-port",
+							Port:     888,
+							Protocol: "TCP",
+						}}),
+				endpoints("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "new-identity",
+					[]corev1.EndpointPort{
+						{
+							Name:     "svc-2-port",
+							Port:     888,
+							Protocol: "TCP",
+						}}),
+			},
+		},
+	} {
+		tc := tt // pin
+		tc.run(t)
+	}
+}
+func TestRemoteGatewayDeletedMirroring(t *testing.T) {
+	for _, tt := range []mirroringTestCase{
+		{
+			description: "removes endpoint subsets when gateway is deleted",
+			environment: gatewayDeleted,
+			expectedLocalServices: []*corev1.Service{
+				mirroredService("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
+					[]corev1.ServicePort{
+						{
+							Name:     "svc-1-port",
+							Protocol: "TCP",
+							Port:     8081,
+						},
+					}),
+
+				mirroredService("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion", []corev1.ServicePort{
+					{
+						Name:     "svc-2-port",
+						Protocol: "TCP",
+						Port:     8082,
+					},
+				}),
+			},
+			expectedLocalEndpoints: []*corev1.Endpoints{
+				endpoints("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "", nil),
+				endpoints("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "", nil),
+			},
+		},
+	} {
+		tc := tt // pin
+		tc.run(t)
+	}
+}
+
+func TestClusterUnregisteredMirroring(t *testing.T) {
+	for _, tt := range []mirroringTestCase{
+		{
+			description: "unregisters cluster and cleans up all mirrored resources",
+			environment: clusterUnregistered,
+		},
+	} {
+		tc := tt // pin
+		tc.run(t)
+	}
+}
+
+func TestGcOrphanedServicesMirroring(t *testing.T) {
+	for _, tt := range []mirroringTestCase{
+		{
+			description: "deletes mirrored resources that are no longer present on the remote cluster",
+			environment: gcTriggered,
+			expectedLocalServices: []*corev1.Service{
+				mirroredService("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "", nil),
+			},
+
+			expectedLocalEndpoints: []*corev1.Endpoints{
+				endpoints("test-service-1-remote", "test-namespace", "", "", "", "", nil),
+			},
+		},
+	} {
+		tc := tt // pin
+		tc.run(t)
+	}
+}
+
+func onAddOrUpdateTestCases(isAdd bool) []mirroringTestCase {
+
+	testType := "ADD"
+	if !isAdd {
+		testType = "UPDATE"
+	}
+
+	return []mirroringTestCase{
+		{
+			description: fmt.Sprintf("enqueue a RemoteServiceCreated event when this is not a gateway and we have the needed annotations (%s)", testType),
+			environment: onAddOrUpdateExportedSvc(isAdd),
+			expectedEventsInQueue: []interface{}{&RemoteServiceCreated{
+				service: remoteService("test-service", "test-namespace", "gateway", "gateway-ns", "resVersion", nil),
+				gatewayData: &gatewayMetadata{
+					Name:      "gateway",
+					Namespace: "gateway-ns",
+				},
+			}},
+		},
+		{
+			description: fmt.Sprintf("enqueue a ConsiderGatewayUpdateDispatch event if this is clearly not a mirrorable service (%s)", testType),
+			environment: onAddOrUpdateNonExportedSvc(isAdd),
+			expectedEventsInQueue: []interface{}{&ConsiderGatewayUpdateDispatch{
+				maybeGateway: remoteService("test-service", "test-namespace", "", "", "resVersion", nil),
+			}},
+		},
+		{
+			description: fmt.Sprintf("enqueue a RemoteServiceUpdated event if this is a service that we have already mirrored and its res version is different (%s)", testType),
+			environment: onAddOrUpdateRemoteServiceUpdated(isAdd),
+			expectedEventsInQueue: []interface{}{&RemoteServiceUpdated{
+				localService:   mirroredService("test-service-remote", "test-namespace", "gateway", "gateway-ns", "pastResourceVersion", "gatewayResVersion", nil),
+				localEndpoints: endpoints("test-service-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "", nil),
+				remoteUpdate:   remoteService("test-service", "test-namespace", "gateway", "gateway-ns", "currentResVersion", nil),
+				gatewayData: gatewayMetadata{
+					Name:      "gateway",
+					Namespace: "gateway-ns",
+				},
+			}},
+			expectedLocalServices: []*corev1.Service{
+				mirroredService("test-service-remote", "test-namespace", "gateway", "gateway-ns", "pastResourceVersion", "gatewayResVersion", nil),
+			},
+			expectedLocalEndpoints: []*corev1.Endpoints{
+				endpoints("test-service-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "", nil),
+			},
+		},
+		{
+			description: fmt.Sprintf("not enqueue any events as this update does not really tell us anything new (res version is the same...) (%s)", testType),
+			environment: onAddOrUpdateSameResVersion(isAdd),
+			expectedLocalServices: []*corev1.Service{
+				mirroredService("test-service-remote", "test-namespace", "gateway", "gateway-ns", "currentResVersion", "gatewayResVersion", nil),
+			},
+			expectedLocalEndpoints: []*corev1.Endpoints{
+				endpoints("test-service-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "", nil),
+			},
+		},
+		{
+			description: fmt.Sprintf("enqueue RemoteServiceDeleted event as this service is not mirrorable anymore (%s)", testType),
+			environment: serviceNotExportedAnymore(isAdd),
+			expectedEventsInQueue: []interface{}{&RemoteServiceDeleted{
+				Name:      "test-service",
+				Namespace: "test-namespace",
+			}},
+
+			expectedLocalServices: []*corev1.Service{
+				mirroredService("test-service-remote", "test-namespace", "gateway", "gateway-ns", "currentResVersion", "gatewayResVersion", nil),
+			},
+			expectedLocalEndpoints: []*corev1.Endpoints{
+				endpoints("test-service-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "", nil),
+			},
+		},
+	}
+}
+
+func TestOnAdd(t *testing.T) {
+	for _, tt := range onAddOrUpdateTestCases(true) {
+		tc := tt // pin
+		tc.run(t)
+	}
+}
+
+func TestOnUpdate(t *testing.T) {
+	for _, tt := range onAddOrUpdateTestCases(false) {
+		tc := tt // pin
+		tc.run(t)
+	}
+}
+
+func TestOnDelete(t *testing.T) {
+	for _, tt := range []mirroringTestCase{
+		{
+			description: "enqueues a RemoteServiceDeleted because there is gateway metadata present on the service",
+			environment: onDeleteWithGatewayMetadata,
+			expectedEventsInQueue: []interface{}{
+				&RemoteServiceDeleted{
+					Name:      "test-service",
+					Namespace: "test-namespace",
+					GatewayData: gatewayMetadata{
+						Name:      "gateway",
+						Namespace: "gateway-ns",
+					},
+				},
+			},
+		},
+		{
+			description: "enqueues a RemoteGatewayDeleted because there is no gateway metadata present on the service",
+			environment: onDeleteNoGatewayMetadata,
+			expectedEventsInQueue: []interface{}{
+				&RemoteGatewayDeleted{
+					gatewayData: gatewayMetadata{
+						Name:      "gateway",
+						Namespace: "test-namespace",
+					},
+				},
+			},
+		},
+	} {
+		tc := tt // pin
+		tc.run(t)
+	}
+}
diff --git a/controller/cmd/service-mirror/cluster_watcher_probe_events_test.go b/controller/cmd/service-mirror/cluster_watcher_probe_events_test.go
new file mode 100644
index 000000000..2c201c1b1
--- /dev/null
+++ b/controller/cmd/service-mirror/cluster_watcher_probe_events_test.go
@@ -0,0 +1,214 @@
+package servicemirror
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/util/workqueue"
+)
+
+type BufferingProbeEventSink struct {
+	events []interface{}
+}
+
+func (s *BufferingProbeEventSink) send(event interface{}) {
+	s.events = append(s.events, event)
+}
+
+type probeEventsTestCase struct {
+	description                      string
+	environment                      *testEnvironment
+	expectedEventsSentToProbeManager []interface{}
+}
+
+func (tc *probeEventsTestCase) run(t *testing.T) {
+	t.Run(tc.description, func(t *testing.T) {
+		q := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
+		probeEventsSink := &BufferingProbeEventSink{}
+		_, err := tc.environment.runEnvironment(probeEventsSink, q)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		expectedNumEvents := len(tc.expectedEventsSentToProbeManager)
+		actualNumEvents := len(probeEventsSink.events)
+
+		if expectedNumEvents != actualNumEvents {
+			t.Fatalf("Was expecting %d events but got %d", expectedNumEvents, actualNumEvents)
+		}
+
+		for i, ev := range tc.expectedEventsSentToProbeManager {
+			evInQueue := probeEventsSink.events[i]
+			if !reflect.DeepEqual(ev, evInQueue) {
+				t.Fatalf("was expecting to see event %T but got %T", ev, evInQueue)
+			}
+		}
+	})
+}
+
+func gatewaySpec(name, namespace, clustername, address, resVersion, identity string, incomingPort, probePort uint32, probePath string, probePeriod uint32) GatewaySpec {
+	return GatewaySpec{
+		gatewayName:      name,
+		gatewayNamespace: namespace,
+		clusterName:      clustername,
+		addresses: []v1.EndpointAddress{
+			{
+				IP: address,
+			},
+		},
+		incomingPort:    incomingPort,
+		resourceVersion: resVersion,
+		identity:        identity,
+		ProbeConfig: &ProbeConfig{
+			path:            probePath,
+			port:            probePort,
+			periodInSeconds: probePeriod,
+		},
+	}
+}
+
+func TestRemoteServiceCreatedProbeEvents(t *testing.T) {
+	for _, tt := range []probeEventsTestCase{
+		{
+			description:                      "do not send event if gateway cannot be resolved",
+			environment:                      serviceCreateWithMissingGateway,
+			expectedEventsSentToProbeManager: []interface{}{},
+		},
+		{
+			description:                      "do not send event if gateway has wrong spec",
+			environment:                      createServiceWrongGatewaySpec,
+			expectedEventsSentToProbeManager: []interface{}{},
+		},
+		{
+			description: "create service and endpoints when gateway can be resolved",
+			environment: createServiceOkeGatewaySpec,
+			expectedEventsSentToProbeManager: []interface{}{
+				&MirroredServicePaired{
+					serviceName:      fmt.Sprintf("service-one-%s", clusterName),
+					serviceNamespace: "ns1",
+					GatewaySpec:      gatewaySpec("existing-gateway", "existing-namespace", clusterName, "192.0.2.127", "222", "gateway-identity", 888, defaultProbePort, defaultProbePath, defaultProbePeriod),
+				},
+			},
+		},
+	} {
+		tc := tt // pin
+		tc.run(t)
+	}
+}
+
+func TestRemoteServiceDeletedProbeEvents(t *testing.T) {
+	for _, tt := range []probeEventsTestCase{
+		{
+			description: "send a service unpaired event",
+			environment: deleteMirroredService,
+			expectedEventsSentToProbeManager: []interface{}{
+				&MirroredServiceUnpaired{
+					serviceName:      fmt.Sprintf("test-service-remote-to-delete-%s", clusterName),
+					serviceNamespace: "test-namespace-to-delete",
+					gatewayName:      "gateway",
+					gatewayNs:        "gateway-ns",
+					clusterName:      clusterName,
+				},
+			},
+		},
+	} {
+		tc := tt // pin
+		tc.run(t)
+	}
+}
+
+func TestRemoteServiceUpdatedProbeEvents(t *testing.T) {
+	for _, tt := range []probeEventsTestCase{
+		{
+			description: "unpairs from old and pairs to new gateway",
+			environment: updateServiceToNewGateway,
+			expectedEventsSentToProbeManager: []interface{}{
+				&MirroredServiceUnpaired{
+					serviceName:      fmt.Sprintf("test-service-%s", clusterName),
+					serviceNamespace: "test-namespace",
+					gatewayName:      "gateway",
+					gatewayNs:        "gateway-ns",
+					clusterName:      clusterName,
+				},
+				&MirroredServicePaired{
+					serviceName:      fmt.Sprintf("test-service-%s", clusterName),
+					serviceNamespace: "test-namespace",
+					GatewaySpec:      gatewaySpec("gateway-new", "gateway-ns", clusterName, "0.0.0.0", "currentGatewayResVersion", "", 999, defaultProbePort, defaultProbePath, defaultProbePeriod),
+				},
+			},
+		},
+		{
+			description: "does not send event when gateway assignment does not change",
+			environment: updateServiceWithChangedPorts,
+		},
+	} {
+		tc := tt // pin
+		tc.run(t)
+	}
+}
+
+func TestRemoteGatewayUpdatedProbeEvents(t *testing.T) {
+	for _, tt := range []probeEventsTestCase{
+		{
+			description: "sends gateway updated when endpoints ports",
+			environment: remoteGatewayUpdated,
+			expectedEventsSentToProbeManager: []interface{}{
+				&GatewayUpdated{
+					GatewaySpec: gatewaySpec("gateway", "gateway-ns", clusterName, "0.0.0.0", "currentGatewayResVersion", "", 999, defaultProbePort, defaultProbePath, defaultProbePeriod),
+				},
+			},
+		},
+
+		{
+			description: "sends gateway updated when address changes",
+			environment: gatewayAddressChanged,
+			expectedEventsSentToProbeManager: []interface{}{
+				&GatewayUpdated{
+					GatewaySpec: gatewaySpec("gateway", "gateway-ns", "some-cluster", "0.0.0.1", "currentGatewayResVersion", "", 888, 1, "/p", 222),
+				},
+			},
+		},
+		{
+			description: "sends gateway updated when identity changes",
+			environment: gatewayIdentityChanged,
+			expectedEventsSentToProbeManager: []interface{}{
+				&GatewayUpdated{
+					GatewaySpec: gatewaySpec("gateway", "gateway-ns", clusterName, "0.0.0.0", "currentGatewayResVersion", "new-identity", 888, defaultProbePort, defaultProbePath, defaultProbePeriod),
+				},
+			},
+		},
+		{
+			description: "sends gateway updated when probe changes",
+			environment: gatewayProbeConfigChanged,
+			expectedEventsSentToProbeManager: []interface{}{
+				&GatewayUpdated{
+					GatewaySpec: gatewaySpec("gateway", "gateway-ns", clusterName, "0.0.0.0", "currentGatewayResVersion", "identity", 888, defaultProbePort, "/new-path", defaultProbePeriod),
+				},
+			},
+		},
+	} {
+		tc := tt // pin
+		tc.run(t)
+	}
+}
+
+func TestRemoteGatewayDeletedProbeEvents(t *testing.T) {
+	for _, tt := range []probeEventsTestCase{
+		{
+			description: "sends a gateway deleted event to the probe manager",
+			environment: gatewayDeleted,
+			expectedEventsSentToProbeManager: []interface{}{
+				&GatewayDeleted{
+					gatewayName: "gateway",
+					gatewayNs:   "gateway-ns",
+					clusterName: clusterName,
+				},
+			},
+		},
+	} {
+		tc := tt // pin
+		tc.run(t)
+	}
+}
diff --git a/controller/cmd/service-mirror/cluster_watcher_test.go b/controller/cmd/service-mirror/cluster_watcher_test.go
deleted file mode 100644
index 1637258e2..000000000
--- a/controller/cmd/service-mirror/cluster_watcher_test.go
+++ /dev/null
@@ -1,1120 +0,0 @@
-package servicemirror
-
-import (
-	"fmt"
-	"reflect"
-	"testing"
-
-	"github.com/linkerd/linkerd2/controller/k8s"
-	logging "github.com/sirupsen/logrus"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/util/workqueue"
-)
-
-const (
-	clusterName   = "remote"
-	clusterDomain = "cluster.local"
-)
-
-type testCase struct {
-	testDescription        string
-	events                 []interface{}
-	remoteResources        []string
-	localResources         []string
-	expectedLocalServices  []*corev1.Service
-	expectedLocalEndpoints []*corev1.Endpoints
-	expectedEventsInQueue  []interface{}
-}
-
-type NoOpProbeEventSink struct{}
-
-func (s *NoOpProbeEventSink) send(event interface{}) {}
-
-func runTestCase(tc *testCase, t *testing.T) {
-	t.Run(tc.testDescription, func(t *testing.T) {
-		remoteAPI, err := k8s.NewFakeAPI(tc.remoteResources...)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		localAPI, err := k8s.NewFakeAPI(tc.localResources...)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		remoteAPI.Sync(nil)
-		localAPI.Sync(nil)
-
-		q := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
-
-		watcher := RemoteClusterServiceWatcher{
-			clusterName:     clusterName,
-			clusterDomain:   clusterDomain,
-			remoteAPIClient: remoteAPI,
-			localAPIClient:  localAPI,
-			stopper:         nil,
-			log:             logging.WithFields(logging.Fields{"cluster": clusterName}),
-			eventsQueue:     q,
-			requeueLimit:    0,
-			probeEventsSink: &NoOpProbeEventSink{},
-		}
-
-		for _, ev := range tc.events {
-			q.Add(ev)
-		}
-
-		for range tc.events {
-			watcher.processNextEvent()
-		}
-
-		localAPI.Sync(nil)
-		remoteAPI.Sync(nil)
-
-		if tc.expectedLocalServices == nil {
-			// ensure the are no local services
-			services, err := localAPI.Client.CoreV1().Services(corev1.NamespaceAll).List(metav1.ListOptions{})
-			if err != nil {
-				t.Fatal(err)
-			}
-			if len(services.Items) > 0 {
-				t.Fatalf("Was expecting no local services but instead found %v", services.Items)
-
-			}
-		} else {
-			for _, expected := range tc.expectedLocalServices {
-				actual, err := localAPI.Client.CoreV1().Services(expected.Namespace).Get(expected.Name, metav1.GetOptions{})
-				if err != nil {
-					t.Fatalf("Could not find mirrored service with name %s", expected.Name)
-				}
-
-				if err := diffServices(expected, actual); err != nil {
-					t.Fatal(err)
-				}
-			}
-		}
-
-		if tc.expectedLocalEndpoints == nil {
-			// ensure the are no local endpoints
-			endpoints, err := localAPI.Client.CoreV1().Endpoints(corev1.NamespaceAll).List(metav1.ListOptions{})
-			if err != nil {
-				t.Fatal(err)
-			}
-			if len(endpoints.Items) > 0 {
-				t.Fatalf("Was expecting no local endpoints but instead found %d", len(endpoints.Items))
-
-			}
-		} else {
-			for _, expected := range tc.expectedLocalEndpoints {
-				actual, err := localAPI.Client.CoreV1().Endpoints(expected.Namespace).Get(expected.Name, metav1.GetOptions{})
-				if err != nil {
-					t.Fatalf("Could not find endpoints with name %s", expected.Name)
-				}
-
-				if err := diffEndpoints(expected, actual); err != nil {
-					t.Fatal(err)
-				}
-			}
-		}
-
-		expectedNumEvents := len(tc.expectedEventsInQueue)
-		actualNumEvents := watcher.eventsQueue.Len()
-
-		if expectedNumEvents != actualNumEvents {
-			t.Fatalf("Was expecting %d events but got %d", expectedNumEvents, actualNumEvents)
-		}
-
-		for _, ev := range tc.expectedEventsInQueue {
-			evInQueue, _ := watcher.eventsQueue.Get()
-			if !reflect.DeepEqual(ev, evInQueue) {
-				t.Fatalf("was expecting to see event %T but got %T", ev, evInQueue)
-			}
-		}
-	})
-}
-
-func TestRemoteServiceCreated(t *testing.T) {
-	for _, tt := range []testCase{
-		{
-			testDescription: "create service and endpoints when gateway cannot be resolved",
-			events: []interface{}{
-				&RemoteServiceCreated{
-					service: remoteService("service-one", "ns1", "missing-gateway", "missing-namespace", "111", nil),
-					gatewayData: &gatewayMetadata{
-						Name:      "missing-gateway",
-						Namespace: "missing-namespace",
-					},
-				},
-			},
-			expectedLocalServices: []*corev1.Service{
-				mirroredService("service-one-remote", "ns1", "missing-gateway", "missing-namespace", "111", "", nil),
-			},
-			expectedLocalEndpoints: []*corev1.Endpoints{
-				endpoints("service-one-remote", "ns1", "missing-gateway", "missing-namespace", "", "", nil),
-			},
-		},
-		{
-			testDescription: "create service and endpoints without subsets when gateway spec is wrong",
-			events: []interface{}{
-				&RemoteServiceCreated{
-					service: remoteService("service-one", "ns1", "existing-gateway", "existing-namespace",
-						"111", []corev1.ServicePort{
-							{
-								Name:     "port1",
-								Protocol: "TCP",
-								Port:     555,
-							},
-							{
-								Name:     "port2",
-								Protocol: "TCP",
-								Port:     666,
-							},
-						}),
-
-					gatewayData: &gatewayMetadata{
-						Name:      "existing-gateway",
-						Namespace: "existing-namespace",
-					},
-				},
-			},
-			expectedLocalServices: []*corev1.Service{
-				mirroredService("service-one-remote", "ns1", "existing-gateway", "existing-namespace", "111", "",
-					[]corev1.ServicePort{
-						{
-							Name:     "port1",
-							Protocol: "TCP",
-							Port:     555,
-						},
-						{
-							Name:     "port2",
-							Protocol: "TCP",
-							Port:     666,
-						},
-					}),
-			},
-			expectedLocalEndpoints: []*corev1.Endpoints{
-				endpoints("service-one-remote", "ns1", "existing-gateway", "existing-namespace", "", "", nil),
-			},
-			remoteResources: []string{
-				gatewayAsYaml("existing-gateway", "existing-namespace", "222", "192.0.2.127", "incoming-port-wrong", 888, "", t),
-			},
-		},
-		{
-			testDescription: "create service and endpoints when gateway can be resolved",
-			events: []interface{}{
-				&RemoteServiceCreated{
-
-					service: remoteService("service-one", "ns1", "existing-gateway", "existing-namespace", "111", []corev1.ServicePort{
-						{
-							Name:     "port1",
-							Protocol: "TCP",
-							Port:     555,
-						},
-						{
-							Name:     "port2",
-							Protocol: "TCP",
-							Port:     666,
-						},
-					}),
-					gatewayData: &gatewayMetadata{
-						Name:      "existing-gateway",
-						Namespace: "existing-namespace",
-					},
-				},
-			},
-			expectedLocalServices: []*corev1.Service{
-				mirroredService(
-					"service-one-remote",
-					"ns1",
-					"existing-gateway",
-					"existing-namespace",
-					"111",
-					"222",
-					[]corev1.ServicePort{
-						{
-							Name:     "port1",
-							Protocol: "TCP",
-							Port:     555,
-						},
-						{
-							Name:     "port2",
-							Protocol: "TCP",
-							Port:     666,
-						},
-					}),
-			},
-			expectedLocalEndpoints: []*corev1.Endpoints{
-				endpoints("service-one-remote", "ns1", "existing-gateway", "existing-namespace", "192.0.2.127", "gateway-identity", []corev1.EndpointPort{
-					{
-						Name:     "port1",
-						Port:     888,
-						Protocol: "TCP",
-					},
-					{
-						Name:     "port2",
-						Port:     888,
-						Protocol: "TCP",
-					},
-				}),
-			},
-			remoteResources: []string{
-				gatewayAsYaml("existing-gateway", "existing-namespace", "222", "192.0.2.127", "incoming-port", 888, "gateway-identity", t),
-			},
-		},
-	} {
-		tc := tt // pin
-		runTestCase(&tc, t)
-	}
-}
-
-func TestRemoteServiceDeleted(t *testing.T) {
-	for _, tt := range []testCase{
-		{
-			testDescription: "deletes locally mirrored service",
-			events: []interface{}{
-				&RemoteServiceDeleted{
-					Name:      "test-service-remote-to-delete",
-					Namespace: "test-namespace-to-delete",
-					GatewayData: gatewayMetadata{
-						Name:      "gateway",
-						Namespace: "gateway-ns",
-					},
-				},
-			},
-
-			localResources: []string{
-				mirroredServiceAsYaml("test-service-remote-to-delete-remote", "test-namespace-to-delete", "", "", "", "", nil, t),
-				endpointsAsYaml("test-service-remote-to-delete-remote", "test-namespace-to-delete", "", "", "", "gateway-identity", nil, t),
-			},
-		},
-	} {
-		tc := tt // pin
-		runTestCase(&tc, t)
-	}
-}
-
-func TestRemoteServiceUpdated(t *testing.T) {
-	for _, tt := range []testCase{
-		{
-			testDescription: "update to new gateway",
-			events: []interface{}{
-				&RemoteServiceUpdated{
-					remoteUpdate: remoteService("test-service", "test-namespace", "gateway-new", "gateway-ns", "currentServiceResVersion", []corev1.ServicePort{
-						{
-							Name:     "port1",
-							Protocol: "TCP",
-							Port:     111,
-						},
-						{
-							Name:     "port2",
-							Protocol: "TCP",
-							Port:     222,
-						},
-					}),
-					localService: mirroredService("test-service-remote", "test-namespace", "gateway", "gateway-ns", "pastServiceResVersion", "pastGatewayResVersion", []corev1.ServicePort{
-						{
-							Name:     "port1",
-							Protocol: "TCP",
-							Port:     111,
-						},
-						{
-							Name:     "port2",
-							Protocol: "TCP",
-							Port:     222,
-						},
-					}),
-					localEndpoints: endpoints("test-service-remote", "test-namespace", "gateway", "gateway-ns", "192.0.2.127", "", []corev1.EndpointPort{
-						{
-							Name:     "port1",
-							Port:     888,
-							Protocol: "TCP",
-						},
-						{
-							Name:     "port2",
-							Port:     888,
-							Protocol: "TCP",
-						},
-					}),
-					gatewayData: gatewayMetadata{
-						Name:      "gateway-new",
-						Namespace: "gateway-ns",
-					},
-				},
-			},
-			remoteResources: []string{
-				gatewayAsYaml("gateway-new", "gateway-ns", "currentGatewayResVersion", "0.0.0.0", "incoming-port", 999, "", t),
-			},
-			localResources: []string{
-				mirroredServiceAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "past", "pastGatewayResVersion", []corev1.ServicePort{
-					{
-						Name:     "port1",
-						Protocol: "TCP",
-						Port:     111,
-					},
-					{
-						Name:     "port2",
-						Protocol: "TCP",
-						Port:     222,
-					},
-				}, t),
-				endpointsAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "192.0.2.127", "", []corev1.EndpointPort{
-					{
-						Name:     "port1",
-						Port:     888,
-						Protocol: "TCP",
-					},
-					{
-						Name:     "port2",
-						Port:     888,
-						Protocol: "TCP",
-					},
-				}, t),
-			},
-			expectedLocalServices: []*corev1.Service{
-				mirroredService(
-					"test-service-remote",
-					"test-namespace",
-					"gateway-new",
-					"gateway-ns",
-					"currentServiceResVersion",
-					"currentGatewayResVersion",
-					[]corev1.ServicePort{
-						{
-							Name:     "port1",
-							Protocol: "TCP",
-							Port:     111,
-						},
-						{
-							Name:     "port3",
-							Protocol: "TCP",
-							Port:     333,
-						},
-					}),
-			},
-			expectedLocalEndpoints: []*corev1.Endpoints{
-				endpoints("test-service-remote", "test-namespace", "gateway-new", "gateway-ns", "0.0.0.0", "", []corev1.EndpointPort{
-					{
-						Name:     "port1",
-						Port:     999,
-						Protocol: "TCP",
-					},
-					{
-						Name:     "port2",
-						Port:     999,
-						Protocol: "TCP",
-					},
-				}),
-			},
-		},
-		{
-			testDescription: "updates service ports on both service and endpoints",
-			events: []interface{}{
-				&RemoteServiceUpdated{
-					remoteUpdate: remoteService("test-service", "test-namespace", "gateway", "gateway-ns", "currentServiceResVersion", []corev1.ServicePort{
-						{
-							Name:     "port1",
-							Protocol: "TCP",
-							Port:     111,
-						},
-						{
-							Name:     "port3",
-							Protocol: "TCP",
-							Port:     333,
-						},
-					}),
-					localService: mirroredService("test-service-remote", "test-namespace", "gateway", "gateway-ns", "pastServiceResVersion", "pastGatewayResVersion", []corev1.ServicePort{
-						{
-							Name:     "port1",
-							Protocol: "TCP",
-							Port:     111,
-						},
-						{
-							Name:     "port2",
-							Protocol: "TCP",
-							Port:     222,
-						},
-					}),
-					localEndpoints: endpoints("test-service-remote", "test-namespace", "gateway", "gateway-ns", "192.0.2.127", "", []corev1.EndpointPort{
-						{
-							Name:     "port1",
-							Port:     888,
-							Protocol: "TCP",
-						},
-						{
-							Name:     "port2",
-							Port:     888,
-							Protocol: "TCP",
-						},
-					}),
-					gatewayData: gatewayMetadata{
-						Name:      "gateway",
-						Namespace: "gateway-ns",
-					},
-				},
-			},
-			remoteResources: []string{
-				gatewayAsYaml("gateway", "gateway-ns", "currentGatewayResVersion", "192.0.2.127", "incoming-port", 888, "", t),
-			},
-			localResources: []string{
-				mirroredServiceAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "past", "pastGatewayResVersion", []corev1.ServicePort{
-					{
-						Name:     "port1",
-						Protocol: "TCP",
-						Port:     111,
-					},
-					{
-						Name:     "port2",
-						Protocol: "TCP",
-						Port:     222,
-					},
-					{
-						Name:     "port3",
-						Protocol: "TCP",
-						Port:     333,
-					},
-				}, t),
-				endpointsAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "192.0.2.127", "", []corev1.EndpointPort{
-					{
-						Name:     "port1",
-						Port:     888,
-						Protocol: "TCP",
-					},
-					{
-						Name:     "port2",
-						Port:     888,
-						Protocol: "TCP",
-					},
-					{
-						Name:     "port3",
-						Port:     888,
-						Protocol: "TCP",
-					},
-				}, t),
-			},
-			expectedLocalServices: []*corev1.Service{
-				mirroredService("test-service-remote", "test-namespace", "gateway", "gateway-ns", "currentServiceResVersion", "currentGatewayResVersion",
-					[]corev1.ServicePort{
-						{
-							Name:     "port1",
-							Protocol: "TCP",
-							Port:     111,
-						},
-						{
-							Name:     "port3",
-							Protocol: "TCP",
-							Port:     333,
-						},
-					}),
-			},
-
-			expectedLocalEndpoints: []*corev1.Endpoints{
-				endpoints("test-service-remote", "test-namespace", "gateway", "gateway-ns", "192.0.2.127", "", []corev1.EndpointPort{
-					{
-						Name:     "port1",
-						Port:     888,
-						Protocol: "TCP",
-					},
-					{
-						Name:     "port3",
-						Port:     888,
-						Protocol: "TCP",
-					},
-				}),
-			},
-		},
-	} {
-		tc := tt // pin
-		runTestCase(&tc, t)
-	}
-}
-
-func TestRemoteGatewayUpdated(t *testing.T) {
-	for _, tt := range []testCase{
-		{
-			testDescription: "endpoints ports are updated on gateway change",
-			events: []interface{}{
-				&RemoteGatewayUpdated{
-
-					gatewaySpec: GatewaySpec{
-						gatewayName:      "gateway",
-						gatewayNamespace: "gateway-ns",
-						clusterName:      "remote",
-						addresses:        []corev1.EndpointAddress{{IP: "0.0.0.0"}},
-						incomingPort:     999,
-						resourceVersion:  "currentGatewayResVersion",
-						ProbeConfig:      nil,
-					},
-					affectedServices: []*corev1.Service{
-						mirroredService("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
-							[]corev1.ServicePort{
-								{
-									Name:     "svc-1-port",
-									Protocol: "TCP",
-									Port:     8081,
-								},
-							}),
-
-						mirroredService("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion", []corev1.ServicePort{
-							{
-								Name:     "svc-2-port",
-								Protocol: "TCP",
-								Port:     8082,
-							},
-						}),
-					},
-				},
-			},
-
-			expectedLocalServices: []*corev1.Service{
-				mirroredService("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "currentGatewayResVersion",
-					[]corev1.ServicePort{
-						{
-							Name:     "svc-1-port",
-							Protocol: "TCP",
-							Port:     8081,
-						},
-					}),
-
-				mirroredService("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "currentGatewayResVersion", []corev1.ServicePort{
-					{
-						Name:     "svc-2-port",
-						Protocol: "TCP",
-						Port:     8082,
-					},
-				}),
-			},
-			expectedLocalEndpoints: []*corev1.Endpoints{
-				endpoints("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
-					[]corev1.EndpointPort{
-						{
-							Name:     "svc-1-port",
-							Port:     999,
-							Protocol: "TCP",
-						}}),
-				endpoints("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
-					[]corev1.EndpointPort{
-						{
-							Name:     "svc-2-port",
-							Port:     999,
-							Protocol: "TCP",
-						}}),
-			},
-			localResources: []string{
-				mirroredServiceAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
-					[]corev1.ServicePort{
-						{
-							Name:     "svc-1-port",
-							Protocol: "TCP",
-							Port:     8081,
-						},
-					}, t),
-				endpointsAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
-					[]corev1.EndpointPort{
-						{
-							Name:     "svc-1-port",
-							Port:     888,
-							Protocol: "TCP",
-						}}, t),
-				mirroredServiceAsYaml("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion", []corev1.ServicePort{
-					{
-						Name:     "svc-2-port",
-						Protocol: "TCP",
-						Port:     8082,
-					},
-				}, t),
-				endpointsAsYaml("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
-					[]corev1.EndpointPort{
-						{
-							Name:     "svc-2-port",
-							Port:     888,
-							Protocol: "TCP",
-						}}, t),
-			},
-		},
-
-		{
-			testDescription: "endpoints addresses are updated on gateway change",
-			events: []interface{}{
-				&RemoteGatewayUpdated{
-					gatewaySpec: GatewaySpec{
-						gatewayName:      "gateway",
-						gatewayNamespace: "gateway-ns",
-						clusterName:      "remote",
-						addresses:        []corev1.EndpointAddress{{IP: "0.0.0.1"}},
-						incomingPort:     888,
-						resourceVersion:  "currentGatewayResVersion",
-					},
-					affectedServices: []*corev1.Service{
-						mirroredService("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
-							[]corev1.ServicePort{
-								{
-									Name:     "svc-1-port",
-									Protocol: "TCP",
-									Port:     8081,
-								},
-							}),
-						mirroredService("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion", []corev1.ServicePort{
-							{
-								Name:     "svc-2-port",
-								Protocol: "TCP",
-								Port:     8082,
-							},
-						}),
-					},
-				},
-			},
-			expectedLocalServices: []*corev1.Service{
-				mirroredService("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "currentGatewayResVersion",
-					[]corev1.ServicePort{
-						{
-							Name:     "svc-1-port",
-							Protocol: "TCP",
-							Port:     8081,
-						},
-					}),
-				mirroredService("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "currentGatewayResVersion", []corev1.ServicePort{
-					{
-						Name:     "svc-2-port",
-						Protocol: "TCP",
-						Port:     8082,
-					},
-				}),
-			},
-			expectedLocalEndpoints: []*corev1.Endpoints{
-				endpoints("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.1", "",
-					[]corev1.EndpointPort{
-						{
-							Name:     "svc-1-port",
-							Port:     888,
-							Protocol: "TCP",
-						}}),
-				endpoints("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.1", "",
-					[]corev1.EndpointPort{
-						{
-							Name:     "svc-2-port",
-							Port:     888,
-							Protocol: "TCP",
-						}}),
-			},
-
-			localResources: []string{
-				mirroredServiceAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
-					[]corev1.ServicePort{
-						{
-							Name:     "svc-1-port",
-							Protocol: "TCP",
-							Port:     8081,
-						},
-					}, t),
-				endpointsAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
-					[]corev1.EndpointPort{
-						{
-							Name:     "svc-1-port",
-							Port:     888,
-							Protocol: "TCP",
-						}}, t),
-				mirroredServiceAsYaml("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion", []corev1.ServicePort{
-					{
-						Name:     "svc-2-port",
-						Protocol: "TCP",
-						Port:     8082,
-					},
-				}, t),
-				endpointsAsYaml("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
-					[]corev1.EndpointPort{
-						{
-							Name:     "svc-2-port",
-							Port:     888,
-							Protocol: "TCP",
-						}}, t),
-			},
-		},
-
-		{
-			testDescription: "identity is updated on gateway change",
-			events: []interface{}{
-				&RemoteGatewayUpdated{
-					gatewaySpec: GatewaySpec{
-						gatewayName:      "gateway",
-						gatewayNamespace: "gateway-ns",
-						clusterName:      "",
-						addresses:        []corev1.EndpointAddress{{IP: "0.0.0.0"}},
-						incomingPort:     888,
-						resourceVersion:  "currentGatewayResVersion",
-						identity:         "new-identity",
-						ProbeConfig:      nil,
-					},
-					affectedServices: []*corev1.Service{
-						mirroredService("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
-							[]corev1.ServicePort{
-								{
-									Name:     "svc-1-port",
-									Protocol: "TCP",
-									Port:     8081,
-								},
-							}),
-						mirroredService("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion", []corev1.ServicePort{
-							{
-								Name:     "svc-2-port",
-								Protocol: "TCP",
-								Port:     8082,
-							},
-						}),
-					},
-				},
-			},
-			expectedLocalServices: []*corev1.Service{
-				mirroredService("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "currentGatewayResVersion",
-					[]corev1.ServicePort{
-						{
-							Name:     "svc-1-port",
-							Protocol: "TCP",
-							Port:     8081,
-						},
-					}),
-				mirroredService("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "currentGatewayResVersion", []corev1.ServicePort{
-					{
-						Name:     "svc-2-port",
-						Protocol: "TCP",
-						Port:     8082,
-					},
-				}),
-			},
-			expectedLocalEndpoints: []*corev1.Endpoints{
-				endpoints("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "new-identity",
-					[]corev1.EndpointPort{
-						{
-							Name:     "svc-1-port",
-							Port:     888,
-							Protocol: "TCP",
-						}}),
-				endpoints("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "new-identity",
-					[]corev1.EndpointPort{
-						{
-							Name:     "svc-2-port",
-							Port:     888,
-							Protocol: "TCP",
-						}}),
-			},
-
-			localResources: []string{
-				mirroredServiceAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
-					[]corev1.ServicePort{
-						{
-							Name:     "svc-1-port",
-							Protocol: "TCP",
-							Port:     8081,
-						},
-					}, t),
-				endpointsAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
-					[]corev1.EndpointPort{
-						{
-							Name:     "svc-1-port",
-							Port:     888,
-							Protocol: "TCP",
-						}}, t),
-				mirroredServiceAsYaml("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion", []corev1.ServicePort{
-					{
-						Name:     "svc-2-port",
-						Protocol: "TCP",
-						Port:     8082,
-					},
-				}, t),
-				endpointsAsYaml("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
-					[]corev1.EndpointPort{
-						{
-							Name:     "svc-2-port",
-							Port:     888,
-							Protocol: "TCP",
-						}}, t),
-			},
-		},
-	} {
-		tc := tt // pin
-		runTestCase(&tc, t)
-	}
-}
-func TestRemoteGatewayDeleted(t *testing.T) {
-	for _, tt := range []testCase{
-		{
-			testDescription: "removes endpoint subsets when gateway is deleted",
-			events: []interface{}{
-				&RemoteGatewayDeleted{
-					gatewayData: gatewayMetadata{
-						Name:      "gateway",
-						Namespace: "gateway-ns",
-					},
-				},
-			},
-			expectedLocalServices: []*corev1.Service{
-				mirroredService("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
-					[]corev1.ServicePort{
-						{
-							Name:     "svc-1-port",
-							Protocol: "TCP",
-							Port:     8081,
-						},
-					}),
-
-				mirroredService("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion", []corev1.ServicePort{
-					{
-						Name:     "svc-2-port",
-						Protocol: "TCP",
-						Port:     8082,
-					},
-				}),
-			},
-			expectedLocalEndpoints: []*corev1.Endpoints{
-				endpoints("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "", nil),
-				endpoints("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "", nil),
-			},
-			localResources: []string{
-				mirroredServiceAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
-					[]corev1.ServicePort{
-						{
-							Name:     "svc-1-port",
-							Protocol: "TCP",
-							Port:     8081,
-						},
-					}, t),
-				endpointsAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
-					[]corev1.EndpointPort{
-						{
-							Name:     "svc-1-port",
-							Port:     888,
-							Protocol: "TCP",
-						}}, t),
-				mirroredServiceAsYaml("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion", []corev1.ServicePort{
-					{
-						Name:     "svc-2-port",
-						Protocol: "TCP",
-						Port:     8082,
-					},
-				}, t),
-				endpointsAsYaml("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
-					[]corev1.EndpointPort{
-						{
-							Name:     "svc-2-port",
-							Port:     888,
-							Protocol: "TCP",
-						}}, t),
-			},
-		},
-	} {
-		tc := tt // pin
-		runTestCase(&tc, t)
-	}
-}
-
-func TestClusterUnregistered(t *testing.T) {
-	for _, tt := range []testCase{
-		{
-			testDescription: "unregisters cluster and cleans up all mirrored resources",
-			events: []interface{}{
-				&ClusterUnregistered{},
-			},
-			localResources: []string{
-				mirroredServiceAsYaml("test-service-1-remote", "test-namespace", "", "", "", "", nil, t),
-				endpointsAsYaml("test-service-1-remote", "test-namespace", "", "", "", "", nil, t),
-				mirroredServiceAsYaml("test-service-2-remote", "test-namespace", "", "", "", "", nil, t),
-				endpointsAsYaml("test-service-2-remote", "test-namespace", "", "", "", "", nil, t),
-			},
-		},
-	} {
-		tc := tt // pin
-		runTestCase(&tc, t)
-	}
-}
-
-func TestGcOrphanedServices(t *testing.T) {
-	for _, tt := range []testCase{
-		{
-			testDescription: "deletes mirrored resources that are no longer present on the remote cluster",
-			events: []interface{}{
-				&OprhanedServicesGcTriggered{},
-			},
-			localResources: []string{
-				mirroredServiceAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "", nil, t),
-				endpointsAsYaml("test-service-1-remote", "test-namespace", "", "", "", "", nil, t),
-				mirroredServiceAsYaml("test-service-2-remote", "test-namespace", "", "", "", "", nil, t),
-				endpointsAsYaml("test-service-2-remote", "test-namespace", "", "", "", "", nil, t),
-			},
-			remoteResources: []string{
-				remoteServiceAsYaml("test-service-1", "test-namespace", "gateway", "gateway-ns", "", nil, t),
-			},
-
-			expectedLocalServices: []*corev1.Service{
-				mirroredService("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "", nil),
-			},
-
-			expectedLocalEndpoints: []*corev1.Endpoints{
-				endpoints("test-service-1-remote", "test-namespace", "", "", "", "", nil),
-			},
-		},
-	} {
-		tc := tt // pin
-		runTestCase(&tc, t)
-	}
-}
-
-// the following tests ensure that onAdd, onUpdate and onDelete result in
-// queueing more specific events to be processed
-
-func onAddOrUpdateEvent(isAdd bool, svc *corev1.Service) interface{} {
-	if isAdd {
-		return &OnAddCalled{svc: svc}
-	}
-	return &OnUpdateCalled{svc: svc}
-}
-
-func onAddOrUpdateTestCases(t *testing.T, isAdd bool) []testCase {
-
-	testType := "ADD"
-	if !isAdd {
-		testType = "UPDATE"
-	}
-
-	return []testCase{
-		{
-			testDescription: fmt.Sprintf("enqueue a RemoteServiceCreated event when this is not a gateway and we have the needed annotations (%s)", testType),
-			events: []interface{}{
-				onAddOrUpdateEvent(isAdd, remoteService("test-service", "test-namespace", "gateway", "gateway-ns", "resVersion", nil)),
-			},
-			expectedEventsInQueue: []interface{}{&RemoteServiceCreated{
-				service: remoteService("test-service", "test-namespace", "gateway", "gateway-ns", "resVersion", nil),
-				gatewayData: &gatewayMetadata{
-					Name:      "gateway",
-					Namespace: "gateway-ns",
-				},
-			}},
-		},
-		{
-			testDescription: fmt.Sprintf("enqueue a ConsiderGatewayUpdateDispatch event if this is clearly not a mirrorable service (%s)", testType),
-			events: []interface{}{
-				onAddOrUpdateEvent(isAdd, remoteService("test-service", "test-namespace", "", "", "resVersion", nil)),
-			},
-			expectedEventsInQueue: []interface{}{&ConsiderGatewayUpdateDispatch{
-				maybeGateway: remoteService("test-service", "test-namespace", "", "", "resVersion", nil),
-			}},
-		},
-		{
-			testDescription: fmt.Sprintf("enqueue a RemoteServiceUpdated event if this is a service that we have already mirrored and its res version is different (%s)", testType),
-			events: []interface{}{
-				onAddOrUpdateEvent(isAdd, remoteService("test-service", "test-namespace", "gateway", "gateway-ns", "currentResVersion", nil)),
-			},
-			localResources: []string{
-				mirroredServiceAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "pastResourceVersion", "gatewayResVersion", nil, t),
-				endpointsAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "", nil, t),
-			},
-			expectedEventsInQueue: []interface{}{&RemoteServiceUpdated{
-				localService:   mirroredService("test-service-remote", "test-namespace", "gateway", "gateway-ns", "pastResourceVersion", "gatewayResVersion", nil),
-				localEndpoints: endpoints("test-service-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "", nil),
-				remoteUpdate:   remoteService("test-service", "test-namespace", "gateway", "gateway-ns", "currentResVersion", nil),
-				gatewayData: gatewayMetadata{
-					Name:      "gateway",
-					Namespace: "gateway-ns",
-				},
-			}},
-			expectedLocalServices: []*corev1.Service{
-				mirroredService("test-service-remote", "test-namespace", "gateway", "gateway-ns", "pastResourceVersion", "gatewayResVersion", nil),
-			},
-			expectedLocalEndpoints: []*corev1.Endpoints{
-				endpoints("test-service-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "", nil),
-			},
-		},
-		{
-			testDescription: fmt.Sprintf("not enqueue any events as this update does not really tell us anything new (res version is the same...) (%s)", testType),
-			events: []interface{}{
-				onAddOrUpdateEvent(isAdd, remoteService("test-service", "test-namespace", "gateway", "gateway-ns", "currentResVersion", nil)),
-			},
-			localResources: []string{
-				mirroredServiceAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "currentResVersion", "gatewayResVersion", nil, t),
-				endpointsAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "", nil, t),
-			},
-			expectedLocalServices: []*corev1.Service{
-				mirroredService("test-service-remote", "test-namespace", "gateway", "gateway-ns", "currentResVersion", "gatewayResVersion", nil),
-			},
-			expectedLocalEndpoints: []*corev1.Endpoints{
-				endpoints("test-service-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "", nil),
-			},
-		},
-		{
-			testDescription: fmt.Sprintf("enqueue RemoteServiceDeleted event as this service is not mirrorable anymore (%s)", testType),
-			events: []interface{}{
-				onAddOrUpdateEvent(isAdd, remoteService("test-service", "test-namespace", "", "gateway-ns", "currentResVersion", nil)),
-			},
-			localResources: []string{
-				mirroredServiceAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "currentResVersion", "gatewayResVersion", nil, t),
-				endpointsAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "", nil, t),
-			},
-			expectedEventsInQueue: []interface{}{&RemoteServiceDeleted{
-				Name:      "test-service",
-				Namespace: "test-namespace",
-			}},
-
-			expectedLocalServices: []*corev1.Service{
-				mirroredService("test-service-remote", "test-namespace", "gateway", "gateway-ns", "currentResVersion", "gatewayResVersion", nil),
-			},
-			expectedLocalEndpoints: []*corev1.Endpoints{
-				endpoints("test-service-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "", nil),
-			},
-		},
-	}
-}
-
-func TestOnAdd(t *testing.T) {
-	for _, tt := range onAddOrUpdateTestCases(t, true) {
-		tc := tt // pin
-		runTestCase(&tc, t)
-	}
-}
-
-func TestOnUpdate(t *testing.T) {
-	for _, tt := range onAddOrUpdateTestCases(t, false) {
-		tc := tt // pin
-		runTestCase(&tc, t)
-	}
-}
-
-func TestOnDelete(t *testing.T) {
-	for _, tt := range []testCase{
-		{
-			testDescription: "enqueues a RemoteServiceDeleted because there is gateway metadata present on the service",
-			events: []interface{}{
-				&OnDeleteCalled{
-					svc: remoteService("test-service", "test-namespace", "gateway", "gateway-ns", "currentResVersion", nil),
-				},
-			},
-
-			expectedEventsInQueue: []interface{}{
-				&RemoteServiceDeleted{
-					Name:      "test-service",
-					Namespace: "test-namespace",
-					GatewayData: gatewayMetadata{
-						Name:      "gateway",
-						Namespace: "gateway-ns",
-					},
-				},
-			},
-		},
-		{
-			testDescription: "enqueues a RemoteGatewayDeleted because there is no gateway metadata present on the service",
-			events: []interface{}{
-				&OnDeleteCalled{
-					svc: remoteService("gateway", "test-namespace", "", "", "currentResVersion", nil),
-				},
-			},
-			expectedEventsInQueue: []interface{}{
-				&RemoteGatewayDeleted{
-					gatewayData: gatewayMetadata{
-						Name:      "gateway",
-						Namespace: "test-namespace",
-					},
-				},
-			},
-		},
-	} {
-		tc := tt // pin
-		runTestCase(&tc, t)
-	}
-}
diff --git a/controller/cmd/service-mirror/cluster_watcher_test_util.go b/controller/cmd/service-mirror/cluster_watcher_test_util.go
index 17869dbcb..5c3ef5556 100644
--- a/controller/cmd/service-mirror/cluster_watcher_test_util.go
+++ b/controller/cmd/service-mirror/cluster_watcher_test_util.go
@@ -2,16 +2,721 @@ package servicemirror
 
 import (
 	"fmt"
+	"log"
 	"reflect"
 	"strings"
-	"testing"
 
 	"github.com/ghodss/yaml"
+	"github.com/linkerd/linkerd2/controller/k8s"
 	consts "github.com/linkerd/linkerd2/pkg/k8s"
+	logging "github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/workqueue"
 )
 
+const (
+	clusterName        = "remote"
+	clusterDomain      = "cluster.local"
+	defaultProbePath   = "/probe"
+	defaultProbePort   = 12345
+	defaultProbePeriod = 60
+)
+
+type testEnvironment struct {
+	events          []interface{}
+	remoteResources []string
+	localResources  []string
+}
+
+func (te *testEnvironment) runEnvironment(probeEventSync ProbeEventSink, watcherQueue workqueue.RateLimitingInterface) (*k8s.API, error) {
+	remoteAPI, err := k8s.NewFakeAPI(te.remoteResources...)
+	if err != nil {
+		return nil, err
+	}
+
+	localAPI, err := k8s.NewFakeAPI(te.localResources...)
+	if err != nil {
+		return nil, err
+	}
+
+	remoteAPI.Sync(nil)
+	localAPI.Sync(nil)
+
+	watcher := RemoteClusterServiceWatcher{
+		clusterName:     clusterName,
+		clusterDomain:   clusterDomain,
+		remoteAPIClient: remoteAPI,
+		localAPIClient:  localAPI,
+		stopper:         nil,
+		log:             logging.WithFields(logging.Fields{"cluster": clusterName}),
+		eventsQueue:     watcherQueue,
+		requeueLimit:    0,
+		probeEventsSink: probeEventSync,
+	}
+
+	for _, ev := range te.events {
+		watcherQueue.Add(ev)
+	}
+
+	for range te.events {
+		watcher.processNextEvent()
+	}
+
+	localAPI.Sync(nil)
+	remoteAPI.Sync(nil)
+
+	return localAPI, nil
+}
+
+var serviceCreateWithMissingGateway = &testEnvironment{
+	events: []interface{}{
+		&RemoteServiceCreated{
+			service: remoteService("service-one", "ns1", "missing-gateway", "missing-namespace", "111", nil),
+			gatewayData: &gatewayMetadata{
+				Name:      "missing-gateway",
+				Namespace: "missing-namespace",
+			},
+		},
+	},
+}
+
+var createServiceWrongGatewaySpec = &testEnvironment{
+	events: []interface{}{
+		&RemoteServiceCreated{
+			service: remoteService("service-one", "ns1", "existing-gateway", "existing-namespace",
+				"111", []corev1.ServicePort{
+					{
+						Name:     "port1",
+						Protocol: "TCP",
+						Port:     555,
+					},
+					{
+						Name:     "port2",
+						Protocol: "TCP",
+						Port:     666,
+					},
+				}),
+
+			gatewayData: &gatewayMetadata{
+				Name:      "existing-gateway",
+				Namespace: "existing-namespace",
+			},
+		},
+	},
+	remoteResources: []string{
+		gatewayAsYaml("existing-gateway", "existing-namespace", "222", "192.0.2.127", "incoming-port-wrong", 888, "", 111, "/path", 666),
+	},
+}
+
+var createServiceOkeGatewaySpec = &testEnvironment{
+	events: []interface{}{
+		&RemoteServiceCreated{
+			service: remoteService("service-one", "ns1", "existing-gateway", "existing-namespace", "111", []corev1.ServicePort{
+				{
+					Name:     "port1",
+					Protocol: "TCP",
+					Port:     555,
+				},
+				{
+					Name:     "port2",
+					Protocol: "TCP",
+					Port:     666,
+				},
+			}),
+			gatewayData: &gatewayMetadata{
+				Name:      "existing-gateway",
+				Namespace: "existing-namespace",
+			},
+		},
+	},
+	remoteResources: []string{
+		gatewayAsYaml("existing-gateway", "existing-namespace", "222", "192.0.2.127", "incoming-port", 888, "gateway-identity", defaultProbePort, defaultProbePath, defaultProbePeriod),
+	},
+}
+
+var deleteMirroredService = &testEnvironment{
+	events: []interface{}{
+		&RemoteServiceDeleted{
+			Name:      "test-service-remote-to-delete",
+			Namespace: "test-namespace-to-delete",
+			GatewayData: gatewayMetadata{
+				Name:      "gateway",
+				Namespace: "gateway-ns",
+			},
+		},
+	},
+	localResources: []string{
+		mirroredServiceAsYaml("test-service-remote-to-delete-remote", "test-namespace-to-delete", "", "", "", "", nil),
+		endpointsAsYaml("test-service-remote-to-delete-remote", "test-namespace-to-delete", "", "", "", "gateway-identity", nil),
+	},
+}
+
+var updateServiceToNewGateway = &testEnvironment{
+	events: []interface{}{
+		&RemoteServiceUpdated{
+			remoteUpdate: remoteService("test-service", "test-namespace", "gateway-new", "gateway-ns", "currentServiceResVersion", []corev1.ServicePort{
+				{
+					Name:     "port1",
+					Protocol: "TCP",
+					Port:     111,
+				},
+				{
+					Name:     "port2",
+					Protocol: "TCP",
+					Port:     222,
+				},
+			}),
+			localService: mirroredService("test-service-remote", "test-namespace", "gateway", "gateway-ns", "pastServiceResVersion", "pastGatewayResVersion", []corev1.ServicePort{
+				{
+					Name:     "port1",
+					Protocol: "TCP",
+					Port:     111,
+				},
+				{
+					Name:     "port2",
+					Protocol: "TCP",
+					Port:     222,
+				},
+			}),
+			localEndpoints: endpoints("test-service-remote", "test-namespace", "gateway", "gateway-ns", "192.0.2.127", "", []corev1.EndpointPort{
+				{
+					Name:     "port1",
+					Port:     888,
+					Protocol: "TCP",
+				},
+				{
+					Name:     "port2",
+					Port:     888,
+					Protocol: "TCP",
+				},
+			}),
+			gatewayData: gatewayMetadata{
+				Name:      "gateway-new",
+				Namespace: "gateway-ns",
+			},
+		},
+	},
+	remoteResources: []string{
+		gatewayAsYaml("gateway-new", "gateway-ns", "currentGatewayResVersion", "0.0.0.0", "incoming-port", 999, "", defaultProbePort, defaultProbePath, defaultProbePeriod),
+	},
+	localResources: []string{
+		mirroredServiceAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "past", "pastGatewayResVersion", []corev1.ServicePort{
+			{
+				Name:     "port1",
+				Protocol: "TCP",
+				Port:     111,
+			},
+			{
+				Name:     "port2",
+				Protocol: "TCP",
+				Port:     222,
+			},
+		}),
+		endpointsAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "192.0.2.127", "", []corev1.EndpointPort{
+			{
+				Name:     "port1",
+				Port:     888,
+				Protocol: "TCP",
+			},
+			{
+				Name:     "port2",
+				Port:     888,
+				Protocol: "TCP",
+			},
+		}),
+	},
+}
+
+var updateServiceWithChangedPorts = &testEnvironment{
+	events: []interface{}{
+		&RemoteServiceUpdated{
+			remoteUpdate: remoteService("test-service", "test-namespace", "gateway", "gateway-ns", "currentServiceResVersion", []corev1.ServicePort{
+				{
+					Name:     "port1",
+					Protocol: "TCP",
+					Port:     111,
+				},
+				{
+					Name:     "port3",
+					Protocol: "TCP",
+					Port:     333,
+				},
+			}),
+			localService: mirroredService("test-service-remote", "test-namespace", "gateway", "gateway-ns", "pastServiceResVersion", "pastGatewayResVersion", []corev1.ServicePort{
+				{
+					Name:     "port1",
+					Protocol: "TCP",
+					Port:     111,
+				},
+				{
+					Name:     "port2",
+					Protocol: "TCP",
+					Port:     222,
+				},
+			}),
+			localEndpoints: endpoints("test-service-remote", "test-namespace", "gateway", "gateway-ns", "192.0.2.127", "", []corev1.EndpointPort{
+				{
+					Name:     "port1",
+					Port:     888,
+					Protocol: "TCP",
+				},
+				{
+					Name:     "port2",
+					Port:     888,
+					Protocol: "TCP",
+				},
+			}),
+			gatewayData: gatewayMetadata{
+				Name:      "gateway",
+				Namespace: "gateway-ns",
+			},
+		},
+	},
+	remoteResources: []string{
+		gatewayAsYaml("gateway", "gateway-ns", "currentGatewayResVersion", "192.0.2.127", "incoming-port", 888, "", defaultProbePort, defaultProbePath, defaultProbePeriod),
+	},
+	localResources: []string{
+		mirroredServiceAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "past", "pastGatewayResVersion", []corev1.ServicePort{
+			{
+				Name:     "port1",
+				Protocol: "TCP",
+				Port:     111,
+			},
+			{
+				Name:     "port2",
+				Protocol: "TCP",
+				Port:     222,
+			},
+			{
+				Name:     "port3",
+				Protocol: "TCP",
+				Port:     333,
+			},
+		}),
+		endpointsAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "192.0.2.127", "", []corev1.EndpointPort{
+			{
+				Name:     "port1",
+				Port:     888,
+				Protocol: "TCP",
+			},
+			{
+				Name:     "port2",
+				Port:     888,
+				Protocol: "TCP",
+			},
+			{
+				Name:     "port3",
+				Port:     888,
+				Protocol: "TCP",
+			},
+		}),
+	},
+}
+
+var remoteGatewayUpdated = &testEnvironment{
+	events: []interface{}{
+		&RemoteGatewayUpdated{
+			gatewaySpec: GatewaySpec{
+				gatewayName:      "gateway",
+				gatewayNamespace: "gateway-ns",
+				clusterName:      "remote",
+				addresses:        []corev1.EndpointAddress{{IP: "0.0.0.0"}},
+				incomingPort:     999,
+				resourceVersion:  "currentGatewayResVersion",
+				ProbeConfig: &ProbeConfig{
+					path:            defaultProbePath,
+					port:            defaultProbePort,
+					periodInSeconds: defaultProbePeriod,
+				},
+			},
+			affectedServices: []*corev1.Service{
+				mirroredService("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
+					[]corev1.ServicePort{
+						{
+							Name:     "svc-1-port",
+							Protocol: "TCP",
+							Port:     8081,
+						},
+					}),
+
+				mirroredService("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion", []corev1.ServicePort{
+					{
+						Name:     "svc-2-port",
+						Protocol: "TCP",
+						Port:     8082,
+					},
+				}),
+			},
+		},
+	},
+	localResources: []string{
+		mirroredServiceAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
+			[]corev1.ServicePort{
+				{
+					Name:     "svc-1-port",
+					Protocol: "TCP",
+					Port:     8081,
+				},
+			}),
+		endpointsAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
+			[]corev1.EndpointPort{
+				{
+					Name:     "svc-1-port",
+					Port:     888,
+					Protocol: "TCP",
+				}}),
+		mirroredServiceAsYaml("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion", []corev1.ServicePort{
+			{
+				Name:     "svc-2-port",
+				Protocol: "TCP",
+				Port:     8082,
+			},
+		}),
+		endpointsAsYaml("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
+			[]corev1.EndpointPort{
+				{
+					Name:     "svc-2-port",
+					Port:     888,
+					Protocol: "TCP",
+				}}),
+	},
+}
+
+var gatewayAddressChanged = &testEnvironment{
+	events: []interface{}{
+		&RemoteGatewayUpdated{
+			gatewaySpec: GatewaySpec{
+				gatewayName:      "gateway",
+				gatewayNamespace: "gateway-ns",
+				clusterName:      "some-cluster",
+				addresses:        []corev1.EndpointAddress{{IP: "0.0.0.1"}},
+				incomingPort:     888,
+				resourceVersion:  "currentGatewayResVersion",
+				ProbeConfig: &ProbeConfig{
+					path:            "/p",
+					port:            1,
+					periodInSeconds: 222,
+				},
+			},
+			affectedServices: []*corev1.Service{
+				mirroredService("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
+					[]corev1.ServicePort{
+						{
+							Name:     "svc-1-port",
+							Protocol: "TCP",
+							Port:     8081,
+						},
+					}),
+				mirroredService("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion", []corev1.ServicePort{
+					{
+						Name:     "svc-2-port",
+						Protocol: "TCP",
+						Port:     8082,
+					},
+				}),
+			},
+		},
+	},
+	localResources: []string{
+		mirroredServiceAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
+			[]corev1.ServicePort{
+				{
+					Name:     "svc-1-port",
+					Protocol: "TCP",
+					Port:     8081,
+				},
+			}),
+		endpointsAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
+			[]corev1.EndpointPort{
+				{
+					Name:     "svc-1-port",
+					Port:     888,
+					Protocol: "TCP",
+				}}),
+		mirroredServiceAsYaml("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion", []corev1.ServicePort{
+			{
+				Name:     "svc-2-port",
+				Protocol: "TCP",
+				Port:     8082,
+			},
+		}),
+		endpointsAsYaml("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
+			[]corev1.EndpointPort{
+				{
+					Name:     "svc-2-port",
+					Port:     888,
+					Protocol: "TCP",
+				}}),
+	},
+}
+
+var gatewayIdentityChanged = &testEnvironment{
+	events: []interface{}{
+		&RemoteGatewayUpdated{
+			gatewaySpec: GatewaySpec{
+				gatewayName:      "gateway",
+				gatewayNamespace: "gateway-ns",
+				clusterName:      clusterName,
+				addresses:        []corev1.EndpointAddress{{IP: "0.0.0.0"}},
+				incomingPort:     888,
+				resourceVersion:  "currentGatewayResVersion",
+				identity:         "new-identity",
+				ProbeConfig: &ProbeConfig{
+					path:            defaultProbePath,
+					port:            defaultProbePort,
+					periodInSeconds: defaultProbePeriod,
+				},
+			},
+			affectedServices: []*corev1.Service{
+				mirroredService("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
+					[]corev1.ServicePort{
+						{
+							Name:     "svc-1-port",
+							Protocol: "TCP",
+							Port:     8081,
+						},
+					}),
+				mirroredService("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion", []corev1.ServicePort{
+					{
+						Name:     "svc-2-port",
+						Protocol: "TCP",
+						Port:     8082,
+					},
+				}),
+			},
+		},
+	},
+	localResources: []string{
+		mirroredServiceAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
+			[]corev1.ServicePort{
+				{
+					Name:     "svc-1-port",
+					Protocol: "TCP",
+					Port:     8081,
+				},
+			}),
+		endpointsAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
+			[]corev1.EndpointPort{
+				{
+					Name:     "svc-1-port",
+					Port:     888,
+					Protocol: "TCP",
+				}}),
+		mirroredServiceAsYaml("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion", []corev1.ServicePort{
+			{
+				Name:     "svc-2-port",
+				Protocol: "TCP",
+				Port:     8082,
+			},
+		}),
+		endpointsAsYaml("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
+			[]corev1.EndpointPort{
+				{
+					Name:     "svc-2-port",
+					Port:     888,
+					Protocol: "TCP",
+				}}),
+	},
+}
+
+var gatewayProbeConfigChanged = &testEnvironment{
+	events: []interface{}{
+		&RemoteGatewayUpdated{
+			gatewaySpec: GatewaySpec{
+				gatewayName:      "gateway",
+				gatewayNamespace: "gateway-ns",
+				clusterName:      clusterName,
+				addresses:        []corev1.EndpointAddress{{IP: "0.0.0.0"}},
+				incomingPort:     888,
+				resourceVersion:  "currentGatewayResVersion",
+				identity:         "identity",
+				ProbeConfig: &ProbeConfig{
+					path:            "/new-path",
+					port:            defaultProbePort,
+					periodInSeconds: defaultProbePeriod,
+				},
+			},
+			affectedServices: []*corev1.Service{
+				mirroredService("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
+					[]corev1.ServicePort{
+						{
+							Name:     "svc-1-port",
+							Protocol: "TCP",
+							Port:     8081,
+						},
+					}),
+			},
+		},
+	},
+	localResources: []string{
+		mirroredServiceAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
+			[]corev1.ServicePort{
+				{
+					Name:     "svc-1-port",
+					Protocol: "TCP",
+					Port:     8081,
+				},
+			}),
+		endpointsAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
+			[]corev1.EndpointPort{
+				{
+					Name:     "svc-1-port",
+					Port:     888,
+					Protocol: "TCP",
+				}}),
+	},
+}
+
+var gatewayDeleted = &testEnvironment{
+	events: []interface{}{
+		&RemoteGatewayDeleted{
+			gatewayData: gatewayMetadata{
+				Name:      "gateway",
+				Namespace: "gateway-ns",
+			},
+		},
+	},
+	localResources: []string{
+		mirroredServiceAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion",
+			[]corev1.ServicePort{
+				{
+					Name:     "svc-1-port",
+					Protocol: "TCP",
+					Port:     8081,
+				},
+			}),
+		endpointsAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
+			[]corev1.EndpointPort{
+				{
+					Name:     "svc-1-port",
+					Port:     888,
+					Protocol: "TCP",
+				}}),
+		mirroredServiceAsYaml("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "", "pastGatewayResVersion", []corev1.ServicePort{
+			{
+				Name:     "svc-2-port",
+				Protocol: "TCP",
+				Port:     8082,
+			},
+		}),
+		endpointsAsYaml("test-service-2-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "",
+			[]corev1.EndpointPort{
+				{
+					Name:     "svc-2-port",
+					Port:     888,
+					Protocol: "TCP",
+				}}),
+	},
+}
+
+var clusterUnregistered = &testEnvironment{
+	events: []interface{}{
+		&ClusterUnregistered{},
+	},
+	localResources: []string{
+		mirroredServiceAsYaml("test-service-1-remote", "test-namespace", "", "", "", "", nil),
+		endpointsAsYaml("test-service-1-remote", "test-namespace", "", "", "", "", nil),
+		mirroredServiceAsYaml("test-service-2-remote", "test-namespace", "", "", "", "", nil),
+		endpointsAsYaml("test-service-2-remote", "test-namespace", "", "", "", "", nil),
+	},
+}
+
+var gcTriggered = &testEnvironment{
+	events: []interface{}{
+		&OprhanedServicesGcTriggered{},
+	},
+	localResources: []string{
+		mirroredServiceAsYaml("test-service-1-remote", "test-namespace", "gateway", "gateway-ns", "", "", nil),
+		endpointsAsYaml("test-service-1-remote", "test-namespace", "", "", "", "", nil),
+		mirroredServiceAsYaml("test-service-2-remote", "test-namespace", "", "", "", "", nil),
+		endpointsAsYaml("test-service-2-remote", "test-namespace", "", "", "", "", nil),
+	},
+	remoteResources: []string{
+		remoteServiceAsYaml("test-service-1", "test-namespace", "gateway", "gateway-ns", "", nil),
+	},
+}
+
+func onAddOrUpdateExportedSvc(isAdd bool) *testEnvironment {
+	return &testEnvironment{
+		events: []interface{}{
+			onAddOrUpdateEvent(isAdd, remoteService("test-service", "test-namespace", "gateway", "gateway-ns", "resVersion", nil)),
+		},
+	}
+
+}
+
+func onAddOrUpdateNonExportedSvc(isAdd bool) *testEnvironment {
+	return &testEnvironment{
+		events: []interface{}{
+			onAddOrUpdateEvent(isAdd, remoteService("test-service", "test-namespace", "", "", "resVersion", nil)),
+		},
+	}
+
+}
+
+func onAddOrUpdateRemoteServiceUpdated(isAdd bool) *testEnvironment {
+	return &testEnvironment{
+		events: []interface{}{
+			onAddOrUpdateEvent(isAdd, remoteService("test-service", "test-namespace", "gateway", "gateway-ns", "currentResVersion", nil)),
+		},
+		localResources: []string{
+			mirroredServiceAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "pastResourceVersion", "gatewayResVersion", nil),
+			endpointsAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "", nil),
+		},
+	}
+}
+
+func onAddOrUpdateSameResVersion(isAdd bool) *testEnvironment {
+	return &testEnvironment{
+		events: []interface{}{
+			onAddOrUpdateEvent(isAdd, remoteService("test-service", "test-namespace", "gateway", "gateway-ns", "currentResVersion", nil)),
+		},
+		localResources: []string{
+			mirroredServiceAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "currentResVersion", "gatewayResVersion", nil),
+			endpointsAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "", nil),
+		},
+	}
+}
+
+func serviceNotExportedAnymore(isAdd bool) *testEnvironment {
+	return &testEnvironment{
+		events: []interface{}{
+			onAddOrUpdateEvent(isAdd, remoteService("test-service", "test-namespace", "", "gateway-ns", "currentResVersion", nil)),
+		},
+		localResources: []string{
+			mirroredServiceAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "currentResVersion", "gatewayResVersion", nil),
+			endpointsAsYaml("test-service-remote", "test-namespace", "gateway", "gateway-ns", "0.0.0.0", "", nil),
+		},
+	}
+}
+
+var onDeleteWithGatewayMetadata = &testEnvironment{
+	events: []interface{}{
+		&OnDeleteCalled{
+			svc: remoteService("test-service", "test-namespace", "gateway", "gateway-ns", "currentResVersion", nil),
+		},
+	},
+}
+
+var onDeleteNoGatewayMetadata = &testEnvironment{
+	events: []interface{}{
+		&OnDeleteCalled{
+			svc: remoteService("gateway", "test-namespace", "", "", "currentResVersion", nil),
+		},
+	},
+}
+
+// the following tests ensure that onAdd, onUpdate and onDelete result in
+// queueing more specific events to be processed
+
+func onAddOrUpdateEvent(isAdd bool, svc *corev1.Service) interface{} {
+	if isAdd {
+		return &OnAddCalled{svc: svc}
+	}
+	return &OnUpdateCalled{svc: svc}
+}
+
 func diffServices(expected, actual *corev1.Service) error {
 	if expected.Name != actual.Name {
 		return fmt.Errorf("was expecting service with name %s but was %s", expected.Name, actual.Name)
@@ -80,12 +785,12 @@ func remoteService(name, namespace, gtwName, gtwNs, resourceVersion string, port
 	}
 }
 
-func remoteServiceAsYaml(name, namespace, gtwName, gtwNs, resourceVersion string, ports []corev1.ServicePort, t *testing.T) string {
+func remoteServiceAsYaml(name, namespace, gtwName, gtwNs, resourceVersion string, ports []corev1.ServicePort) string {
 	svc := remoteService(name, namespace, gtwName, gtwNs, resourceVersion, ports)
 
 	bytes, err := yaml.Marshal(svc)
 	if err != nil {
-		t.Fatal(err)
+		log.Fatal(err)
 	}
 	return string(bytes)
 }
@@ -121,17 +826,17 @@ func mirroredService(name, namespace, gtwName, gtwNs, resourceVersion, gatewayRe
 	}
 }
 
-func mirroredServiceAsYaml(name, namespace, gtwName, gtwNs, resourceVersion, gatewayResourceVersion string, ports []corev1.ServicePort, t *testing.T) string {
+func mirroredServiceAsYaml(name, namespace, gtwName, gtwNs, resourceVersion, gatewayResourceVersion string, ports []corev1.ServicePort) string {
 	svc := mirroredService(name, namespace, gtwName, gtwNs, resourceVersion, gatewayResourceVersion, ports)
 
 	bytes, err := yaml.Marshal(svc)
 	if err != nil {
-		t.Fatal(err)
+		log.Fatal(err)
 	}
 	return string(bytes)
 }
 
-func gateway(name, namespace, resourceVersion, ip, portName string, port int32, identity string) *corev1.Service {
+func gateway(name, namespace, resourceVersion, ip, portName string, port int32, identity string, probePort int, probePath string, probePeriod int) *corev1.Service {
 	svc := corev1.Service{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Service",
@@ -142,7 +847,10 @@ func gateway(name, namespace, resourceVersion, ip, portName string, port int32,
 			Namespace:       namespace,
 			ResourceVersion: resourceVersion,
 			Annotations: map[string]string{
-				consts.GatewayIdentity: identity,
+				consts.GatewayIdentity:    identity,
+				consts.GatewayProbePath:   probePath,
+				consts.GatewayProbePeriod: fmt.Sprint(probePeriod),
+				consts.GatewayProbePort:   fmt.Sprint(probePort),
 			},
 		},
 		Spec: corev1.ServiceSpec{
@@ -162,12 +870,12 @@ func gateway(name, namespace, resourceVersion, ip, portName string, port int32,
 	return &svc
 }
 
-func gatewayAsYaml(name, namespace, resourceVersion, ip, portName string, port int32, identity string, t *testing.T) string {
-	gtw := gateway(name, namespace, resourceVersion, ip, portName, port, identity)
+func gatewayAsYaml(name, namespace, resourceVersion, ip, portName string, port int32, identity string, probePort int, probePath string, probePeriod int) string {
+	gtw := gateway(name, namespace, resourceVersion, ip, portName, port, identity, probePort, probePath, probePeriod)
 
 	bytes, err := yaml.Marshal(gtw)
 	if err != nil {
-		t.Fatal(err)
+		log.Fatal(err)
 	}
 	return string(bytes)
 }
@@ -215,12 +923,12 @@ func endpoints(name, namespace, gtwName, gtwNs, gatewayIP string, gatewayIdentit
 	return endpoints
 }
 
-func endpointsAsYaml(name, namespace, gtwName, gtwNs, gatewayIP, gatewayIdentity string, ports []corev1.EndpointPort, t *testing.T) string {
+func endpointsAsYaml(name, namespace, gtwName, gtwNs, gatewayIP, gatewayIdentity string, ports []corev1.EndpointPort) string {
 	ep := endpoints(name, namespace, gtwName, gtwNs, gatewayIP, gatewayIdentity, ports)
 
 	bytes, err := yaml.Marshal(ep)
 	if err != nil {
-		t.Fatal(err)
+		log.Fatal(err)
 	}
 	return string(bytes)
 }
diff --git a/pkg/charts/linkerd2/values_test.go b/pkg/charts/linkerd2/values_test.go
index 2805270a1..0c65178e8 100644
--- a/pkg/charts/linkerd2/values_test.go
+++ b/pkg/charts/linkerd2/values_test.go
@@ -14,24 +14,27 @@ func TestNewValues(t *testing.T) {
 	testVersion := "linkerd-dev"
 
 	expected := &Values{
-		Stage:                       "",
-		ControllerImage:             "gcr.io/linkerd-io/controller",
-		ControllerImageVersion:      testVersion,
-		WebImage:                    "gcr.io/linkerd-io/web",
-		PrometheusImage:             "prom/prometheus:v2.15.2",
-		GrafanaImage:                "gcr.io/linkerd-io/grafana",
-		ControllerReplicas:          1,
-		ControllerLogLevel:          "info",
-		PrometheusLogLevel:          "info",
-		ControllerUID:               2103,
-		EnableH2Upgrade:             true,
-		EnablePodAntiAffinity:       false,
-		WebhookFailurePolicy:        "Ignore",
-		OmitWebhookSideEffects:      false,
-		RestrictDashboardPrivileges: false,
-		DisableHeartBeat:            false,
-		HeartbeatSchedule:           "0 0 * * *",
-		InstallNamespace:            true,
+		Stage:                         "",
+		ControllerImage:               "gcr.io/linkerd-io/controller",
+		ControllerImageVersion:        testVersion,
+		WebImage:                      "gcr.io/linkerd-io/web",
+		PrometheusImage:               "prom/prometheus:v2.15.2",
+		GrafanaImage:                  "gcr.io/linkerd-io/grafana",
+		ControllerReplicas:            1,
+		ControllerLogLevel:            "info",
+		PrometheusLogLevel:            "info",
+		PrometheusExtraArgs:           map[string]string{},
+		PrometheusAlertmanagers:       []interface{}{},
+		PrometheusRuleConfigMapMounts: []PrometheusRuleConfigMapMount{},
+		ControllerUID:                 2103,
+		EnableH2Upgrade:               true,
+		EnablePodAntiAffinity:         false,
+		WebhookFailurePolicy:          "Ignore",
+		OmitWebhookSideEffects:        false,
+		RestrictDashboardPrivileges:   false,
+		DisableHeartBeat:              false,
+		HeartbeatSchedule:             "0 0 * * *",
+		InstallNamespace:              true,
 		Global: &Global{
 			Namespace:                "linkerd",
 			ClusterDomain:            "cluster.local",
@@ -148,6 +151,12 @@ func TestNewValues(t *testing.T) {
 
 	t.Run("HA", func(t *testing.T) {
 		actual, err := NewValues(true)
+
+		// workaround for mergo, which resets these to []interface{}(nil)
+		// and []PrometheusRuleConfigMapMount(nil)
+		actual.PrometheusAlertmanagers = []interface{}{}
+		actual.PrometheusRuleConfigMapMounts = []PrometheusRuleConfigMapMount{}
+
 		if err != nil {
 			t.Fatalf("Unexpected error: %v\n", err)
 		}
diff --git a/pkg/healthcheck/healthcheck_test.go b/pkg/healthcheck/healthcheck_test.go
index 9c45c1e1e..a19d9141b 100644
--- a/pkg/healthcheck/healthcheck_test.go
+++ b/pkg/healthcheck/healthcheck_test.go
@@ -13,6 +13,7 @@ import (
 
 	"github.com/linkerd/linkerd2/pkg/issuercerts"
 	"github.com/linkerd/linkerd2/pkg/tls"
+	"github.com/linkerd/linkerd2/testutil"
 
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/protobuf/ptypes/duration"
@@ -2254,7 +2255,7 @@ func TestLinkerdPreInstallGlobalResourcesChecks(t *testing.T) {
 			"pre-linkerd-global-resources no PodSecurityPolicies exist",
 		}
 		if !reflect.DeepEqual(observer.results, expected) {
-			t.Errorf("Mismatch result.\nExpected: %v\n Actual: %v\n", expected, observer.results)
+			testutil.AnnotatedErrorf(t, "Mistmatch result", "Mismatch result.\nExpected: %v\n Actual: %v\n", expected, observer.results)
 		}
 	})
 
@@ -2302,12 +2303,12 @@ metadata:
 		hc.kubeAPI, err = k8s.NewFakeAPI(resources...)
 		hc.ControlPlaneNamespace = "test-ns"
 		if err != nil {
-			t.Fatalf("Unexpected error: %s", err)
+			testutil.AnnotatedFatalf(t, "Unexpected error", "Unexpected error: %s", err)
 		}
 
 		observer := newObserver()
 		if hc.RunChecks(observer.resultFn) {
-			t.Errorf("Expect RunChecks to return false")
+			testutil.Error(t, "Expect RunChecks to return false")
 		}
 
 		expected := []string{
diff --git a/pkg/inject/inject_test.go b/pkg/inject/inject_test.go
index e7630e952..bd949ed28 100644
--- a/pkg/inject/inject_test.go
+++ b/pkg/inject/inject_test.go
@@ -8,6 +8,7 @@ import (
 	l5dcharts "github.com/linkerd/linkerd2/pkg/charts/linkerd2"
 	"github.com/linkerd/linkerd2/pkg/k8s"
 	"github.com/linkerd/linkerd2/pkg/version"
+	"github.com/linkerd/linkerd2/testutil"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -300,7 +301,7 @@ func TestConfigAccessors(t *testing.T) {
 			t.Run("identityContext", func(t *testing.T) {
 				expected := testCase.expected.identityContext
 				if actual := resourceConfig.identityContext(); !reflect.DeepEqual(expected, actual) {
-					t.Errorf("Expected: %+v Actual: %+v", expected, actual)
+					testutil.Errorf(t, "Expected: %+v Actual: %+v", expected, actual)
 				}
 			})
 
diff --git a/test/get/get_test.go b/test/get/get_test.go
index 3450e6f24..1474d6a5f 100644
--- a/test/get/get_test.go
+++ b/test/get/get_test.go
@@ -105,6 +105,28 @@ func TestCliGet(t *testing.T) {
 			t.Fatalf("Pod output check failed:\n%s\nCommand output:\n%s", err, out)
 		}
 	})
+
+	t.Run("get pods from the default namespace of current context", func(t *testing.T) {
+		out, err := TestHelper.Kubectl("", "config", "set-context", "--namespace="+TestHelper.GetLinkerdNamespace(), "--current")
+		if err != nil {
+			t.Fatalf("Unexpected error: %v output:\n%s", err, out)
+		}
+
+		out, stderr, err = TestHelper.LinkerdRun("get", "pods")
+		if err != nil {
+			t.Fatalf("Unexpected error: %v output:\n%s\n%s", err, out, stderr)
+		}
+
+		err = checkPodOutput(out, linkerdPods, "linkerd-heartbeat", TestHelper.GetLinkerdNamespace())
+		if err != nil {
+			t.Fatalf("Pod output check failed:\n%s\nCommand output:\n%s", err, out)
+		}
+
+		out, err = TestHelper.Kubectl("", "config", "set-context", "--namespace=default", "--current")
+		if err != nil {
+			t.Fatalf("Unexpected error: %v output:\n%s", err, out)
+		}
+	})
 }
 
 func checkPodOutput(cmdOutput string, expectedPodCounts map[string]int, optionalPod string, namespace string) error {
diff --git a/test/tap/tap_test.go b/test/tap/tap_test.go
index 0e8c8f360..b2f697d2e 100644
--- a/test/tap/tap_test.go
+++ b/test/tap/tap_test.go
@@ -116,6 +116,28 @@ func TestCliTap(t *testing.T) {
 		}
 	})
 
+	t.Run("tap a deployment using context namespace", func(t *testing.T) {
+		out, err := TestHelper.Kubectl("", "config", "set-context", "--namespace="+prefixedNs, "--current")
+		if err != nil {
+			t.Fatalf("Unexpected error: %v output:\n%s", err, out)
+		}
+
+		events, err := tap("deploy/t1")
+		if err != nil {
+			t.Fatal(err.Error())
+		}
+
+		err = validateExpected(events, expectedT1)
+		if err != nil {
+			t.Fatal(err.Error())
+		}
+
+		out, err = TestHelper.Kubectl("", "config", "set-context", "--namespace=default", "--current")
+		if err != nil {
+			t.Fatalf("Unexpected error: %v output:\n%s", err, out)
+		}
+	})
+
 	t.Run("tap a disabled deployment", func(t *testing.T) {
 		out, stderr, err := TestHelper.LinkerdRun("tap", "deploy/t4", "--namespace", prefixedNs)
 		if out != "" {
diff --git a/testutil/annotations.go b/testutil/annotations.go
new file mode 100644
index 000000000..e82856b7f
--- /dev/null
+++ b/testutil/annotations.go
@@ -0,0 +1,97 @@
+package testutil
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+const (
+	envFlag  = "GH_ANNOTATION"
+	rootPath = "/linkerd2/"
+)
+
+func echoAnnotation(args ...interface{}) {
+	if _, ok := os.LookupEnv(envFlag); ok {
+		_, fileName, fileLine, ok := runtime.Caller(2)
+		if !ok {
+			panic("Couldn't recover runtime info")
+		}
+		fileName = fileName[strings.LastIndex(fileName, rootPath)+len(rootPath):]
+		for _, arg := range args {
+			fmt.Printf("::error file=%s,line=%d::%s\n", fileName, fileLine, arg)
+		}
+	}
+}
+
+// Error is a wrapper around t.Error()
+// args are passed to t.Error(args) and each arg will be sent to stdout formatted
+// as a Github annotation when the envFlag environment variable is set
+func Error(t *testing.T, args ...interface{}) {
+	t.Helper()
+	echoAnnotation(args...)
+	t.Error(args...)
+}
+
+// AnnotatedError is similar to Error() but it also admits a msg string that
+// will be used as the Github annotation
+func AnnotatedError(t *testing.T, msg string, args ...interface{}) {
+	t.Helper()
+	echoAnnotation(msg)
+	t.Error(args...)
+}
+
+// Errorf is a wrapper around t.Errorf()
+// format and args are passed to t.Errorf(format, args) and the formatted
+// message will be sent to stdout as a Github annotation when the envFlag
+// environment variable is set
+func Errorf(t *testing.T, format string, args ...interface{}) {
+	t.Helper()
+	echoAnnotation(fmt.Sprintf(format, args...))
+	t.Errorf(format, args...)
+}
+
+// AnnotatedErrorf is similar to Errorf() but it also admits a msg string that
+// will be used as the Github annotation
+func AnnotatedErrorf(t *testing.T, msg, format string, args ...interface{}) {
+	t.Helper()
+	echoAnnotation(msg)
+	t.Errorf(format, args...)
+}
+
+// Fatal is a wrapper around t.Fatal()
+// args are passed to t.Fatal(args) and each arg will be sent to stdout formatted
+// as a Github annotation when the envFlag environment variable is set
+func Fatal(t *testing.T, args ...interface{}) {
+	t.Helper()
+	echoAnnotation(args)
+	t.Fatal(args...)
+}
+
+// AnnotatedFatal is similar to Fatal() but it also admits a msg string that
+// will be used as the Github annotation
+func AnnotatedFatal(t *testing.T, msg, format string, args ...interface{}) {
+	t.Helper()
+	echoAnnotation(msg)
+	t.Fatal(args...)
+}
+
+// Fatalf is a wrapper around t.Errorf()
+// format and args are passed to t.Fatalf(format, args) and the formatted
+// message will be sent to stdout as a Github annotation when the envFlag
+// environment variable is set
+func Fatalf(t *testing.T, format string, args ...interface{}) {
+	t.Helper()
+	echoAnnotation(fmt.Sprintf(format, args...))
+	t.Fatalf(format, args...)
+}
+
+// AnnotatedFatalf is similar to Fatalf() but it also admits a msg string that
+// will be used as the Github annotation
+func AnnotatedFatalf(t *testing.T, msg, format string, args ...interface{}) {
+	t.Helper()
+	echoAnnotation(msg)
+	t.Fatalf(format, args...)
+}
diff --git a/testutil/annotations_test.go b/testutil/annotations_test.go
new file mode 100644
index 000000000..56a8a75d0
--- /dev/null
+++ b/testutil/annotations_test.go
@@ -0,0 +1,69 @@
+package testutil
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	os.Setenv(envFlag, "true")
+	os.Exit(m.Run())
+}
+
+func redirectStdout(t *testing.T) (*os.File, chan string) {
+	origStdout := os.Stdout
+	newStdout, w, pipeErr := os.Pipe()
+	if pipeErr != nil {
+		t.Fatalf("error creating os.Pipe(): %s", pipeErr)
+	}
+	os.Stdout = w
+
+	// retrieve the payload sent to newStdout in a separate goroutine
+	// to avoid blocking
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, newStdout)
+		outC <- buf.String()
+	}()
+
+	return origStdout, outC
+}
+
+func restoreStdout(outC chan string, origStdout *os.File) string {
+	os.Stdout.Close()
+	out := <-outC
+	os.Stdout = origStdout
+	return out
+}
+
+func TestError(t *testing.T) {
+	msg := "This is an error"
+
+	// redirect stdout temporarily to catch the Github annotation output
+	origStdout, outC := redirectStdout(t)
+	Error(&testing.T{}, msg)
+	out := restoreStdout(outC, origStdout)
+
+	if strings.TrimSpace(out) != "::error file=testutil/annotations_test.go,line=48::This is an error" {
+		t.Fatalf("unexpected stdout content: %s", out)
+	}
+}
+
+func TestAnnotatedErrorf(t *testing.T) {
+	msgFormat := "This is a detailed error: %s"
+	str := "foobar"
+	msgDesc := "This is a generic error"
+
+	// redirect stdout temporarily to catch the Github annotation output
+	origStdout, outC := redirectStdout(t)
+	AnnotatedErrorf(&testing.T{}, msgDesc, msgFormat, str)
+	out := restoreStdout(outC, origStdout)
+
+	if strings.TrimSpace(out) != "::error file=testutil/annotations_test.go,line=63::This is a generic error" {
+		t.Fatalf("unexpected stdout content: %s", out)
+	}
+}