Repository: submariner-io/lighthouse
Pull request #450 (2021-02-03 16:52:45)

0.8.1 backports
PR backports for 0.8.1

* #424 (4924193f) During cleanup use proper namespace for EPSlices verification (#424)
* #407 (a8ab4aa5) Add e2e to test DNS request to a specific cluster (#407)
* #422 (28632565) Add e2e to verify round-robin for Service discovery (#422)
* #430 (e990e6de) Fix issues in e2e (#430)
* #429 (9be744d9) E2e to verify lighthouse return cluster-ip from healthy service (#429)
* #435 (8f8fd426) Use the PR base branch as reference when linting (#435)
* #427 (fe9fbbe7) Remove fetching of cluster IDs in E2E (#427)
* #442 (6520558f) Use env var for ClusterId in lighthouse-coredns (#442)
* #443 (d6330df7)  Include port numbers in ServiceImports (#443)
* Bump E2E timeout to 45min
* Bump shipyard image to release-0.8
* Reclaim additional free space, the old way.


Patch:
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
index 4922204..67e051e 100644
--- a/.github/workflows/linting.yml
+++ b/.github/workflows/linting.yml
@@ -51,6 +51,7 @@ jobs:
         with:
           config-file: ".markdownlinkcheck.json"
           check-modified-files-only: "yes"
+          base-branch: ${{ github.base_ref }}
 
   markdownlint:
     name: Markdown
diff --git a/Dockerfile.dapper b/Dockerfile.dapper
index 2413737..38b3ae7 100644
--- a/Dockerfile.dapper
+++ b/Dockerfile.dapper
@@ -1,4 +1,4 @@
-FROM quay.io/submariner/shipyard-dapper-base:0.8.0
+FROM quay.io/submariner/shipyard-dapper-base:release-0.8
 
 ENV DAPPER_ENV="REPO TAG QUAY_USERNAME QUAY_PASSWORD GITHUB_SHA BUILD_ARGS CLUSTERS_ARGS DEPLOY_ARGS RELEASE_ARGS" \
     DAPPER_SOURCE=/go/src/github.com/submariner-io/lighthouse DAPPER_DOCKER_SOCKET=true
diff --git a/Makefile b/Makefile
index be81e55..c475c7f 100644
--- a/Makefile
+++ b/Makefile
@@ -8,7 +8,7 @@ TARGETS := $(shell ls -p scripts | grep -v -e / -e deploy)
 CLUSTER_SETTINGS_FLAG = --cluster_settings $(DAPPER_SOURCE)/scripts/cluster_settings
 override CLUSTERS_ARGS += $(CLUSTER_SETTINGS_FLAG)
 override DEPLOY_ARGS += $(CLUSTER_SETTINGS_FLAG)
-override E2E_ARGS += cluster1 cluster2
+override E2E_ARGS += cluster1 cluster2 cluster3
 override UNIT_TEST_ARGS += test/e2e
 
 # Process extra flags from the `using=a,b,c` optional flag
diff --git a/pkg/agent/controller/agent.go b/pkg/agent/controller/agent.go
index 06499f6..2c7b537 100644
--- a/pkg/agent/controller/agent.go
+++ b/pkg/agent/controller/agent.go
@@ -196,15 +196,10 @@ func (a *Controller) Start(stopCh <-chan struct{}) error {
 }
 
 func (a *Controller) serviceExportToServiceImport(obj runtime.Object, op syncer.Operation) (runtime.Object, bool) {
-	if op == syncer.Update {
-		return nil, false
-	}
-
 	svcExport := obj.(*mcsv1a1.ServiceExport)
-	serviceImport := a.newServiceImport(svcExport)
 
 	if op == syncer.Delete {
-		return serviceImport, false
+		return a.newServiceImport(svcExport), false
 	}
 
 	obj, found, err := a.serviceSyncer.GetResource(svcExport.Name, svcExport.Namespace)
@@ -225,6 +220,10 @@ func (a *Controller) serviceExportToServiceImport(obj runtime.Object, op syncer.
 		return nil, true
 	}
 
+	if op == syncer.Update && getLastExportConditionReason(svcExport) != serviceUnavailable {
+		return nil, false
+	}
+
 	svc := obj.(*corev1.Service)
 
 	svcType, ok := getServiceImportType(svc)
@@ -255,13 +254,15 @@ func (a *Controller) serviceExportToServiceImport(obj runtime.Object, op syncer.
 		return nil, true
 	}
 
+	serviceImport := a.newServiceImport(svcExport)
+
 	serviceImport.Spec = mcsv1a1.ServiceImportSpec{
 		Ports:                 []mcsv1a1.ServicePort{},
 		Type:                  svcType,
 		SessionAffinityConfig: new(corev1.SessionAffinityConfig),
 	}
 
-	ips, err := a.getIPsForService(svc, svcType)
+	ips, ports, err := a.getIPsAndPortsForService(svc, svcType)
 	if err != nil {
 		// Failed to get ips for some reason, requeue
 		a.updateExportedServiceStatus(svcExport.Name, svcExport.Namespace, mcsv1a1.ServiceExportValid,
@@ -280,6 +281,7 @@ func (a *Controller) serviceExportToServiceImport(obj runtime.Object, op syncer.
 
 	if svcType == mcsv1a1.ClusterSetIP {
 		serviceImport.Spec.IPs = ips
+		serviceImport.Spec.Ports = ports
 		/* We also store the clusterIP in an annotation as an optimization to recover it in case the IPs are
 		cleared out when here's no backing Endpoint pods.
 		*/
@@ -292,6 +294,15 @@ func (a *Controller) serviceExportToServiceImport(obj runtime.Object, op syncer.
 	return serviceImport, false
 }
 
+func getLastExportConditionReason(svcExport *mcsv1a1.ServiceExport) string {
+	numCond := len(svcExport.Status.Conditions)
+	if numCond > 0 && svcExport.Status.Conditions[numCond-1].Reason != nil {
+		return *svcExport.Status.Conditions[numCond-1].Reason
+	}
+
+	return ""
+}
+
 func getServiceImportType(service *corev1.Service) (mcsv1a1.ServiceImportType, bool) {
 	if service.Spec.Type != "" && service.Spec.Type != corev1.ServiceTypeClusterIP {
 		return "", false
@@ -305,7 +316,7 @@ func getServiceImportType(service *corev1.Service) (mcsv1a1.ServiceImportType, b
 }
 
 func (a *Controller) onSuccessfulServiceImportSync(synced runtime.Object, op syncer.Operation) {
-	if op != syncer.Create {
+	if op == syncer.Delete {
 		return
 	}
 
@@ -349,6 +360,9 @@ func (a *Controller) serviceToRemoteServiceImport(obj runtime.Object, op syncer.
 
 func (a *Controller) updateExportedServiceStatus(name, namespace string, condType mcsv1a1.ServiceExportConditionType,
 	status corev1.ConditionStatus, reason, msg string) {
+	klog.V(log.DEBUG).Infof("updateExportedServiceStatus for (%s/%s) - Type: %q, Status: %q, Reason: %q, Message: %q",
+		namespace, name, condType, status, reason, msg)
+
 	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
 		toUpdate, err := a.getServiceExport(name, namespace)
 		if apierrors.IsNotFound(err) {
@@ -369,7 +383,7 @@ func (a *Controller) updateExportedServiceStatus(name, namespace string, condTyp
 
 		numCond := len(toUpdate.Status.Conditions)
 		if numCond > 0 && serviceExportConditionEqual(&toUpdate.Status.Conditions[numCond-1], &exportCondition) {
-			klog.V(log.DEBUG).Infof("Last ServiceExportCondition for (%s/%s) is equal - not updating status: %#v",
+			klog.V(log.TRACE).Infof("Last ServiceExportCondition for (%s/%s) is equal - not updating status: %#v",
 				namespace, name, toUpdate.Status.Conditions[numCond-1])
 			return nil
 		}
@@ -433,27 +447,35 @@ func (a *Controller) newServiceImport(svcExport *mcsv1a1.ServiceExport) *mcsv1a1
 	}
 }
 
-func (a *Controller) getIPsForService(service *corev1.Service, siType mcsv1a1.ServiceImportType) ([]string, error) {
+func (a *Controller) getIPsAndPortsForService(service *corev1.Service, siType mcsv1a1.ServiceImportType) (
+	[]string, []mcsv1a1.ServicePort, error) {
+	mcsPort := mcsv1a1.ServicePort{}
+	if len(service.Spec.Ports) > 0 {
+		mcsPort.Name = service.Spec.Ports[0].Name
+		mcsPort.Protocol = service.Spec.Ports[0].Protocol
+		mcsPort.Port = service.Spec.Ports[0].Port
+	}
+
 	if siType == mcsv1a1.ClusterSetIP {
 		mcsIp := getGlobalIpFromService(service)
 		if mcsIp == "" {
 			mcsIp = service.Spec.ClusterIP
 		}
 
-		return []string{mcsIp}, nil
+		return []string{mcsIp}, []mcsv1a1.ServicePort{mcsPort}, nil
 	}
 
 	endpoint, err := a.kubeClientSet.CoreV1().Endpoints(service.Namespace).Get(service.Name, metav1.GetOptions{})
 	if err != nil {
 		if !apierrors.IsNotFound(err) {
 			klog.Errorf("Error retrieving Endpoints for Service (%s/%s): %v", service.Namespace, service.Name, err)
-			return nil, errors.WithMessage(err, "Error retrieving the Endpoints for the Service")
+			return nil, nil, errors.WithMessage(err, "Error retrieving the Endpoints for the Service")
 		}
 
-		return make([]string, 0), nil
+		return make([]string, 0), make([]mcsv1a1.ServicePort, 0), nil
 	}
 
-	return getIPsFromEndpoint(endpoint), nil
+	return getIPsFromEndpoint(endpoint), []mcsv1a1.ServicePort{mcsPort}, nil
 }
 
 func (a *Controller) getObjectNameWithClusterId(name, namespace string) string {
diff --git a/pkg/gateway/controller.go b/pkg/gateway/controller.go
index 08ec026..344c88a 100644
--- a/pkg/gateway/controller.go
+++ b/pkg/gateway/controller.go
@@ -2,6 +2,7 @@ package gateway
 
 import (
 	"fmt"
+	"os"
 	"sync/atomic"
 
 	"github.com/submariner-io/admiral/pkg/log"
@@ -41,8 +42,13 @@ func NewController() *Controller {
 		stopCh:           make(chan struct{}),
 		gatewayAvailable: true,
 	}
+
 	controller.clusterStatusMap.Store(make(map[string]bool))
-	controller.localClusterID.Store("")
+
+	localClusterId := os.Getenv("SUBMARINER_CLUSTERID")
+
+	klog.Infof("Setting localClusterID from env: %q", localClusterId)
+	controller.localClusterID.Store(localClusterId)
 
 	return controller
 }
diff --git a/scripts/cluster_settings b/scripts/cluster_settings
index 68cba0e..5f61cbd 100644
--- a/scripts/cluster_settings
+++ b/scripts/cluster_settings
@@ -1,10 +1,12 @@
 . "${SCRIPTS_DIR}"/lib/source_only
 
 # We need a minimal setup to verify that lighthouse works
-clusters=('cluster1' 'cluster2')
+clusters=('cluster1' 'cluster2' 'cluster3')
 cluster_nodes['cluster1']="control-plane worker worker"
 cluster_nodes['cluster2']="control-plane worker worker"
+cluster_nodes['cluster3']="control-plane worker"
 
-cluster_cni=( ['cluster1']="weave" ['cluster2']="weave" )
+cluster_cni=( ['cluster1']="weave" ['cluster2']="weave" ['cluster3']="weave" )
+
+cluster_subm=( ['cluster1']="true" ['cluster2']="true"  ['cluster3']="true" )
 
-cluster_subm=( ['cluster1']="true" ['cluster2']="true" )


Test patch:
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index d3da417..45b797a 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -7,7 +7,7 @@ on:
 jobs:
   e2e:
     name: E2E
-    timeout-minutes: 30
+    timeout-minutes: 45
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -20,8 +20,8 @@ jobs:
 
       - name: Reclaim free space
         run: |
-          sudo swapoff -a
-          sudo rm -f /swapfile
+          # cleanup 30GB+ of tools we don't need for our CI
+          rm -rf /opt/ghc /usr/share/dotnet /usr/share/swift
           df -h
           free -h
 
diff --git a/.github/workflows/upgrade-e2e.yml b/.github/workflows/upgrade-e2e.yml
index 18af893..fb59e16 100644
--- a/.github/workflows/upgrade-e2e.yml
+++ b/.github/workflows/upgrade-e2e.yml
@@ -14,10 +14,10 @@ jobs:
     steps:
       - uses: actions/checkout@master
 
-      - name: Reclaim free space!
+      - name: Reclaim free space
         run: |
-          sudo swapoff -a
-          sudo rm -f /swapfile
+          # cleanup 30GB+ of tools we don't need for our CI
+          rm -rf /opt/ghc /usr/share/dotnet /usr/share/swift
           df -h
           free -h
 
diff --git a/pkg/agent/controller/agent_test.go b/pkg/agent/controller/agent_test.go
index 66d777d..59affa5 100644
--- a/pkg/agent/controller/agent_test.go
+++ b/pkg/agent/controller/agent_test.go
@@ -86,8 +86,8 @@ var _ = Describe("ServiceImport syncing", func() {
 		})
 	})
 
-	When("an exported Service is deleted while the ServiceExport still exists", func() {
-		It("should delete the ServiceImport", func() {
+	When("an exported Service is deleted and recreated while the ServiceExport still exists", func() {
+		It("should delete and recreate the ServiceImport", func() {
 			t.createService()
 			t.createServiceExport()
 			nextStatusIndex := t.awaitServiceExported(t.service.Spec.ClusterIP, 0)
@@ -95,6 +95,9 @@ var _ = Describe("ServiceImport syncing", func() {
 			t.deleteService()
 			t.awaitServiceUnexported()
 			t.awaitServiceUnavailableStatus(nextStatusIndex)
+
+			t.createService()
+			t.awaitServiceExported(t.service.Spec.ClusterIP, nextStatusIndex+1)
 		})
 	})
 
diff --git a/test/e2e/discovery/headless_services.go b/test/e2e/discovery/headless_services.go
index 9341fef..117b956 100644
--- a/test/e2e/discovery/headless_services.go
+++ b/test/e2e/discovery/headless_services.go
@@ -43,6 +43,16 @@ var _ = Describe("[discovery] Test Headless Service Discovery Across Clusters",
 			}
 		})
 	})
+
+	When("a pod tries to resolve a headless service in a specific remote cluster by its cluster name", func() {
+		It("should resolve the backing pod IPs from the specified remote cluster", func() {
+			if !framework.TestContext.GlobalnetEnabled {
+				RunHeadlessDiscoveryClusterNameTest(f)
+			} else {
+				framework.Skipf("Globalnet is enabled, skipping the test...")
+			}
+		})
+	})
 })
 
 func RunHeadlessDiscoveryTest(f *lhframework.Framework) {
@@ -65,12 +75,16 @@ func RunHeadlessDiscoveryTest(f *lhframework.Framework) {
 
 	ipList := f.GetEndpointIPs(framework.ClusterB, nginxHeadlessClusterB.Name, nginxHeadlessClusterB.Namespace)
 
-	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipList, checkedDomains, true)
+	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipList, checkedDomains,
+		"", true)
+	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipList, checkedDomains,
+		clusterBName, true)
 
 	f.DeleteServiceExport(framework.ClusterB, nginxHeadlessClusterB.Name, nginxHeadlessClusterB.Namespace)
 	f.AwaitServiceImportCount(framework.ClusterA, nginxHeadlessClusterB.Name, nginxHeadlessClusterB.Namespace, 0)
 
-	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipList, checkedDomains, false)
+	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipList, checkedDomains,
+		"", false)
 }
 
 func RunHeadlessDiscoveryLocalAndRemoteTest(f *lhframework.Framework) {
@@ -103,13 +117,16 @@ func RunHeadlessDiscoveryLocalAndRemoteTest(f *lhframework.Framework) {
 	ipListA := f.GetEndpointIPs(framework.ClusterA, nginxHeadlessClusterA.Name, nginxHeadlessClusterA.Namespace)
 	ipList := append(ipListB, ipListA...)
 
-	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipList, checkedDomains, true)
+	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipList, checkedDomains,
+		"", true)
 
 	f.DeleteServiceExport(framework.ClusterB, nginxHeadlessClusterB.Name, nginxHeadlessClusterB.Namespace)
 	f.AwaitServiceImportCount(framework.ClusterA, nginxHeadlessClusterB.Name, nginxHeadlessClusterB.Namespace, 1)
 
-	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipListB, checkedDomains, false)
-	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipListA, checkedDomains, true)
+	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipListB, checkedDomains,
+		"", false)
+	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipListA, checkedDomains,
+		"", true)
 }
 
 func RunHeadlessPodsAvailabilityTest(f *lhframework.Framework) {
@@ -133,22 +150,67 @@ func RunHeadlessPodsAvailabilityTest(f *lhframework.Framework) {
 	netshootPodList := f.NewNetShootDeployment(framework.ClusterA)
 
 	ipList := f.AwaitEndpointIPs(framework.ClusterB, nginxHeadlessClusterB.Name, nginxHeadlessClusterB.Namespace, 3)
-	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipList, checkedDomains, true)
+	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipList, checkedDomains,
+		"", true)
 
 	f.SetNginxReplicaSet(framework.ClusterB, 0)
 	ipList = f.AwaitEndpointIPs(framework.ClusterB, nginxHeadlessClusterB.Name, nginxHeadlessClusterB.Namespace, 0)
-	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipList, checkedDomains, false)
+	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipList, checkedDomains,
+		"", false)
 
 	f.SetNginxReplicaSet(framework.ClusterB, 2)
 	ipList = f.AwaitEndpointIPs(framework.ClusterB, nginxHeadlessClusterB.Name, nginxHeadlessClusterB.Namespace, 2)
-	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipList, checkedDomains, true)
+	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipList, checkedDomains,
+		"", true)
+}
+
+func RunHeadlessDiscoveryClusterNameTest(f *lhframework.Framework) {
+	clusterAName := framework.TestContext.ClusterIDs[framework.ClusterA]
+	clusterBName := framework.TestContext.ClusterIDs[framework.ClusterB]
+
+	By(fmt.Sprintf("Creating an Nginx Deployment on on %q", clusterAName))
+	f.NewNginxDeployment(framework.ClusterA)
+
+	By(fmt.Sprintf("Creating a Nginx Headless Service on %q", clusterAName))
+
+	nginxHeadlessClusterA := f.NewNginxHeadlessService(framework.ClusterA)
+
+	f.NewServiceExport(framework.ClusterA, nginxHeadlessClusterA.Name, nginxHeadlessClusterA.Namespace)
+	f.AwaitServiceExportedStatusCondition(framework.ClusterA, nginxHeadlessClusterA.Name, nginxHeadlessClusterA.Namespace)
+
+	By(fmt.Sprintf("Creating an Nginx Deployment on on %q", clusterBName))
+	f.NewNginxDeployment(framework.ClusterB)
+
+	By(fmt.Sprintf("Creating a Nginx Headless Service on %q", clusterBName))
+
+	nginxHeadlessClusterB := f.NewNginxHeadlessService(framework.ClusterB)
+
+	f.NewServiceExport(framework.ClusterB, nginxHeadlessClusterB.Name, nginxHeadlessClusterB.Namespace)
+	f.AwaitServiceExportedStatusCondition(framework.ClusterB, nginxHeadlessClusterB.Name, nginxHeadlessClusterB.Namespace)
+
+	By(fmt.Sprintf("Creating a Netshoot Deployment on %q", clusterAName))
+
+	netshootPodList := f.NewNetShootDeployment(framework.ClusterA)
+
+	ipListClusterA := f.GetEndpointIPs(framework.ClusterA, nginxHeadlessClusterA.Name, nginxHeadlessClusterA.Namespace)
+	ipListClusterB := f.GetEndpointIPs(framework.ClusterB, nginxHeadlessClusterB.Name, nginxHeadlessClusterB.Namespace)
+
+	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterA, netshootPodList, ipListClusterA, checkedDomains,
+		clusterAName, true)
+	verifyHeadlessIpsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipListClusterB, checkedDomains,
+		clusterBName, true)
 }
 
 func verifyHeadlessIpsWithDig(f *framework.Framework, cluster framework.ClusterIndex, service *corev1.Service, targetPod *corev1.PodList,
-	ipList, domains []string, shouldContain bool) {
+	ipList, domains []string, clusterName string, shouldContain bool) {
 	cmd := []string{"dig", "+short"}
+	var clusterDNSName string
+	if clusterName != "" {
+		clusterDNSName = clusterName + "."
+	}
+
 	for i := range domains {
-		cmd = append(cmd, service.Name+"."+f.Namespace+".svc."+domains[i])
+		cmd = append(cmd, clusterDNSName+service.Name+"."+f.Namespace+".svc."+domains[i])
 	}
 
 	op := "are"
diff --git a/test/e2e/discovery/service_discovery.go b/test/e2e/discovery/service_discovery.go
index 0ab6c0f..bc29713 100644
--- a/test/e2e/discovery/service_discovery.go
+++ b/test/e2e/discovery/service_discovery.go
@@ -2,10 +2,12 @@ package discovery
 
 import (
 	"fmt"
+	"math"
 	"strconv"
 	"strings"
 
 	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 	lhframework "github.com/submariner-io/lighthouse/test/e2e/framework"
 	"github.com/submariner-io/shipyard/test/e2e/framework"
 	corev1 "k8s.io/api/core/v1"
@@ -53,6 +55,40 @@ var _ = Describe("[discovery] Test Service Discovery Across Clusters", func() {
 		})
 	})
 
+	When("a pod tries to resolve a service in a specific remote cluster by its cluster name", func() {
+		It("should resolve the service on the specified cluster", func() {
+			RunServiceDiscoveryClusterNameTest(f)
+		})
+	})
+
+	When("a pod tries to resolve a service multiple times", func() {
+		It("should resolve the service from both the clusters in a round robin fashion", func() {
+			RunServiceDiscoveryRoundRobinTest(f)
+		})
+	})
+
+	When("one of the clusters with a service is not healthy", func() {
+		var healthCheckIP, endpointName string
+
+		BeforeEach(func() {
+			if len(framework.TestContext.ClusterIDs) < 3 {
+				Skip("Only two clusters are deployed and hence skipping the test")
+				return
+			}
+
+			randomIP := "192.168.1.5"
+			endpointName, healthCheckIP = f.GetHealthCheckIPInfo(framework.ClusterC)
+			f.SetHealthCheckIP(framework.ClusterC, randomIP, endpointName)
+		})
+
+		It("should not resolve that cluster's service IP", func() {
+			RunServicesClusterAvailabilityMutliClusterTest(f)
+		})
+
+		AfterEach(func() {
+			f.SetHealthCheckIP(framework.ClusterC, healthCheckIP, endpointName)
+		})
+	})
 })
 
 func RunServiceDiscoveryTest(f *lhframework.Framework) {
@@ -75,21 +111,24 @@ func RunServiceDiscoveryTest(f *lhframework.Framework) {
 
 	netshootPodList := f.NewNetShootDeployment(framework.ClusterA)
 
-	if svc, err := f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace); err == nil {
-		nginxServiceClusterB = svc
-		f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterB)
-		f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
-		f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
-	}
+	svc, err := f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+	Expect(err).NotTo(HaveOccurred())
+
+	nginxServiceClusterB = svc
+	f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterB)
+	f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
+	f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
 
-	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains, true)
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains,
+		"", true)
 
 	f.DeleteServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
 	f.AwaitServiceImportDelete(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
 
 	f.DeleteService(framework.ClusterB, nginxServiceClusterB.Name)
 
-	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains, false)
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains,
+		"", false)
 }
 
 func RunServiceDiscoveryLocalTest(f *lhframework.Framework) {
@@ -121,25 +160,28 @@ func RunServiceDiscoveryLocalTest(f *lhframework.Framework) {
 	clusterADomain := getClusterDomain(f.Framework, framework.ClusterA, netshootPodList)
 
 	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterA, nginxServiceClusterA, netshootPodList,
-		[]string{clusterADomain}, true)
+		[]string{clusterADomain}, "", true)
 
 	f.DeleteService(framework.ClusterA, nginxServiceClusterA.Name)
 
-	if svc, err := f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace); err == nil {
-		nginxServiceClusterB = svc
-		f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterB)
-		f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
-		f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
-	}
+	svc, err := f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+	Expect(err).NotTo(HaveOccurred())
+
+	nginxServiceClusterB = svc
+	f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterB)
+	f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
+	f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
 
-	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains, true)
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains,
+		"", true)
 
 	f.DeleteServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
 	f.AwaitServiceImportDelete(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
 
 	f.DeleteService(framework.ClusterB, nginxServiceClusterB.Name)
 
-	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains, false)
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains,
+		"", false)
 }
 
 func RunServiceExportTest(f *lhframework.Framework) {
@@ -161,19 +203,22 @@ func RunServiceExportTest(f *lhframework.Framework) {
 
 	netshootPodList := f.NewNetShootDeployment(framework.ClusterA)
 
-	if svc, err := f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace); err == nil {
-		nginxServiceClusterB = svc
-		f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterB)
-		f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
-		f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
-	}
+	svc, err := f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+	Expect(err).NotTo(HaveOccurred())
+
+	nginxServiceClusterB = svc
+	f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterB)
+	f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
+	f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
 
-	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains, true)
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains,
+		"", true)
 
 	f.DeleteServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
 	f.AwaitServiceImportDelete(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
 
-	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains, false)
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains,
+		"", false)
 }
 
 func RunServicesPodAvailabilityTest(f *lhframework.Framework) {
@@ -195,26 +240,37 @@ func RunServicesPodAvailabilityTest(f *lhframework.Framework) {
 
 	netshootPodList := f.NewNetShootDeployment(framework.ClusterA)
 
-	if svc, err := f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace); err == nil {
-		nginxServiceClusterB = svc
-		f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterB)
-		f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
-		f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
-	}
+	svc, err := f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+	Expect(err).NotTo(HaveOccurred())
 
-	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains, true)
+	nginxServiceClusterB = svc
+	f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterB)
+	f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
+	f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
+
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains,
+		"", true)
 	f.SetNginxReplicaSet(framework.ClusterB, 0)
-	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains, false)
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains,
+		"", false)
 	f.SetNginxReplicaSet(framework.ClusterB, 2)
-	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains, true)
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains,
+		"", true)
 }
 
 func RunServicesPodAvailabilityMutliClusterTest(f *lhframework.Framework) {
+	if len(framework.TestContext.ClusterIDs) < 3 {
+		Skip("Only two clusters are deployed and hence skipping the test")
+		return
+	}
+
 	clusterAName := framework.TestContext.ClusterIDs[framework.ClusterA]
 	clusterBName := framework.TestContext.ClusterIDs[framework.ClusterB]
+	clusterCName := framework.TestContext.ClusterIDs[framework.ClusterC]
 
-	By(fmt.Sprintf("Creating an Nginx Deployment on on %q", clusterBName))
+	By(fmt.Sprintf("Creating an Nginx Deployment on %q", clusterBName))
 	f.NewNginxDeployment(framework.ClusterB)
+
 	By(fmt.Sprintf("Creating a Nginx Service on %q", clusterBName))
 
 	nginxServiceClusterB := f.NewNginxService(framework.ClusterB)
@@ -224,51 +280,228 @@ func RunServicesPodAvailabilityMutliClusterTest(f *lhframework.Framework) {
 
 	f.AwaitServiceExportedStatusCondition(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
 
+	By(fmt.Sprintf("Creating an Nginx Deployment on %q", clusterCName))
+	f.NewNginxDeployment(framework.ClusterC)
+
+	By(fmt.Sprintf("Creating a Nginx Service on %q", clusterCName))
+
+	nginxServiceClusterC := f.Framework.NewNginxService(framework.ClusterC)
+
+	f.AwaitGlobalnetIP(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+	f.NewServiceExport(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+
 	By(fmt.Sprintf("Creating a Netshoot Deployment on %q", clusterAName))
 
 	netshootPodList := f.NewNetShootDeployment(framework.ClusterA)
 
-	if svc, err := f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace); err == nil {
-		nginxServiceClusterB = svc
-		f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterB)
-		f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
-		f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
-	}
+	svc, err := f.GetService(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+	Expect(err).NotTo(HaveOccurred())
+
+	nginxServiceClusterC = svc
+	f.AwaitServiceImportIP(framework.ClusterC, nginxServiceClusterC)
+	f.AwaitEndpointSlices(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace, 2, 2)
+
+	svc, err = f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+	Expect(err).NotTo(HaveOccurred())
+
+	nginxServiceClusterB = svc
+	f.AwaitServiceImportIP(framework.ClusterB, nginxServiceClusterB)
+	f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 2, 2)
+
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains,
+		"", true)
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterC, nginxServiceClusterC, netshootPodList, checkedDomains,
+		"", true)
 
-	By(fmt.Sprintf("Creating an Nginx Deployment on on %q", clusterAName))
+	f.SetNginxReplicaSet(framework.ClusterC, 0)
+
+	f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace, 2, 1)
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterA, nginxServiceClusterC, netshootPodList, checkedDomains,
+		"", false)
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains,
+		"", true)
+
+	f.SetNginxReplicaSet(framework.ClusterB, 0)
+	f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace, 2, 0)
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterA, nginxServiceClusterC, netshootPodList, checkedDomains,
+		"", false)
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains,
+		"", false)
+}
+
+func RunServiceDiscoveryClusterNameTest(f *lhframework.Framework) {
+	clusterAName := framework.TestContext.ClusterIDs[framework.ClusterA]
+	clusterBName := framework.TestContext.ClusterIDs[framework.ClusterB]
+
+	By(fmt.Sprintf("Creating an Nginx Deployment on %q", clusterAName))
 	f.NewNginxDeployment(framework.ClusterA)
+
 	By(fmt.Sprintf("Creating a Nginx Service on %q", clusterAName))
 
-	nginxServiceClusterA := f.NewNginxService(framework.ClusterA)
+	nginxServiceClusterA := f.Framework.NewNginxService(framework.ClusterA)
 
 	f.AwaitGlobalnetIP(framework.ClusterA, nginxServiceClusterA.Name, nginxServiceClusterA.Namespace)
 	f.NewServiceExport(framework.ClusterA, nginxServiceClusterA.Name, nginxServiceClusterA.Namespace)
 
-	f.AwaitServiceExportedStatusCondition(framework.ClusterA, nginxServiceClusterA.Name, nginxServiceClusterA.Namespace)
+	By(fmt.Sprintf("Creating an Nginx Deployment on %q", clusterBName))
+	f.NewNginxDeployment(framework.ClusterB)
+
+	By(fmt.Sprintf("Creating a Nginx Service on %q", clusterBName))
+
+	nginxServiceClusterB := f.NewNginxService(framework.ClusterB)
+
+	f.AwaitGlobalnetIP(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+	f.NewServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+
+	f.AwaitServiceExportedStatusCondition(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+
+	By(fmt.Sprintf("Creating a Netshoot Deployment on %q", clusterAName))
+
+	netshootPodList := f.NewNetShootDeployment(framework.ClusterA)
+
+	svc, err := f.GetService(framework.ClusterA, nginxServiceClusterA.Name, nginxServiceClusterA.Namespace)
+	Expect(err).NotTo(HaveOccurred())
+
+	nginxServiceClusterA = svc
+	f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterA)
+	f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterA.Name, nginxServiceClusterA.Namespace, 2, 2)
+
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterA, nginxServiceClusterA, netshootPodList, checkedDomains,
+		clusterAName, true)
+
+	svc, err = f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+	Expect(err).NotTo(HaveOccurred())
+
+	nginxServiceClusterB = svc
+	f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterB)
+	f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 2, 2)
+
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains,
+		clusterBName, true)
+}
+
+func RunServiceDiscoveryRoundRobinTest(f *lhframework.Framework) {
+	if len(framework.TestContext.ClusterIDs) < 3 {
+		Skip("Only two clusters are deployed and hence skipping the test")
+		return
+	}
+
+	clusterAName := framework.TestContext.ClusterIDs[framework.ClusterA]
+	clusterBName := framework.TestContext.ClusterIDs[framework.ClusterB]
+	clusterCName := framework.TestContext.ClusterIDs[framework.ClusterC]
+
+	By(fmt.Sprintf("Creating an Nginx Deployment on %q", clusterBName))
+	f.NewNginxDeployment(framework.ClusterB)
 
-	if svc, err := f.GetService(framework.ClusterA, nginxServiceClusterA.Name, nginxServiceClusterA.Namespace); err == nil {
-		nginxServiceClusterA = svc
-		f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterA)
-		f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterA.Name, nginxServiceClusterA.Namespace, 2, 2)
+	By(fmt.Sprintf("Creating a Nginx Service on %q", clusterBName))
+
+	nginxServiceClusterB := f.NewNginxService(framework.ClusterB)
+
+	f.AwaitGlobalnetIP(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+	f.NewServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+
+	f.AwaitServiceExportedStatusCondition(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+
+	By(fmt.Sprintf("Creating an Nginx Deployment on %q", clusterCName))
+	f.NewNginxDeployment(framework.ClusterC)
+
+	By(fmt.Sprintf("Creating a Nginx Service on %q", clusterCName))
+
+	nginxServiceClusterC := f.Framework.NewNginxService(framework.ClusterC)
+
+	f.AwaitGlobalnetIP(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+	f.NewServiceExport(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+
+	By(fmt.Sprintf("Creating a Netshoot Deployment on %q", clusterAName))
+
+	netshootPodList := f.NewNetShootDeployment(framework.ClusterA)
+
+	svc, err := f.GetService(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+	Expect(err).NotTo(HaveOccurred())
+
+	nginxServiceClusterC = svc
+	f.AwaitServiceImportIP(framework.ClusterC, nginxServiceClusterC)
+	f.AwaitEndpointSlices(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace, 2, 2)
+
+	svc, err = f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+	Expect(err).NotTo(HaveOccurred())
+
+	nginxServiceClusterB = svc
+	f.AwaitServiceImportIP(framework.ClusterB, nginxServiceClusterB)
+	f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 2, 2)
+
+	var serviceIPList []string
+
+	serviceIPClusterB, ok := nginxServiceClusterB.Annotations[submarinerIpamGlobalIp]
+	if !ok {
+		serviceIPClusterB = nginxServiceClusterB.Spec.ClusterIP
 	}
 
-	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterA, nginxServiceClusterA, netshootPodList, checkedDomains, true)
-	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains, false)
+	serviceIPList = append(serviceIPList, serviceIPClusterB)
 
-	f.SetNginxReplicaSet(framework.ClusterA, 0)
+	serviceIPClusterC, ok := nginxServiceClusterC.Annotations[submarinerIpamGlobalIp]
+	if !ok {
+		serviceIPClusterC = nginxServiceClusterC.Spec.ClusterIP
+	}
 
-	f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterA.Name, nginxServiceClusterA.Namespace, 2, 1)
-	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterA, nginxServiceClusterA, netshootPodList, checkedDomains, false)
-	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains, true)
+	serviceIPList = append(serviceIPList, serviceIPClusterC)
 
-	f.SetNginxReplicaSet(framework.ClusterB, 0)
-	f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterA.Name, nginxServiceClusterA.Namespace, 2, 0)
-	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterA, nginxServiceClusterA, netshootPodList, checkedDomains, false)
-	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains, false)
+	verifyRoundRobinWithDig(f.Framework, framework.ClusterA, nginxServiceClusterB.Name, serviceIPList, netshootPodList, checkedDomains)
+}
+
+func RunServicesClusterAvailabilityMutliClusterTest(f *lhframework.Framework) {
+	clusterBName := framework.TestContext.ClusterIDs[framework.ClusterB]
+	clusterCName := framework.TestContext.ClusterIDs[framework.ClusterC]
+
+	By(fmt.Sprintf("Creating an Nginx Deployment on on %q", clusterBName))
+	f.NewNginxDeployment(framework.ClusterB)
+	By(fmt.Sprintf("Creating a Nginx Service on %q", clusterBName))
+
+	nginxServiceClusterB := f.NewNginxService(framework.ClusterB)
+
+	f.AwaitGlobalnetIP(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+	f.NewServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+
+	f.AwaitServiceExportedStatusCondition(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+
+	By(fmt.Sprintf("Creating a Netshoot Deployment on %q", clusterCName))
+
+	netshootPodList := f.NewNetShootDeployment(framework.ClusterA)
+
+	svc, err := f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+	Expect(err).NotTo(HaveOccurred())
+
+	nginxServiceClusterB = svc
+	f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterB)
+	f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
+	f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
+
+	By(fmt.Sprintf("Creating an Nginx Deployment on on %q", clusterCName))
+	f.NewNginxDeployment(framework.ClusterC)
+	By(fmt.Sprintf("Creating a Nginx Service on %q", clusterCName))
+
+	nginxServiceClusterC := f.NewNginxService(framework.ClusterC)
+
+	f.AwaitGlobalnetIP(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+	f.NewServiceExport(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+
+	f.AwaitServiceExportedStatusCondition(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+
+	svc, err = f.GetService(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+	Expect(err).NotTo(HaveOccurred())
+
+	nginxServiceClusterC = svc
+	f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterC)
+	f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace, 2, 2)
+
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList,
+		checkedDomains, "", true)
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterC, nginxServiceClusterC, netshootPodList,
+		checkedDomains, "", false)
 }
 
 func verifyServiceIpWithDig(f *framework.Framework, srcCluster, targetCluster framework.ClusterIndex, service *corev1.Service,
-	targetPod *corev1.PodList, domains []string, shouldContain bool) {
+	targetPod *corev1.PodList, domains []string, clusterName string, shouldContain bool) {
 	var serviceIP string
 	var ok bool
 
@@ -278,8 +511,14 @@ func verifyServiceIpWithDig(f *framework.Framework, srcCluster, targetCluster fr
 	}
 
 	cmd := []string{"dig", "+short"}
+
+	var clusterDNSName string
+	if clusterName != "" {
+		clusterDNSName = clusterName + "."
+	}
+
 	for i := range domains {
-		cmd = append(cmd, service.Name+"."+f.Namespace+".svc."+domains[i])
+		cmd = append(cmd, clusterDNSName+service.Name+"."+f.Namespace+".svc."+domains[i])
 	}
 
 	op := "is"
@@ -317,6 +556,58 @@ func verifyServiceIpWithDig(f *framework.Framework, srcCluster, targetCluster fr
 	})
 }
 
+func verifyRoundRobinWithDig(f *framework.Framework, srcCluster framework.ClusterIndex, serviceName string, serviceIPList []string,
+	targetPod *corev1.PodList, domains []string) {
+	cmd := []string{"dig", "+short"}
+
+	for i := range domains {
+		cmd = append(cmd, serviceName+"."+f.Namespace+".svc."+domains[i])
+	}
+
+	serviceIPMap := make(map[string]int)
+
+	By(fmt.Sprintf("Executing %q to verify IPs %q for service %q are discoverable in a"+
+		" round-robin fashion", strings.Join(cmd, " "), serviceIPList, serviceName))
+
+	var retIPs []string
+
+	for count := 0; count < 10; count++ {
+		framework.AwaitUntil("verify if service IP is discoverable", func() (interface{}, error) {
+			stdout, _, err := f.ExecWithOptions(framework.ExecOptions{
+				Command:       cmd,
+				Namespace:     f.Namespace,
+				PodName:       targetPod.Items[0].Name,
+				ContainerName: targetPod.Items[0].Spec.Containers[0].Name,
+				CaptureStdout: true,
+				CaptureStderr: true,
+			}, srcCluster)
+			if err != nil {
+				return nil, err
+			}
+
+			return stdout, nil
+		}, func(result interface{}) (bool, string, error) {
+			for _, serviceIP := range serviceIPList {
+				if strings.Contains(result.(string), serviceIP) {
+					serviceIPMap[serviceIP]++
+					retIPs = append(retIPs, serviceIP)
+					break
+				}
+			}
+
+			return true, "", nil
+		})
+	}
+
+	By(fmt.Sprintf("Service IP %q was returned %d times and Service IP %q was returned %d times - "+
+		"verifying the difference between them is within the threshold", serviceIPList[0], serviceIPMap[serviceIPList[0]],
+		serviceIPList[1], serviceIPMap[serviceIPList[1]]))
+
+	Expect(int(math.Abs(float64(serviceIPMap[serviceIPList[0]]-serviceIPMap[serviceIPList[1]]))) < 3).To(BeTrue(),
+		"Service IPs were not returned in proper round-robin fashion: Expected IPs: %v,"+
+			" Returned IPs: %v, IP Counts: %v", serviceIPList, retIPs, serviceIPMap)
+}
+
 func getClusterDomain(f *framework.Framework, cluster framework.ClusterIndex, targetPod *corev1.PodList) string {
 	/*
 		Kubernetes adds --cluster-domain config to all pods' /etc/resolve.conf exactly as follows:
diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go
index fc97fa7..21778f9 100644
--- a/test/e2e/framework/framework.go
+++ b/test/e2e/framework/framework.go
@@ -4,6 +4,11 @@ import (
 	"fmt"
 	"strings"
 
+	"k8s.io/client-go/dynamic"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	lhconstants "github.com/submariner-io/lighthouse/pkg/constants"
@@ -34,6 +39,7 @@ type Framework struct {
 }
 
 var MCSClients []*mcsClientset.Clientset
+var EndpointClients []dynamic.ResourceInterface
 
 func init() {
 	framework.AddBeforeSuite(beforeSuite)
@@ -46,10 +52,11 @@ func NewFramework(baseName string) *Framework {
 	BeforeEach(f.BeforeEach)
 
 	AfterEach(func() {
+		namespace := f.Namespace
 		f.AfterEach()
 
-		f.AwaitEndpointSlices(framework.ClusterB, "", f.Namespace, 0, 0)
-		f.AwaitEndpointSlices(framework.ClusterA, "", f.Namespace, 0, 0)
+		f.AwaitEndpointSlices(framework.ClusterB, "", namespace, 0, 0)
+		f.AwaitEndpointSlices(framework.ClusterA, "", namespace, 0, 0)
 	})
 
 	return f
@@ -60,21 +67,7 @@ func beforeSuite() {
 
 	for _, restConfig := range framework.RestConfigs {
 		MCSClients = append(MCSClients, createLighthouseClient(restConfig))
-	}
-
-	framework.By("Fetching ClusterIDs")
-
-	for idx, kubeClient := range framework.KubeClients {
-		agentDeployment, err :=
-			kubeClient.AppsV1().Deployments(framework.TestContext.SubmarinerNamespace).Get("submariner-lighthouse-agent", metav1.GetOptions{})
-		Expect(err).To(Succeed())
-
-		for _, envVar := range agentDeployment.Spec.Template.Spec.Containers[0].Env {
-			if envVar.Name == "SUBMARINER_CLUSTERID" {
-				framework.TestContext.ClusterIDs[idx] = envVar.Value
-				break
-			}
-		}
+		EndpointClients = append(EndpointClients, createEndpointClientSet(restConfig))
 	}
 
 	framework.DetectGlobalnet()
@@ -87,6 +80,18 @@ func createLighthouseClient(restConfig *rest.Config) *mcsClientset.Clientset {
 	return clientSet
 }
 
+func createEndpointClientSet(restConfig *rest.Config) dynamic.ResourceInterface {
+	clientSet, err := dynamic.NewForConfig(restConfig)
+	Expect(err).To(Not(HaveOccurred()))
+
+	gvr, _ := schema.ParseResourceArg("endpoints.v1.submariner.io")
+	endpointsClient := clientSet.Resource(*gvr).Namespace("submariner-operator")
+	_, err = endpointsClient.List(metav1.ListOptions{})
+	Expect(err).To(Not(HaveOccurred()))
+
+	return endpointsClient
+}
+
 func (f *Framework) NewServiceExport(cluster framework.ClusterIndex, name, namespace string) *mcsv1a1.ServiceExport {
 	nginxServiceExport := mcsv1a1.ServiceExport{
 		ObjectMeta: metav1.ObjectMeta{
@@ -415,3 +420,44 @@ func (f *Framework) SetNginxStatefulSetReplicas(cluster framework.ClusterIndex,
 
 	return result
 }
+
+func (f *Framework) GetHealthCheckIPInfo(cluster framework.ClusterIndex) (endpointName, healthCheckIP string) {
+	framework.AwaitUntil("Get healthCheckIP", func() (interface{}, error) {
+		unstructuredEndpointList, err := EndpointClients[cluster].List(metav1.ListOptions{})
+		return unstructuredEndpointList, err
+	}, func(result interface{}) (bool, string, error) {
+		unstructuredEndpointList := result.(*unstructured.UnstructuredList)
+		for _, endpoint := range unstructuredEndpointList.Items {
+			By(fmt.Sprintf("Getting the endpoint %s, for cluster %s", endpoint.GetName(), framework.TestContext.ClusterIDs[cluster]))
+
+			if strings.Contains(endpoint.GetName(), framework.TestContext.ClusterIDs[cluster]) {
+				endpointName = endpoint.GetName()
+
+				var found bool
+				var err error
+				healthCheckIP, found, err = unstructured.NestedString(endpoint.Object, "spec", "healthCheckIP")
+
+				if err != nil {
+					return false, "", err
+				}
+
+				if !found {
+					return false, fmt.Sprintf("HealthcheckIP not found in %#v ", endpoint), nil
+				}
+			}
+		}
+		return true, "", nil
+	})
+
+	return endpointName, healthCheckIP
+}
+
+func (f *Framework) SetHealthCheckIP(cluster framework.ClusterIndex, ip, endpointName string) {
+	By(fmt.Sprintf("Setting health check IP cluster %q to %v", framework.TestContext.ClusterIDs[cluster], ip))
+	patch := fmt.Sprintf(`{"spec":{"healthCheckIP":%q}}`, ip)
+
+	framework.AwaitUntil("set healthCheckIP", func() (interface{}, error) {
+		endpoint, err := EndpointClients[cluster].Patch(endpointName, types.MergePatchType, []byte(patch), metav1.PatchOptions{})
+		return endpoint, err
+	}, framework.NoopCheckResult)
+}