Repository: Azure/ARO-RP
Pull request #2649 (2023-01-19 16:38:34)

Update subnet generation for local dev
### Which issue this PR addresses:

Since the introduction of aks-vnet and dev-vpn-vnet. Dev-vnet has /16 address space here https://github.com/Azure/ARO-RP/pull/2238.

Development clusters use address space of dev-vnet to create subnets, changing the generatesubnets function to allow clusters to be created within the /16 address space. This limits the number of clusters we can create to 128 
<!--
Please include a link to the ADO work item as well as any GitHub issues.

Usage: `Fixes #<GitHub issue number>`, or `Fixes (paste link of issue)`.
-->

### What this PR does / why we need it:

<!--
Include a brief summary of what the PR is intended to accomplish and how the PR
does it. (2-3 sentences)
-->

### Test plan for issue:

<!--
How did you test that this PR works?

- Are there unit tests?
- Are there integration/e2e tests?
- If it is not possible to write automated tests, explain why and document how
  to manually test and verify the feature.
-->

### Is there any documentation that needs to be updated for this PR?

<!--
- If yes and the docs are in GitHub, include doc updates in the PR.
- If yes and the docs are not in GitHub (i.e. ADO wiki), include a link to the
  docs.
- If no, explain why (e.g. "tech debt cleanup, N/A").
-->


karanmagdani: This will also facilitate removal of proxy here: https://github.com/Azure/ARO-RP/pull/2396
karanmagdani: CI is failing with 
```
##[warning]Failed to download task '1ESHostedPoolValidation'. Error Unable to read data from the transport connection: Connection reset by peer.
##[warning]Inner Exception: Connection reset by peer
##[warning]Back off 22.994 seconds before retry.
```

Patch:
diff --git a/.pipelines/ci.yml b/.pipelines/ci.yml
index 61a10f7f0..545fe81b5 100644
--- a/.pipelines/ci.yml
+++ b/.pipelines/ci.yml
@@ -33,7 +33,7 @@ variables:
 jobs:
 - job: Python_Unit_Tests
   pool:
-    name: aro-ci-1es
+    name: 1es-aro-ci-pool
   variables:
     HOME: $(Agent.BuildDirectory)
   steps:
@@ -47,7 +47,7 @@ jobs:
 
 - job: Golang_Unit_Tests
   pool:
-    name: aro-ci-1es
+    name: 1es-aro-ci-pool
   variables:
     GOCACHE: /tmp/gocache
   steps:
@@ -110,7 +110,7 @@ jobs:
 
 - job: Lint_Admin_Portal
   pool:
-    name: aro-ci-1es
+    name: 1es-aro-ci-pool
   steps:
   - template: ./templates/template-checkout.yml
   - script: |
diff --git a/pkg/operator/controllers/machinehealthcheck/staticresources/machinehealthcheck.yaml b/pkg/operator/controllers/machinehealthcheck/staticresources/machinehealthcheck.yaml
index caf285367..86ff3f3dd 100644
--- a/pkg/operator/controllers/machinehealthcheck/staticresources/machinehealthcheck.yaml
+++ b/pkg/operator/controllers/machinehealthcheck/staticresources/machinehealthcheck.yaml
@@ -21,3 +21,4 @@ spec:
     timeout: "300s"
     status: "Unknown"
   maxUnhealthy: "1"
+  nodeStartupTimeout: "20m"
diff --git a/pkg/util/cluster/cluster.go b/pkg/util/cluster/cluster.go
index 952cfa2ff..1fc57cd69 100644
--- a/pkg/util/cluster/cluster.go
+++ b/pkg/util/cluster/cluster.go
@@ -339,12 +339,15 @@ func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName str
 
 func (c *Cluster) generateSubnets() (vnetPrefix string, masterSubnet string, workerSubnet string) {
 	// pick a random 23 in range [10.3.0.0, 10.127.255.0]
-	// 10.0.0.0 is used by dev-vnet to host CI
-	// 10.1.0.0 is used by rp-vnet to host Proxy VM
-	// 10.2.0.0 is used by dev-vpn-vnet to host VirtualNetworkGateway
+	// 10.0.0.0/16 is used by dev-vnet to host CI
+	// 10.1.0.0/24 is used by rp-vnet to host Proxy VM
+	// 10.2.0.0/24 is used by dev-vpn-vnet to host VirtualNetworkGateway
 	var x, y int
 	rand.Seed(time.Now().UnixNano())
-	for x == 0 && y == 0 {
+	// Local Dev clusters are limited to /16 dev-vnet
+	if !c.ci {
+		x, y = 0, 2*rand.Intn(128)
+	} else {
 		x, y = rand.Intn((124))+3, 2*rand.Intn(128)
 	}
 	vnetPrefix = fmt.Sprintf("10.%d.%d.0/23", x, y)
diff --git a/python/az/aro/azext_aro/custom.py b/python/az/aro/azext_aro/custom.py
index 38d75a36e..f65230a23 100644
--- a/python/az/aro/azext_aro/custom.py
+++ b/python/az/aro/azext_aro/custom.py
@@ -12,7 +12,7 @@ from azure.cli.core.commands.client_factory import get_mgmt_service_client
 from azure.cli.core.commands.client_factory import get_subscription_id
 from azure.cli.core.profiles import ResourceType
 from azure.cli.core.util import sdk_no_wait
-from azure.cli.core.azclierror import FileOperationError, ResourceNotFoundError, UnauthorizedError
+from azure.cli.core.azclierror import FileOperationError, ResourceNotFoundError, UnauthorizedError, ValidationError
 from azext_aro._aad import AADManager
 from azext_aro._rbac import assign_role_to_resource, \
     has_role_assignment_on_resource
@@ -278,13 +278,19 @@ def generate_random_id():
     return random_id
 
 
-def get_network_resources_from_subnets(cli_ctx, subnets):
+def get_network_resources_from_subnets(cli_ctx, subnets, fail):
     network_client = get_mgmt_service_client(cli_ctx, ResourceType.MGMT_NETWORK)
 
     subnet_resources = set()
     for sn in subnets:
         sid = parse_resource_id(sn)
 
+        if 'resource_group' not in sid or 'name' not in sid or 'resource_name' not in sid:
+            if fail:
+                raise ValidationError(f"""(ValidationError) Failed to validate subnet '{sn}'.
+                    Please retry, if issue persists: raise azure support ticket""")
+            logger.info("Failed to validate subnet '%s'", sn)
+
         subnet = network_client.subnets.get(resource_group_name=sid['resource_group'],
                                             virtual_network_name=sid['name'],
                                             subnet_name=sid['resource_name'])
@@ -298,7 +304,7 @@ def get_network_resources_from_subnets(cli_ctx, subnets):
     return subnet_resources
 
 
-def get_cluster_network_resources(cli_ctx, oc):
+def get_cluster_network_resources(cli_ctx, oc, fail):
     master_subnet = oc.master_profile.subnet_id
     worker_subnets = set()
 
@@ -316,11 +322,11 @@ def get_cluster_network_resources(cli_ctx, oc):
         name=master_parts['name'],
     )
 
-    return get_network_resources(cli_ctx, worker_subnets | {master_subnet}, vnet)
+    return get_network_resources(cli_ctx, worker_subnets | {master_subnet}, vnet, fail)
 
 
-def get_network_resources(cli_ctx, subnets, vnet):
-    subnet_resources = get_network_resources_from_subnets(cli_ctx, subnets)
+def get_network_resources(cli_ctx, subnets, vnet, fail):
+    subnet_resources = get_network_resources_from_subnets(cli_ctx, subnets, fail)
 
     resources = set()
     resources.add(vnet)
@@ -421,7 +427,7 @@ def resolve_rp_client_id():
 def ensure_resource_permissions(cli_ctx, oc, fail, sp_obj_ids):
     try:
         # Get cluster resources we need to assign permissions on, sort to ensure the same order of operations
-        resources = {ROLE_NETWORK_CONTRIBUTOR: sorted(get_cluster_network_resources(cli_ctx, oc)),
+        resources = {ROLE_NETWORK_CONTRIBUTOR: sorted(get_cluster_network_resources(cli_ctx, oc, fail)),
                      ROLE_READER: sorted(get_disk_encryption_resources(oc))}
     except (CloudError, HttpOperationError) as e:
         if fail:


Test patch:
diff --git a/test/e2e/adminapi_cluster_get.go b/test/e2e/adminapi_cluster_get.go
index 6aa6c9562..e8b28b444 100644
--- a/test/e2e/adminapi_cluster_get.go
+++ b/test/e2e/adminapi_cluster_get.go
@@ -14,13 +14,11 @@ var _ = Describe("[Admin API] Get cluster action", func() {
 	BeforeEach(skipIfNotInDevelopmentEnv)
 
 	It("must return single cluster with admin fields", func(ctx context.Context) {
-		resourceID := resourceIDFromEnv()
-
 		By("requesting the cluster document via RP admin API")
-		oc := adminGetCluster(Default, ctx, resourceID)
+		oc := adminGetCluster(Default, ctx, clusterResourceID)
 
 		By("checking that we received the expected cluster")
-		Expect(oc.ID).To(Equal(resourceID))
+		Expect(oc.ID).To(Equal(clusterResourceID))
 
 		By("checking that fields available only in Admin API have values")
 		// Note: some fields will have empty values
diff --git a/test/e2e/adminapi_cluster_getlogs.go b/test/e2e/adminapi_cluster_getlogs.go
index 6f26a4a0f..eb6bd2f76 100644
--- a/test/e2e/adminapi_cluster_getlogs.go
+++ b/test/e2e/adminapi_cluster_getlogs.go
@@ -70,7 +70,7 @@ func testGetPodLogsOK(ctx context.Context, containerName, podName, namespace str
 		"podname":   []string{podName},
 	}
 	var logs string
-	resp, err := adminRequest(ctx, http.MethodGet, "/admin"+resourceIDFromEnv()+"/kubernetespodlogs", params, nil, &logs)
+	resp, err := adminRequest(ctx, http.MethodGet, "/admin"+clusterResourceID+"/kubernetespodlogs", params, nil, &logs)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(resp.StatusCode).To(Equal(http.StatusOK))
 
@@ -87,7 +87,7 @@ func testGetPodLogsFromCustomerNamespaceForbidden(ctx context.Context, container
 	}
 
 	var logs string
-	resp, err := adminRequest(ctx, http.MethodGet, "/admin"+resourceIDFromEnv()+"/kubernetespodlogs", params, nil, logs)
+	resp, err := adminRequest(ctx, http.MethodGet, "/admin"+clusterResourceID+"/kubernetespodlogs", params, nil, logs)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(resp.StatusCode).To(Equal(http.StatusForbidden))
 
diff --git a/test/e2e/adminapi_cluster_list.go b/test/e2e/adminapi_cluster_list.go
index c4ee6139b..52ac7aa90 100644
--- a/test/e2e/adminapi_cluster_list.go
+++ b/test/e2e/adminapi_cluster_list.go
@@ -17,23 +17,17 @@ var _ = Describe("[Admin API] List clusters action", func() {
 	BeforeEach(skipIfNotInDevelopmentEnv)
 
 	It("must return list of clusters with admin fields", func(ctx context.Context) {
-		resourceID := resourceIDFromEnv()
-
-		testAdminClustersList(Default, ctx, "/admin/providers/Microsoft.RedHatOpenShift/openShiftClusters", resourceID)
+		testAdminClustersList(Default, ctx, "/admin/providers/Microsoft.RedHatOpenShift/openShiftClusters", clusterResourceID)
 	})
 
 	It("must return list of clusters with admin fields by subscription", func(ctx context.Context) {
-		resourceID := resourceIDFromEnv()
-
 		path := fmt.Sprintf("/subscriptions/%s/providers/Microsoft.RedHatOpenShift/openShiftClusters", _env.SubscriptionID())
-		testAdminClustersList(Default, ctx, path, resourceID)
+		testAdminClustersList(Default, ctx, path, clusterResourceID)
 	})
 
 	It("must return list of clusters with admin fields by resource group", func(ctx context.Context) {
-		resourceID := resourceIDFromEnv()
-
 		path := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.RedHatOpenShift/openShiftClusters", _env.SubscriptionID(), vnetResourceGroup)
-		testAdminClustersList(Default, ctx, path, resourceID)
+		testAdminClustersList(Default, ctx, path, clusterResourceID)
 	})
 })
 
diff --git a/test/e2e/adminapi_cluster_update.go b/test/e2e/adminapi_cluster_update.go
index 838e53f73..f7d7e4ede 100644
--- a/test/e2e/adminapi_cluster_update.go
+++ b/test/e2e/adminapi_cluster_update.go
@@ -19,10 +19,9 @@ var _ = Describe("[Admin API] Cluster admin update action", func() {
 
 	It("must run cluster update operation on a cluster", func(ctx context.Context) {
 		var oc = &admin.OpenShiftCluster{}
-		resourceID := resourceIDFromEnv()
 
 		By("triggering the update via RP admin API")
-		resp, err := adminRequest(ctx, http.MethodPatch, resourceID, nil, json.RawMessage("{}"), oc)
+		resp, err := adminRequest(ctx, http.MethodPatch, clusterResourceID, nil, json.RawMessage("{}"), oc)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(resp.StatusCode).To(Equal(http.StatusOK))
 
@@ -32,7 +31,7 @@ var _ = Describe("[Admin API] Cluster admin update action", func() {
 
 		By("waiting for the update to complete")
 		Eventually(func(g Gomega, ctx context.Context) {
-			oc = adminGetCluster(g, ctx, resourceID)
+			oc = adminGetCluster(g, ctx, clusterResourceID)
 
 			g.Expect(oc.Properties.ProvisioningState).To(Equal(admin.ProvisioningStateSucceeded))
 		}).WithContext(ctx).Should(Succeed())
diff --git a/test/e2e/adminapi_csr.go b/test/e2e/adminapi_csr.go
index 92ec92d18..af7c44f6a 100644
--- a/test/e2e/adminapi_csr.go
+++ b/test/e2e/adminapi_csr.go
@@ -65,7 +65,7 @@ func testCSRApproveOK(ctx context.Context, objName, namespace string) {
 	params := url.Values{
 		"csrName": []string{objName},
 	}
-	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+resourceIDFromEnv()+"/approvecsr", params, nil, nil)
+	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+clusterResourceID+"/approvecsr", params, nil, nil)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(resp.StatusCode).To(Equal(http.StatusOK))
 
@@ -87,7 +87,7 @@ func testCSRApproveOK(ctx context.Context, objName, namespace string) {
 
 func testCSRMassApproveOK(ctx context.Context, namePrefix, namespace string, csrCount int) {
 	By("approving all CSRs via RP admin API")
-	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+resourceIDFromEnv()+"/approvecsr", nil, nil, nil)
+	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+clusterResourceID+"/approvecsr", nil, nil, nil)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(resp.StatusCode).To(Equal(http.StatusOK))
 
diff --git a/test/e2e/adminapi_drainnode.go b/test/e2e/adminapi_drainnode.go
index 32f4901e5..faca81ea6 100644
--- a/test/e2e/adminapi_drainnode.go
+++ b/test/e2e/adminapi_drainnode.go
@@ -64,7 +64,7 @@ func testCordonNodeOK(ctx context.Context, nodeName string) {
 		"shouldCordon": []string{"true"},
 		"vmName":       []string{nodeName},
 	}
-	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+resourceIDFromEnv()+"/cordonnode", params, nil, nil)
+	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+clusterResourceID+"/cordonnode", params, nil, nil)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(resp.StatusCode).To(Equal(http.StatusOK))
 
@@ -81,7 +81,7 @@ func testUncordonNodeOK(ctx context.Context, nodeName string) {
 		"shouldCordon": []string{"false"},
 		"vmName":       []string{nodeName},
 	}
-	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+resourceIDFromEnv()+"/cordonnode", params, nil, nil)
+	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+clusterResourceID+"/cordonnode", params, nil, nil)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(resp.StatusCode).To(Equal(http.StatusOK))
 
@@ -97,7 +97,7 @@ func testDrainNodeOK(ctx context.Context, nodeName string, drainer *drain.Helper
 	params := url.Values{
 		"vmName": []string{nodeName},
 	}
-	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+resourceIDFromEnv()+"/drainnode", params, nil, nil)
+	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+clusterResourceID+"/drainnode", params, nil, nil)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(resp.StatusCode).To(Equal(http.StatusOK))
 }
diff --git a/test/e2e/adminapi_kubernetesobjects.go b/test/e2e/adminapi_kubernetesobjects.go
index aad192ed8..c6e3c3dc8 100644
--- a/test/e2e/adminapi_kubernetesobjects.go
+++ b/test/e2e/adminapi_kubernetesobjects.go
@@ -106,7 +106,7 @@ func testSecretOperationsForbidden(objName, namespace string) {
 		By("creating a new secret via RP admin API")
 		obj := mockSecret(objName, namespace)
 		var cloudErr api.CloudError
-		resp, err := adminRequest(ctx, http.MethodPost, "/admin"+resourceIDFromEnv()+"/kubernetesobjects", nil, obj, &cloudErr)
+		resp, err := adminRequest(ctx, http.MethodPost, "/admin"+clusterResourceID+"/kubernetesobjects", nil, obj, &cloudErr)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(resp.StatusCode).To(Equal(http.StatusForbidden))
 		Expect(cloudErr.Code).To(Equal(api.CloudErrorCodeForbidden))
@@ -120,7 +120,7 @@ func testSecretOperationsForbidden(objName, namespace string) {
 			"name":      []string{objName},
 		}
 		var cloudErr api.CloudError
-		resp, err := adminRequest(ctx, http.MethodGet, "/admin"+resourceIDFromEnv()+"/kubernetesobjects", params, nil, &cloudErr)
+		resp, err := adminRequest(ctx, http.MethodGet, "/admin"+clusterResourceID+"/kubernetesobjects", params, nil, &cloudErr)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(resp.StatusCode).To(Equal(http.StatusForbidden))
 
@@ -135,7 +135,7 @@ func testSecretOperationsForbidden(objName, namespace string) {
 			"namespace": []string{namespace},
 		}
 		var cloudErr api.CloudError
-		resp, err := adminRequest(ctx, http.MethodGet, "/admin"+resourceIDFromEnv()+"/kubernetesobjects", params, nil, &cloudErr)
+		resp, err := adminRequest(ctx, http.MethodGet, "/admin"+clusterResourceID+"/kubernetesobjects", params, nil, &cloudErr)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(resp.StatusCode).To(Equal(http.StatusForbidden))
 
@@ -151,7 +151,7 @@ func testSecretOperationsForbidden(objName, namespace string) {
 			"name":      []string{objName},
 		}
 		var cloudErr api.CloudError
-		resp, err := adminRequest(ctx, http.MethodDelete, "/admin"+resourceIDFromEnv()+"/kubernetesobjects", params, nil, &cloudErr)
+		resp, err := adminRequest(ctx, http.MethodDelete, "/admin"+clusterResourceID+"/kubernetesobjects", params, nil, &cloudErr)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(resp.StatusCode).To(Equal(http.StatusForbidden))
 		Expect(cloudErr.Code).To(Equal(api.CloudErrorCodeForbidden))
@@ -161,7 +161,7 @@ func testSecretOperationsForbidden(objName, namespace string) {
 func testConfigMapCreateOK(ctx context.Context, objName, namespace string) {
 	By("creating a new object via RP admin API")
 	obj := mockConfigMap(objName, namespace)
-	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+resourceIDFromEnv()+"/kubernetesobjects", nil, obj, nil)
+	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+clusterResourceID+"/kubernetesobjects", nil, obj, nil)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(resp.StatusCode).To(Equal(http.StatusOK))
 
@@ -181,7 +181,7 @@ func testConfigMapGetOK(ctx context.Context, objName, namespace string) {
 		"name":      []string{objName},
 	}
 	var obj corev1.ConfigMap
-	resp, err := adminRequest(ctx, http.MethodGet, "/admin"+resourceIDFromEnv()+"/kubernetesobjects", params, nil, &obj)
+	resp, err := adminRequest(ctx, http.MethodGet, "/admin"+clusterResourceID+"/kubernetesobjects", params, nil, &obj)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(resp.StatusCode).To(Equal(http.StatusOK))
 
@@ -200,7 +200,7 @@ func testConfigMapListOK(ctx context.Context, objName, namespace string) {
 		"namespace": []string{namespace},
 	}
 	var obj corev1.ConfigMapList
-	resp, err := adminRequest(ctx, http.MethodGet, "/admin"+resourceIDFromEnv()+"/kubernetesobjects", params, nil, &obj)
+	resp, err := adminRequest(ctx, http.MethodGet, "/admin"+clusterResourceID+"/kubernetesobjects", params, nil, &obj)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(resp.StatusCode).To(Equal(http.StatusOK))
 
@@ -217,7 +217,7 @@ func testConfigMapUpdateOK(ctx context.Context, objName, namespace string) {
 	obj := mockConfigMap(objName, namespace)
 	obj.Data["key"] = "new_value"
 
-	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+resourceIDFromEnv()+"/kubernetesobjects", nil, obj, nil)
+	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+clusterResourceID+"/kubernetesobjects", nil, obj, nil)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(resp.StatusCode).To(Equal(http.StatusOK))
 
@@ -238,7 +238,7 @@ func testConfigMapForceDeleteForbidden(ctx context.Context, objName, namespace s
 		"force":     []string{"true"},
 	}
 	var cloudErr api.CloudError
-	resp, err := adminRequest(ctx, http.MethodDelete, "/admin"+resourceIDFromEnv()+"/kubernetesobjects", params, nil, &cloudErr)
+	resp, err := adminRequest(ctx, http.MethodDelete, "/admin"+clusterResourceID+"/kubernetesobjects", params, nil, &cloudErr)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(resp.StatusCode).To(Equal(http.StatusForbidden))
 	Expect(cloudErr.Code).To(Equal(api.CloudErrorCodeForbidden))
@@ -252,7 +252,7 @@ func testConfigMapDeleteOK(ctx context.Context, objName, namespace string) {
 		"name":      []string{objName},
 		"force":     []string{"false"},
 	}
-	resp, err := adminRequest(ctx, http.MethodDelete, "/admin"+resourceIDFromEnv()+"/kubernetesobjects", params, nil, nil)
+	resp, err := adminRequest(ctx, http.MethodDelete, "/admin"+clusterResourceID+"/kubernetesobjects", params, nil, nil)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(resp.StatusCode).To(Equal(http.StatusOK))
 
@@ -269,7 +269,7 @@ func testConfigMapCreateOrUpdateForbidden(ctx context.Context, operation, objNam
 	By(operation + " a new object via RP admin API")
 	obj := mockConfigMap(objName, namespace)
 	var cloudErr api.CloudError
-	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+resourceIDFromEnv()+"/kubernetesobjects", nil, obj, &cloudErr)
+	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+clusterResourceID+"/kubernetesobjects", nil, obj, &cloudErr)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(resp.StatusCode).To(Equal(http.StatusForbidden))
 	Expect(cloudErr.Code).To(Equal(api.CloudErrorCodeForbidden))
@@ -284,7 +284,7 @@ func testConfigMapDeleteForbidden(ctx context.Context, objName, namespace string
 		"force":     []string{"false"},
 	}
 	var cloudErr api.CloudError
-	resp, err := adminRequest(ctx, http.MethodDelete, "/admin"+resourceIDFromEnv()+"/kubernetesobjects", params, nil, &cloudErr)
+	resp, err := adminRequest(ctx, http.MethodDelete, "/admin"+clusterResourceID+"/kubernetesobjects", params, nil, &cloudErr)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(resp.StatusCode).To(Equal(http.StatusForbidden))
 	Expect(cloudErr.Code).To(Equal(api.CloudErrorCodeForbidden))
@@ -293,7 +293,7 @@ func testConfigMapDeleteForbidden(ctx context.Context, objName, namespace string
 func testPodCreateOK(ctx context.Context, containerName, objName, namespace string) {
 	By("creating a new pod via RP admin API")
 	obj := mockPod(containerName, objName, namespace, "")
-	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+resourceIDFromEnv()+"/kubernetesobjects", nil, obj, nil)
+	resp, err := adminRequest(ctx, http.MethodPost, "/admin"+clusterResourceID+"/kubernetesobjects", nil, obj, nil)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(resp.StatusCode).To(Equal(http.StatusOK))
 
@@ -313,7 +313,7 @@ func testPodForceDeleteOK(ctx context.Context, objName, namespace string) {
 		"name":      []string{objName},
 		"force":     []string{"true"},
 	}
-	resp, err := adminRequest(ctx, http.MethodDelete, "/admin"+resourceIDFromEnv()+"/kubernetesobjects", params, nil, nil)
+	resp, err := adminRequest(ctx, http.MethodDelete, "/admin"+clusterResourceID+"/kubernetesobjects", params, nil, nil)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(resp.StatusCode).To(Equal(http.StatusOK))
 
diff --git a/test/e2e/adminapi_redeployvm.go b/test/e2e/adminapi_redeployvm.go
index 5244c7c68..e5edfc608 100644
--- a/test/e2e/adminapi_redeployvm.go
+++ b/test/e2e/adminapi_redeployvm.go
@@ -31,8 +31,6 @@ var _ = Describe("[Admin API] VM redeploy action", func() {
 	BeforeEach(skipIfNotInDevelopmentEnv)
 
 	It("must trigger a selected VM to redeploy", func(ctx context.Context) {
-		resourceID := resourceIDFromEnv()
-
 		By("getting the resource group where the VM instances live in")
 		oc, err := clients.OpenshiftClustersv20200430.Get(ctx, vnetResourceGroup, clusterName)
 		Expect(err).NotTo(HaveOccurred())
@@ -50,7 +48,7 @@ var _ = Describe("[Admin API] VM redeploy action", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		By("triggering VM redeployment via RP Admin API")
-		resp, err := adminRequest(ctx, http.MethodPost, "/admin"+resourceID+"/redeployvm", url.Values{"vmName": []string{*vm.Name}}, nil, nil)
+		resp, err := adminRequest(ctx, http.MethodPost, "/admin"+clusterResourceID+"/redeployvm", url.Values{"vmName": []string{*vm.Name}}, nil, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(resp.StatusCode).To(Equal(http.StatusOK))
 
diff --git a/test/e2e/adminapi_resources.go b/test/e2e/adminapi_resources.go
index f103cc0de..f83747545 100644
--- a/test/e2e/adminapi_resources.go
+++ b/test/e2e/adminapi_resources.go
@@ -23,8 +23,6 @@ var _ = Describe("[Admin API] List Azure resources action", func() {
 	BeforeEach(skipIfNotInDevelopmentEnv)
 
 	It("must list Azure resources for a cluster", func(ctx context.Context) {
-		resourceID := resourceIDFromEnv()
-
 		By("getting the resource group where cluster resources live in")
 		oc, err := clients.OpenshiftClustersv20200430.Get(ctx, vnetResourceGroup, clusterName)
 		Expect(err).NotTo(HaveOccurred())
@@ -77,7 +75,7 @@ var _ = Describe("[Admin API] List Azure resources action", func() {
 
 		By("getting the actual Azure resource IDs via RP admin API")
 		var actualResources []mgmtfeatures.GenericResourceExpanded
-		resp, err := adminRequest(ctx, http.MethodGet, "/admin"+resourceID+"/resources", nil, nil, &actualResources)
+		resp, err := adminRequest(ctx, http.MethodGet, "/admin"+clusterResourceID+"/resources", nil, nil, &actualResources)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(resp.StatusCode).To(Equal(http.StatusOK))
 
diff --git a/test/e2e/hive.go b/test/e2e/hive.go
new file mode 100644
index 000000000..d57e4d295
--- /dev/null
+++ b/test/e2e/hive.go
@@ -0,0 +1,69 @@
+package e2e
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/Azure/ARO-RP/pkg/api/admin"
+	"github.com/Azure/ARO-RP/pkg/hive"
+)
+
+var (
+	clusterPlatformLabelKey string = "hive.openshift.io/cluster-platform"
+	clusterRegionLabelKey   string = "hive.openshift.io/cluster-region"
+
+	controlPlaneAPIURLOverride = func(clusterDomain string, clusterLocation string) string {
+		if !strings.ContainsRune(clusterDomain, '.') {
+			clusterDomain += "." + clusterLocation + "." + os.Getenv("PARENT_DOMAIN_NAME")
+		}
+
+		return fmt.Sprintf("api-int.%s:6443", clusterDomain)
+	}
+)
+
+var _ = Describe("Hive-managed ARO cluster", func() {
+	var adminAPICluster *admin.OpenShiftCluster
+
+	BeforeEach(func(ctx context.Context) {
+		adminAPICluster = adminGetCluster(Default, ctx, clusterResourceID)
+
+		skipIfNotHiveManagedCluster(adminAPICluster)
+	})
+
+	It("has been properly created/adopted by Hive", func(ctx context.Context) {
+		By("verifying that a corresponding ClusterDeployment object exists in the expected namespace in the Hive cluster")
+		cd, err := clients.Hive.HiveV1().ClusterDeployments(adminAPICluster.Properties.HiveProfile.Namespace).Get(ctx, hive.ClusterDeploymentName, metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred())
+
+		By("verifying that the ClusterDeployment object has the expected name and labels")
+		Expect(cd.ObjectMeta).NotTo(BeNil())
+		Expect(cd.ObjectMeta.Name).To(Equal(hive.ClusterDeploymentName))
+		Expect(cd.ObjectMeta.Labels).Should(HaveKey(clusterPlatformLabelKey))
+		Expect(cd.ObjectMeta.Labels[clusterPlatformLabelKey]).To(Equal("azure"))
+		Expect(cd.ObjectMeta.Labels).Should(HaveKey(clusterRegionLabelKey))
+		Expect(cd.ObjectMeta.Labels[clusterRegionLabelKey]).To(Equal(adminAPICluster.Location))
+
+		By("verifying that the ClusterDeployment object spec correctly includes the ARO cluster's Azure region and RG name")
+		Expect(cd.Spec).NotTo(BeNil())
+		Expect(cd.Spec.ClusterName).To(Equal(adminAPICluster.Name))
+		Expect(cd.Spec.Platform).NotTo(BeNil())
+		Expect(cd.Spec.Platform.Azure).NotTo(BeNil())
+		Expect(cd.Spec.Platform.Azure.BaseDomainResourceGroupName).To(Equal(adminAPICluster.Properties.ClusterProfile.ResourceGroupID))
+		Expect(cd.Spec.Platform.Azure.Region).To(Equal(adminAPICluster.Location))
+
+		By("verifying that the ClusterDeployment object spec includes the expected ControlPlaneConfig overrides")
+		Expect(cd.Spec.ControlPlaneConfig).NotTo(BeNil())
+		Expect(cd.Spec.ControlPlaneConfig.APIServerIPOverride).To(Equal(adminAPICluster.Properties.NetworkProfile.APIServerPrivateEndpointIP))
+		Expect(cd.Spec.ControlPlaneConfig.APIURLOverride).To(Equal(controlPlaneAPIURLOverride(adminAPICluster.Properties.ClusterProfile.Domain, adminAPICluster.Location)))
+	})
+})
diff --git a/test/e2e/operator.go b/test/e2e/operator.go
index 25a805ce0..d6e5f947e 100644
--- a/test/e2e/operator.go
+++ b/test/e2e/operator.go
@@ -311,9 +311,14 @@ var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() {
 })
 
 var _ = Describe("ARO Operator - MUO Deployment", func() {
+	const (
+		managedUpgradeOperatorNamespace  = "openshift-managed-upgrade-operator"
+		managedUpgradeOperatorDeployment = "managed-upgrade-operator"
+	)
+
 	It("must be deployed by default with FIPS crypto mandated", func(ctx context.Context) {
 		By("getting MUO pods")
-		pods, err := clients.Kubernetes.CoreV1().Pods("openshift-managed-upgrade-operator").List(ctx, metav1.ListOptions{
+		pods, err := clients.Kubernetes.CoreV1().Pods(managedUpgradeOperatorNamespace).List(ctx, metav1.ListOptions{
 			LabelSelector: "name=managed-upgrade-operator",
 		})
 		Expect(err).NotTo(HaveOccurred())
@@ -321,12 +326,39 @@ var _ = Describe("ARO Operator - MUO Deployment", func() {
 
 		By("verifying that MUO has FIPS crypto mandated by reading logs")
 		Eventually(func(g Gomega, ctx context.Context) {
-			b, err := clients.Kubernetes.CoreV1().Pods("openshift-managed-upgrade-operator").GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{}).DoRaw(ctx)
+			b, err := clients.Kubernetes.CoreV1().Pods(managedUpgradeOperatorNamespace).GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{}).DoRaw(ctx)
 			g.Expect(err).NotTo(HaveOccurred())
 
 			g.Expect(string(b)).To(ContainSubstring(`msg="FIPS crypto mandated: true"`))
 		}).WithContext(ctx).Should(Succeed())
 	})
+
+	It("must be restored if deleted", func(ctx context.Context) {
+		deleteMUODeployment := func(ctx context.Context) error {
+			return clients.Kubernetes.
+				AppsV1().
+				Deployments(managedUpgradeOperatorNamespace).
+				Delete(ctx, managedUpgradeOperatorDeployment, metav1.DeleteOptions{})
+		}
+
+		muoDeploymentExists := func(g Gomega, ctx context.Context) {
+			_, err := clients.Kubernetes.
+				AppsV1().
+				Deployments(managedUpgradeOperatorNamespace).
+				Get(ctx, managedUpgradeOperatorDeployment, metav1.GetOptions{})
+
+			g.Expect(err).ToNot(HaveOccurred())
+		}
+
+		By("waiting for the MUO deployment to be ready")
+		Eventually(muoDeploymentExists).WithContext(ctx).Should(Succeed())
+
+		By("deleting the MUO deployment")
+		Expect(deleteMUODeployment(ctx)).Should(Succeed())
+
+		By("waiting for the MUO deployment to be reconciled")
+		Eventually(muoDeploymentExists).WithContext(ctx).Should(Succeed())
+	})
 })
 
 var _ = Describe("ARO Operator - ImageConfig Reconciler", func() {
diff --git a/test/e2e/setup.go b/test/e2e/setup.go
index c234b3991..770a2930f 100644
--- a/test/e2e/setup.go
+++ b/test/e2e/setup.go
@@ -23,6 +23,7 @@ import (
 	configclient "github.com/openshift/client-go/config/clientset/versioned"
 	machineclient "github.com/openshift/client-go/machine/clientset/versioned"
 	projectclient "github.com/openshift/client-go/project/clientset/versioned"
+	hiveclient "github.com/openshift/hive/pkg/client/clientset/versioned"
 	mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned"
 	"github.com/sirupsen/logrus"
 	"github.com/tebeka/selenium"
@@ -32,6 +33,7 @@ import (
 	"k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/client-go/tools/clientcmd/api/latest"
 
+	"github.com/Azure/ARO-RP/pkg/api/admin"
 	"github.com/Azure/ARO-RP/pkg/env"
 	"github.com/Azure/ARO-RP/pkg/hive"
 	aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned"
@@ -60,16 +62,17 @@ type clientSet struct {
 	NetworkSecurityGroups network.SecurityGroupsClient
 	Subnet                network.SubnetsClient
 
+	RestConfig         *rest.Config
+	HiveRestConfig     *rest.Config
+	Kubernetes         kubernetes.Interface
+	MachineAPI         machineclient.Interface
+	MachineConfig      mcoclient.Interface
+	AROClusters        aroclient.Interface
+	ConfigClient       configclient.Interface
+	Project            projectclient.Interface
+	Hive               hiveclient.Interface
 	HiveAKS            kubernetes.Interface
 	HiveClusterManager hive.ClusterManager
-
-	RestConfig    *rest.Config
-	Kubernetes    kubernetes.Interface
-	MachineAPI    machineclient.Interface
-	MachineConfig mcoclient.Interface
-	AROClusters   aroclient.Interface
-	ConfigClient  configclient.Interface
-	Project       projectclient.Interface
 }
 
 var (
@@ -77,6 +80,7 @@ var (
 	_env              env.Core
 	vnetResourceGroup string
 	clusterName       string
+	clusterResourceID string
 	clients           *clientSet
 
 	dockerSucceeded bool
@@ -96,6 +100,12 @@ func skipIfDockerNotWorking() {
 	}
 }
 
+func skipIfNotHiveManagedCluster(adminAPICluster *admin.OpenShiftCluster) {
+	if adminAPICluster.Properties.HiveProfile == (admin.HiveProfile{}) {
+		Skip("skipping tests because this ARO cluster has not been created/adopted by Hive")
+	}
+}
+
 func SaveScreenshotAndExit(wd selenium.WebDriver, e error) {
 	log.Infof("Error : %s", e.Error())
 	log.Info("Taking Screenshot and saving page source")
@@ -295,17 +305,23 @@ func newClientSet(ctx context.Context) (*clientSet, error) {
 		return nil, err
 	}
 
-	aksCfg, err := liveCfg.HiveRestConfig(ctx, 1)
+	hiveShard := 1
+	hiveRestConfig, err := liveCfg.HiveRestConfig(ctx, hiveShard)
+	if err != nil {
+		return nil, err
+	}
+
+	hiveClientSet, err := hiveclient.NewForConfig(hiveRestConfig)
 	if err != nil {
 		return nil, err
 	}
 
-	hiveAKS, err := kubernetes.NewForConfig(aksCfg)
+	hiveAKS, err := kubernetes.NewForConfig(hiveRestConfig)
 	if err != nil {
 		return nil, err
 	}
 
-	hiveCM, err := hive.NewFromConfig(log, _env, aksCfg)
+	hiveCM, err := hive.NewFromConfig(log, _env, hiveRestConfig)
 	if err != nil {
 		return nil, err
 	}
@@ -323,16 +339,17 @@ func newClientSet(ctx context.Context) (*clientSet, error) {
 		Subnet:                network.NewSubnetsClient(_env.Environment(), _env.SubscriptionID(), authorizer),
 		NetworkSecurityGroups: network.NewSecurityGroupsClient(_env.Environment(), _env.SubscriptionID(), authorizer),
 
+		RestConfig:         restconfig,
+		HiveRestConfig:     hiveRestConfig,
+		Kubernetes:         cli,
+		MachineAPI:         machineapicli,
+		MachineConfig:      mcocli,
+		AROClusters:        arocli,
+		Project:            projectcli,
+		ConfigClient:       configcli,
+		Hive:               hiveClientSet,
 		HiveAKS:            hiveAKS,
 		HiveClusterManager: hiveCM,
-
-		RestConfig:    restconfig,
-		Kubernetes:    cli,
-		MachineAPI:    machineapicli,
-		MachineConfig: mcocli,
-		AROClusters:   arocli,
-		Project:       projectcli,
-		ConfigClient:  configcli,
 	}, nil
 }
 
@@ -419,6 +436,8 @@ func setup(ctx context.Context) error {
 		}
 	}
 
+	clusterResourceID = resourceIDFromEnv()
+
 	clients, err = newClientSet(ctx)
 	if err != nil {
 		return err