Repository: Azure/ARO-RP
Pull request #2437 (2022-10-04 10:26:03)

Removes "Node check" E2E
### Which issue this PR addresses:

Part of [WI 13667437](https://msazure.visualstudio.com/AzureRedHatOpenShift/_workitems/edit/13667437).

### What this PR does / why we need it:

This test seems to be unnecessary as it looks like something which should be tested as part of OCP.

### Test plan for issue:

Run e2e

### Is there any documentation that needs to be updated for this PR?

No, just test removal



Patch:
diff --git a/.github/workflows/golint.yml b/.github/workflows/golint.yml
index c2ccc6c2b..5e9e7d7f3 100644
--- a/.github/workflows/golint.yml
+++ b/.github/workflows/golint.yml
@@ -27,7 +27,7 @@ jobs:
         uses: golangci/golangci-lint-action@v3
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          version: v1.45.2
+          version: v1.49.0
           args: -v --timeout 15m
           # Optional: working directory, useful for monorepos
           #working-directory: pkg
diff --git a/.golangci.yml b/.golangci.yml
index 7ba36ac32..740585546 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -29,7 +29,6 @@ linters:
   enable:
     - gofmt
     - unused
-    - deadcode
     - gosimple
     - govet
     - staticcheck
diff --git a/docs/prepare-a-shared-rp-development-environment.md b/docs/prepare-a-shared-rp-development-environment.md
index 781bf920d..2b976ecb7 100644
--- a/docs/prepare-a-shared-rp-development-environment.md
+++ b/docs/prepare-a-shared-rp-development-environment.md
@@ -456,6 +456,8 @@ each of the bash functions below.
    deploy_rp_dev
    # Deploy the proxy and VPN
    deploy_env_dev
+   # Deploy AKS resources for Hive
+   deploy_aks_dev
    ```
 
    If you encounter a "VirtualNetworkGatewayCannotUseStandardPublicIP" error
diff --git a/hack/devtools/deploy-shared-env.sh b/hack/devtools/deploy-shared-env.sh
index 8fe72b538..4705cbccd 100644
--- a/hack/devtools/deploy-shared-env.sh
+++ b/hack/devtools/deploy-shared-env.sh
@@ -48,6 +48,7 @@ deploy_env_dev_ci() {
             "proxyImage=arointsvc.azurecr.io/proxy:latest" \
             "proxyImageAuth=$(jq -r '.auths["arointsvc.azurecr.io"].auth' <<<$PULL_SECRET)" \
             "proxyKey=$(base64 -w0 <secrets/proxy.key)" \
+            "vpnCACertificate=$(base64 -w0 <secrets/vpn-ca.crt)" \
             "sshPublicKey=$(<secrets/proxy_id_rsa.pub)" >/dev/null
 }
 
@@ -77,8 +78,7 @@ deploy_aks_dev() {
         --parameters \
             "adminObjectId=$ADMIN_OBJECT_ID" \
             "dnsZone=$DOMAIN_NAME" \
-            "sshRSAPublicKey=$(<secrets/proxy_id_rsa.pub)" \
-            "vpnCACertificate=$(base64 -w0 <secrets/vpn-ca.crt)" >/dev/null
+            "sshRSAPublicKey=$(<secrets/proxy_id_rsa.pub)" >/dev/null
 }
 
 deploy_env_dev_override() {
diff --git a/pkg/api/admin/openshiftcluster.go b/pkg/api/admin/openshiftcluster.go
index ad8b79fdb..c23e10965 100644
--- a/pkg/api/admin/openshiftcluster.go
+++ b/pkg/api/admin/openshiftcluster.go
@@ -84,6 +84,7 @@ type MaintenanceTask string
 const (
 	MaintenanceTaskEverything MaintenanceTask = "Everything"
 	MaintenanceTaskOperator   MaintenanceTask = "OperatorUpdate"
+	MaintenanceTaskRenewCerts MaintenanceTask = "CertificatesRenewal"
 )
 
 // Operator feature flags
diff --git a/pkg/api/openshiftcluster.go b/pkg/api/openshiftcluster.go
index fc8232f82..9ebf27ff5 100644
--- a/pkg/api/openshiftcluster.go
+++ b/pkg/api/openshiftcluster.go
@@ -173,6 +173,7 @@ type MaintenanceTask string
 const (
 	MaintenanceTaskEverything MaintenanceTask = "Everything"
 	MaintenanceTaskOperator   MaintenanceTask = "OperatorUpdate"
+	MaintenanceTaskRenewCerts MaintenanceTask = "CertificatesRenewal"
 )
 
 // Cluster-scoped flags
diff --git a/pkg/cluster/arooperator.go b/pkg/cluster/arooperator.go
index 638947516..3b85ee7c6 100644
--- a/pkg/cluster/arooperator.go
+++ b/pkg/cluster/arooperator.go
@@ -49,3 +49,7 @@ func (m *manager) ensureAROOperatorRunningDesiredVersion(ctx context.Context) (b
 	}
 	return true, nil
 }
+
+func (m *manager) renewMDSDCertificate(ctx context.Context) error {
+	return m.aroOperatorDeployer.RenewMDSDCertificate(ctx)
+}
diff --git a/pkg/cluster/install.go b/pkg/cluster/install.go
index c908d8bdc..f9fe872d1 100644
--- a/pkg/cluster/install.go
+++ b/pkg/cluster/install.go
@@ -37,6 +37,7 @@ func (m *manager) adminUpdate() []steps.Step {
 	task := m.doc.OpenShiftCluster.Properties.MaintenanceTask
 	isEverything := task == api.MaintenanceTaskEverything || task == ""
 	isOperator := task == api.MaintenanceTaskOperator
+	isRenewCerts := task == api.MaintenanceTaskRenewCerts
 
 	// Generic fix-up or setup actions that are fairly safe to always take, and
 	// don't require a running cluster
@@ -73,20 +74,50 @@ func (m *manager) adminUpdate() []steps.Step {
 			steps.Action(m.fixSREKubeconfig),
 			steps.Action(m.fixUserAdminKubeconfig),
 			steps.Action(m.createOrUpdateRouterIPFromCluster),
+		)
+	}
+
+	if isEverything || isRenewCerts {
+		toRun = append(toRun,
 			steps.Action(m.fixMCSCert),
 			steps.Action(m.fixMCSUserData),
+		)
+	}
+
+	if isEverything {
+		toRun = append(toRun,
 			steps.Action(m.ensureGatewayUpgrade),
+		)
+	}
+
+	if isEverything || isRenewCerts {
+		toRun = append(toRun,
 			steps.Action(m.configureAPIServerCertificate),
 			steps.Action(m.configureIngressCertificate),
+		)
+	}
+
+	if isEverything {
+		toRun = append(toRun,
 			steps.Action(m.populateRegistryStorageAccountName),
 			steps.Action(m.ensureMTUSize),
 		)
 	}
 
+	if isEverything || isOperator || isRenewCerts {
+		toRun = append(toRun,
+			steps.Action(m.initializeOperatorDeployer))
+	}
+
+	if isRenewCerts {
+		toRun = append(toRun,
+			steps.Action(m.renewMDSDCertificate),
+		)
+	}
+
 	// Update the ARO Operator
 	if isEverything || isOperator {
 		toRun = append(toRun,
-			steps.Action(m.initializeOperatorDeployer), // depends on kube clients
 			steps.Action(m.ensureAROOperator),
 			steps.Condition(m.aroDeploymentReady, 20*time.Minute, true),
 			steps.Condition(m.ensureAROOperatorRunningDesiredVersion, 5*time.Minute, true),
diff --git a/pkg/cluster/nsg.go b/pkg/cluster/nsg.go
index 6421ca2a7..dec1258c5 100644
--- a/pkg/cluster/nsg.go
+++ b/pkg/cluster/nsg.go
@@ -31,7 +31,7 @@ func (m *manager) clusterNSG(infraID, location string) *arm.Resource {
 					SourceAddressPrefix:      to.StringPtr("*"),
 					DestinationAddressPrefix: to.StringPtr("*"),
 					Access:                   mgmtnetwork.SecurityRuleAccessAllow,
-					Priority:                 to.Int32Ptr(101),
+					Priority:                 to.Int32Ptr(120),
 					Direction:                mgmtnetwork.SecurityRuleDirectionInbound,
 				},
 				Name: to.StringPtr("apiserver_in"),
diff --git a/pkg/deploy/assets/aks-development.json b/pkg/deploy/assets/aks-development.json
index 591822ab6..3e6272e39 100644
--- a/pkg/deploy/assets/aks-development.json
+++ b/pkg/deploy/assets/aks-development.json
@@ -168,20 +168,6 @@
                 "description": "Specifies the address prefix of the subnet hosting the pods of the AKS cluster."
             }
         },
-        "gatewaySubnetName": {
-            "type": "string",
-            "defaultValue": "GatewaySubnet",
-            "metadata": {
-                "description": "Subnet name that will contain the App Service Environment"
-            }
-        },
-        "gatewaySubnetPrefix": {
-            "type": "string",
-            "defaultValue": "10.128.0.0/24",
-            "metadata": {
-                "description": "Subnet address prefix"
-            }
-        },
         "serviceCidr": {
             "type": "string",
             "defaultValue": "10.130.0.0/16",
@@ -265,10 +251,6 @@
             "metadata": {
                 "description": "Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster."
             }
-        },
-        "vpnCACertificate": {
-            "type": "string",
-            "defaultValue": ""
         }
     },
     "variables": {
@@ -353,12 +335,6 @@
                                 }
                             ]
                         }
-                    },
-                    {
-                        "name": "[parameters('gatewaySubnetName')]",
-                        "properties": {
-                            "addressPrefix": "[parameters('gatewaySubnetPrefix')]"
-                        }
                     }
                 ]
             }
@@ -436,15 +412,39 @@
             }
         },
         {
-            "name": "aks-vpn-pip",
-            "type": "Microsoft.Network/publicIPAddresses",
+            "type": "Microsoft.Network/virtualNetworks/virtualNetworkPeerings",
+            "name": "[concat('dev-vpn-vnet/peering-', parameters('vnetName'))]",
+            "apiVersion": "2021-12-01",
             "location": "[resourceGroup().location]",
-            "apiVersion": "2020-08-01",
-            "sku": {
-                "name": "Basic"
-            },
+            "dependsOn": [
+                "[variables('vnetId')]"
+            ],
+            "properties": {
+                "allowVirtualNetworkAccess": true,
+                "allowForwardedTraffic": true,
+                "allowGatewayTransit": true,
+                "useRemoteGateways": false,
+                "remoteVirtualNetwork": {
+                    "id": "[variables('vnetId')]"
+                }
+            }
+        },
+        {
+            "type": "Microsoft.Network/virtualNetworks/virtualNetworkPeerings",
+            "name": "[concat(parameters('vnetName'), '/peering-dev-vpn-vnet')]",
+            "apiVersion": "2021-12-01",
+            "location": "[resourceGroup().location]",
+            "dependsOn": [
+                "[variables('vnetId')]"
+            ],
             "properties": {
-                "publicIPAllocationMethod": "Dynamic"
+                "allowVirtualNetworkAccess": true,
+                "allowForwardedTraffic": true,
+                "allowGatewayTransit": false,
+                "useRemoteGateways": true,
+                "remoteVirtualNetwork": {
+                    "id": "[resourceId('Microsoft.Network/virtualNetworks', 'dev-vpn-vnet')]"
+                }
             }
         },
         {
@@ -612,55 +612,6 @@
                 ],
                 "enableSoftDelete": true
             }
-        },
-        {
-            "type": "Microsoft.Network/virtualNetworkGateways",
-            "name": "aks-vpn",
-            "apiVersion": "2020-08-01",
-            "location": "[resourceGroup().location]",
-            "dependsOn": [
-                "[resourceId('Microsoft.Network/publicIPAddresses', 'aks-vpn-pip')]",
-                "[variables('vnetId')]",
-                "[parameters('aksClusterName')]"
-            ],
-            "properties": {
-                "ipConfigurations": [
-                    {
-                        "properties": {
-                            "subnet": {
-                                "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('vnetName'), parameters('gatewaySubnetName'))]"
-                            },
-                            "publicIPAddress": {
-                                "id": "[resourceId('Microsoft.Network/publicIPAddresses', 'aks-vpn-pip')]"
-                            }
-                        },
-                        "name": "default"
-                    }
-                ],
-                "vpnType": "RouteBased",
-                "sku": {
-                    "name": "VpnGw1",
-                    "tier": "VpnGw1"
-                },
-                "vpnClientConfiguration": {
-                    "vpnClientAddressPool": {
-                        "addressPrefixes": [
-                            "192.168.254.0/24"
-                        ]
-                    },
-                    "vpnClientRootCertificates": [
-                        {
-                            "properties": {
-                                "publicCertData": "[parameters('vpnCACertificate')]"
-                            },
-                            "name": "aks-vpn-ca"
-                        }
-                    ],
-                    "vpnClientProtocols": [
-                        "OpenVPN"
-                    ]
-                }
-            }
         }
     ]
 }
diff --git a/pkg/operator/deploy/deploy.go b/pkg/operator/deploy/deploy.go
index 5e0dc7069..2ce418004 100644
--- a/pkg/operator/deploy/deploy.go
+++ b/pkg/operator/deploy/deploy.go
@@ -50,6 +50,7 @@ type Operator interface {
 	CreateOrUpdate(context.Context) error
 	IsReady(context.Context) (bool, error)
 	IsRunningDesiredVersion(context.Context) (bool, error)
+	RenewMDSDCertificate(context.Context) error
 }
 
 type operator struct {
@@ -364,6 +365,31 @@ func (o *operator) CreateOrUpdate(ctx context.Context) error {
 	return nil
 }
 
+func (o *operator) RenewMDSDCertificate(ctx context.Context) error {
+	key, cert := o.env.ClusterGenevaLoggingSecret()
+	gcsKeyBytes, err := utiltls.PrivateKeyAsBytes(key)
+	if err != nil {
+		return err
+	}
+	gcsCertBytes, err := utiltls.CertAsBytes(cert)
+	if err != nil {
+		return err
+	}
+
+	s, err := o.kubernetescli.CoreV1().Secrets(pkgoperator.Namespace).Get(ctx, pkgoperator.SecretName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	s.Data["gcscert.pem"] = gcsCertBytes
+	s.Data["gcskey.pem"] = gcsKeyBytes
+
+	_, err = o.kubernetescli.CoreV1().Secrets(pkgoperator.Namespace).Update(ctx, s, metav1.UpdateOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (o *operator) IsReady(ctx context.Context) (bool, error) {
 	ok, err := ready.CheckDeploymentIsReady(ctx, o.kubernetescli.AppsV1().Deployments(pkgoperator.Namespace), "aro-operator-master")()
 	if !ok || err != nil {
diff --git a/pkg/portal/middleware/aad.go b/pkg/portal/middleware/aad.go
index dd59e85e1..c506ffa5c 100644
--- a/pkg/portal/middleware/aad.go
+++ b/pkg/portal/middleware/aad.go
@@ -16,6 +16,7 @@ import (
 
 	"github.com/Azure/go-autorest/autorest/adal"
 	"github.com/gorilla/mux"
+	"github.com/gorilla/securecookie"
 	"github.com/gorilla/sessions"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/oauth2"
@@ -141,7 +142,18 @@ func (a *aad) AAD(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		session, err := a.store.Get(r, SessionName)
 		if err != nil {
-			a.internalServerError(w, err)
+			cookieError, ok := err.(securecookie.Error)
+			if ok && cookieError != nil && cookieError.IsDecode() {
+				cookie := &http.Cookie{
+					Name:    SessionName,
+					Path:    "/",
+					Expires: time.Unix(0, 0),
+				}
+				http.SetCookie(w, cookie)
+				http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
+			} else {
+				a.internalServerError(w, err)
+			}
 			return
 		}
 
diff --git a/pkg/util/azureclient/mgmt/network/virtualnetworkpeerings.go b/pkg/util/azureclient/mgmt/network/virtualnetworkpeerings.go
index 6946402d9..a7673d411 100644
--- a/pkg/util/azureclient/mgmt/network/virtualnetworkpeerings.go
+++ b/pkg/util/azureclient/mgmt/network/virtualnetworkpeerings.go
@@ -4,6 +4,8 @@ package network
 // Licensed under the Apache License 2.0.
 
 import (
+	"context"
+
 	mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network"
 	"github.com/Azure/go-autorest/autorest"
 
@@ -11,6 +13,8 @@ import (
 )
 
 type VirtualNetworkPeeringsClient interface {
+	CreateOrUpdate(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string, virtualNetworkPeeringParameters mgmtnetwork.VirtualNetworkPeering) (result mgmtnetwork.VirtualNetworkPeeringsCreateOrUpdateFuture, err error)
+	Delete(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string) (result mgmtnetwork.VirtualNetworkPeeringsDeleteFuture, err error)
 	VirtualNetworkPeeringsAddons
 }
 
diff --git a/pkg/util/azureclient/mgmt/network/virtualnetworkpeerings_addons.go b/pkg/util/azureclient/mgmt/network/virtualnetworkpeerings_addons.go
index 112e36a90..3ffd88887 100644
--- a/pkg/util/azureclient/mgmt/network/virtualnetworkpeerings_addons.go
+++ b/pkg/util/azureclient/mgmt/network/virtualnetworkpeerings_addons.go
@@ -5,12 +5,24 @@ package network
 
 import (
 	"context"
+
+	mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network"
 )
 
 type VirtualNetworkPeeringsAddons interface {
+	CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string, virtualNetworkPeeringParameters mgmtnetwork.VirtualNetworkPeering) (err error)
 	DeleteAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string) (err error)
 }
 
+func (c *virtualNetworkPeeringsClient) CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string, virtualNetworkPeeringParameters mgmtnetwork.VirtualNetworkPeering) (err error) {
+	future, err := c.CreateOrUpdate(ctx, resourceGroupName, virtualNetworkName, virtualNetworkPeeringName, virtualNetworkPeeringParameters)
+	if err != nil {
+		return err
+	}
+
+	return future.WaitForCompletionRef(ctx, c.Client)
+}
+
 func (c *virtualNetworkPeeringsClient) DeleteAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string) (err error) {
 	future, err := c.VirtualNetworkPeeringsClient.Delete(ctx, resourceGroupName, virtualNetworkName, virtualNetworkPeeringName)
 	if err != nil {
diff --git a/pkg/util/cluster/cluster.go b/pkg/util/cluster/cluster.go
index 7245f2ca0..a8543938f 100644
--- a/pkg/util/cluster/cluster.go
+++ b/pkg/util/cluster/cluster.go
@@ -20,6 +20,7 @@ import (
 	mgmtauthorization "github.com/Azure/azure-sdk-for-go/services/preview/authorization/mgmt/2018-09-01-preview/authorization"
 	mgmtfeatures "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features"
 	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/go-autorest/autorest/azure/auth"
 	"github.com/Azure/go-autorest/autorest/to"
 	"github.com/sirupsen/logrus"
@@ -46,9 +47,10 @@ import (
 )
 
 type Cluster struct {
-	log *logrus.Entry
-	env env.Core
-	ci  bool
+	log          *logrus.Entry
+	env          env.Core
+	ci           bool
+	ciParentVnet string
 
 	deployments                       features.DeploymentsClient
 	groups                            features.ResourceGroupsClient
@@ -63,6 +65,7 @@ type Cluster struct {
 	routetables                       network.RouteTablesClient
 	roleassignments                   authorization.RoleAssignmentsClient
 	peerings                          network.VirtualNetworkPeeringsClient
+	ciParentVnetPeerings              network.VirtualNetworkPeeringsClient
 	vaultsClient                      keyvaultclient.VaultsClient
 }
 
@@ -121,6 +124,18 @@ func New(log *logrus.Entry, environment env.Core, ci bool) (*Cluster, error) {
 		vaultsClient:                      keyvaultclient.NewVaultsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
 	}
 
+	if ci && env.IsLocalDevelopmentMode() {
+		// Only peer if CI=true and RP_MODE=development
+		c.ciParentVnet = fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/dev-vpn-vnet", c.env.SubscriptionID(), c.env.ResourceGroup())
+
+		r, err := azure.ParseResourceID(c.ciParentVnet)
+		if err != nil {
+			return nil, err
+		}
+
+		c.ciParentVnetPeerings = network.NewVirtualNetworkPeeringsClient(environment.Environment(), r.SubscriptionID, authorizer)
+	}
+
 	return c, nil
 }
 
@@ -308,6 +323,14 @@ func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName str
 		if err != nil {
 			return err
 		}
+
+		if env.IsLocalDevelopmentMode() {
+			c.log.Info("peering subnets to CI infra")
+			err = c.peerSubnetsToCI(ctx, vnetResourceGroup, clusterName)
+			if err != nil {
+				return err
+			}
+		}
 	}
 
 	c.log.Info("done")
@@ -315,13 +338,15 @@ func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName str
 }
 
 func (c *Cluster) generateSubnets() (vnetPrefix string, masterSubnet string, workerSubnet string) {
-	// pick a random /23 in the range [10.0.2.0, 10.128.0.0).  10.0.0.0 is used
-	// by dev-vnet to host CI; 10.128.0.0+ is used for pods.
+	// pick a random 23 in range [10.3.0.0, 10.127.255.0]
+	// 10.0.0.0 is used by dev-vnet to host CI
+	// 10.1.0.0 is used by rp-vnet to host Proxy VM
+	// 10.2.0.0 is used by dev-vpn-vnet to host VirtualNetworkGateway
 	var x, y int
+	rand.Seed(time.Now().UnixNano())
 	for x == 0 && y == 0 {
-		x, y = rand.Intn(128), 2*rand.Intn(128)
+		x, y = rand.Intn((124))+3, 2*rand.Intn(128)
 	}
-
 	vnetPrefix = fmt.Sprintf("10.%d.%d.0/23", x, y)
 	masterSubnet = fmt.Sprintf("10.%d.%d.0/24", x, y)
 	workerSubnet = fmt.Sprintf("10.%d.%d.0/24", x, y+1)
@@ -359,6 +384,16 @@ func (c *Cluster) Delete(ctx context.Context, vnetResourceGroup, clusterName str
 				errs = append(errs, err)
 			}
 		}
+		// Only delete peering if CI=true and RP_MODE=development
+		if env.IsLocalDevelopmentMode() {
+			r, err := azure.ParseResourceID(c.ciParentVnet)
+			if err == nil {
+				err = c.ciParentVnetPeerings.DeleteAndWait(ctx, r.ResourceGroup, r.ResourceName, vnetResourceGroup+"-peer")
+			}
+			if err != nil {
+				errs = append(errs, err)
+			}
+		}
 	} else {
 		// Deleting the deployment does not clean up the associated resources
 		c.log.Info("deleting deployment")
@@ -574,3 +609,41 @@ func (c *Cluster) deleteRoleAssignments(ctx context.Context, vnetResourceGroup,
 
 	return nil
 }
+
+func (c *Cluster) peerSubnetsToCI(ctx context.Context, vnetResourceGroup, clusterName string) error {
+	cluster := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/dev-vnet", c.env.SubscriptionID(), vnetResourceGroup)
+
+	r, err := azure.ParseResourceID(c.ciParentVnet)
+	if err != nil {
+		return err
+	}
+
+	clusterProp := &mgmtnetwork.VirtualNetworkPeeringPropertiesFormat{
+		RemoteVirtualNetwork: &mgmtnetwork.SubResource{
+			ID: &c.ciParentVnet,
+		},
+		AllowVirtualNetworkAccess: to.BoolPtr(true),
+		AllowForwardedTraffic:     to.BoolPtr(true),
+		UseRemoteGateways:         to.BoolPtr(true),
+	}
+	rpProp := &mgmtnetwork.VirtualNetworkPeeringPropertiesFormat{
+		RemoteVirtualNetwork: &mgmtnetwork.SubResource{
+			ID: &cluster,
+		},
+		AllowVirtualNetworkAccess: to.BoolPtr(true),
+		AllowForwardedTraffic:     to.BoolPtr(true),
+		AllowGatewayTransit:       to.BoolPtr(true),
+	}
+
+	err = c.peerings.CreateOrUpdateAndWait(ctx, vnetResourceGroup, "dev-vnet", r.ResourceGroup+"-peer", mgmtnetwork.VirtualNetworkPeering{VirtualNetworkPeeringPropertiesFormat: clusterProp})
+	if err != nil {
+		return err
+	}
+
+	err = c.ciParentVnetPeerings.CreateOrUpdateAndWait(ctx, r.ResourceGroup, r.ResourceName, vnetResourceGroup+"-peer", mgmtnetwork.VirtualNetworkPeering{VirtualNetworkPeeringPropertiesFormat: rpProp})
+	if err != nil {
+		return err
+	}
+
+	return err
+}
diff --git a/pkg/util/mocks/azureclient/mgmt/network/network.go b/pkg/util/mocks/azureclient/mgmt/network/network.go
index b6912a8a1..15a1b9fa3 100644
--- a/pkg/util/mocks/azureclient/mgmt/network/network.go
+++ b/pkg/util/mocks/azureclient/mgmt/network/network.go
@@ -591,6 +591,50 @@ func (m *MockVirtualNetworkPeeringsClient) EXPECT() *MockVirtualNetworkPeeringsC
 	return m.recorder
 }
 
+// CreateOrUpdate mocks base method.
+func (m *MockVirtualNetworkPeeringsClient) CreateOrUpdate(arg0 context.Context, arg1, arg2, arg3 string, arg4 network.VirtualNetworkPeering) (network.VirtualNetworkPeeringsCreateOrUpdateFuture, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "CreateOrUpdate", arg0, arg1, arg2, arg3, arg4)
+	ret0, _ := ret[0].(network.VirtualNetworkPeeringsCreateOrUpdateFuture)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// CreateOrUpdate indicates an expected call of CreateOrUpdate.
+func (mr *MockVirtualNetworkPeeringsClientMockRecorder) CreateOrUpdate(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdate", reflect.TypeOf((*MockVirtualNetworkPeeringsClient)(nil).CreateOrUpdate), arg0, arg1, arg2, arg3, arg4)
+}
+
+// CreateOrUpdateAndWait mocks base method.
+func (m *MockVirtualNetworkPeeringsClient) CreateOrUpdateAndWait(arg0 context.Context, arg1, arg2, arg3 string, arg4 network.VirtualNetworkPeering) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "CreateOrUpdateAndWait", arg0, arg1, arg2, arg3, arg4)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// CreateOrUpdateAndWait indicates an expected call of CreateOrUpdateAndWait.
+func (mr *MockVirtualNetworkPeeringsClientMockRecorder) CreateOrUpdateAndWait(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateAndWait", reflect.TypeOf((*MockVirtualNetworkPeeringsClient)(nil).CreateOrUpdateAndWait), arg0, arg1, arg2, arg3, arg4)
+}
+
+// Delete mocks base method.
+func (m *MockVirtualNetworkPeeringsClient) Delete(arg0 context.Context, arg1, arg2, arg3 string) (network.VirtualNetworkPeeringsDeleteFuture, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Delete", arg0, arg1, arg2, arg3)
+	ret0, _ := ret[0].(network.VirtualNetworkPeeringsDeleteFuture)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Delete indicates an expected call of Delete.
+func (mr *MockVirtualNetworkPeeringsClientMockRecorder) Delete(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockVirtualNetworkPeeringsClient)(nil).Delete), arg0, arg1, arg2, arg3)
+}
+
 // DeleteAndWait mocks base method.
 func (m *MockVirtualNetworkPeeringsClient) DeleteAndWait(arg0 context.Context, arg1, arg2, arg3 string) error {
 	m.ctrl.T.Helper()
diff --git a/pkg/util/mocks/operator/deploy/deploy.go b/pkg/util/mocks/operator/deploy/deploy.go
index 5d5009d70..4b42b5609 100644
--- a/pkg/util/mocks/operator/deploy/deploy.go
+++ b/pkg/util/mocks/operator/deploy/deploy.go
@@ -77,3 +77,17 @@ func (mr *MockOperatorMockRecorder) IsRunningDesiredVersion(arg0 interface{}) *g
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsRunningDesiredVersion", reflect.TypeOf((*MockOperator)(nil).IsRunningDesiredVersion), arg0)
 }
+
+// RenewMDSDCertificate mocks base method.
+func (m *MockOperator) RenewMDSDCertificate(arg0 context.Context) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "RenewMDSDCertificate", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// RenewMDSDCertificate indicates an expected call of RenewMDSDCertificate.
+func (mr *MockOperatorMockRecorder) RenewMDSDCertificate(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RenewMDSDCertificate", reflect.TypeOf((*MockOperator)(nil).RenewMDSDCertificate), arg0)
+}


Test patch:
diff --git a/.pipelines/e2e.yml b/.pipelines/e2e.yml
index c1167206b..1c86207e4 100644
--- a/.pipelines/e2e.yml
+++ b/.pipelines/e2e.yml
@@ -43,6 +43,9 @@ jobs:
       az account set -s $AZURE_SUBSCRIPTION_ID
 
       set -x
+
+      export PRIVATE_CLUSTER=true
+
       . ./hack/e2e/run-rp-and-e2e.sh
       trap 'set +e; kill_rp; clean_e2e_db; kill_vpn' EXIT
 
@@ -55,5 +58,8 @@ jobs:
       register_sub
 
       export CI=true
+
       make test-e2e
+
+
   - template: ./templates/template-az-cli-logout.yml
diff --git a/hack/e2e/run-rp-and-e2e.sh b/hack/e2e/run-rp-and-e2e.sh
index e6e3cdfe5..0bf6a3ed5 100755
--- a/hack/e2e/run-rp-and-e2e.sh
+++ b/hack/e2e/run-rp-and-e2e.sh
@@ -76,6 +76,16 @@ clean_e2e_db(){
         --resource-group $RESOURCEGROUP >/dev/null
 }
 
+run_vpn() {
+    sudo openvpn --config secrets/$VPN --daemon --writepid vpnpid
+    sleep 10
+}
+
+kill_vpn() {
+    while read pid; do sudo kill $pid; done < vpnpid
+}
+
+
 # TODO: CLUSTER and is also recalculated in multiple places
 # in the billing pipelines :-(
 
diff --git a/pkg/api/admin/openshiftcluster_validatestatic.go b/pkg/api/admin/openshiftcluster_validatestatic.go
index c4346edf3..cad10cbf2 100644
--- a/pkg/api/admin/openshiftcluster_validatestatic.go
+++ b/pkg/api/admin/openshiftcluster_validatestatic.go
@@ -29,7 +29,7 @@ func (sv openShiftClusterStaticValidator) validateDelta(oc, current *OpenShiftCl
 		return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodePropertyChangeNotAllowed, err.Target, err.Message)
 	}
 
-	if !(oc.Properties.MaintenanceTask == "" || oc.Properties.MaintenanceTask == MaintenanceTaskEverything || oc.Properties.MaintenanceTask == MaintenanceTaskOperator) {
+	if !(oc.Properties.MaintenanceTask == "" || oc.Properties.MaintenanceTask == MaintenanceTaskEverything || oc.Properties.MaintenanceTask == MaintenanceTaskOperator || oc.Properties.MaintenanceTask == MaintenanceTaskRenewCerts) {
 		return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "properties.maintenanceTask", "Invalid enum parameter.")
 	}
 
diff --git a/pkg/cluster/adminupdate_test.go b/pkg/cluster/adminupdate_test.go
index f8a5d5155..8f8900a3f 100644
--- a/pkg/cluster/adminupdate_test.go
+++ b/pkg/cluster/adminupdate_test.go
@@ -181,6 +181,30 @@ func TestAdminUpdateSteps(t *testing.T) {
 				"[Action updateProvisionedBy-fm]",
 			},
 		},
+		{
+			name: "Rotate in-cluster MDSD/Ingress/API certs",
+			fixture: func() (*api.OpenShiftClusterDocument, bool) {
+				doc := baseClusterDoc()
+				doc.OpenShiftCluster.Properties.ProvisioningState = api.ProvisioningStateAdminUpdating
+				doc.OpenShiftCluster.Properties.MaintenanceTask = api.MaintenanceTaskRenewCerts
+				return doc, true
+			},
+			shouldRunSteps: []string{
+				"[Action initializeKubernetesClients-fm]",
+				"[Action ensureBillingRecord-fm]",
+				"[Action ensureDefaults-fm]",
+				"[Action fixupClusterSPObjectID-fm]",
+				"[Action fixInfraID-fm]",
+				"[Action startVMs-fm]",
+				"[Condition apiServersReady-fm, timeout 30m0s]",
+				"[Action fixMCSCert-fm]",
+				"[Action fixMCSUserData-fm]",
+				"[Action configureAPIServerCertificate-fm]",
+				"[Action configureIngressCertificate-fm]",
+				"[Action initializeOperatorDeployer-fm]",
+				"[Action renewMDSDCertificate-fm]",
+			},
+		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			doc, adoptViaHive := tt.fixture()
diff --git a/pkg/portal/middleware/aad_test.go b/pkg/portal/middleware/aad_test.go
index 6280db2c2..867a18a39 100644
--- a/pkg/portal/middleware/aad_test.go
+++ b/pkg/portal/middleware/aad_test.go
@@ -137,7 +137,7 @@ func TestAAD(t *testing.T) {
 					},
 				}, nil
 			},
-			wantStatusCode: http.StatusInternalServerError,
+			wantStatusCode: http.StatusForbidden,
 		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
diff --git a/test/e2e/nodes.go b/test/e2e/nodes.go
deleted file mode 100644
index d6daaf0d0..000000000
--- a/test/e2e/nodes.go
+++ /dev/null
@@ -1,57 +0,0 @@
-package e2e
-
-// Copyright (c) Microsoft Corporation.
-// Licensed under the Apache License 2.0.
-
-import (
-	"context"
-	"time"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
-
-	"github.com/Azure/ARO-RP/pkg/util/ready"
-)
-
-const (
-	machineSetsNamespace = "openshift-machine-api"
-)
-
-var _ = Describe("Node check", func() {
-	Specify("node count should match the cluster resource and nodes should be ready", func() {
-		ctx := context.Background()
-		machinesets, err := clients.MachineAPI.MachineV1beta1().MachineSets(machineSetsNamespace).List(ctx, metav1.ListOptions{})
-		Expect(err).NotTo(HaveOccurred())
-		expectedNodeCount := 3 // for masters
-		for _, machineset := range machinesets.Items {
-			expectedNodeCount += int(*machineset.Spec.Replicas)
-		}
-		// another hack: we don't currently instantaneously expect all nodes to
-		// be ready, it could be that the workaround operator is busy rotating
-		// them, which we don't currently wait for on create
-		err = wait.PollImmediate(10*time.Second, 30*time.Minute, func() (bool, error) {
-			nodes, err := clients.Kubernetes.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
-			if err != nil {
-				log.Warn(err)
-				return false, nil // swallow error
-			}
-
-			var nodeCount int
-			for _, node := range nodes.Items {
-				if ready.NodeIsReady(&node) {
-					nodeCount++
-				} else {
-					for _, c := range node.Status.Conditions {
-						log.Warnf("node %s status %s", node.Name, c.String())
-					}
-				}
-			}
-
-			return nodeCount == expectedNodeCount, nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-	})
-})
diff --git a/test/e2e/operator.go b/test/e2e/operator.go
index 915b61f5b..a26f2612a 100644
--- a/test/e2e/operator.go
+++ b/test/e2e/operator.go
@@ -5,11 +5,9 @@ package e2e
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"regexp"
 	"sort"
-	"strconv"
 	"strings"
 	"time"
 
@@ -411,30 +409,6 @@ var _ = Describe("ARO Operator - MUO Deployment", func() {
 	})
 })
 
-var _ = Describe("ARO Operator - MHC Deployment", func() {
-	It("must be deployed and managed by default", func() {
-		mhcIsDeployed := func() (bool, error) {
-			By("getting ARO operator config")
-			co, err := clients.AROClusters.AroV1alpha1().Clusters().Get(context.Background(), "cluster", metav1.GetOptions{})
-			if err != nil {
-				return false, err
-			}
-
-			By("checking ARO operator flags")
-			mhcEnabled, _ := strconv.ParseBool(co.Spec.OperatorFlags.GetWithDefault("aro.machinehealthcheck.enabled", "false"))
-			mhcManaged, _ := strconv.ParseBool(co.Spec.OperatorFlags.GetWithDefault("aro.machinehealthcheck.managed", "false"))
-
-			if mhcEnabled && mhcManaged {
-				return true, nil
-			}
-			return false, errors.New("mhc should be enabled and managed by default")
-		}
-
-		err := wait.PollImmediate(30*time.Second, 10*time.Minute, mhcIsDeployed)
-		Expect(err).NotTo(HaveOccurred())
-	})
-})
-
 var _ = Describe("ARO Operator - ImageConfig Reconciler", func() {
 	const (
 		imageconfigFlag  = "aro.imageconfig.enabled"