Repository: Azure/ARO-RP
Pull request #1973 (2022-02-17 20:44:26)

Egress Lockdown: Enable and Toggle
### Which issue this PR addresses:

<!--
Please include a link to the ADO work item as well as any GitHub issues.

Usage: `Fixes #<GitHub issue number>`, or `Fixes (paste link of issue)`.
-->
Fixes https://msazure.visualstudio.com/AzureRedHatOpenShift/_workitems/edit/13137954/

### What this PR does / why we need it:

<!--
Include a brief summary of what the PR is intended to accomplish and how the PR
does it. (2-3 sentences)
-->
This feature will enable the egress lockdown feature by default on new clusters while also allowing current clusters to be admin-upgraded with the new feature.

### Test plan for issue:

<!--
How did you test that this PR works?

- Are there unit tests?
- Are there integration/e2e tests?
- If it is not possible to write automated tests, explain why and document how
  to manually test and verify the feature.
-->

- [x] Update unit test for default cluster features
- [x] Update unit test for bi-directional toggling of the feature in admin update
- [x] Write new e2e test for internal container registry, since that takes a path from node -> gateway -> storageaccount of the registry
- [x] Manual testing in int-like env for:
    - [x] Current cluster upgrade:
        - [x] Creating cluster w/o egress lockdown
        - [x] Admin patch that cluster with the feature
        - [x] Make sure all is well
   - [x] New default behavior:
        - [x] Creating cluster w/ default (egress lockdown)
        - [x] Make sure all is well
   - [x] Ensure only admins can update the Egress flag

### Is there any documentation that needs to be updated for this PR?

<!--
- If yes and the docs are in GitHub, include doc updates in the PR.
- If yes and the docs are not in GitHub (i.e. ADO wiki), include a link to the
  docs.
- If no, explain why (e.g. "tech debt cleanup, N/A").
-->



Patch:
diff --git a/.pipelines/prod-release-tag.yml b/.pipelines/prod-release-tag.yml
index 0e36b9e6b..bb9f2baf6 100644
--- a/.pipelines/prod-release-tag.yml
+++ b/.pipelines/prod-release-tag.yml
@@ -80,7 +80,6 @@ stages:
       - southcentralus
       - westus
       - westus2
-      - westus3
       configFileName: prod-config.yaml
       azureDevOpsJSONSPN: $(aro-v4-ci-devops-spn)
       vsoProjectID: $(vso-project-id)
diff --git a/.pipelines/prod-release.yml b/.pipelines/prod-release.yml
index 687f815e2..e1aacd527 100644
--- a/.pipelines/prod-release.yml
+++ b/.pipelines/prod-release.yml
@@ -76,7 +76,6 @@ stages:
       - southcentralus
       - westus
       - westus2
-      - westus3
       configFileName: prod-config.yaml
       configVersion: ${{ parameters.configVersion }}
       azureDevOpsJSONSPN: $(aro-v4-ci-devops-spn)
diff --git a/docs/prepare-a-shared-rp-development-environment.md b/docs/prepare-a-shared-rp-development-environment.md
index 15a4988ab..815bdb2b1 100644
--- a/docs/prepare-a-shared-rp-development-environment.md
+++ b/docs/prepare-a-shared-rp-development-environment.md
@@ -541,9 +541,3 @@ Development value: secrets/cluster-logging-int.pem
    ```bash
    vpn_configuration
    ```
-
-
-## Append Resource Group to Subscription Cleaner DenyList
-
-* We have subscription pruning that takes place routinely and need to add our resource group for the shared rp environment to the `denylist` of the cleaner:
-	* [https://github.com/Azure/ARO-RP/blob/e918d1b87be53a3b3cdf18b674768a6480fb56b8/hack/clean/clean.go#L29](https://github.com/Azure/ARO-RP/blob/e918d1b87be53a3b3cdf18b674768a6480fb56b8/hack/clean/clean.go#L29) 
diff --git a/pkg/frontend/openshiftcluster_putorpatch.go b/pkg/frontend/openshiftcluster_putorpatch.go
index 087770e6d..19fece5b4 100644
--- a/pkg/frontend/openshiftcluster_putorpatch.go
+++ b/pkg/frontend/openshiftcluster_putorpatch.go
@@ -84,10 +84,9 @@ func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus.
 				},
 			},
 		}
-		// TODO (BV): reenable gateway on create once we fix bugs
-		// if !f.env.IsLocalDevelopmentMode() /* not local dev or CI */ {
-		// 	doc.OpenShiftCluster.Properties.FeatureProfile.GatewayEnabled = true
-		// }
+		if !f.env.IsLocalDevelopmentMode() /* not local dev or CI */ {
+			doc.OpenShiftCluster.Properties.FeatureProfile.GatewayEnabled = true
+		}
 	}
 
 	doc.CorrelationData = correlationData
diff --git a/pkg/monitor/cluster/cluster.go b/pkg/monitor/cluster/cluster.go
index feadd1e14..fce45a0c0 100644
--- a/pkg/monitor/cluster/cluster.go
+++ b/pkg/monitor/cluster/cluster.go
@@ -12,7 +12,6 @@ import (
 	"github.com/Azure/go-autorest/autorest/azure"
 	configv1 "github.com/openshift/api/config/v1"
 	configclient "github.com/openshift/client-go/config/clientset/versioned"
-	maoclient "github.com/openshift/machine-api-operator/pkg/generated/clientset/versioned"
 	mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned"
 	"github.com/sirupsen/logrus"
 	appsv1 "k8s.io/api/apps/v1"
@@ -35,7 +34,6 @@ type Monitor struct {
 	restconfig *rest.Config
 	cli        kubernetes.Interface
 	configcli  configclient.Interface
-	maocli     maoclient.Interface
 	mcocli     mcoclient.Interface
 	m          metrics.Interface
 	arocli     aroclient.Interface
@@ -72,11 +70,6 @@ func NewMonitor(ctx context.Context, log *logrus.Entry, restConfig *rest.Config,
 		return nil, err
 	}
 
-	maocli, err := maoclient.NewForConfig(restConfig)
-	if err != nil {
-		return nil, err
-	}
-
 	mcocli, err := mcoclient.NewForConfig(restConfig)
 	if err != nil {
 		return nil, err
@@ -97,7 +90,6 @@ func NewMonitor(ctx context.Context, log *logrus.Entry, restConfig *rest.Config,
 		restconfig: restConfig,
 		cli:        cli,
 		configcli:  configcli,
-		maocli:     maocli,
 		mcocli:     mcocli,
 		arocli:     arocli,
 		m:          m,
diff --git a/pkg/monitor/cluster/nodeconditions.go b/pkg/monitor/cluster/nodeconditions.go
index 89a639460..1a11c7c52 100644
--- a/pkg/monitor/cluster/nodeconditions.go
+++ b/pkg/monitor/cluster/nodeconditions.go
@@ -5,13 +5,9 @@ package cluster
 
 import (
 	"context"
-	"encoding/json"
-	"strconv"
 
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	azureproviderv1beta1 "sigs.k8s.io/cluster-api-provider-azure/pkg/apis/azureprovider/v1beta1"
 )
 
 var nodeConditionsExpected = map[corev1.NodeConditionType]corev1.ConditionStatus{
@@ -27,34 +23,27 @@ func (mon *Monitor) emitNodeConditions(ctx context.Context) error {
 		return err
 	}
 
-	spotInstances := mon.getSpotInstances(ctx)
-
 	mon.emitGauge("node.count", int64(len(ns.Items)), nil)
 
 	for _, n := range ns.Items {
-
 		for _, c := range n.Status.Conditions {
 			if c.Status == nodeConditionsExpected[c.Type] {
 				continue
 			}
 
-			_, isSpotInstance := spotInstances[n.Name]
-
 			mon.emitGauge("node.conditions", 1, map[string]string{
-				"nodeName":     n.Name,
-				"status":       string(c.Status),
-				"type":         string(c.Type),
-				"spotInstance": strconv.FormatBool(isSpotInstance),
+				"nodeName": n.Name,
+				"status":   string(c.Status),
+				"type":     string(c.Type),
 			})
 
 			if mon.hourlyRun {
 				mon.log.WithFields(logrus.Fields{
-					"metric":       "node.conditions",
-					"name":         n.Name,
-					"status":       c.Status,
-					"type":         c.Type,
-					"message":      c.Message,
-					"spotInstance": isSpotInstance,
+					"metric":  "node.conditions",
+					"name":    n.Name,
+					"status":  c.Status,
+					"type":    c.Type,
+					"message": c.Message,
 				}).Print()
 			}
 		}
@@ -68,30 +57,3 @@ func (mon *Monitor) emitNodeConditions(ctx context.Context) error {
 
 	return nil
 }
-
-// getSpotInstances returns a map where the keys are the machine name and only exist if the machine is a spot instance
-func (mon *Monitor) getSpotInstances(ctx context.Context) map[string]struct{} {
-	spotInstances := make(map[string]struct{})
-	machines, err := mon.maocli.MachineV1beta1().Machines("openshift-machine-api").List(ctx, metav1.ListOptions{})
-
-	if err != nil {
-		// when this call fails we may report spot vms as non spot until the next successful call
-		mon.log.Error(err)
-		return spotInstances
-	}
-
-	for _, machine := range machines.Items {
-		var spec azureproviderv1beta1.AzureMachineProviderSpec
-		err = json.Unmarshal(machine.Spec.ProviderSpec.Value.Raw, &spec)
-		if err != nil {
-			mon.log.Error(err)
-			continue
-		}
-
-		if spec.SpotVMOptions != nil {
-			spotInstances[machine.Name] = struct{}{}
-		}
-	}
-
-	return spotInstances
-}
diff --git a/pkg/operator/controllers/checker/checker_controller.go b/pkg/operator/controllers/checker/checker_controller.go
index 579cad709..965370423 100644
--- a/pkg/operator/controllers/checker/checker_controller.go
+++ b/pkg/operator/controllers/checker/checker_controller.go
@@ -83,11 +83,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.
 		}
 	}
 
-	// We always requeue here:
-	// * Either immediately (with rate limiting) based on the error
-	//   when err != nil.
-	// * Or based on RequeueAfter when err == nil.
-	return reconcile.Result{RequeueAfter: time.Hour}, err
+	return reconcile.Result{RequeueAfter: time.Hour, Requeue: true}, err
 }
 
 // SetupWithManager setup our manager
diff --git a/pkg/operator/controllers/node/node_controller.go b/pkg/operator/controllers/node/node_controller.go
index 2b53dfc83..879a164fc 100644
--- a/pkg/operator/controllers/node/node_controller.go
+++ b/pkg/operator/controllers/node/node_controller.go
@@ -101,7 +101,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.
 	deadline := t.Add(gracePeriod)
 	now := time.Now()
 	if deadline.After(now) {
-		return reconcile.Result{RequeueAfter: deadline.Sub(now)}, nil
+		return reconcile.Result{
+			RequeueAfter: deadline.Sub(now),
+		}, err
 	}
 
 	// drain the node disabling eviction
diff --git a/pkg/operator/controllers/workaround/workaround_controller.go b/pkg/operator/controllers/workaround/workaround_controller.go
index a99c63614..cd8fef05e 100644
--- a/pkg/operator/controllers/workaround/workaround_controller.go
+++ b/pkg/operator/controllers/workaround/workaround_controller.go
@@ -87,7 +87,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.
 			return reconcile.Result{}, err
 		}
 	}
-	return reconcile.Result{RequeueAfter: time.Hour}, nil
+	return reconcile.Result{RequeueAfter: time.Hour, Requeue: true}, nil
 }
 
 // SetupWithManager setup our manager
diff --git a/pkg/operator/deploy/deploy.go b/pkg/operator/deploy/deploy.go
index 7b08ba15d..fae77da2c 100644
--- a/pkg/operator/deploy/deploy.go
+++ b/pkg/operator/deploy/deploy.go
@@ -175,10 +175,12 @@ func (o *operator) resources() ([]kruntime.Object, error) {
 		},
 	}
 
-	// TODO (BV): reenable gateway once we fix bugs
-	// if o.oc.Properties.NetworkProfile.GatewayPrivateEndpointIP != "" {
-	// 	cluster.Spec.GatewayDomains = append(o.env.GatewayDomains(), o.oc.Properties.ImageRegistryStorageAccountName+".blob."+o.env.Environment().StorageEndpointSuffix)
-	// }
+	if o.oc.Properties.FeatureProfile.GatewayEnabled && o.oc.Properties.NetworkProfile.GatewayPrivateEndpointIP != "" {
+		cluster.Spec.GatewayDomains = append(o.env.GatewayDomains(), o.oc.Properties.ImageRegistryStorageAccountName+".blob."+o.env.Environment().StorageEndpointSuffix)
+	} else {
+		// covers the case of an admin-disable, we need to update dnsmasq on each node
+		cluster.Spec.GatewayDomains = make([]string, 0)
+	}
 
 	// create a secret here for genevalogging, later we will copy it to
 	// the genevalogging namespace.
diff --git a/portal/package-lock.json b/portal/package-lock.json
index caa7fbe7a..c1ece3eca 100644
--- a/portal/package-lock.json
+++ b/portal/package-lock.json
@@ -5700,9 +5700,9 @@
       }
     },
     "url-parse": {
-      "version": "1.5.7",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.7.tgz",
-      "integrity": "sha512-HxWkieX+STA38EDk7CE9MEryFeHCKzgagxlGvsdS7WBImq9Mk+PGwiT56w82WI3aicwJA8REp42Cxo98c8FZMA==",
+      "version": "1.5.3",
+      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.3.tgz",
+      "integrity": "sha512-IIORyIQD9rvj0A4CLWsHkBBJuNqWpFQe224b6j9t/ABmquIS0qDU2pY6kl6AuOrL5OkCXHMCFNe1jBcuAggjvQ==",
       "dev": true,
       "requires": {
         "querystringify": "^2.1.1",


Test patch:
diff --git a/.pipelines/e2e-with-billing-all-regions.yml b/.pipelines/e2e-with-billing-all-regions.yml
index 14dfde24b..d6cd44400 100644
--- a/.pipelines/e2e-with-billing-all-regions.yml
+++ b/.pipelines/e2e-with-billing-all-regions.yml
@@ -68,7 +68,6 @@ stages:
     - southcentralus
     - westus
     - westus2
-    - westus3
 - stage: PhaseThree
   displayName: wait 2 hours to avoid resource threshold
   dependsOn: []
diff --git a/pkg/api/admin/openshiftcluster_validatestatic.go b/pkg/api/admin/openshiftcluster_validatestatic.go
index f94c6a338..bb56358e4 100644
--- a/pkg/api/admin/openshiftcluster_validatestatic.go
+++ b/pkg/api/admin/openshiftcluster_validatestatic.go
@@ -33,9 +33,5 @@ func (sv *openShiftClusterStaticValidator) validateDelta(oc, current *OpenShiftC
 		return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "properties.maintenanceTask", "Invalid enum parameter.")
 	}
 
-	if current.Properties.FeatureProfile.GatewayEnabled && !oc.Properties.FeatureProfile.GatewayEnabled {
-		return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodePropertyChangeNotAllowed, "properties.featureProfile.gatewayEnabled", "Changing property 'properties.featureProfile.gatewayEnabled' is not allowed.")
-	}
-
 	return nil
 }
diff --git a/pkg/api/admin/openshiftcluster_validatestatic_test.go b/pkg/api/admin/openshiftcluster_validatestatic_test.go
index a04448654..a00ca327f 100644
--- a/pkg/api/admin/openshiftcluster_validatestatic_test.go
+++ b/pkg/api/admin/openshiftcluster_validatestatic_test.go
@@ -129,29 +129,34 @@ func TestOpenShiftClusterStaticValidateDelta(t *testing.T) {
 			wantErr: "400: PropertyChangeNotAllowed: properties.lastAdminUpdateError: Changing property 'properties.lastAdminUpdateError' is not allowed.",
 		},
 		{
-			name: "valid gatewayEnabled change",
+			name: "disable gatewayEnabled on enabled clusters is allowed",
 			oc: func() *OpenShiftCluster {
-				return &OpenShiftCluster{}
+				return &OpenShiftCluster{
+					Properties: OpenShiftClusterProperties{
+						FeatureProfile: FeatureProfile{
+							GatewayEnabled: true,
+						},
+					},
+				}
 			},
 			modify: func(oc *OpenShiftCluster) {
-				oc.Properties.FeatureProfile.GatewayEnabled = true
+				oc.Properties.FeatureProfile.GatewayEnabled = false
 			},
 		},
 		{
-			name: "valid gatewayEnabled change",
+			name: "enable gatewayEnabled on disabled clusters is allowed",
 			oc: func() *OpenShiftCluster {
 				return &OpenShiftCluster{
 					Properties: OpenShiftClusterProperties{
 						FeatureProfile: FeatureProfile{
-							GatewayEnabled: true,
+							GatewayEnabled: false,
 						},
 					},
 				}
 			},
 			modify: func(oc *OpenShiftCluster) {
-				oc.Properties.FeatureProfile.GatewayEnabled = false
+				oc.Properties.FeatureProfile.GatewayEnabled = true
 			},
-			wantErr: "400: PropertyChangeNotAllowed: properties.featureProfile.gatewayEnabled: Changing property 'properties.featureProfile.gatewayEnabled' is not allowed.",
 		},
 		{
 			name: "console url change is not allowed",
diff --git a/pkg/frontend/openshiftcluster_putorpatch_test.go b/pkg/frontend/openshiftcluster_putorpatch_test.go
index 6f18785df..adeaefd23 100644
--- a/pkg/frontend/openshiftcluster_putorpatch_test.go
+++ b/pkg/frontend/openshiftcluster_putorpatch_test.go
@@ -719,7 +719,7 @@ func TestPutOrPatchOpenShiftCluster(t *testing.T) {
 								EncryptionAtHost: api.EncryptionAtHostDisabled,
 							},
 							FeatureProfile: api.FeatureProfile{
-								GatewayEnabled: false,
+								GatewayEnabled: true,
 							},
 							OperatorFlags: api.DefaultOperatorFlags.Copy(),
 						},
diff --git a/pkg/monitor/cluster/nodeconditions_test.go b/pkg/monitor/cluster/nodeconditions_test.go
index 6abb2057d..eb8c6c4d0 100644
--- a/pkg/monitor/cluster/nodeconditions_test.go
+++ b/pkg/monitor/cluster/nodeconditions_test.go
@@ -5,19 +5,12 @@ package cluster
 
 import (
 	"context"
-	"encoding/json"
 	"testing"
 
 	"github.com/golang/mock/gomock"
-	machinev1beta1 "github.com/openshift/machine-api-operator/pkg/apis/machine/v1beta1"
-	maoclient "github.com/openshift/machine-api-operator/pkg/generated/clientset/versioned"
-	maofake "github.com/openshift/machine-api-operator/pkg/generated/clientset/versioned/fake"
-	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	kruntime "k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes/fake"
-	azureproviderv1beta1 "sigs.k8s.io/cluster-api-provider-azure/pkg/apis/azureprovider/v1beta1"
 
 	mock_metrics "github.com/Azure/ARO-RP/pkg/util/mocks/metrics"
 )
@@ -25,19 +18,9 @@ import (
 func TestEmitNodeConditions(t *testing.T) {
 	ctx := context.Background()
 
-	provSpec, err := json.Marshal(azureproviderv1beta1.AzureMachineProviderSpec{})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	kubeletVersion := "v1.17.1+9d33dd3"
-
 	cli := fake.NewSimpleClientset(&corev1.Node{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "aro-master-0",
-			Annotations: map[string]string{
-				"machine.openshift.io/machine": "openshift-machine-api/master-0",
-			},
 		},
 		Status: corev1.NodeStatus{
 			Conditions: []corev1.NodeCondition{
@@ -47,15 +30,12 @@ func TestEmitNodeConditions(t *testing.T) {
 				},
 			},
 			NodeInfo: corev1.NodeSystemInfo{
-				KubeletVersion: kubeletVersion,
+				KubeletVersion: "v1.17.1+9d33dd3",
 			},
 		},
 	}, &corev1.Node{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "aro-master-1",
-			Annotations: map[string]string{
-				"machine.openshift.io/machine": "openshift-machine-api/master-1",
-			},
 		},
 		Status: corev1.NodeStatus{
 			Conditions: []corev1.NodeCondition{
@@ -65,38 +45,10 @@ func TestEmitNodeConditions(t *testing.T) {
 				},
 			},
 			NodeInfo: corev1.NodeSystemInfo{
-				KubeletVersion: kubeletVersion,
+				KubeletVersion: "v1.17.1+9d33dd3",
 			},
 		},
 	})
-	maoclient := maofake.NewSimpleClientset(
-		&machinev1beta1.Machine{
-			Spec: machinev1beta1.MachineSpec{
-				ProviderSpec: machinev1beta1.ProviderSpec{
-					Value: &kruntime.RawExtension{
-						Raw: provSpec,
-					},
-				},
-			},
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      "openshift-machine-api/master-0",
-				Namespace: "openshift-machine-api",
-			},
-		},
-		&machinev1beta1.Machine{
-			Spec: machinev1beta1.MachineSpec{
-				ProviderSpec: machinev1beta1.ProviderSpec{
-					Value: &kruntime.RawExtension{
-						Raw: provSpec,
-					},
-				},
-			},
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      "openshift-machine-api/master-1",
-				Namespace: "openshift-machine-api",
-			},
-		},
-	)
 
 	controller := gomock.NewController(t)
 	defer controller.Finish()
@@ -104,171 +56,33 @@ func TestEmitNodeConditions(t *testing.T) {
 	m := mock_metrics.NewMockInterface(controller)
 
 	mon := &Monitor{
-		cli:    cli,
-		maocli: maoclient,
-		m:      m,
+		cli: cli,
+		m:   m,
 	}
 
 	m.EXPECT().EmitGauge("node.count", int64(2), map[string]string{})
 	m.EXPECT().EmitGauge("node.conditions", int64(1), map[string]string{
-		"nodeName":     "aro-master-0",
-		"status":       "True",
-		"type":         "MemoryPressure",
-		"spotInstance": "false",
+		"nodeName": "aro-master-0",
+		"status":   "True",
+		"type":     "MemoryPressure",
 	})
 	m.EXPECT().EmitGauge("node.conditions", int64(1), map[string]string{
-		"nodeName":     "aro-master-1",
-		"status":       "False",
-		"type":         "Ready",
-		"spotInstance": "false",
+		"nodeName": "aro-master-1",
+		"status":   "False",
+		"type":     "Ready",
 	})
 
 	m.EXPECT().EmitGauge("node.kubelet.version", int64(1), map[string]string{
 		"nodeName":       "aro-master-0",
-		"kubeletVersion": kubeletVersion,
+		"kubeletVersion": "v1.17.1+9d33dd3",
 	})
 	m.EXPECT().EmitGauge("node.kubelet.version", int64(1), map[string]string{
 		"nodeName":       "aro-master-1",
-		"kubeletVersion": kubeletVersion,
+		"kubeletVersion": "v1.17.1+9d33dd3",
 	})
 
-	err = mon.emitNodeConditions(ctx)
+	err := mon.emitNodeConditions(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
-
-func TestGetSpotInstances(t *testing.T) {
-	ctx := context.Background()
-
-	spotProvSpec, err := json.Marshal(azureproviderv1beta1.AzureMachineProviderSpec{
-		SpotVMOptions: &azureproviderv1beta1.SpotVMOptions{},
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	provSpec, err := json.Marshal(azureproviderv1beta1.AzureMachineProviderSpec{})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for _, tt := range []struct {
-		name                 string
-		maocli               maoclient.Interface
-		node                 corev1.Node
-		expectedSpotInstance bool
-	}{
-		{
-			name: "node is a spot instance",
-			maocli: maofake.NewSimpleClientset(&machinev1beta1.Machine{
-				Spec: machinev1beta1.MachineSpec{
-					ProviderSpec: machinev1beta1.ProviderSpec{
-						Value: &kruntime.RawExtension{
-							Raw: spotProvSpec,
-						},
-					},
-				},
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      "aro-spot-0",
-					Namespace: "openshift-machine-api",
-				},
-			}),
-			node: corev1.Node{
-				ObjectMeta: metav1.ObjectMeta{
-					Name: "aro-spot-0",
-					Annotations: map[string]string{
-						"machine.openshift.io/machine": "openshift-machine-api/spot-0",
-					},
-				},
-			},
-			expectedSpotInstance: true,
-		},
-		{
-			name: "node is not a spot instance",
-			maocli: maofake.NewSimpleClientset(&machinev1beta1.Machine{
-				Spec: machinev1beta1.MachineSpec{
-					ProviderSpec: machinev1beta1.ProviderSpec{
-						Value: &kruntime.RawExtension{
-							Raw: provSpec,
-						},
-					},
-				},
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      "master-0",
-					Namespace: "openshift-machine-api",
-				},
-			}),
-			node: corev1.Node{
-				ObjectMeta: metav1.ObjectMeta{
-					Name: "aro-master-0",
-					Annotations: map[string]string{
-						"machine.openshift.io/machine": "openshift-machine-api/master-0",
-					},
-				},
-			},
-			expectedSpotInstance: false,
-		},
-		{
-			name: "node is missing annotation",
-			maocli: maofake.NewSimpleClientset(&machinev1beta1.Machine{
-				Spec: machinev1beta1.MachineSpec{
-					ProviderSpec: machinev1beta1.ProviderSpec{
-						Value: &kruntime.RawExtension{
-							Raw: provSpec,
-						},
-					},
-				},
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      "master-0",
-					Namespace: "openshift-machine-api",
-				},
-			}),
-			node: corev1.Node{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:        "aro-master-0",
-					Annotations: map[string]string{},
-				},
-			},
-			expectedSpotInstance: false,
-		},
-		{
-			name: "malformed json in providerSpec",
-			maocli: maofake.NewSimpleClientset(&machinev1beta1.Machine{
-				Spec: machinev1beta1.MachineSpec{
-					ProviderSpec: machinev1beta1.ProviderSpec{
-						Value: &kruntime.RawExtension{
-							Raw: []byte(";df9j"),
-						},
-					},
-				},
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      "aro-spot-1",
-					Namespace: "openshift-machine-api",
-				},
-			}),
-			node: corev1.Node{
-				ObjectMeta: metav1.ObjectMeta{
-					Name: "aro-spot-1",
-					Annotations: map[string]string{
-						"machine.openshift.io/machine": "openshift-machine-api/spot-0",
-					},
-				},
-			},
-			expectedSpotInstance: false,
-		},
-	} {
-		controller := gomock.NewController(t)
-		defer controller.Finish()
-
-		mon := &Monitor{
-			maocli: tt.maocli,
-			log:    logrus.NewEntry(logrus.StandardLogger()),
-		}
-
-		_, isSpotInstance := mon.getSpotInstances(ctx)[tt.node.Name]
-		if isSpotInstance != tt.expectedSpotInstance {
-			t.Fatalf("test %s: isSpotInstance should be %t but got %t", tt.name, tt.expectedSpotInstance, isSpotInstance)
-		}
-	}
-}
diff --git a/pkg/operator/controllers/workaround/workaround_controller_test.go b/pkg/operator/controllers/workaround/workaround_controller_test.go
index 9bd6e774f..bd5c5181b 100644
--- a/pkg/operator/controllers/workaround/workaround_controller_test.go
+++ b/pkg/operator/controllers/workaround/workaround_controller_test.go
@@ -68,7 +68,7 @@ func TestWorkaroundReconciler(t *testing.T) {
 				c := mw.EXPECT().IsRequired(gomock.Any()).Return(true)
 				mw.EXPECT().Ensure(gomock.Any()).After(c).Return(nil)
 			},
-			want: ctrl.Result{RequeueAfter: time.Hour},
+			want: ctrl.Result{Requeue: true, RequeueAfter: time.Hour},
 		},
 		{
 			name: "is not required",
@@ -76,7 +76,7 @@ func TestWorkaroundReconciler(t *testing.T) {
 				c := mw.EXPECT().IsRequired(gomock.Any()).Return(false)
 				mw.EXPECT().Remove(gomock.Any()).After(c).Return(nil)
 			},
-			want: ctrl.Result{RequeueAfter: time.Hour},
+			want: ctrl.Result{Requeue: true, RequeueAfter: time.Hour},
 		},
 		{
 			name: "has error",
diff --git a/test/e2e/adminapi_redeployvm.go b/test/e2e/adminapi_redeployvm.go
index df35a2f24..d9d7762b7 100644
--- a/test/e2e/adminapi_redeployvm.go
+++ b/test/e2e/adminapi_redeployvm.go
@@ -36,7 +36,6 @@ var _ = Describe("[Admin API] VM redeploy action", func() {
 		Expect(err).NotTo(HaveOccurred())
 		Expect(vms).NotTo(HaveLen(0))
 		vm := vms[0]
-		log.Infof("selected vm: %s", *vm.Name)
 
 		By("triggering the redeploy action")
 		clockDrift := -1 * time.Minute
@@ -68,18 +67,19 @@ var _ = Describe("[Admin API] VM redeploy action", func() {
 			var nodeNotReady, rebooted, nodeReady bool
 
 			for _, event := range events.Items {
-				if event.CreationTimestamp.After(nodeKillTime.Time) && event.Regarding.Name == *vm.Name {
-					if !nodeNotReady && event.Reason == "NodeNotReady" {
+				if event.CreationTimestamp.After(nodeKillTime.Time) {
+					if !nodeNotReady &&
+						event.Reason == "NodeNotReady" &&
+						event.Regarding.Name == *vm.Name {
 						nodeNotReady = true
-						log.Info("node entered not ready state")
-					}
-					if !rebooted && event.Reason == "Rebooted" {
+					} else if !rebooted &&
+						event.Reason == "Rebooted" &&
+						event.Regarding.Name == *vm.Name {
 						rebooted = true
-						log.Info("node reboooted")
-					}
-					if !nodeReady && event.Reason == "NodeReady" {
+					} else if !nodeReady &&
+						event.Reason == "NodeReady" &&
+						event.Regarding.Name == *vm.Name {
 						nodeReady = true
-						log.Info("node returned to ready state")
 						break
 					}
 				}
diff --git a/test/e2e/cluster.go b/test/e2e/cluster.go
index 2bcf0a9c0..1ea23478e 100644
--- a/test/e2e/cluster.go
+++ b/test/e2e/cluster.go
@@ -83,6 +83,18 @@ var _ = Describe("Cluster smoke test", func() {
 			return ready.ServiceIsReady(svc)
 		}, 5*time.Minute, 10*time.Second).Should(BeTrue())
 	})
+
+	// mainly we want to test the gateway/egress functionality - this request for the image will travel from
+	// node > gateway > storage account of the registry.
+	Specify("Can access and use the internal container registry", func() {
+		ctx := context.Background()
+		deployName := "internal-registry-deploy"
+		err := createContainerFromInternalContainerRegistryImage(ctx, clients.Kubernetes, deployName)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = wait.PollImmediate(10*time.Second, 5*time.Minute, ready.CheckDeploymentIsReady(ctx, clients.Kubernetes.AppsV1().Deployments(testNamespace), deployName))
+		Expect(err).NotTo(HaveOccurred())
+	})
 })
 
 func createStatefulSet(ctx context.Context, cli kubernetes.Interface) error {
@@ -167,3 +179,40 @@ func createLoadBalancerService(ctx context.Context, cli kubernetes.Interface, na
 	_, err := cli.CoreV1().Services(testNamespace).Create(context.Background(), svc, metav1.CreateOptions{})
 	return err
 }
+
+func createContainerFromInternalContainerRegistryImage(ctx context.Context, cli kubernetes.Interface, name string) error {
+	deploy := &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: testNamespace,
+		},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: to.Int32Ptr(1),
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"app": name,
+				},
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{"app": name},
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name:  "cli",
+							Image: "image-registry.openshift-image-registry.svc:5000/openshift/cli",
+							Command: []string{
+								"/bin/sh",
+								"-c",
+								"while true; do sleep 1; done",
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	_, err := cli.AppsV1().Deployments(testNamespace).Create(context.Background(), deploy, metav1.CreateOptions{})
+	return err
+}