Repository: Azure/ARO-RP
Pull request #2006 (2022-03-16 04:53:56)

update go-cosmosdb version to incorporate the latest change
### Which issue this PR addresses:
Fixes https://msazure.visualstudio.com/AzureRedHatOpenShift/_workitems/edit/12373924

### What this PR does / why we need it:
It updates go-cosmosdb version in go.mod and go.sum to incorporate the latest change

### Test plan for issue:
Make succeeded and did not do any other tests as it is just an update on the go mod files

### Is there any documentation that needs to be updated for this PR?
No, as nothing actually changed


microsoft-cla[bot]: [![CLA assistant check](https://cla.opensource.microsoft.com/pull/badge/not_signed)](https://cla.opensource.microsoft.com/Azure/ARO-RP?pullRequest=2006) <br/>Thank you for your submission, we really appreciate it. Like many open source projects, we ask that you sign our [Contributor License Agreement](https://cla.opensource.microsoft.com/Azure/ARO-RP?pullRequest=2006) before we can accept your contribution.<br/><br/>:x: yjst2012 [sign now](https://cla.opensource.microsoft.com/Azure/ARO-RP?pullRequest=2006)<br/><sub>You have signed the CLA already but the status is still pending? Let us [recheck](https://cla.opensource.microsoft.com/check/Azure/ARO-RP?pullRequest=2006) it.</sub>

Patch:
diff --git a/cmd/aro/operator.go b/cmd/aro/operator.go
index 386916359..49701e892 100644
--- a/cmd/aro/operator.go
+++ b/cmd/aro/operator.go
@@ -23,6 +23,7 @@ import (
 	pkgoperator "github.com/Azure/ARO-RP/pkg/operator"
 	aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/alertwebhook"
+	"github.com/Azure/ARO-RP/pkg/operator/controllers/autosizednodes"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/banner"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/checker"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/clusteroperatoraro"
@@ -210,6 +211,11 @@ func operator(ctx context.Context, log *logrus.Entry) error {
 		if err = (muo.NewReconciler(arocli, kubernetescli, dh)).SetupWithManager(mgr); err != nil {
 			return fmt.Errorf("unable to create controller %s: %v", muo.ControllerName, err)
 		}
+		if err = (autosizednodes.NewReconciler(
+			log.WithField("controller", autosizednodes.ControllerName),
+			mgr)).SetupWithManager(mgr); err != nil {
+			return fmt.Errorf("unable to create controller %s: %v", autosizednodes.ControllerName, err)
+		}
 	}
 
 	if err = (checker.NewReconciler(
diff --git a/go.mod b/go.mod
index e888d42e9..790d9298e 100644
--- a/go.mod
+++ b/go.mod
@@ -57,7 +57,7 @@ require (
 	github.com/gorilla/securecookie v1.1.1
 	github.com/gorilla/sessions v1.2.1
 	github.com/h2non/filetype v1.1.3 // indirect
-	github.com/jewzaam/go-cosmosdb v0.0.0-20211018134417-8d1494e7862f
+	github.com/jewzaam/go-cosmosdb v0.0.0-20220315232836-282b67c5b234
 	github.com/jstemmer/go-junit-report v0.9.1
 	github.com/klauspost/compress v1.14.2 // indirect
 	github.com/libvirt/libvirt-go v7.4.0+incompatible // indirect
diff --git a/go.sum b/go.sum
index dfb8c79b5..8e1ae4b2c 100644
--- a/go.sum
+++ b/go.sum
@@ -1529,8 +1529,8 @@ github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4Fw
 github.com/jen20/awspolicyequivalence v1.1.0/go.mod h1:PV1fS2xyHhCLp83vbgSMFr2drM4GzG61wkz+k4pOG3E=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
-github.com/jewzaam/go-cosmosdb v0.0.0-20211018134417-8d1494e7862f h1:G0wto7wY9OnIuwSPiLfyefcUN4FYEdtvwQMsCQdlDas=
-github.com/jewzaam/go-cosmosdb v0.0.0-20211018134417-8d1494e7862f/go.mod h1:Sxto8tJPhBRIa7jDsuTx4TZAcTYu0NH8E39iWqqMeRs=
+github.com/jewzaam/go-cosmosdb v0.0.0-20220315232836-282b67c5b234 h1:R0Hokq55Hv3SpbXyfZRh8vIrKFAPZ+SQqyTsRc7J/4E=
+github.com/jewzaam/go-cosmosdb v0.0.0-20220315232836-282b67c5b234/go.mod h1:kZxm8EB19+pd3nT92t0aQblXb7K4sHG8cp59cleqdNc=
 github.com/jewzaam/installer-aro v0.9.0-master.0.20220208140934-766bcf74e25c h1:yg14zwP4cFO98qqFlbkNqcouP/J0+zdCUf11qa/J/PU=
 github.com/jewzaam/installer-aro v0.9.0-master.0.20220208140934-766bcf74e25c/go.mod h1:qjvlt/LNPsGbLg2DtyY0VzxfKo+WVwcxJhektwKU+5E=
 github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
diff --git a/pkg/api/defaults.go b/pkg/api/defaults.go
index 1e6d88b69..c468f4bff 100644
--- a/pkg/api/defaults.go
+++ b/pkg/api/defaults.go
@@ -76,5 +76,6 @@ func DefaultOperatorFlags() OperatorFlags {
 		"aro.routefix.enabled":                     flagTrue,
 		"aro.storageaccounts.enabled":              flagTrue,
 		"aro.workaround.enabled":                   flagTrue,
+		"aro.autosizednodes.enable":                flagFalse,
 	}
 }
diff --git a/pkg/cluster/arooperator.go b/pkg/cluster/arooperator.go
index 83bd81973..879361c09 100644
--- a/pkg/cluster/arooperator.go
+++ b/pkg/cluster/arooperator.go
@@ -5,17 +5,23 @@ package cluster
 
 import (
 	"context"
-
-	"github.com/Azure/ARO-RP/pkg/operator/deploy"
 )
 
+func (m *manager) isIngressProfileAvailable() bool {
+	// We try to aqcuire the IngressProfiles data at frontend best effort enrichment time only.
+	// When we start deallocated VMs and wait for the API do become available again, we don't pick
+	// the information up, even though it would be available.
+	return len(m.doc.OpenShiftCluster.Properties.IngressProfiles) != 0
+}
+
 func (m *manager) ensureAROOperator(ctx context.Context) error {
-	dep, err := deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.extensionscli, m.kubernetescli)
-	if err != nil {
-		m.log.Errorf("cannot ensureAROOperator.New: %s", err.Error())
-		return err
+	//ensure the IngressProfile information is available from the cluster which is not the case when the cluster vms were freshly restarted.
+	if !m.isIngressProfileAvailable() {
+		m.log.Error("skip ensureAROOperator")
+		return nil
 	}
-	err = dep.CreateOrUpdate(ctx)
+
+	err := m.aroOperatorDeployer.CreateOrUpdate(ctx)
 	if err != nil {
 		m.log.Errorf("cannot ensureAROOperator.CreateOrUpdate: %s", err.Error())
 	}
@@ -23,9 +29,10 @@ func (m *manager) ensureAROOperator(ctx context.Context) error {
 }
 
 func (m *manager) aroDeploymentReady(ctx context.Context) (bool, error) {
-	dep, err := deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.extensionscli, m.kubernetescli)
-	if err != nil {
-		return false, err
+	if !m.isIngressProfileAvailable() {
+		m.log.Error("skip aroDeploymentReady")
+		// skip and don't retry
+		return true, nil
 	}
-	return dep.IsReady(ctx)
+	return m.aroOperatorDeployer.IsReady(ctx)
 }
diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
index db9a7cc72..877a10c71 100644
--- a/pkg/cluster/cluster.go
+++ b/pkg/cluster/cluster.go
@@ -23,6 +23,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/database"
 	"github.com/Azure/ARO-RP/pkg/env"
 	aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned"
+	"github.com/Azure/ARO-RP/pkg/operator/deploy"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/graphrbac"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/authorization"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/compute"
@@ -90,6 +91,8 @@ type manager struct {
 	securitycli      securityclient.Interface
 	arocli           aroclient.Interface
 	imageregistrycli imageregistryclient.Interface
+
+	aroOperatorDeployer deploy.Operator
 }
 
 // New returns a cluster manager
diff --git a/pkg/cluster/install.go b/pkg/cluster/install.go
index 1355a4058..8980e7c15 100644
--- a/pkg/cluster/install.go
+++ b/pkg/cluster/install.go
@@ -22,6 +22,7 @@ import (
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned"
+	"github.com/Azure/ARO-RP/pkg/operator/deploy"
 	"github.com/Azure/ARO-RP/pkg/util/restconfig"
 	"github.com/Azure/ARO-RP/pkg/util/steps"
 	"github.com/Azure/ARO-RP/pkg/util/version"
@@ -42,6 +43,7 @@ func (m *manager) adminUpdate() []steps.Step {
 	// don't require a running cluster
 	toRun := []steps.Step{
 		steps.Action(m.initializeKubernetesClients), // must be first
+		steps.Action(m.initializeOperatorDeployer),  // depends on kube clients
 		steps.Action(m.ensureBillingRecord),         // belt and braces
 		steps.Action(m.ensureDefaults),
 		steps.Action(m.fixupClusterSPObjectID),
@@ -107,6 +109,7 @@ func (m *manager) Update(ctx context.Context) error {
 	steps := []steps.Step{
 		steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.validateResources)),
 		steps.Action(m.initializeKubernetesClients), // All init steps are first
+		steps.Action(m.initializeOperatorDeployer),  // depends on kube clients
 		steps.Action(m.initializeClusterSPClients),
 		steps.Action(m.clusterSPObjectID),
 		// credentials rotation flow steps
@@ -163,12 +166,14 @@ func (m *manager) Install(ctx context.Context) error {
 			steps.Action(m.createAPIServerPrivateEndpoint),
 			steps.Action(m.createCertificates),
 			steps.Action(m.initializeKubernetesClients),
+			steps.Action(m.initializeOperatorDeployer), // depends on kube clients
 			steps.Condition(m.bootstrapConfigMapReady, 30*time.Minute, true),
 			steps.Action(m.ensureAROOperator),
 			steps.Action(m.incrInstallPhase),
 		},
 		api.InstallPhaseRemoveBootstrap: {
 			steps.Action(m.initializeKubernetesClients),
+			steps.Action(m.initializeOperatorDeployer), // depends on kube clients
 			steps.Action(m.removeBootstrap),
 			steps.Action(m.removeBootstrapIgnition),
 			steps.Action(m.configureAPIServerCertificate),
@@ -296,6 +301,13 @@ func (m *manager) initializeKubernetesClients(ctx context.Context) error {
 	return err
 }
 
+// initializeKubernetesClients initializes clients which are used
+// once the cluster is up later on in the install process.
+func (m *manager) initializeOperatorDeployer(ctx context.Context) (err error) {
+	m.aroOperatorDeployer, err = deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.extensionscli, m.kubernetescli)
+	return
+}
+
 // updateProvisionedBy sets the deploying resource provider version in
 // the cluster document for deployment-tracking purposes.
 func (m *manager) updateProvisionedBy(ctx context.Context) error {
diff --git a/pkg/operator/controllers/autosizednodes/autosizednodes_controller.go b/pkg/operator/controllers/autosizednodes/autosizednodes_controller.go
new file mode 100644
index 000000000..05a0416d8
--- /dev/null
+++ b/pkg/operator/controllers/autosizednodes/autosizednodes_controller.go
@@ -0,0 +1,135 @@
+package autosizednodes
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/coreos/ignition/v2/config/util"
+	mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
+	"github.com/sirupsen/logrus"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+
+	arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
+)
+
+type Reconciler struct {
+	client client.Client
+
+	log *logrus.Entry
+}
+
+const (
+	ControllerName = "AutoSizedNodes"
+
+	ControllerEnabled = "aro.autosizednodes.enabled"
+	configName        = "dynamic-node"
+)
+
+func NewReconciler(log *logrus.Entry, mgr ctrl.Manager) *Reconciler {
+	return &Reconciler{
+		client: mgr.GetClient(),
+
+		log: log,
+	}
+}
+
+func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
+	var aro arov1alpha1.Cluster
+	var err error
+
+	err = r.client.Get(ctx, request.NamespacedName, &aro)
+	if err != nil {
+		err = fmt.Errorf("unable to fetch aro cluster: %w", err)
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	r.log.Infof("Config changed, autoSize: %t\n", aro.Spec.OperatorFlags.GetSimpleBoolean(ControllerEnabled))
+
+	// key is used to locate the object in the etcd
+	key := types.NamespacedName{
+		Name: configName,
+	}
+
+	if !aro.Spec.OperatorFlags.GetSimpleBoolean(ControllerEnabled) {
+		// defaults to deleting the config
+		config := mcv1.KubeletConfig{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: configName,
+			},
+		}
+		err = r.client.Delete(ctx, &config, &client.DeleteOptions{})
+		if err != nil {
+			err = fmt.Errorf("could not delete KubeletConfig: %w", err)
+		}
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	defaultConfig := makeConfig()
+
+	var config mcv1.KubeletConfig
+	err = r.client.Get(ctx, key, &config)
+	if kerrors.IsNotFound(err) {
+		// If config doesn't exist, create a new one
+		err := r.client.Create(ctx, &defaultConfig, &client.CreateOptions{})
+		if err != nil {
+			err = fmt.Errorf("could not create KubeletConfig: %w", err)
+		}
+		return ctrl.Result{}, err
+	}
+	if err != nil {
+		// If error, return it (controller-runtime will requeue for a retry)
+		return ctrl.Result{}, fmt.Errorf("could not fetch KubeletConfig: %w", err)
+	}
+
+	// If already exists, update the spec
+	config.Spec = defaultConfig.Spec
+	err = r.client.Update(ctx, &config, &client.UpdateOptions{})
+	if err != nil {
+		err = fmt.Errorf("could not update KubeletConfig: %w", err)
+	}
+	return ctrl.Result{}, err
+}
+
+// SetupWithManager prepares the controller with info who to watch
+func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
+	clusterPredicate := predicate.NewPredicateFuncs(func(o client.Object) bool {
+		name := o.GetName()
+		return strings.EqualFold(arov1alpha1.SingletonClusterName, name)
+	})
+
+	b := ctrl.NewControllerManagedBy(mgr).
+		For(&arov1alpha1.Cluster{}, builder.WithPredicates(clusterPredicate))
+
+		// Controller adds ControllerManagedBy to KubeletConfit created by this controller.
+		// Any changes will trigger reconcile, but only for that config.
+	return b.
+		Named(ControllerName).
+		Owns(&mcv1.KubeletConfig{}).
+		Complete(r)
+}
+
+func makeConfig() mcv1.KubeletConfig {
+	return mcv1.KubeletConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: configName,
+		},
+		Spec: mcv1.KubeletConfigSpec{
+			AutoSizingReserved: util.BoolToPtr(true),
+			MachineConfigPoolSelector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"pools.operator.machineconfiguration.openshift.io/worker": "",
+				},
+			},
+		},
+	}
+}
diff --git a/pkg/operator/controllers/autosizednodes/doc.go b/pkg/operator/controllers/autosizednodes/doc.go
new file mode 100644
index 000000000..89f784567
--- /dev/null
+++ b/pkg/operator/controllers/autosizednodes/doc.go
@@ -0,0 +1,9 @@
+package autosizednodes
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+// autosizednodes monitors/creates/removes "dynamic-node" KubeletConfig
+// that tells machine-config-operator to turn on auto sized nodes feature
+// the code that is executed by the mco:
+// - https://github.com/openshift/machine-config-operator/blob/fbc4d8e46a7746442f4de3651113d2181d458b12/templates/common/_base/files/kubelet-auto-sizing.yaml
diff --git a/pkg/operator/controllers/workaround/ifreload.go b/pkg/operator/controllers/workaround/ifreload.go
index e1a3e18fe..e78530161 100644
--- a/pkg/operator/controllers/workaround/ifreload.go
+++ b/pkg/operator/controllers/workaround/ifreload.go
@@ -11,6 +11,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 
+	arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
 	"github.com/Azure/ARO-RP/pkg/util/version"
 )
 
@@ -36,7 +37,7 @@ func (*ifReload) Name() string {
 	return "ifReload"
 }
 
-func (i *ifReload) IsRequired(clusterVersion *version.Version) bool {
+func (i *ifReload) IsRequired(clusterVersion *version.Version, cluster *arov1alpha1.Cluster) bool {
 	return clusterVersion.Lt(i.versionFixed)
 }
 
diff --git a/pkg/operator/controllers/workaround/systemreserved.go b/pkg/operator/controllers/workaround/systemreserved.go
index 0f4b5d65c..8fd12f42f 100644
--- a/pkg/operator/controllers/workaround/systemreserved.go
+++ b/pkg/operator/controllers/workaround/systemreserved.go
@@ -14,6 +14,8 @@ import (
 	kruntime "k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 
+	arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
+	"github.com/Azure/ARO-RP/pkg/operator/controllers/autosizednodes"
 	"github.com/Azure/ARO-RP/pkg/util/dynamichelper"
 	"github.com/Azure/ARO-RP/pkg/util/version"
 )
@@ -45,7 +47,10 @@ func (sr *systemreserved) Name() string {
 	return "SystemReserved fix for bz-1857446"
 }
 
-func (sr *systemreserved) IsRequired(clusterVersion *version.Version) bool {
+func (sr *systemreserved) IsRequired(clusterVersion *version.Version, cluster *arov1alpha1.Cluster) bool {
+	if cluster.Spec.OperatorFlags.GetSimpleBoolean(autosizednodes.ControllerEnabled) {
+		return false
+	}
 	return clusterVersion.Lt(sr.versionFixed)
 }
 
diff --git a/pkg/operator/controllers/workaround/workaround.go b/pkg/operator/controllers/workaround/workaround.go
index e3a88db1b..3d0752664 100644
--- a/pkg/operator/controllers/workaround/workaround.go
+++ b/pkg/operator/controllers/workaround/workaround.go
@@ -6,6 +6,7 @@ package workaround
 import (
 	"context"
 
+	arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
 	"github.com/Azure/ARO-RP/pkg/util/version"
 )
 
@@ -14,7 +15,7 @@ type Workaround interface {
 	Name() string
 	// IsRequired returns true when the clusterversion is indicates that the cluster
 	// is effected by the bug that the workaround fixes.
-	IsRequired(clusterVersion *version.Version) bool
+	IsRequired(clusterVersion *version.Version, cluster *arov1alpha1.Cluster) bool
 	// Ensure will apply the workaround to the cluster.
 	Ensure(context.Context) error
 	// Remove will remove the workaround from the cluster
diff --git a/pkg/operator/controllers/workaround/workaround_controller.go b/pkg/operator/controllers/workaround/workaround_controller.go
index c538a7456..7ff96d571 100644
--- a/pkg/operator/controllers/workaround/workaround_controller.go
+++ b/pkg/operator/controllers/workaround/workaround_controller.go
@@ -76,7 +76,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.
 	}
 
 	for _, wa := range r.workarounds {
-		if wa.IsRequired(clusterVersion) {
+		if wa.IsRequired(clusterVersion, instance) {
 			err = wa.Ensure(ctx)
 		} else {
 			err = wa.Remove(ctx)
diff --git a/pkg/operator/deploy/generate.go b/pkg/operator/deploy/generate.go
new file mode 100644
index 000000000..43f129fd6
--- /dev/null
+++ b/pkg/operator/deploy/generate.go
@@ -0,0 +1,8 @@
+package deploy
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+//go:generate rm -rf ../../util/mocks/operator/$GOPACKAGE
+//go:generate go run ../../../vendor/github.com/golang/mock/mockgen -destination=../../util/mocks/operator/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/operator/$GOPACKAGE Operator
+//go:generate go run ../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../util/mocks/operator/$GOPACKAGE/$GOPACKAGE.go
diff --git a/pkg/util/mocks/operator/controllers/workaround/workaround.go b/pkg/util/mocks/operator/controllers/workaround/workaround.go
index 04f21b236..01186df68 100644
--- a/pkg/util/mocks/operator/controllers/workaround/workaround.go
+++ b/pkg/util/mocks/operator/controllers/workaround/workaround.go
@@ -10,6 +10,7 @@ import (
 
 	gomock "github.com/golang/mock/gomock"
 
+	v1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
 	version "github.com/Azure/ARO-RP/pkg/util/version"
 )
 
@@ -51,17 +52,17 @@ func (mr *MockWorkaroundMockRecorder) Ensure(arg0 interface{}) *gomock.Call {
 }
 
 // IsRequired mocks base method.
-func (m *MockWorkaround) IsRequired(arg0 *version.Version) bool {
+func (m *MockWorkaround) IsRequired(arg0 *version.Version, arg1 *v1alpha1.Cluster) bool {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "IsRequired", arg0)
+	ret := m.ctrl.Call(m, "IsRequired", arg0, arg1)
 	ret0, _ := ret[0].(bool)
 	return ret0
 }
 
 // IsRequired indicates an expected call of IsRequired.
-func (mr *MockWorkaroundMockRecorder) IsRequired(arg0 interface{}) *gomock.Call {
+func (mr *MockWorkaroundMockRecorder) IsRequired(arg0, arg1 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsRequired", reflect.TypeOf((*MockWorkaround)(nil).IsRequired), arg0)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsRequired", reflect.TypeOf((*MockWorkaround)(nil).IsRequired), arg0, arg1)
 }
 
 // Name mocks base method.
diff --git a/pkg/util/mocks/operator/deploy/deploy.go b/pkg/util/mocks/operator/deploy/deploy.go
new file mode 100644
index 000000000..211c4622c
--- /dev/null
+++ b/pkg/util/mocks/operator/deploy/deploy.go
@@ -0,0 +1,64 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/Azure/ARO-RP/pkg/operator/deploy (interfaces: Operator)
+
+// Package mock_deploy is a generated GoMock package.
+package mock_deploy
+
+import (
+	context "context"
+	reflect "reflect"
+
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockOperator is a mock of Operator interface.
+type MockOperator struct {
+	ctrl     *gomock.Controller
+	recorder *MockOperatorMockRecorder
+}
+
+// MockOperatorMockRecorder is the mock recorder for MockOperator.
+type MockOperatorMockRecorder struct {
+	mock *MockOperator
+}
+
+// NewMockOperator creates a new mock instance.
+func NewMockOperator(ctrl *gomock.Controller) *MockOperator {
+	mock := &MockOperator{ctrl: ctrl}
+	mock.recorder = &MockOperatorMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockOperator) EXPECT() *MockOperatorMockRecorder {
+	return m.recorder
+}
+
+// CreateOrUpdate mocks base method.
+func (m *MockOperator) CreateOrUpdate(arg0 context.Context) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "CreateOrUpdate", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// CreateOrUpdate indicates an expected call of CreateOrUpdate.
+func (mr *MockOperatorMockRecorder) CreateOrUpdate(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdate", reflect.TypeOf((*MockOperator)(nil).CreateOrUpdate), arg0)
+}
+
+// IsReady mocks base method.
+func (m *MockOperator) IsReady(arg0 context.Context) (bool, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "IsReady", arg0)
+	ret0, _ := ret[0].(bool)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// IsReady indicates an expected call of IsReady.
+func (mr *MockOperatorMockRecorder) IsReady(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsReady", reflect.TypeOf((*MockOperator)(nil).IsReady), arg0)
+}
diff --git a/vendor/github.com/jewzaam/go-cosmosdb/pkg/gencosmosdb/bindata.go b/vendor/github.com/jewzaam/go-cosmosdb/pkg/gencosmosdb/bindata.go
index 3d3713931..b3dc1ea67 100644
--- a/vendor/github.com/jewzaam/go-cosmosdb/pkg/gencosmosdb/bindata.go
+++ b/vendor/github.com/jewzaam/go-cosmosdb/pkg/gencosmosdb/bindata.go
@@ -1,6 +1,5 @@
-// Code generated by go-bindata. (@generated) DO NOT EDIT.
-
- //Package gencosmosdb generated by go-bindata.// sources:
+// Code generated for package gencosmosdb by go-bindata DO NOT EDIT. (@generated)
+// sources:
 // cosmosdb/authorizer.go
 // cosmosdb/collection.go
 // cosmosdb/cosmosdb.go
@@ -28,7 +27,7 @@ import (
 func bindataRead(data []byte, name string) ([]byte, error) {
 	gz, err := gzip.NewReader(bytes.NewBuffer(data))
 	if err != nil {
-		return nil, fmt.Errorf("read %q: %v", name, err)
+		return nil, fmt.Errorf("Read %q: %v", name, err)
 	}
 
 	var buf bytes.Buffer
@@ -36,7 +35,7 @@ func bindataRead(data []byte, name string) ([]byte, error) {
 	clErr := gz.Close()
 
 	if err != nil {
-		return nil, fmt.Errorf("read %q: %v", name, err)
+		return nil, fmt.Errorf("Read %q: %v", name, err)
 	}
 	if clErr != nil {
 		return nil, err
@@ -72,7 +71,7 @@ func (fi bindataFileInfo) Mode() os.FileMode {
 	return fi.mode
 }
 
-// ModTime return file modify time
+// Mode return file modify time
 func (fi bindataFileInfo) ModTime() time.Time {
 	return fi.modTime
 }
diff --git a/vendor/github.com/jewzaam/go-cosmosdb/pkg/gencosmosdb/generate.go b/vendor/github.com/jewzaam/go-cosmosdb/pkg/gencosmosdb/generate.go
index 905d3cde6..309f689a7 100644
--- a/vendor/github.com/jewzaam/go-cosmosdb/pkg/gencosmosdb/generate.go
+++ b/vendor/github.com/jewzaam/go-cosmosdb/pkg/gencosmosdb/generate.go
@@ -1,3 +1,3 @@
 package gencosmosdb
 
-//go:generate go-bindata -pkg gencosmosdb -prefix cosmosdb -nometadata cosmosdb
+//go:generate go run ../../vendor/github.com/go-bindata/go-bindata/go-bindata -pkg gencosmosdb -prefix cosmosdb -nometadata cosmosdb
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 175507630..59ef9cc88 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -612,7 +612,7 @@ github.com/hashicorp/hcl/json/token
 github.com/imdario/mergo
 # github.com/inconshreveable/mousetrap v1.0.0
 github.com/inconshreveable/mousetrap
-# github.com/jewzaam/go-cosmosdb v0.0.0-20211018134417-8d1494e7862f
+# github.com/jewzaam/go-cosmosdb v0.0.0-20220315232836-282b67c5b234
 ## explicit
 github.com/jewzaam/go-cosmosdb/cmd/gencosmosdb
 github.com/jewzaam/go-cosmosdb/pkg/gencosmosdb
@@ -1928,6 +1928,7 @@ sigs.k8s.io/controller-runtime/pkg/certwatcher
 sigs.k8s.io/controller-runtime/pkg/client
 sigs.k8s.io/controller-runtime/pkg/client/apiutil
 sigs.k8s.io/controller-runtime/pkg/client/config
+sigs.k8s.io/controller-runtime/pkg/client/fake/
 sigs.k8s.io/controller-runtime/pkg/cluster
 sigs.k8s.io/controller-runtime/pkg/config
 sigs.k8s.io/controller-runtime/pkg/config/v1alpha1
diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/client.go b/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/client.go
new file mode 100644
index 000000000..ded8a60d3
--- /dev/null
+++ b/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/client.go
@@ -0,0 +1,675 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fake
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilrand "k8s.io/apimachinery/pkg/util/rand"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/testing"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
+	"sigs.k8s.io/controller-runtime/pkg/internal/objectutil"
+)
+
+type versionedTracker struct {
+	testing.ObjectTracker
+	scheme *runtime.Scheme
+}
+
+type fakeClient struct {
+	tracker         versionedTracker
+	scheme          *runtime.Scheme
+	schemeWriteLock sync.Mutex
+}
+
+var _ client.WithWatch = &fakeClient{}
+
+const (
+	maxNameLength          = 63
+	randomLength           = 5
+	maxGeneratedNameLength = maxNameLength - randomLength
+)
+
+// NewFakeClient creates a new fake client for testing.
+// You can choose to initialize it with a slice of runtime.Object.
+//
+// Deprecated: Please use NewClientBuilder instead.
+func NewFakeClient(initObjs ...runtime.Object) client.WithWatch {
+	return NewClientBuilder().WithRuntimeObjects(initObjs...).Build()
+}
+
+// NewFakeClientWithScheme creates a new fake client with the given scheme
+// for testing.
+// You can choose to initialize it with a slice of runtime.Object.
+//
+// Deprecated: Please use NewClientBuilder instead.
+func NewFakeClientWithScheme(clientScheme *runtime.Scheme, initObjs ...runtime.Object) client.WithWatch {
+	return NewClientBuilder().WithScheme(clientScheme).WithRuntimeObjects(initObjs...).Build()
+}
+
+// NewClientBuilder returns a new builder to create a fake client.
+func NewClientBuilder() *ClientBuilder {
+	return &ClientBuilder{}
+}
+
+// ClientBuilder builds a fake client.
+type ClientBuilder struct {
+	scheme             *runtime.Scheme
+	initObject         []client.Object
+	initLists          []client.ObjectList
+	initRuntimeObjects []runtime.Object
+}
+
+// WithScheme sets this builder's internal scheme.
+// If not set, defaults to client-go's global scheme.Scheme.
+func (f *ClientBuilder) WithScheme(scheme *runtime.Scheme) *ClientBuilder {
+	f.scheme = scheme
+	return f
+}
+
+// WithObjects can be optionally used to initialize this fake client with client.Object(s).
+func (f *ClientBuilder) WithObjects(initObjs ...client.Object) *ClientBuilder {
+	f.initObject = append(f.initObject, initObjs...)
+	return f
+}
+
+// WithLists can be optionally used to initialize this fake client with client.ObjectList(s).
+func (f *ClientBuilder) WithLists(initLists ...client.ObjectList) *ClientBuilder {
+	f.initLists = append(f.initLists, initLists...)
+	return f
+}
+
+// WithRuntimeObjects can be optionally used to initialize this fake client with runtime.Object(s).
+func (f *ClientBuilder) WithRuntimeObjects(initRuntimeObjs ...runtime.Object) *ClientBuilder {
+	f.initRuntimeObjects = append(f.initRuntimeObjects, initRuntimeObjs...)
+	return f
+}
+
+// Build builds and returns a new fake client.
+func (f *ClientBuilder) Build() client.WithWatch {
+	if f.scheme == nil {
+		f.scheme = scheme.Scheme
+	}
+
+	tracker := versionedTracker{ObjectTracker: testing.NewObjectTracker(f.scheme, scheme.Codecs.UniversalDecoder()), scheme: f.scheme}
+	for _, obj := range f.initObject {
+		if err := tracker.Add(obj); err != nil {
+			panic(fmt.Errorf("failed to add object %v to fake client: %w", obj, err))
+		}
+	}
+	for _, obj := range f.initLists {
+		if err := tracker.Add(obj); err != nil {
+			panic(fmt.Errorf("failed to add list %v to fake client: %w", obj, err))
+		}
+	}
+	for _, obj := range f.initRuntimeObjects {
+		if err := tracker.Add(obj); err != nil {
+			panic(fmt.Errorf("failed to add runtime object %v to fake client: %w", obj, err))
+		}
+	}
+	return &fakeClient{
+		tracker: tracker,
+		scheme:  f.scheme,
+	}
+}
+
+const trackerAddResourceVersion = "999"
+
+func (t versionedTracker) Add(obj runtime.Object) error {
+	var objects []runtime.Object
+	if meta.IsListType(obj) {
+		var err error
+		objects, err = meta.ExtractList(obj)
+		if err != nil {
+			return err
+		}
+	} else {
+		objects = []runtime.Object{obj}
+	}
+	for _, obj := range objects {
+		accessor, err := meta.Accessor(obj)
+		if err != nil {
+			return fmt.Errorf("failed to get accessor for object: %w", err)
+		}
+		if accessor.GetResourceVersion() == "" {
+			// We use a "magic" value of 999 here because this field
+			// is parsed as uint and and 0 is already used in Update.
+			// As we can't go lower, go very high instead so this can
+			// be recognized
+			accessor.SetResourceVersion(trackerAddResourceVersion)
+		}
+		if err := t.ObjectTracker.Add(obj); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (t versionedTracker) Create(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return fmt.Errorf("failed to get accessor for object: %v", err)
+	}
+	if accessor.GetName() == "" {
+		return apierrors.NewInvalid(
+			obj.GetObjectKind().GroupVersionKind().GroupKind(),
+			accessor.GetName(),
+			field.ErrorList{field.Required(field.NewPath("metadata.name"), "name is required")})
+	}
+	if accessor.GetResourceVersion() != "" {
+		return apierrors.NewBadRequest("resourceVersion can not be set for Create requests")
+	}
+	accessor.SetResourceVersion("1")
+	if err := t.ObjectTracker.Create(gvr, obj, ns); err != nil {
+		accessor.SetResourceVersion("")
+		return err
+	}
+	return nil
+}
+
+func (t versionedTracker) Update(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return fmt.Errorf("failed to get accessor for object: %v", err)
+	}
+
+	if accessor.GetName() == "" {
+		return apierrors.NewInvalid(
+			obj.GetObjectKind().GroupVersionKind().GroupKind(),
+			accessor.GetName(),
+			field.ErrorList{field.Required(field.NewPath("metadata.name"), "name is required")})
+	}
+
+	gvk := obj.GetObjectKind().GroupVersionKind()
+	if gvk.Empty() {
+		gvk, err = apiutil.GVKForObject(obj, t.scheme)
+		if err != nil {
+			return err
+		}
+	}
+
+	oldObject, err := t.ObjectTracker.Get(gvr, ns, accessor.GetName())
+	if err != nil {
+		// If the resource is not found and the resource allows create on update, issue a
+		// create instead.
+		if apierrors.IsNotFound(err) && allowsCreateOnUpdate(gvk) {
+			return t.Create(gvr, obj, ns)
+		}
+		return err
+	}
+
+	oldAccessor, err := meta.Accessor(oldObject)
+	if err != nil {
+		return err
+	}
+
+	// If the new object does not have the resource version set and it allows unconditional update,
+	// default it to the resource version of the existing resource
+	if accessor.GetResourceVersion() == "" && allowsUnconditionalUpdate(gvk) {
+		accessor.SetResourceVersion(oldAccessor.GetResourceVersion())
+	}
+	if accessor.GetResourceVersion() != oldAccessor.GetResourceVersion() {
+		return apierrors.NewConflict(gvr.GroupResource(), accessor.GetName(), errors.New("object was modified"))
+	}
+	if oldAccessor.GetResourceVersion() == "" {
+		oldAccessor.SetResourceVersion("0")
+	}
+	intResourceVersion, err := strconv.ParseUint(oldAccessor.GetResourceVersion(), 10, 64)
+	if err != nil {
+		return fmt.Errorf("can not convert resourceVersion %q to int: %v", oldAccessor.GetResourceVersion(), err)
+	}
+	intResourceVersion++
+	accessor.SetResourceVersion(strconv.FormatUint(intResourceVersion, 10))
+	if !accessor.GetDeletionTimestamp().IsZero() && len(accessor.GetFinalizers()) == 0 {
+		return t.ObjectTracker.Delete(gvr, accessor.GetNamespace(), accessor.GetName())
+	}
+	return t.ObjectTracker.Update(gvr, obj, ns)
+}
+
+func (c *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error {
+	gvr, err := getGVRFromObject(obj, c.scheme)
+	if err != nil {
+		return err
+	}
+	o, err := c.tracker.Get(gvr, key.Namespace, key.Name)
+	if err != nil {
+		return err
+	}
+
+	gvk, err := apiutil.GVKForObject(obj, c.scheme)
+	if err != nil {
+		return err
+	}
+	ta, err := meta.TypeAccessor(o)
+	if err != nil {
+		return err
+	}
+	ta.SetKind(gvk.Kind)
+	ta.SetAPIVersion(gvk.GroupVersion().String())
+
+	j, err := json.Marshal(o)
+	if err != nil {
+		return err
+	}
+	decoder := scheme.Codecs.UniversalDecoder()
+	_, _, err = decoder.Decode(j, nil, obj)
+	return err
+}
+
+func (c *fakeClient) Watch(ctx context.Context, list client.ObjectList, opts ...client.ListOption) (watch.Interface, error) {
+	gvk, err := apiutil.GVKForObject(list, c.scheme)
+	if err != nil {
+		return nil, err
+	}
+
+	if strings.HasSuffix(gvk.Kind, "List") {
+		gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
+	}
+
+	listOpts := client.ListOptions{}
+	listOpts.ApplyOptions(opts)
+
+	gvr, _ := meta.UnsafeGuessKindToResource(gvk)
+	return c.tracker.Watch(gvr, listOpts.Namespace)
+}
+
+func (c *fakeClient) List(ctx context.Context, obj client.ObjectList, opts ...client.ListOption) error {
+	gvk, err := apiutil.GVKForObject(obj, c.scheme)
+	if err != nil {
+		return err
+	}
+
+	originalKind := gvk.Kind
+
+	if strings.HasSuffix(gvk.Kind, "List") {
+		gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
+	}
+
+	if _, isUnstructuredList := obj.(*unstructured.UnstructuredList); isUnstructuredList && !c.scheme.Recognizes(gvk) {
+		// We need tor register the ListKind with UnstructuredList:
+		// https://github.com/kubernetes/kubernetes/blob/7b2776b89fb1be28d4e9203bdeec079be903c103/staging/src/k8s.io/client-go/dynamic/fake/simple.go#L44-L51
+		c.schemeWriteLock.Lock()
+		c.scheme.AddKnownTypeWithName(gvk.GroupVersion().WithKind(gvk.Kind+"List"), &unstructured.UnstructuredList{})
+		c.schemeWriteLock.Unlock()
+	}
+
+	listOpts := client.ListOptions{}
+	listOpts.ApplyOptions(opts)
+
+	gvr, _ := meta.UnsafeGuessKindToResource(gvk)
+	o, err := c.tracker.List(gvr, gvk, listOpts.Namespace)
+	if err != nil {
+		return err
+	}
+
+	ta, err := meta.TypeAccessor(o)
+	if err != nil {
+		return err
+	}
+	ta.SetKind(originalKind)
+	ta.SetAPIVersion(gvk.GroupVersion().String())
+
+	j, err := json.Marshal(o)
+	if err != nil {
+		return err
+	}
+	decoder := scheme.Codecs.UniversalDecoder()
+	_, _, err = decoder.Decode(j, nil, obj)
+	if err != nil {
+		return err
+	}
+
+	if listOpts.LabelSelector != nil {
+		objs, err := meta.ExtractList(obj)
+		if err != nil {
+			return err
+		}
+		filteredObjs, err := objectutil.FilterWithLabels(objs, listOpts.LabelSelector)
+		if err != nil {
+			return err
+		}
+		err = meta.SetList(obj, filteredObjs)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *fakeClient) Scheme() *runtime.Scheme {
+	return c.scheme
+}
+
+func (c *fakeClient) RESTMapper() meta.RESTMapper {
+	// TODO: Implement a fake RESTMapper.
+	return nil
+}
+
+func (c *fakeClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
+	createOptions := &client.CreateOptions{}
+	createOptions.ApplyOptions(opts)
+
+	for _, dryRunOpt := range createOptions.DryRun {
+		if dryRunOpt == metav1.DryRunAll {
+			return nil
+		}
+	}
+
+	gvr, err := getGVRFromObject(obj, c.scheme)
+	if err != nil {
+		return err
+	}
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return err
+	}
+
+	if accessor.GetName() == "" && accessor.GetGenerateName() != "" {
+		base := accessor.GetGenerateName()
+		if len(base) > maxGeneratedNameLength {
+			base = base[:maxGeneratedNameLength]
+		}
+		accessor.SetName(fmt.Sprintf("%s%s", base, utilrand.String(randomLength)))
+	}
+
+	return c.tracker.Create(gvr, obj, accessor.GetNamespace())
+}
+
+func (c *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error {
+	gvr, err := getGVRFromObject(obj, c.scheme)
+	if err != nil {
+		return err
+	}
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return err
+	}
+	delOptions := client.DeleteOptions{}
+	delOptions.ApplyOptions(opts)
+
+	return c.deleteObject(gvr, accessor)
+}
+
+func (c *fakeClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error {
+	gvk, err := apiutil.GVKForObject(obj, c.scheme)
+	if err != nil {
+		return err
+	}
+
+	dcOptions := client.DeleteAllOfOptions{}
+	dcOptions.ApplyOptions(opts)
+
+	gvr, _ := meta.UnsafeGuessKindToResource(gvk)
+	o, err := c.tracker.List(gvr, gvk, dcOptions.Namespace)
+	if err != nil {
+		return err
+	}
+
+	objs, err := meta.ExtractList(o)
+	if err != nil {
+		return err
+	}
+	filteredObjs, err := objectutil.FilterWithLabels(objs, dcOptions.LabelSelector)
+	if err != nil {
+		return err
+	}
+	for _, o := range filteredObjs {
+		accessor, err := meta.Accessor(o)
+		if err != nil {
+			return err
+		}
+		err = c.deleteObject(gvr, accessor)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *fakeClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error {
+	updateOptions := &client.UpdateOptions{}
+	updateOptions.ApplyOptions(opts)
+
+	for _, dryRunOpt := range updateOptions.DryRun {
+		if dryRunOpt == metav1.DryRunAll {
+			return nil
+		}
+	}
+
+	gvr, err := getGVRFromObject(obj, c.scheme)
+	if err != nil {
+		return err
+	}
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return err
+	}
+	return c.tracker.Update(gvr, obj, accessor.GetNamespace())
+}
+
+func (c *fakeClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
+	patchOptions := &client.PatchOptions{}
+	patchOptions.ApplyOptions(opts)
+
+	for _, dryRunOpt := range patchOptions.DryRun {
+		if dryRunOpt == metav1.DryRunAll {
+			return nil
+		}
+	}
+
+	gvr, err := getGVRFromObject(obj, c.scheme)
+	if err != nil {
+		return err
+	}
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return err
+	}
+	data, err := patch.Data(obj)
+	if err != nil {
+		return err
+	}
+
+	reaction := testing.ObjectReaction(c.tracker)
+	handled, o, err := reaction(testing.NewPatchAction(gvr, accessor.GetNamespace(), accessor.GetName(), patch.Type(), data))
+	if err != nil {
+		return err
+	}
+	if !handled {
+		panic("tracker could not handle patch method")
+	}
+
+	gvk, err := apiutil.GVKForObject(obj, c.scheme)
+	if err != nil {
+		return err
+	}
+	ta, err := meta.TypeAccessor(o)
+	if err != nil {
+		return err
+	}
+	ta.SetKind(gvk.Kind)
+	ta.SetAPIVersion(gvk.GroupVersion().String())
+
+	j, err := json.Marshal(o)
+	if err != nil {
+		return err
+	}
+	decoder := scheme.Codecs.UniversalDecoder()
+	_, _, err = decoder.Decode(j, nil, obj)
+	return err
+}
+
+func (c *fakeClient) Status() client.StatusWriter {
+	return &fakeStatusWriter{client: c}
+}
+
+func (c *fakeClient) deleteObject(gvr schema.GroupVersionResource, accessor metav1.Object) error {
+	old, err := c.tracker.Get(gvr, accessor.GetNamespace(), accessor.GetName())
+	if err == nil {
+		oldAccessor, err := meta.Accessor(old)
+		if err == nil {
+			if len(oldAccessor.GetFinalizers()) > 0 {
+				now := metav1.Now()
+				oldAccessor.SetDeletionTimestamp(&now)
+				return c.tracker.Update(gvr, old, accessor.GetNamespace())
+			}
+		}
+	}
+
+	//TODO: implement propagation
+	return c.tracker.Delete(gvr, accessor.GetNamespace(), accessor.GetName())
+}
+
+func getGVRFromObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersionResource, error) {
+	gvk, err := apiutil.GVKForObject(obj, scheme)
+	if err != nil {
+		return schema.GroupVersionResource{}, err
+	}
+	gvr, _ := meta.UnsafeGuessKindToResource(gvk)
+	return gvr, nil
+}
+
+type fakeStatusWriter struct {
+	client *fakeClient
+}
+
+func (sw *fakeStatusWriter) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error {
+	// TODO(droot): This results in full update of the obj (spec + status). Need
+	// a way to update status field only.
+	return sw.client.Update(ctx, obj, opts...)
+}
+
+func (sw *fakeStatusWriter) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
+	// TODO(droot): This results in full update of the obj (spec + status). Need
+	// a way to update status field only.
+	return sw.client.Patch(ctx, obj, patch, opts...)
+}
+
+func allowsUnconditionalUpdate(gvk schema.GroupVersionKind) bool {
+	switch gvk.Group {
+	case "apps":
+		switch gvk.Kind {
+		case "ControllerRevision", "DaemonSet", "Deployment", "ReplicaSet", "StatefulSet":
+			return true
+		}
+	case "autoscaling":
+		switch gvk.Kind {
+		case "HorizontalPodAutoscaler":
+			return true
+		}
+	case "batch":
+		switch gvk.Kind {
+		case "CronJob", "Job":
+			return true
+		}
+	case "certificates":
+		switch gvk.Kind {
+		case "Certificates":
+			return true
+		}
+	case "flowcontrol":
+		switch gvk.Kind {
+		case "FlowSchema", "PriorityLevelConfiguration":
+			return true
+		}
+	case "networking":
+		switch gvk.Kind {
+		case "Ingress", "IngressClass", "NetworkPolicy":
+			return true
+		}
+	case "policy":
+		switch gvk.Kind {
+		case "PodSecurityPolicy":
+			return true
+		}
+	case "rbac":
+		switch gvk.Kind {
+		case "ClusterRole", "ClusterRoleBinding", "Role", "RoleBinding":
+			return true
+		}
+	case "scheduling":
+		switch gvk.Kind {
+		case "PriorityClass":
+			return true
+		}
+	case "settings":
+		switch gvk.Kind {
+		case "PodPreset":
+			return true
+		}
+	case "storage":
+		switch gvk.Kind {
+		case "StorageClass":
+			return true
+		}
+	case "":
+		switch gvk.Kind {
+		case "ConfigMap", "Endpoint", "Event", "LimitRange", "Namespace", "Node",
+			"PersistentVolume", "PersistentVolumeClaim", "Pod", "PodTemplate",
+			"ReplicationController", "ResourceQuota", "Secret", "Service",
+			"ServiceAccount", "EndpointSlice":
+			return true
+		}
+	}
+
+	return false
+}
+
+func allowsCreateOnUpdate(gvk schema.GroupVersionKind) bool {
+	switch gvk.Group {
+	case "coordination":
+		switch gvk.Kind {
+		case "Lease":
+			return true
+		}
+	case "node":
+		switch gvk.Kind {
+		case "RuntimeClass":
+			return true
+		}
+	case "rbac":
+		switch gvk.Kind {
+		case "ClusterRole", "ClusterRoleBinding", "Role", "RoleBinding":
+			return true
+		}
+	case "":
+		switch gvk.Kind {
+		case "Endpoint", "Event", "LimitRange", "Service":
+			return true
+		}
+	}
+
+	return false
+}
diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/doc.go b/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/doc.go
new file mode 100644
index 000000000..7d680690d
--- /dev/null
+++ b/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/doc.go
@@ -0,0 +1,39 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+Package fake provides a fake client for testing.
+
+A fake client is backed by its simple object store indexed by GroupVersionResource.
+You can create a fake client with optional objects.
+
+	client := NewFakeClientWithScheme(scheme, initObjs...) // initObjs is a slice of runtime.Object
+
+You can invoke the methods defined in the Client interface.
+
+When in doubt, it's almost always better not to use this package and instead use
+envtest.Environment with a real client and API server.
+
+WARNING: ⚠️ Current Limitations / Known Issues with the fake Client ⚠️
+- This client does not have a way to inject specific errors to test handled vs. unhandled errors.
+- There is some support for sub resources which can cause issues with tests if you're trying to update
+  e.g. metadata and status in the same reconcile.
+- No OpeanAPI validation is performed when creating or updating objects.
+- ObjectMeta's `Generation` and `ResourceVersion` don't behave properly, Patch or Update
+operations that rely on these fields will fail, or give false positives.
+
+*/
+package fake


Test patch:
diff --git a/pkg/cluster/adminupdate_test.go b/pkg/cluster/adminupdate_test.go
index fa1ea56b3..52fc01893 100644
--- a/pkg/cluster/adminupdate_test.go
+++ b/pkg/cluster/adminupdate_test.go
@@ -41,6 +41,7 @@ func TestAdminUpdateSteps(t *testing.T) {
 			},
 			shouldRunSteps: []string{
 				"[Action initializeKubernetesClients-fm]",
+				"[Action initializeOperatorDeployer-fm]",
 				"[Action ensureBillingRecord-fm]",
 				"[Action ensureDefaults-fm]",
 				"[Action fixupClusterSPObjectID-fm]",
@@ -61,6 +62,7 @@ func TestAdminUpdateSteps(t *testing.T) {
 			},
 			shouldRunSteps: []string{
 				"[Action initializeKubernetesClients-fm]",
+				"[Action initializeOperatorDeployer-fm]",
 				"[Action ensureBillingRecord-fm]",
 				"[Action ensureDefaults-fm]",
 				"[Action fixupClusterSPObjectID-fm]",
@@ -99,6 +101,7 @@ func TestAdminUpdateSteps(t *testing.T) {
 			},
 			shouldRunSteps: []string{
 				"[Action initializeKubernetesClients-fm]",
+				"[Action initializeOperatorDeployer-fm]",
 				"[Action ensureBillingRecord-fm]",
 				"[Action ensureDefaults-fm]",
 				"[Action fixupClusterSPObjectID-fm]",
diff --git a/pkg/cluster/arooperator_test.go b/pkg/cluster/arooperator_test.go
new file mode 100644
index 000000000..5832d2f41
--- /dev/null
+++ b/pkg/cluster/arooperator_test.go
@@ -0,0 +1,204 @@
+package cluster
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/sirupsen/logrus"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	mock_deploy "github.com/Azure/ARO-RP/pkg/util/mocks/operator/deploy"
+)
+
+func TestEnsureAROOperator(t *testing.T) {
+	ctx := context.Background()
+
+	const (
+		key = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName1"
+	)
+
+	for _, tt := range []struct {
+		name    string
+		doc     *api.OpenShiftClusterDocument
+		mocks   func(*mock_deploy.MockOperator)
+		wantErr string
+	}{
+		{
+			name: "create/update success",
+			doc: &api.OpenShiftClusterDocument{
+				Key: strings.ToLower(key),
+				OpenShiftCluster: &api.OpenShiftCluster{
+					ID: key,
+					Properties: api.OpenShiftClusterProperties{
+						IngressProfiles: []api.IngressProfile{
+							{
+								Visibility: api.VisibilityPublic,
+								Name:       "default",
+							},
+						},
+						ProvisioningState: api.ProvisioningStateAdminUpdating,
+						ClusterProfile: api.ClusterProfile{
+							Version: "4.8.18",
+						},
+						NetworkProfile: api.NetworkProfile{
+							APIServerPrivateEndpointIP: "1.2.3.4",
+						},
+					},
+				},
+			},
+			mocks: func(dep *mock_deploy.MockOperator) {
+				dep.EXPECT().
+					CreateOrUpdate(gomock.Any()).
+					Return(nil)
+			},
+		},
+		{
+			name: "create/update failure",
+			doc: &api.OpenShiftClusterDocument{
+				Key: strings.ToLower(key),
+				OpenShiftCluster: &api.OpenShiftCluster{
+					ID: key,
+					Properties: api.OpenShiftClusterProperties{
+						IngressProfiles: []api.IngressProfile{
+							{
+								Visibility: api.VisibilityPublic,
+								Name:       "default",
+							},
+						},
+						ProvisioningState: api.ProvisioningStateAdminUpdating,
+						ClusterProfile: api.ClusterProfile{
+							Version: "4.8.18",
+						},
+						NetworkProfile: api.NetworkProfile{
+							APIServerPrivateEndpointIP: "1.2.3.4",
+						},
+					},
+				},
+			},
+			mocks: func(dep *mock_deploy.MockOperator) {
+				dep.EXPECT().
+					CreateOrUpdate(gomock.Any()).
+					Return(errors.New("Mock return: CreateFailed"))
+			},
+
+			wantErr: "Mock return: CreateFailed",
+		},
+		{
+			name: "enriched data not available - skip",
+
+			doc: &api.OpenShiftClusterDocument{
+				Key: strings.ToLower(key),
+				OpenShiftCluster: &api.OpenShiftCluster{
+					ID: key,
+					Properties: api.OpenShiftClusterProperties{
+						IngressProfiles:   nil,
+						ProvisioningState: api.ProvisioningStateAdminUpdating,
+					},
+				},
+			},
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			controller := gomock.NewController(t)
+			defer controller.Finish()
+
+			dep := mock_deploy.NewMockOperator(controller)
+			if tt.mocks != nil {
+				tt.mocks(dep)
+			}
+
+			m := &manager{
+				log: logrus.NewEntry(logrus.StandardLogger()),
+				doc: tt.doc,
+
+				aroOperatorDeployer: dep,
+			}
+
+			err := m.ensureAROOperator(ctx)
+			if err != nil && err.Error() != tt.wantErr ||
+				err == nil && tt.wantErr != "" {
+				t.Error(err)
+			}
+		})
+	}
+}
+
+func TestAroDeploymentReady(t *testing.T) {
+	ctx := context.Background()
+
+	const (
+		key = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName1"
+	)
+
+	for _, tt := range []struct {
+		name    string
+		doc     *api.OpenShiftClusterDocument
+		mocks   func(*mock_deploy.MockOperator)
+		wantRes bool
+	}{
+		{
+			name: "create/update success",
+			doc: &api.OpenShiftClusterDocument{
+				Key: strings.ToLower(key),
+				OpenShiftCluster: &api.OpenShiftCluster{
+					ID: key,
+					Properties: api.OpenShiftClusterProperties{
+						IngressProfiles: []api.IngressProfile{
+							{
+								Visibility: api.VisibilityPublic,
+								Name:       "default",
+							},
+						},
+					},
+				},
+			},
+			mocks: func(dep *mock_deploy.MockOperator) {
+				dep.EXPECT().
+					IsReady(gomock.Any()).
+					Return(true, nil)
+			},
+			wantRes: true,
+		},
+		{
+			name: "enriched data not available - skip",
+			doc: &api.OpenShiftClusterDocument{
+				Key: strings.ToLower(key),
+				OpenShiftCluster: &api.OpenShiftCluster{
+					ID: key,
+					Properties: api.OpenShiftClusterProperties{
+						IngressProfiles: nil,
+					},
+				},
+			},
+			wantRes: true,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			controller := gomock.NewController(t)
+			defer controller.Finish()
+
+			dep := mock_deploy.NewMockOperator(controller)
+			if tt.mocks != nil {
+				tt.mocks(dep)
+			}
+
+			m := &manager{
+				log:                 logrus.NewEntry(logrus.StandardLogger()),
+				doc:                 tt.doc,
+				aroOperatorDeployer: dep,
+			}
+
+			ok, err := m.aroDeploymentReady(ctx)
+			if err != nil || ok != tt.wantRes {
+				t.Error(err)
+			}
+
+		})
+	}
+}
diff --git a/pkg/operator/controllers/autosizednodes/autosizednodes_controller_test.go b/pkg/operator/controllers/autosizednodes/autosizednodes_controller_test.go
new file mode 100644
index 000000000..7deb7c62f
--- /dev/null
+++ b/pkg/operator/controllers/autosizednodes/autosizednodes_controller_test.go
@@ -0,0 +1,129 @@
+package autosizednodes
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"reflect"
+	"strconv"
+	"testing"
+
+	"github.com/coreos/ignition/v2/config/util"
+	"github.com/google/go-cmp/cmp"
+	mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
+	"github.com/sirupsen/logrus"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	// This "_" import is counterintuitive but is required to initialize the scheme
+	// ARO unfortunately relies on implicit import and its side effect for this
+	arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
+	_ "github.com/Azure/ARO-RP/pkg/util/scheme"
+)
+
+func TestAutosizednodesReconciler(t *testing.T) {
+	aro := func(autoSizeEnabled bool) *arov1alpha1.Cluster {
+		return &arov1alpha1.Cluster{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "aro",
+				Namespace: "openshift-azure-operator",
+			},
+			Spec: arov1alpha1.ClusterSpec{
+				OperatorFlags: arov1alpha1.OperatorFlags{
+					ControllerEnabled: strconv.FormatBool(autoSizeEnabled),
+				},
+			},
+		}
+	}
+
+	emptyConfig := mcv1.KubeletConfig{}
+	config := makeConfig()
+
+	tests := []struct {
+		name       string
+		wantGetErr error
+		client     client.Client
+		wantConfig *mcv1.KubeletConfig
+	}{
+		{
+			name:       "is not needed",
+			client:     fake.NewClientBuilder().WithRuntimeObjects(aro(false)).Build(),
+			wantConfig: &emptyConfig,
+			wantGetErr: kerrors.NewNotFound(mcv1.Resource("kubeletconfigs"), "dynamic-node"),
+		},
+		{
+			name:       "is needed and not present already",
+			client:     fake.NewClientBuilder().WithRuntimeObjects(aro(true)).Build(),
+			wantConfig: &config,
+			wantGetErr: nil,
+		},
+		{
+			name:       "is needed and present already",
+			client:     fake.NewClientBuilder().WithRuntimeObjects(aro(true), &config).Build(),
+			wantConfig: &config,
+		},
+		{
+			name:       "is not needed and is present",
+			client:     fake.NewClientBuilder().WithRuntimeObjects(aro(false), &config).Build(),
+			wantConfig: &emptyConfig,
+			wantGetErr: kerrors.NewNotFound(mcv1.Resource("kubeletconfigs"), "dynamic-node"),
+		},
+		{
+			name: "is needed and config got modified",
+			client: fake.NewClientBuilder().WithRuntimeObjects(
+				aro(true),
+				&mcv1.KubeletConfig{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: configName,
+					},
+					Spec: mcv1.KubeletConfigSpec{
+						AutoSizingReserved: util.BoolToPtr(false),
+						MachineConfigPoolSelector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								"pools.operator.machineconfiguration.openshift.io/worker": "",
+							},
+						},
+					},
+				}).Build(),
+			wantConfig: &config,
+			wantGetErr: nil,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx := context.Background()
+			r := Reconciler{
+				client: test.client,
+				log:    logrus.NewEntry(logrus.StandardLogger()),
+			}
+			result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "openshift-azure-operator", Name: "aro"}})
+			if err != nil {
+				t.Error(err)
+			}
+
+			key := types.NamespacedName{
+				Name: configName,
+			}
+			var c mcv1.KubeletConfig
+
+			err = r.client.Get(ctx, key, &c)
+			if err != nil && err.Error() != test.wantGetErr.Error() || err == nil && test.wantGetErr != nil {
+				t.Error(err)
+			}
+
+			if !reflect.DeepEqual(test.wantConfig.Spec, c.Spec) {
+				t.Error(cmp.Diff(test.wantConfig.Spec, c.Spec))
+			}
+
+			if result != (ctrl.Result{}) {
+				t.Error("reconcile returned an unexpected result")
+			}
+		})
+	}
+}
diff --git a/pkg/operator/controllers/workaround/workaround_controller_test.go b/pkg/operator/controllers/workaround/workaround_controller_test.go
index 96990accf..c88f780a5 100644
--- a/pkg/operator/controllers/workaround/workaround_controller_test.go
+++ b/pkg/operator/controllers/workaround/workaround_controller_test.go
@@ -65,7 +65,7 @@ func TestWorkaroundReconciler(t *testing.T) {
 		{
 			name: "is required",
 			mocker: func(mw *mock_workaround.MockWorkaround) {
-				c := mw.EXPECT().IsRequired(gomock.Any()).Return(true)
+				c := mw.EXPECT().IsRequired(gomock.Any(), gomock.Any()).Return(true)
 				mw.EXPECT().Ensure(gomock.Any()).After(c).Return(nil)
 			},
 			want: ctrl.Result{RequeueAfter: time.Hour},
@@ -73,7 +73,7 @@ func TestWorkaroundReconciler(t *testing.T) {
 		{
 			name: "is not required",
 			mocker: func(mw *mock_workaround.MockWorkaround) {
-				c := mw.EXPECT().IsRequired(gomock.Any()).Return(false)
+				c := mw.EXPECT().IsRequired(gomock.Any(), gomock.Any()).Return(false)
 				mw.EXPECT().Remove(gomock.Any()).After(c).Return(nil)
 			},
 			want: ctrl.Result{RequeueAfter: time.Hour},
@@ -81,7 +81,7 @@ func TestWorkaroundReconciler(t *testing.T) {
 		{
 			name: "has error",
 			mocker: func(mw *mock_workaround.MockWorkaround) {
-				mw.EXPECT().IsRequired(gomock.Any()).Return(true)
+				mw.EXPECT().IsRequired(gomock.Any(), gomock.Any()).Return(true)
 				mw.EXPECT().Name().Return("test").AnyTimes()
 				mw.EXPECT().Ensure(gomock.Any()).Return(fmt.Errorf("oops"))
 			},