Repository: Azure/ARO-RP
Pull request #2171 (2022-06-10 10:54:17)

Hive AKS development environment deploy
### Which issue this PR addresses:

Deploys Hive into the AKS development environments.

### What this PR does / why we need it:

In order for us to be able to iterate on the AKS and Hive configs we need a common shared environment to work on. In this PR there are "starter" configs that we can use as a starting point to get the proverbial ball rolling.

### Test plan for issue:

This has been deployed to quite a few test AKS clusters in preparation for the final push to the dev environments.

### Is there any documentation that needs to be updated for this PR?

Added in this PR.




Patch:
diff --git a/.gitignore b/.gitignore
index b7c35b9eb..1ca2eb098 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,5 @@ gomock_reflect_*
 /portal/v1/node_modules/
 /portal/v2/node_modules/
 .idea*
+/hack/hive-config/crds
+/hack/hive-config/hive-deployment.yaml
diff --git a/.pipelines/mirror-aro-to-int.yml b/.pipelines/mirror-aro-to-int.yml
index 14d71ad71..de8c01871 100644
--- a/.pipelines/mirror-aro-to-int.yml
+++ b/.pipelines/mirror-aro-to-int.yml
@@ -37,8 +37,14 @@ jobs:
       ## set the variable
       echo "##vso[task.setvariable variable=TAG]${TAG}"
   - script: |
+      USERNAME=`echo "$(aro-v4-ci-pd-pull)" | base64 -d | cut -d':' -f1`
+      PASSWORD=`echo "$(aro-v4-ci-pd-pull)" | base64 -d | cut -d':' -f2-`
+
       az acr login --name "${{variables.rpIntImageAcr}}"
       az acr import \
+        --force \
         --name "${{variables.rpIntImageAcr}}" \
-        --source "${{variables.rpProdImageAcr}}.azurecr.io/aro:${TAG}"
+        --source "${{variables.rpProdImageAcr}}.azurecr.io/aro:${TAG}" \
+        --username "$USERNAME" \
+        --password "$PASSWORD"
   - template: ./templates/template-az-cli-logout.yml
diff --git a/cmd/aro/mirror.go b/cmd/aro/mirror.go
index 559c3d6a7..1b8406b05 100644
--- a/cmd/aro/mirror.go
+++ b/cmd/aro/mirror.go
@@ -146,6 +146,11 @@ func mirror(ctx context.Context, log *logrus.Entry) error {
 		"registry.redhat.io/rhel8/support-tools:latest",
 		"registry.redhat.io/openshift4/ose-tools-rhel7:latest",
 		"registry.redhat.io/openshift4/ose-tools-rhel8:latest",
+		"registry.access.redhat.com/ubi7/ubi-minimal:latest",
+		"registry.access.redhat.com/ubi8/ubi-minimal:latest",
+		"registry.access.redhat.com/ubi8/nodejs-14:latest",
+		"registry.access.redhat.com/ubi7/go-toolset:1.16.12",
+		"registry.access.redhat.com/ubi8/go-toolset:1.17.7",
 	} {
 		log.Printf("mirroring %s -> %s", ref, pkgmirror.Dest(dstAcr+acrDomainSuffix, ref))
 		err = pkgmirror.Copy(ctx, pkgmirror.Dest(dstAcr+acrDomainSuffix, ref), ref, dstAuth, srcAuthRedhat)
@@ -155,6 +160,21 @@ func mirror(ctx context.Context, log *logrus.Entry) error {
 		}
 	}
 
+	for _, ref := range []string{
+		// Managed Upgrade Operator
+		"quay.io/app-sre/managed-upgrade-operator:v0.1.831-986b967",
+
+		// Hive
+		"quay.io/app-sre/hive:2383a88",
+	} {
+		log.Printf("mirroring %s -> %s", ref, pkgmirror.Dest(dstAcr+acrDomainSuffix, ref))
+		err = pkgmirror.Copy(ctx, pkgmirror.Dest(dstAcr+acrDomainSuffix, ref), ref, dstAuth, srcAuthQuay)
+		if err != nil {
+			log.Errorf("%s: %s\n", ref, err)
+			errorOccurred = true
+		}
+	}
+
 	log.Print("done")
 
 	if errorOccurred {
diff --git a/docs/hive.md b/docs/hive.md
new file mode 100644
index 000000000..f1dad9654
--- /dev/null
+++ b/docs/hive.md
@@ -0,0 +1,37 @@
+# Hive
+
+## Version
+
+Update the HIVE_IMAGE_COMMIT_HASH in `hack/hive-generate-config.sh` with the latest commit sha of the Hive image you are deploying. The commit sha is used to specify the image tag and also used during config generation to checkout the correct version of the config files. The config files are subsequently used by the `hack/hive-dev-install.sh` script during iunstallation or during config updates.
+
+## Generating config
+
+In order to generate config for a dev environment you need to ensure you have the correct `LOCATION` is set in your env file. Once this is done you can simply run the config generation script.
+
+```bash
+# source your environment file
+. ./env
+# run the config generation
+./hack/hive-generate-config.sh
+```
+
+This will download the latest source, reset to the hash specified in HIVE_IMAGE_COMMIT_HASH, and build the config using kustomise.
+
+## Installing
+
+Ensure you have the latest AKS kubeconfig:
+```bash
+# get the AKS kubeconfig
+make aks.kubeconfig
+```
+
+Set KUBECONFIG to the aks.kubeconfig file, for example:
+```bash
+export KUBECONFIG="$PWD/aks.kubeconfig"
+```
+
+Installing then simply requires the running of the install script.
+
+```bash
+./hack/hive-dev-install.sh
+```
diff --git a/hack/hive-config/cluster-image-sets/4.10.15.yaml b/hack/hive-config/cluster-image-sets/4.10.15.yaml
new file mode 100644
index 000000000..5dbff8143
--- /dev/null
+++ b/hack/hive-config/cluster-image-sets/4.10.15.yaml
@@ -0,0 +1,6 @@
+apiVersion: hive.openshift.io/v1
+kind: ClusterImageSet
+metadata:
+  name: openshift-v4.10.15
+spec:
+  releaseImage: quay.io/openshift-release-dev/ocp-release@sha256:ddcb70ce04a01ce487c0f4ad769e9e36a10c8c832a34307c1b1eb8e03a5b7ddb
diff --git a/hack/hive-config/hive-config.yaml b/hack/hive-config/hive-config.yaml
new file mode 100644
index 000000000..6c2bad6d8
--- /dev/null
+++ b/hack/hive-config/hive-config.yaml
@@ -0,0 +1,15 @@
+apiVersion: hive.openshift.io/v1
+kind: HiveConfig
+metadata:
+  name: hive
+spec:
+  logLevel: debug
+  targetNamespace: HIVE_OPERATOR_NS
+  deleteProtection: enabled
+  disabledControllers:
+  - remoteingress
+  failedProvisionConfig:
+    retryReasons:
+    - UnknownError
+  globalPullSecretRef:
+    name: hive-global-pull-secret
diff --git a/hack/hive-dev-install.sh b/hack/hive-dev-install.sh
new file mode 100755
index 000000000..313f66b34
--- /dev/null
+++ b/hack/hive-dev-install.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+HIVE_OPERATOR_NS="hive"
+KUBECTL=$( which kubectl 2> /dev/null || which oc 2> /dev/null)
+
+function cleanup {
+	[ -f "$(pwd)/kubectl" ] && rm -f "$(pwd)/kubectl"
+}
+
+function download_tmp_kubectl {
+	curl -sLO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+	if [ $? -ne 0 ]; then
+		echo ": error downloading kubectl"
+		exit 1
+	fi
+	chmod 755 kubectl
+	KUBECTL="$(pwd)/kubectl"
+}
+
+function verify_tools {
+	if [ ! -z "$KUBECTL" ]; then
+		return
+	fi
+	echo -n "kubectl or oc not detected, downloading"
+	download_tmp_kubectl
+	echo ", done."
+
+	if [ $( $KUBECTL get nodes 2>/dev/null | wc -l ) -eq 0 ]; then
+		echo "unable to connect to the cluster"
+		exit 1
+	fi
+}
+
+set -e
+trap cleanup EXIT
+
+if [ ! -f go.mod ] || [ ! -d ".git" ]; then
+	echo "this script must by run from the repo's root directory"
+	exit 1
+fi
+
+if [ ! -f "./hack/hive-config/hive-deployment.yaml" ] || [ ! -d "./hack/hive-config/crds" ] ; then
+	echo "hive config is missing, generating config, please rerun this script afterwards"
+	./hack/hive-generate-config.sh
+	if [ $? -ne 0 ]; then
+		echo "error generating the hive configs"
+		exit 1
+	fi
+fi
+
+if [ -z "$PULL_SECRET" ]; then
+	echo "global pull secret variable required, please source ./env"
+	exit
+fi
+
+verify_tools
+
+if [ $( $KUBECTL get namespace $HIVE_OPERATOR_NS -o yaml 2>/dev/null | wc -l ) -ne 0 ]; then
+	echo "hive is already installed in the namespace"
+	echo -n "would you like to reapply the configs? (y/N): "
+	read answer
+	if [[ "$answer" != "y" ]]; then
+		exit
+	fi
+else
+	$KUBECTL create namespace $HIVE_OPERATOR_NS
+fi
+
+$KUBECTL apply -f ./hack/hive-config/crds
+$KUBECTL apply -f ./hack/hive-config/hive-deployment.yaml
+$KUBECTL apply -f ./hack/hive-config/cluster-image-sets
+
+echo "$PULL_SECRET" > /tmp/.tmp-secret
+# Using dry-run allows updates to work seamlessly
+$KUBECTL create secret generic hive-global-pull-secret --from-file=.dockerconfigjson=/tmp/.tmp-secret --type=kubernetes.io/dockerconfigjson --namespace $HIVE_OPERATOR_NS -o yaml --dry-run=client | $KUBECTL apply -f - 2>/dev/null
+rm -f /tmp/.tmp-secret
+
+sed "s/HIVE_OPERATOR_NS/$HIVE_OPERATOR_NS/g" hack/hive-config/hive-config.yaml | $KUBECTL apply -f -
+
+echo -e "\nHive is installed."
diff --git a/hack/hive-generate-config.sh b/hack/hive-generate-config.sh
new file mode 100755
index 000000000..ce4b9d6e0
--- /dev/null
+++ b/hack/hive-generate-config.sh
@@ -0,0 +1,94 @@
+#!/bin/bash
+
+# This is the commit sha that the image was built from and ensures we use the correct configs for the release
+HIVE_IMAGE_COMMIT_HASH="2383a88"
+
+# For now we'll use the quay hive image, but this will change to an ACR once the quay.io -> ACR mirroring is setup
+# Note: semi-scientific way to get the latest image: `podman search --list-tags --limit 10000 quay.io/app-sre/hive | tail -n1`
+HIVE_IMAGE="quay.io/app-sre/hive:${HIVE_IMAGE_COMMIT_HASH}"
+
+HIVE_OPERATOR_NS="hive"
+
+# This version is specified in the hive repo and is the only hard dependency for this script
+# https://github.com/openshift/hive/blob/master/vendor/github.com/openshift/build-machinery-go/make/targets/openshift/kustomize.mk#L7
+KUSTOMIZE_VERSION=4.1.3
+KUSTOMIZE=$( which kustomize 2>/dev/null )
+TMPDIR=$( mktemp -d )
+
+function cleanup {
+	popd >& /dev/null
+	[ -d "$TMPDIR" ] && rm -fr "$TMPDIR"
+}
+
+function verify_kustomize {
+	if [ ! -z "$KUSTOMIZE" ]; then
+		return
+	fi
+	echo -n "kustomize not detected, downloading "
+	curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/kustomize/v${KUSTOMIZE_VERSION}/hack/install_kustomize.sh" | bash -s "$KUSTOMIZE_VERSION" "$TMPDIR"
+	if [ $? -ne 0 ]; then
+		echo "error downloading kustomize"
+		exit 1
+	fi
+	KUSTOMIZE="${TMPDIR}/kustomize"
+}
+
+function hive_repo_clone {
+	echo -n "Cloning hive repo into tmp for config generation"
+	CLONE_ERROR=$(git clone https://github.com/openshift/hive.git "$TMPDIR" 2>/dev/null )
+	if [ $? -ne 0 ]; then
+		echo ": error cloning the hive repo: ${CLONE_ERROR}"
+		exit 1
+	fi
+	echo ", done."
+}
+
+function hive_repo_hash_checkout {
+	# go into $TMPDIR and checkout the commit the image was built with
+	pushd $TMPDIR >& /dev/null
+	git reset --hard $HIVE_IMAGE_COMMIT_HASH
+	if [ $? -ne 0 ] || [[ $( git rev-parse --short=${#HIVE_IMAGE_COMMIT_HASH} HEAD ) != ${HIVE_IMAGE_COMMIT_HASH} ]]; then
+		echo "error resetting the hive repo to the correct git hash '${HIVE_IMAGE_COMMIT_HASH}'"
+		exit 1
+	fi
+}
+
+function generate_hive_config {
+	# Create the hive operator install config using kustomize
+	mkdir -p overlays/deploy
+	cp overlays/template/kustomization.yaml overlays/deploy
+	pushd overlays/deploy >& /dev/null
+	$KUSTOMIZE edit set image registry.ci.openshift.org/openshift/hive-v4.0:hive=$HIVE_IMAGE
+	$KUSTOMIZE edit set namespace $HIVE_OPERATOR_NS
+	popd >& /dev/null
+
+	$KUSTOMIZE build overlays/deploy > hive-deployment.yaml
+
+	# return to the repo directory to copy the generated config from $TMPDIR
+	popd >& /dev/null
+	mv "$TMPDIR/hive-deployment.yaml" ./hack/hive-config/
+
+	if [ -d ./hack/hive-config/crds ]; then
+		rm -fr ./hack/hive-config/crds
+	fi
+	cp -R "$TMPDIR/config/crds" ./hack/hive-config/
+}
+
+set -e
+trap cleanup EXIT
+
+if [ ! -f go.mod ] || [ ! -d ".git" ]; then
+	echo "this script must by run from the repo's root directory"
+	exit 1
+fi
+if [[ ! "$TMPDIR" || ! -d "$TMPDIR" ]]; then
+	echo "could not create temp working dir"
+	exit 1
+fi
+
+hive_repo_clone
+hive_repo_hash_checkout
+verify_kustomize
+generate_hive_config
+
+echo -e "\nHive config generated."
diff --git a/hack/validate-imports/allowed-import-names.yaml b/hack/validate-imports/allowed-import-names.yaml
index 12e6b51b0..a637e9f6e 100644
--- a/hack/validate-imports/allowed-import-names.yaml
+++ b/hack/validate-imports/allowed-import-names.yaml
@@ -22,6 +22,9 @@ allowedImportNames:
     - utildiscovery
   github.com/Azure/ARO-RP/pkg/util/embed:
     - utilembed
+  github.com/Azure/ARO-RP/pkg/util/machine:
+    - utilmachine
+    - ""
   github.com/Azure/ARO-RP/pkg/util/namespace:
     - utilnamespace
     - ""
diff --git a/pkg/cluster/createdat.go b/pkg/cluster/createdat.go
deleted file mode 100644
index 826de0076..000000000
--- a/pkg/cluster/createdat.go
+++ /dev/null
@@ -1,32 +0,0 @@
-package cluster
-
-// Copyright (c) Microsoft Corporation.
-// Licensed under the Apache License 2.0.
-
-import (
-	"context"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	"github.com/Azure/ARO-RP/pkg/api"
-)
-
-// populateCreatedAt updates DB document with cluster creation date from the default namespace
-// This is a temporary piece of code to populate the relevant field for older clusters.
-// TODO(mikalai): Remove the function after a round of admin updates
-func (m *manager) populateCreatedAt(ctx context.Context) error {
-	if !m.doc.OpenShiftCluster.Properties.CreatedAt.IsZero() {
-		return nil
-	}
-
-	ns, err := m.kubernetescli.CoreV1().Namespaces().Get(ctx, "default", metav1.GetOptions{})
-	if err != nil {
-		return err
-	}
-
-	m.doc, err = m.db.PatchWithLease(ctx, m.doc.Key, func(doc *api.OpenShiftClusterDocument) error {
-		doc.OpenShiftCluster.Properties.CreatedAt = ns.CreationTimestamp.Time
-		return nil
-	})
-	return err
-}
diff --git a/pkg/cluster/delete.go b/pkg/cluster/delete.go
index a0f0a0299..8c58c0a51 100644
--- a/pkg/cluster/delete.go
+++ b/pkg/cluster/delete.go
@@ -54,7 +54,7 @@ func (m *manager) deleteNic(ctx context.Context, nicName string) error {
 
 	if nic.ProvisioningState == mgmtnetwork.Failed {
 		m.log.Printf("NIC '%s' is in a Failed provisioning state, attempting to reconcile prior to deletion.", *nic.ID)
-		err := m.interfaces.CreateOrUpdateAndWait(ctx, resourceGroup, *nic.Name, mgmtnetwork.Interface{})
+		err := m.interfaces.CreateOrUpdateAndWait(ctx, resourceGroup, *nic.Name, nic)
 		if err != nil {
 			return err
 		}
diff --git a/pkg/cluster/deploystorage.go b/pkg/cluster/deploystorage.go
index dabe55aaf..e6fb43ccc 100644
--- a/pkg/cluster/deploystorage.go
+++ b/pkg/cluster/deploystorage.go
@@ -5,8 +5,6 @@ package cluster
 
 import (
 	"context"
-	"crypto/rand"
-	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -18,15 +16,9 @@ import (
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/go-autorest/autorest/to"
-	"github.com/openshift/installer/pkg/asset/installconfig"
-	"github.com/openshift/installer/pkg/asset/releaseimage"
-	"github.com/openshift/installer/pkg/asset/targets"
-	"github.com/openshift/installer/pkg/asset/templates/content/bootkube"
 	utilrand "k8s.io/apimachinery/pkg/util/rand"
 
 	"github.com/Azure/ARO-RP/pkg/api"
-	"github.com/Azure/ARO-RP/pkg/bootstraplogging"
-	"github.com/Azure/ARO-RP/pkg/cluster/graph"
 	"github.com/Azure/ARO-RP/pkg/env"
 	"github.com/Azure/ARO-RP/pkg/util/arm"
 	"github.com/Azure/ARO-RP/pkg/util/stringutils"
@@ -156,77 +148,6 @@ func (m *manager) deployStorageTemplate(ctx context.Context) error {
 	return arm.DeployTemplate(ctx, m.log, m.deployments, resourceGroup, "storage", t, nil)
 }
 
-// applyInstallConfigCustomisations modifies the InstallConfig and creates
-// parent assets, then regenerates the InstallConfig for use for Ignition
-// generation, etc.
-func (m *manager) applyInstallConfigCustomisations(ctx context.Context, installConfig *installconfig.InstallConfig, image *releaseimage.Image) (graph.Graph, error) {
-	clusterID := &installconfig.ClusterID{
-		UUID:    m.doc.ID,
-		InfraID: m.doc.OpenShiftCluster.Properties.InfraID,
-	}
-
-	bootstrapLoggingConfig, err := bootstraplogging.GetConfig(m.env, m.doc)
-	if err != nil {
-		return nil, err
-	}
-
-	httpSecret := make([]byte, 64)
-	_, err = rand.Read(httpSecret)
-	if err != nil {
-		return nil, err
-	}
-
-	imageRegistryConfig := &bootkube.AROImageRegistryConfig{
-		AccountName:   m.doc.OpenShiftCluster.Properties.ImageRegistryStorageAccountName,
-		ContainerName: "image-registry",
-		HTTPSecret:    hex.EncodeToString(httpSecret),
-	}
-
-	dnsConfig := &bootkube.ARODNSConfig{
-		APIIntIP:  m.doc.OpenShiftCluster.Properties.APIServerProfile.IntIP,
-		IngressIP: m.doc.OpenShiftCluster.Properties.IngressProfiles[0].IP,
-	}
-
-	if m.doc.OpenShiftCluster.Properties.NetworkProfile.GatewayPrivateEndpointIP != "" {
-		dnsConfig.GatewayPrivateEndpointIP = m.doc.OpenShiftCluster.Properties.NetworkProfile.GatewayPrivateEndpointIP
-		dnsConfig.GatewayDomains = append(m.env.GatewayDomains(), m.doc.OpenShiftCluster.Properties.ImageRegistryStorageAccountName+".blob."+m.env.Environment().StorageEndpointSuffix)
-	}
-
-	g := graph.Graph{}
-	g.Set(installConfig, image, clusterID, bootstrapLoggingConfig, dnsConfig, imageRegistryConfig)
-
-	m.log.Print("resolving graph")
-	for _, a := range targets.Cluster {
-		err = g.Resolve(a)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	// Handle MTU3900 feature flag
-	if m.doc.OpenShiftCluster.Properties.NetworkProfile.MTUSize == api.MTU3900 {
-		m.log.Printf("applying feature flag %s", api.FeatureFlagMTU3900)
-		if err = m.overrideEthernetMTU(g); err != nil {
-			return nil, err
-		}
-	}
-
-	return g, nil
-}
-
-func (m *manager) persistGraph(ctx context.Context, g graph.Graph) error {
-	resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/')
-	clusterStorageAccountName := "cluster" + m.doc.OpenShiftCluster.Properties.StorageSuffix
-
-	exists, err := m.graph.Exists(ctx, resourceGroup, clusterStorageAccountName)
-	if err != nil || exists {
-		return err
-	}
-
-	// the graph is quite big, so we store it in a storage account instead of in cosmosdb
-	return m.graph.Save(ctx, resourceGroup, clusterStorageAccountName, g)
-}
-
 func (m *manager) attachNSGs(ctx context.Context) error {
 	for _, subnetID := range []string{
 		m.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID,
diff --git a/pkg/cluster/fixmcsuserdata.go b/pkg/cluster/fixmcsuserdata.go
index fccce111d..80e7641d7 100644
--- a/pkg/cluster/fixmcsuserdata.go
+++ b/pkg/cluster/fixmcsuserdata.go
@@ -5,7 +5,6 @@ package cluster
 
 import (
 	"context"
-	"fmt"
 	"net"
 	"net/url"
 	"strings"
@@ -14,10 +13,10 @@ import (
 	"github.com/ugorji/go/codec"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/util/retry"
 
 	"github.com/Azure/ARO-RP/pkg/api"
+	utilmachine "github.com/Azure/ARO-RP/pkg/util/machine"
 	_ "github.com/Azure/ARO-RP/pkg/util/scheme"
 )
 
@@ -82,16 +81,11 @@ func getUserDataSecretReference(objMeta *metav1.ObjectMeta, spec *machinev1beta1
 		return nil, nil
 	}
 
-	o, _, err := scheme.Codecs.UniversalDeserializer().Decode(spec.ProviderSpec.Value.Raw, nil, nil)
+	machineProviderSpec, err := utilmachine.UnmarshalAzureProviderSpec(spec.Name, utilmachine.Machine, spec.ProviderSpec.Value.Raw)
 	if err != nil {
 		return nil, err
 	}
 
-	machineProviderSpec, ok := o.(*machinev1beta1.AzureMachineProviderSpec)
-	if !ok {
-		return nil, fmt.Errorf("failed to read provider spec: %T", o)
-	}
-
 	if machineProviderSpec.UserDataSecret == nil {
 		return nil, nil
 	}
diff --git a/pkg/cluster/install.go b/pkg/cluster/install.go
index 343c0ca7e..4c2a9b1fe 100644
--- a/pkg/cluster/install.go
+++ b/pkg/cluster/install.go
@@ -14,14 +14,12 @@ import (
 	operatorclient "github.com/openshift/client-go/operator/clientset/versioned"
 	samplesclient "github.com/openshift/client-go/samples/clientset/versioned"
 	securityclient "github.com/openshift/client-go/security/clientset/versioned"
-	"github.com/openshift/installer/pkg/asset/installconfig"
-	"github.com/openshift/installer/pkg/asset/releaseimage"
 	mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned"
 	extensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/client-go/kubernetes"
 
 	"github.com/Azure/ARO-RP/pkg/api"
-	"github.com/Azure/ARO-RP/pkg/cluster/graph"
+	"github.com/Azure/ARO-RP/pkg/installer"
 	aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned"
 	"github.com/Azure/ARO-RP/pkg/operator/deploy"
 	"github.com/Azure/ARO-RP/pkg/util/restconfig"
@@ -83,8 +81,6 @@ func (m *manager) adminUpdate() []steps.Step {
 			steps.Action(m.configureIngressCertificate),
 			steps.Action(m.populateRegistryStorageAccountName),
 			steps.Action(m.ensureMTUSize),
-			steps.Action(m.populateCreatedAt), // TODO(mikalai): Remove after a round of admin updates
-
 		)
 	}
 
@@ -124,14 +120,14 @@ func (m *manager) Update(ctx context.Context) error {
 	return m.runSteps(ctx, steps)
 }
 
+// callInstaller initialises and calls the Installer code. This will later be replaced with a call into Hive.
+func (m *manager) callInstaller(ctx context.Context) error {
+	i := installer.NewInstaller(m.log, m.env, m.doc, m.subscriptionDoc, m.fpAuthorizer, m.deployments, m.graph)
+	return i.Install(ctx)
+}
+
 // Install installs an ARO cluster
 func (m *manager) Install(ctx context.Context) error {
-	var (
-		installConfig *installconfig.InstallConfig
-		image         *releaseimage.Image
-		g             graph.Graph
-	)
-
 	steps := map[api.InstallPhase][]steps.Step{
 		api.InstallPhaseBootstrap: {
 			steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.validateResources)),
@@ -139,11 +135,7 @@ func (m *manager) Install(ctx context.Context) error {
 			steps.Action(m.ensureInfraID),
 			steps.Action(m.generateSSHKey),
 			steps.Action(m.populateMTUSize),
-			steps.Action(func(ctx context.Context) error {
-				var err error
-				installConfig, image, err = m.generateInstallConfig(ctx)
-				return err
-			}),
+
 			steps.Action(m.createDNS),
 			steps.Action(m.initializeClusterSPClients), // must run before clusterSPObjectID
 			steps.Action(m.clusterSPObjectID),
@@ -151,28 +143,22 @@ func (m *manager) Install(ctx context.Context) error {
 			steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.enableServiceEndpoints)),
 			steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.setMasterSubnetPolicies)),
 			steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.deployStorageTemplate)),
+			steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.attachNSGs)),
 			steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.updateAPIIPEarly)),
 			steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.createOrUpdateRouterIPEarly)),
 			steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.ensureGatewayCreate)),
-			steps.Action(func(ctx context.Context) error {
-				var err error
-				// Applies ARO-specific customisations to the InstallConfig
-				g, err = m.applyInstallConfigCustomisations(ctx, installConfig, image)
-				return err
-			}),
-			steps.Action(func(ctx context.Context) error {
-				// saves the graph to storage account
-				return m.persistGraph(ctx, g)
-			}),
-			steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.attachNSGs)),
-			steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.generateKubeconfigs)),
-			steps.Action(m.ensureBillingRecord),
-			steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.deployResourceTemplate)),
 			steps.Action(m.createAPIServerPrivateEndpoint),
 			steps.Action(m.createCertificates),
+
+			// Run installer. For M5/M6 we will persist the graph inside the
+			// installer code since it's easier, but in the future, this data
+			// should be collected from Hive's outputs where needed.
+			steps.Action(m.callInstaller),
+
+			steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.generateKubeconfigs)),
+			steps.Action(m.ensureBillingRecord),
 			steps.Action(m.initializeKubernetesClients),
 			steps.Action(m.initializeOperatorDeployer), // depends on kube clients
-			steps.Condition(m.bootstrapConfigMapReady, 30*time.Minute, true),
 			steps.Condition(m.apiServersReady, 30*time.Minute, true),
 			steps.Action(m.ensureAROOperator),
 			steps.Action(m.incrInstallPhase),
diff --git a/pkg/installer/condition.go b/pkg/installer/condition.go
new file mode 100644
index 000000000..0fb147a5b
--- /dev/null
+++ b/pkg/installer/condition.go
@@ -0,0 +1,21 @@
+package installer
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// condition functions should return an error only if it's not retryable
+// if a condition function encounters a retryable error it should return false, nil.
+
+func (m *manager) bootstrapConfigMapReady(ctx context.Context) (bool, error) {
+	cm, err := m.kubernetescli.CoreV1().ConfigMaps("kube-system").Get(ctx, "bootstrap", metav1.GetOptions{})
+	if err != nil && m.env.IsLocalDevelopmentMode() {
+		m.log.Printf("bootstrapConfigMapReady condition error %s", err)
+	}
+	return err == nil && cm.Data["status"] == "complete", nil
+}
diff --git a/pkg/installer/custominstallconfig.go b/pkg/installer/custominstallconfig.go
new file mode 100644
index 000000000..f038dda24
--- /dev/null
+++ b/pkg/installer/custominstallconfig.go
@@ -0,0 +1,77 @@
+package installer
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+
+	"github.com/openshift/installer/pkg/asset/installconfig"
+	"github.com/openshift/installer/pkg/asset/releaseimage"
+	"github.com/openshift/installer/pkg/asset/targets"
+	"github.com/openshift/installer/pkg/asset/templates/content/bootkube"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	"github.com/Azure/ARO-RP/pkg/bootstraplogging"
+	"github.com/Azure/ARO-RP/pkg/cluster/graph"
+)
+
+// applyInstallConfigCustomisations modifies the InstallConfig and creates
+// parent assets, then regenerates the InstallConfig for use for Ignition
+// generation, etc.
+func (m *manager) applyInstallConfigCustomisations(ctx context.Context, installConfig *installconfig.InstallConfig, image *releaseimage.Image) (graph.Graph, error) {
+	clusterID := &installconfig.ClusterID{
+		UUID:    m.doc.ID,
+		InfraID: m.doc.OpenShiftCluster.Properties.InfraID,
+	}
+
+	bootstrapLoggingConfig, err := bootstraplogging.GetConfig(m.env, m.doc)
+	if err != nil {
+		return nil, err
+	}
+
+	httpSecret := make([]byte, 64)
+	_, err = rand.Read(httpSecret)
+	if err != nil {
+		return nil, err
+	}
+
+	imageRegistryConfig := &bootkube.AROImageRegistryConfig{
+		AccountName:   m.doc.OpenShiftCluster.Properties.ImageRegistryStorageAccountName,
+		ContainerName: "image-registry",
+		HTTPSecret:    hex.EncodeToString(httpSecret),
+	}
+
+	dnsConfig := &bootkube.ARODNSConfig{
+		APIIntIP:  m.doc.OpenShiftCluster.Properties.APIServerProfile.IntIP,
+		IngressIP: m.doc.OpenShiftCluster.Properties.IngressProfiles[0].IP,
+	}
+
+	if m.doc.OpenShiftCluster.Properties.NetworkProfile.GatewayPrivateEndpointIP != "" {
+		dnsConfig.GatewayPrivateEndpointIP = m.doc.OpenShiftCluster.Properties.NetworkProfile.GatewayPrivateEndpointIP
+		dnsConfig.GatewayDomains = append(m.env.GatewayDomains(), m.doc.OpenShiftCluster.Properties.ImageRegistryStorageAccountName+".blob."+m.env.Environment().StorageEndpointSuffix)
+	}
+
+	g := graph.Graph{}
+	g.Set(installConfig, image, clusterID, bootstrapLoggingConfig, dnsConfig, imageRegistryConfig)
+
+	m.log.Print("resolving graph")
+	for _, a := range targets.Cluster {
+		err = g.Resolve(a)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Handle MTU3900 feature flag
+	if m.doc.OpenShiftCluster.Properties.NetworkProfile.MTUSize == api.MTU3900 {
+		m.log.Printf("applying feature flag %s", api.FeatureFlagMTU3900)
+		if err = m.overrideEthernetMTU(g); err != nil {
+			return nil, err
+		}
+	}
+
+	return g, nil
+}
diff --git a/pkg/cluster/deployresources.go b/pkg/installer/deployresources.go
similarity index 99%
rename from pkg/cluster/deployresources.go
rename to pkg/installer/deployresources.go
index da9a5e948..f07538bf0 100644
--- a/pkg/cluster/deployresources.go
+++ b/pkg/installer/deployresources.go
@@ -1,4 +1,4 @@
-package cluster
+package installer
 
 // Copyright (c) Microsoft Corporation.
 // Licensed under the Apache License 2.0.
diff --git a/pkg/cluster/deployresources_resources.go b/pkg/installer/deployresources_resources.go
similarity index 99%
rename from pkg/cluster/deployresources_resources.go
rename to pkg/installer/deployresources_resources.go
index 12db8fd0e..71864133c 100644
--- a/pkg/cluster/deployresources_resources.go
+++ b/pkg/installer/deployresources_resources.go
@@ -1,4 +1,4 @@
-package cluster
+package installer
 
 // Copyright (c) Microsoft Corporation.
 // Licensed under the Apache License 2.0.
diff --git a/pkg/cluster/generateconfig.go b/pkg/installer/generateconfig.go
similarity index 99%
rename from pkg/cluster/generateconfig.go
rename to pkg/installer/generateconfig.go
index bf3f0612b..39aaf6da6 100644
--- a/pkg/cluster/generateconfig.go
+++ b/pkg/installer/generateconfig.go
@@ -1,4 +1,4 @@
-package cluster
+package installer
 
 // Copyright (c) Microsoft Corporation.
 // Licensed under the Apache License 2.0.
diff --git a/pkg/installer/graph.go b/pkg/installer/graph.go
new file mode 100644
index 000000000..e49f21c2e
--- /dev/null
+++ b/pkg/installer/graph.go
@@ -0,0 +1,24 @@
+package installer
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	"github.com/Azure/ARO-RP/pkg/cluster/graph"
+	"github.com/Azure/ARO-RP/pkg/util/stringutils"
+)
+
+func (m *manager) persistGraph(ctx context.Context, g graph.Graph) error {
+	resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/')
+	clusterStorageAccountName := "cluster" + m.doc.OpenShiftCluster.Properties.StorageSuffix
+
+	exists, err := m.graph.Exists(ctx, resourceGroup, clusterStorageAccountName)
+	if err != nil || exists {
+		return err
+	}
+
+	// the graph is quite big, so we store it in a storage account instead of in cosmosdb
+	return m.graph.Save(ctx, resourceGroup, clusterStorageAccountName, g)
+}
diff --git a/pkg/installer/install.go b/pkg/installer/install.go
new file mode 100644
index 000000000..fb11fb9d3
--- /dev/null
+++ b/pkg/installer/install.go
@@ -0,0 +1,94 @@
+package installer
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"time"
+
+	"github.com/openshift/installer/pkg/asset/installconfig"
+	"github.com/openshift/installer/pkg/asset/kubeconfig"
+	"github.com/openshift/installer/pkg/asset/releaseimage"
+	"github.com/pkg/errors"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+
+	"github.com/Azure/ARO-RP/pkg/cluster/graph"
+	"github.com/Azure/ARO-RP/pkg/util/restconfig"
+	"github.com/Azure/ARO-RP/pkg/util/steps"
+	"github.com/Azure/ARO-RP/pkg/util/stringutils"
+)
+
+func (m *manager) Install(ctx context.Context) error {
+	var (
+		installConfig *installconfig.InstallConfig
+		image         *releaseimage.Image
+		g             graph.Graph
+	)
+
+	s := []steps.Step{
+		steps.Action(func(ctx context.Context) error {
+			var err error
+			installConfig, image, err = m.generateInstallConfig(ctx)
+			return err
+		}),
+
+		steps.Action(func(ctx context.Context) error {
+			var err error
+			// Applies ARO-specific customisations to the InstallConfig
+			g, err = m.applyInstallConfigCustomisations(ctx, installConfig, image)
+			return err
+		}),
+		steps.Action(func(ctx context.Context) error {
+			return m.persistGraph(ctx, g)
+		}),
+		steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.deployResourceTemplate)),
+		steps.Action(m.initializeKubernetesClients),
+		steps.Condition(m.bootstrapConfigMapReady, 30*time.Minute, true),
+	}
+
+	err := steps.Run(ctx, m.log, 10*time.Second, s)
+	return err
+}
+
+// initializeKubernetesClients initializes clients using the Installer-generated
+// kubeconfig.
+func (m *manager) initializeKubernetesClients(ctx context.Context) error {
+	resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/')
+	account := "cluster" + m.doc.OpenShiftCluster.Properties.StorageSuffix
+
+	// Load the installer's generated kubeconfig from the graph
+	pg, err := m.graph.LoadPersisted(ctx, resourceGroup, account)
+	if err != nil {
+		return err
+	}
+
+	var adminInternalClient *kubeconfig.AdminInternalClient
+	err = pg.Get(&adminInternalClient)
+	if err != nil {
+		return err
+	}
+
+	// must not proceed if PrivateEndpointIP is not set.  In
+	// k8s.io/client-go/transport/cache.go, k8s caches our transport, and it
+	// can't tell if data in the restconfig.Dial closure has changed.  We don't
+	// want it to cache a transport that can never work.
+	if m.doc.OpenShiftCluster.Properties.NetworkProfile.APIServerPrivateEndpointIP == "" {
+		return errors.New("privateEndpointIP is empty")
+	}
+
+	config, err := clientcmd.Load(adminInternalClient.File.Data)
+	if err != nil {
+		return err
+	}
+
+	r, err := clientcmd.NewDefaultClientConfig(*config, &clientcmd.ConfigOverrides{}).ClientConfig()
+	if err != nil {
+		return err
+	}
+	r.Dial = restconfig.DialContext(m.env, m.doc.OpenShiftCluster)
+
+	m.kubernetescli, err = kubernetes.NewForConfig(r)
+	return err
+}
diff --git a/pkg/installer/manager.go b/pkg/installer/manager.go
new file mode 100644
index 000000000..2eb63d137
--- /dev/null
+++ b/pkg/installer/manager.go
@@ -0,0 +1,48 @@
+package installer
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	"github.com/sirupsen/logrus"
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	"github.com/Azure/ARO-RP/pkg/cluster/graph"
+	"github.com/Azure/ARO-RP/pkg/env"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/features"
+	"github.com/Azure/ARO-RP/pkg/util/refreshable"
+)
+
+type manager struct {
+	log *logrus.Entry
+	env env.Interface
+
+	doc             *api.OpenShiftClusterDocument
+	subscriptionDoc *api.SubscriptionDocument
+	fpAuthorizer    refreshable.Authorizer
+
+	deployments features.DeploymentsClient
+
+	graph graph.Manager
+
+	kubernetescli kubernetes.Interface
+}
+
+type Interface interface {
+	Install(ctx context.Context) error
+}
+
+func NewInstaller(log *logrus.Entry, _env env.Interface, doc *api.OpenShiftClusterDocument, subscriptionDoc *api.SubscriptionDocument, fpAuthorizer refreshable.Authorizer, deployments features.DeploymentsClient, g graph.Manager) Interface {
+	return &manager{
+		log:             log,
+		env:             _env,
+		doc:             doc,
+		subscriptionDoc: subscriptionDoc,
+		fpAuthorizer:    fpAuthorizer,
+		deployments:     deployments,
+		graph:           g,
+	}
+}
diff --git a/pkg/cluster/overridemtu.go b/pkg/installer/overridemtu.go
similarity index 99%
rename from pkg/cluster/overridemtu.go
rename to pkg/installer/overridemtu.go
index 7fa017997..7eed9d9e2 100644
--- a/pkg/cluster/overridemtu.go
+++ b/pkg/installer/overridemtu.go
@@ -1,4 +1,4 @@
-package cluster
+package installer
 
 // Copyright (c) Microsoft Corporation.
 // Licensed under the Apache License 2.0.
diff --git a/pkg/operator/controllers/machine/machine.go b/pkg/operator/controllers/machine/machine.go
index 5e26890e3..901c03b01 100644
--- a/pkg/operator/controllers/machine/machine.go
+++ b/pkg/operator/controllers/machine/machine.go
@@ -5,16 +5,14 @@ package machine
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
-	"strings"
 
 	machinev1beta1 "github.com/openshift/api/machine/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes/scheme"
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/api/validate"
+	utilmachine "github.com/Azure/ARO-RP/pkg/util/machine"
 	_ "github.com/Azure/ARO-RP/pkg/util/scheme"
 )
 
@@ -39,27 +37,9 @@ func (r *Reconciler) machineValid(ctx context.Context, machine *machinev1beta1.M
 		return []error{fmt.Errorf("machine %s: provider spec missing", machine.Name)}
 	}
 
-	var machineProviderSpec *machinev1beta1.AzureMachineProviderSpec
-
-	if strings.Contains(string(machine.Spec.ProviderSpec.Value.Raw), "azureproviderconfig.openshift.io") {
-		machineProviderSpec = &machinev1beta1.AzureMachineProviderSpec{}
-		err := json.Unmarshal(machine.Spec.ProviderSpec.Value.Raw, machineProviderSpec)
-		if err != nil {
-			return []error{fmt.Errorf("machine %s: failed to unmarshal the 'azureproviderconfig.openshift.io' provider spec: %q", machine.Name, err.Error())}
-		}
-	} else {
-		o, _, err := scheme.Codecs.UniversalDeserializer().Decode(machine.Spec.ProviderSpec.Value.Raw, nil, nil)
-		if err != nil {
-			return []error{err}
-		}
-
-		var ok bool
-		machineProviderSpec, ok = o.(*machinev1beta1.AzureMachineProviderSpec)
-		if !ok {
-			// This should never happen: codecs uses scheme that has only one registered type
-			// and if something is wrong with the provider spec - decoding should fail
-			return []error{fmt.Errorf("machine %s: failed to read provider spec: %T", machine.Name, o)}
-		}
+	machineProviderSpec, err := utilmachine.UnmarshalAzureProviderSpec(machine.Name, utilmachine.Machine, machine.Spec.ProviderSpec.Value.Raw)
+	if err != nil {
+		errs = append(errs, err)
 	}
 
 	// Validate VM size in machine provider spec
diff --git a/pkg/operator/controllers/muo/muo_controller.go b/pkg/operator/controllers/muo/muo_controller.go
index d71fe31cb..63103cd30 100644
--- a/pkg/operator/controllers/muo/muo_controller.go
+++ b/pkg/operator/controllers/muo/muo_controller.go
@@ -18,8 +18,10 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
 	aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned"
@@ -160,12 +162,21 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.
 
 // SetupWithManager setup our manager
 func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
+	pullSecretPredicate := predicate.NewPredicateFuncs(func(o client.Object) bool {
+		return (o.GetName() == pullSecretName.Name && o.GetNamespace() == pullSecretName.Namespace)
+	})
+
 	aroClusterPredicate := predicate.NewPredicateFuncs(func(o client.Object) bool {
 		return o.GetName() == arov1alpha1.SingletonClusterName
 	})
 
-	builder := ctrl.NewControllerManagedBy(mgr).
-		For(&arov1alpha1.Cluster{}, builder.WithPredicates(aroClusterPredicate))
+	muoBuilder := ctrl.NewControllerManagedBy(mgr).
+		For(&arov1alpha1.Cluster{}, builder.WithPredicates(aroClusterPredicate)).
+		Watches(
+			&source.Kind{Type: &corev1.Secret{}},
+			&handler.EnqueueRequestForObject{},
+			builder.WithPredicates(pullSecretPredicate),
+		)
 
 	resources, err := r.deployer.Template(&config.MUODeploymentConfig{}, staticFiles)
 	if err != nil {
@@ -175,11 +186,11 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
 	for _, i := range resources {
 		o, ok := i.(client.Object)
 		if ok {
-			builder.Owns(o)
+			muoBuilder.Owns(o)
 		}
 	}
 
-	return builder.
+	return muoBuilder.
 		WithEventFilter(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.AnnotationChangedPredicate{}, predicate.LabelChangedPredicate{})).
 		Named(ControllerName).
 		Complete(r)
diff --git a/pkg/util/clusterdata/worker_profiles_task.go b/pkg/util/clusterdata/worker_profiles_task.go
index 1dc15bd6f..549c7d353 100644
--- a/pkg/util/clusterdata/worker_profiles_task.go
+++ b/pkg/util/clusterdata/worker_profiles_task.go
@@ -9,14 +9,13 @@ import (
 	"sort"
 
 	"github.com/Azure/go-autorest/autorest/azure"
-	machinev1beta1 "github.com/openshift/api/machine/v1beta1"
 	machineclient "github.com/openshift/client-go/machine/clientset/versioned"
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 
 	"github.com/Azure/ARO-RP/pkg/api"
+	utilmachine "github.com/Azure/ARO-RP/pkg/util/machine"
 	_ "github.com/Azure/ARO-RP/pkg/util/scheme"
 )
 
@@ -75,17 +74,12 @@ func (ef *workerProfilesEnricherTask) FetchData(ctx context.Context, callbacks c
 			continue
 		}
 
-		o, _, err := scheme.Codecs.UniversalDeserializer().Decode(machineset.Spec.Template.Spec.ProviderSpec.Value.Raw, nil, nil)
-		if err != nil {
-			ef.log.Info(err)
-			continue
-		}
+		machineProviderSpec, err := utilmachine.UnmarshalAzureProviderSpec(machineset.Name, utilmachine.MachineSet, machineset.Spec.Template.Spec.ProviderSpec.Value.Raw)
 
-		machineProviderSpec, ok := o.(*machinev1beta1.AzureMachineProviderSpec)
-		if !ok {
-			// This should never happen: codecs uses scheme that has only one registered type
-			// and if something is wrong with the provider spec - decoding should fail
-			ef.log.Infof("failed to read provider spec from the machine set %q: %T", machineset.Name, o)
+		if err != nil {
+			// If this happens, the azure machine provider spec type/apiversion may have been updated and
+			// we need to handle it appropriately
+			ef.log.Info(err.Error())
 			continue
 		}
 
diff --git a/pkg/util/machine/azureproviderspec.go b/pkg/util/machine/azureproviderspec.go
new file mode 100644
index 000000000..50064aac8
--- /dev/null
+++ b/pkg/util/machine/azureproviderspec.go
@@ -0,0 +1,51 @@
+package machine
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	machinev1beta1 "github.com/openshift/api/machine/v1beta1"
+	"k8s.io/client-go/kubernetes/scheme"
+
+	_ "github.com/Azure/ARO-RP/pkg/util/scheme"
+)
+
+type MachineType string
+
+const (
+	MachineSet        MachineType = "machineset"
+	Machine           MachineType = "machine"
+	azureProviderSpec             = "azureproviderconfig.openshift.io"
+)
+
+// UnmarshalAzureProviderSpec unmarshals an Azure Provider Spec set on machines or machinesets
+// it contains backward compatibility for the older azureproviderconfig which is no longer in use
+// in OCP 4.10+
+func UnmarshalAzureProviderSpec(name string, mType MachineType, rawProviderSpec []byte) (*machinev1beta1.AzureMachineProviderSpec, error) {
+	if strings.Contains(string(rawProviderSpec), azureProviderSpec) {
+		machineProviderSpec := &machinev1beta1.AzureMachineProviderSpec{}
+		err := json.Unmarshal(rawProviderSpec, machineProviderSpec)
+		if err != nil {
+			return nil, fmt.Errorf("%s %s: failed to unmarshal the %q provider spec: %q", mType, name, azureProviderSpec, err.Error())
+		}
+		return machineProviderSpec, nil
+	}
+
+	o, _, err := scheme.Codecs.UniversalDeserializer().Decode(rawProviderSpec, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	machineProviderSpec, ok := o.(*machinev1beta1.AzureMachineProviderSpec)
+	if !ok {
+		// If this happens, the azure machine provider spec type/apiversion may have been updated and
+		// we need to handle it appropriately
+		return nil, fmt.Errorf("%s %s: failed to read provider spec: %T", mType, name, o)
+	}
+
+	return machineProviderSpec, nil
+}


Test patch:
diff --git a/pkg/cluster/adminupdate_test.go b/pkg/cluster/adminupdate_test.go
index 2f3fc4212..02f6174a8 100644
--- a/pkg/cluster/adminupdate_test.go
+++ b/pkg/cluster/adminupdate_test.go
@@ -86,7 +86,6 @@ func TestAdminUpdateSteps(t *testing.T) {
 				"[Action configureIngressCertificate-fm]",
 				"[Action populateRegistryStorageAccountName-fm]",
 				"[Action ensureMTUSize-fm]",
-				"[Action populateCreatedAt-fm]",
 				"[Action ensureAROOperator-fm]",
 				"[Condition aroDeploymentReady-fm, timeout 20m0s]",
 				"[Action updateProvisionedBy-fm]",
@@ -126,7 +125,6 @@ func TestAdminUpdateSteps(t *testing.T) {
 				"[Action configureIngressCertificate-fm]",
 				"[Action populateRegistryStorageAccountName-fm]",
 				"[Action ensureMTUSize-fm]",
-				"[Action populateCreatedAt-fm]",
 				"[Action ensureAROOperator-fm]",
 				"[Condition aroDeploymentReady-fm, timeout 20m0s]",
 				"[Action updateProvisionedBy-fm]",
diff --git a/pkg/cluster/createdat_test.go b/pkg/cluster/createdat_test.go
deleted file mode 100644
index 033744f21..000000000
--- a/pkg/cluster/createdat_test.go
+++ /dev/null
@@ -1,136 +0,0 @@
-package cluster
-
-// Copyright (c) Microsoft Corporation.
-// Licensed under the Apache License 2.0.
-
-import (
-	"context"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/sirupsen/logrus"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes/fake"
-
-	"github.com/Azure/ARO-RP/pkg/api"
-	testdatabase "github.com/Azure/ARO-RP/test/database"
-)
-
-func TestPopulateCreatedAt(t *testing.T) {
-	ctx := context.Background()
-	resourceID := "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourceGroup/providers/microsoft.redhatopenshift/openshiftclusters/resourceName"
-	mockCreationTimestamp := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)
-
-	for _, tt := range []struct {
-		name     string
-		ns       *corev1.Namespace
-		doc      *api.OpenShiftClusterDocument
-		wantTime time.Time
-		wantErr  string
-	}{
-		{
-			name: "doc without timestamp",
-			ns: &corev1.Namespace{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:              "default",
-					CreationTimestamp: metav1.NewTime(mockCreationTimestamp),
-				},
-			},
-			doc: &api.OpenShiftClusterDocument{
-				Key: strings.ToLower(resourceID),
-				OpenShiftCluster: &api.OpenShiftCluster{
-					ID: resourceID,
-					Properties: api.OpenShiftClusterProperties{
-						ProvisioningState: api.ProvisioningStateAdminUpdating,
-					},
-				},
-			},
-			wantTime: mockCreationTimestamp,
-		},
-		{
-			name: "doc with timestamp",
-			ns: &corev1.Namespace{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:              "default",
-					CreationTimestamp: metav1.NewTime(mockCreationTimestamp),
-				},
-			},
-			doc: &api.OpenShiftClusterDocument{
-				Key: strings.ToLower(resourceID),
-				OpenShiftCluster: &api.OpenShiftCluster{
-					ID: resourceID,
-					Properties: api.OpenShiftClusterProperties{
-						CreatedAt:         time.Date(1970, 1, 1, 0, 0, 0, 1, time.UTC),
-						ProvisioningState: api.ProvisioningStateAdminUpdating,
-					},
-				},
-			},
-			wantTime: time.Date(1970, 1, 1, 0, 0, 0, 1, time.UTC),
-		},
-		{
-			name: "default namespace doesn't exist",
-			ns:   &corev1.Namespace{},
-			doc: &api.OpenShiftClusterDocument{
-				Key: strings.ToLower(resourceID),
-				OpenShiftCluster: &api.OpenShiftCluster{
-					ID: resourceID,
-					Properties: api.OpenShiftClusterProperties{
-						ProvisioningState: api.ProvisioningStateAdminUpdating,
-					},
-				},
-			},
-			wantErr: "namespaces \"default\" not found",
-		},
-	} {
-		t.Run(tt.name, func(t *testing.T) {
-			fakeOpenShiftClustersDatabase, _ := testdatabase.NewFakeOpenShiftClusters()
-			fixture := testdatabase.NewFixture().WithOpenShiftClusters(fakeOpenShiftClustersDatabase)
-			fixture.AddOpenShiftClusterDocuments(tt.doc)
-			err := fixture.Create()
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			clusterdoc, err := fakeOpenShiftClustersDatabase.Dequeue(ctx)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			m := &manager{
-				log:           logrus.NewEntry(logrus.StandardLogger()),
-				kubernetescli: fake.NewSimpleClientset(tt.ns),
-				doc:           clusterdoc,
-				db:            fakeOpenShiftClustersDatabase,
-			}
-
-			err = m.populateCreatedAt(ctx)
-			if err != nil && err.Error() != tt.wantErr ||
-				err == nil && tt.wantErr != "" {
-				t.Fatal(err)
-			}
-
-			if err == nil {
-				if tt.wantErr != "" {
-					t.Error(err)
-				}
-
-				doc, err := fakeOpenShiftClustersDatabase.Get(ctx, strings.ToLower(resourceID))
-				if err != nil {
-					t.Fatal(err)
-				}
-				if tt.wantTime != doc.OpenShiftCluster.Properties.CreatedAt {
-					t.Error(doc.OpenShiftCluster.Properties.CreatedAt)
-				}
-				if tt.wantTime != m.doc.OpenShiftCluster.Properties.CreatedAt {
-					t.Error(m.doc.OpenShiftCluster.Properties.CreatedAt)
-				}
-			} else {
-				if err.Error() != tt.wantErr {
-					t.Error(err)
-				}
-			}
-		})
-	}
-}
diff --git a/pkg/cluster/delete_test.go b/pkg/cluster/delete_test.go
index de008c14d..5978fce6f 100644
--- a/pkg/cluster/delete_test.go
+++ b/pkg/cluster/delete_test.go
@@ -52,7 +52,7 @@ func TestDeleteNic(t *testing.T) {
 			mocks: func(networkInterfaces *mock_network.MockInterfacesClient) {
 				nic.InterfacePropertiesFormat.ProvisioningState = mgmtnetwork.Failed
 				networkInterfaces.EXPECT().Get(gomock.Any(), clusterRG, nicName, "").Return(nic, nil)
-				networkInterfaces.EXPECT().CreateOrUpdateAndWait(gomock.Any(), clusterRG, nicName, gomock.Any()).Return(nil)
+				networkInterfaces.EXPECT().CreateOrUpdateAndWait(gomock.Any(), clusterRG, nicName, nic).Return(nil)
 				networkInterfaces.EXPECT().DeleteAndWait(gomock.Any(), clusterRG, nicName).Return(nil)
 			},
 		},
@@ -61,7 +61,7 @@ func TestDeleteNic(t *testing.T) {
 			mocks: func(networkInterfaces *mock_network.MockInterfacesClient) {
 				nic.InterfacePropertiesFormat.ProvisioningState = mgmtnetwork.Failed
 				networkInterfaces.EXPECT().Get(gomock.Any(), clusterRG, nicName, "").Return(nic, nil)
-				networkInterfaces.EXPECT().CreateOrUpdateAndWait(gomock.Any(), clusterRG, nicName, gomock.Any()).Return(fmt.Errorf("Failed to update"))
+				networkInterfaces.EXPECT().CreateOrUpdateAndWait(gomock.Any(), clusterRG, nicName, nic).Return(fmt.Errorf("Failed to update"))
 			},
 			wantErr: "Failed to update",
 		},
diff --git a/pkg/cluster/deployresources_test.go b/pkg/installer/deployresources_test.go
similarity index 99%
rename from pkg/cluster/deployresources_test.go
rename to pkg/installer/deployresources_test.go
index ba26eac02..0b7224d25 100644
--- a/pkg/cluster/deployresources_test.go
+++ b/pkg/installer/deployresources_test.go
@@ -1,4 +1,4 @@
-package cluster
+package installer
 
 // Copyright (c) Microsoft Corporation.
 // Licensed under the Apache License 2.0.
diff --git a/pkg/util/machine/azureproviderspec_test.go b/pkg/util/machine/azureproviderspec_test.go
new file mode 100644
index 000000000..3a6bbeddd
--- /dev/null
+++ b/pkg/util/machine/azureproviderspec_test.go
@@ -0,0 +1,154 @@
+package machine
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"bytes"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/Azure/go-autorest/autorest/to"
+	"github.com/davecgh/go-spew/spew"
+	machinev1beta1 "github.com/openshift/api/machine/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	kjson "k8s.io/apimachinery/pkg/runtime/serializer/json"
+	"k8s.io/client-go/kubernetes/scheme"
+	azureproviderv1beta1 "sigs.k8s.io/cluster-api-provider-azure/pkg/apis/azureprovider/v1beta1"
+)
+
+type testData struct {
+	name         string
+	machineType  MachineType
+	providerSpec []byte
+	wantErr      string
+}
+
+var machineAzureProviderSpec = &machinev1beta1.AzureMachineProviderSpec{
+	Image: machinev1beta1.Image{
+		Publisher: "azureopenshift",
+		Offer:     "aro4",
+		SKU:       "aro_499",
+		Version:   "499.99.30000610",
+	},
+	Location:             "neversayneverland",
+	NetworkResourceGroup: "networkRG",
+	OSDisk: machinev1beta1.OSDisk{
+		DiskSizeGB: int32(9001),
+		OSType:     "Linux",
+		ManagedDisk: machinev1beta1.ManagedDiskParameters{
+			StorageAccountType: "Premium_LRS",
+		},
+	},
+	PublicIP:           false,
+	PublicLoadBalancer: "",
+	ResourceGroup:      "myRG",
+	VMSize:             "Standard_D8s_v3",
+	Vnet:               "myVnet",
+	Zone:               to.StringPtr("2"),
+}
+
+var azAzureProviderSpec = &azureproviderv1beta1.AzureMachineProviderSpec{
+	Image: azureproviderv1beta1.Image{
+		Publisher: "azureopenshift",
+		Offer:     "aro4",
+		SKU:       "aro_499",
+		Version:   "499.99.30000610",
+	},
+	Location:             "neversayneverland",
+	NetworkResourceGroup: "networkRG",
+	OSDisk: azureproviderv1beta1.OSDisk{
+		DiskSizeGB: int32(9001),
+		OSType:     "Linux",
+		ManagedDisk: azureproviderv1beta1.ManagedDiskParameters{
+			StorageAccountType: "Premium_LRS",
+		},
+	},
+	PublicIP:           false,
+	PublicLoadBalancer: "",
+	ResourceGroup:      "myRG",
+	VMSize:             "Standard_D8s_v3",
+	Vnet:               "myVnet",
+	Zone:               to.StringPtr("2"),
+}
+
+func marshalAzureMachineProviderSpec(t *testing.T, spec kruntime.Object) []byte {
+	serializer := kjson.NewSerializerWithOptions(
+		kjson.DefaultMetaFactory, scheme.Scheme, scheme.Scheme,
+		kjson.SerializerOptions{Yaml: false},
+	)
+
+	json := scheme.Codecs.CodecForVersions(serializer, nil, schema.GroupVersions(scheme.Scheme.PrioritizedVersionsAllGroups()), nil)
+
+	buf := &bytes.Buffer{}
+	err := json.Encode(spec, buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return buf.Bytes()
+}
+
+func TestUnmarshalAzureProviderSpec(t *testing.T) {
+	machineName := "myMachine"
+
+	// Register old type so we can encode the CR to bytes for use only in test
+	scheme.Scheme.AddKnownTypes(azureproviderv1beta1.SchemeGroupVersion, &azureproviderv1beta1.AzureMachineProviderSpec{})
+
+	machineProviderSpecBytes := marshalAzureMachineProviderSpec(t, machineAzureProviderSpec)
+	azProviderSpecBytes := marshalAzureMachineProviderSpec(t, azAzureProviderSpec)
+
+	for _, tt := range []testData{
+		{
+			name:         "valid - machine.openshift.io provider spec",
+			providerSpec: machineProviderSpecBytes,
+		},
+		{
+			name:         "valid - azureproviderconfig.openshift.io provider spec",
+			providerSpec: azProviderSpecBytes,
+		},
+		{
+			name:         "fail - azureproviderconfig.openshift.io invalid json",
+			providerSpec: []byte("azureproviderconfig.openshift.io"),
+			machineType:  MachineSet,
+			wantErr:      fmt.Sprintf("%s %s: failed to unmarshal the \"azureproviderconfig.openshift.io\" provider spec: %q", MachineSet, machineName, "invalid character 'a' looking for beginning of value"),
+		},
+		{
+			name:         "fail - unable to decode type",
+			providerSpec: []byte(`{"apiVersion": "invalid/v1beta1","kind": "AzureMachineProviderSpec"}`),
+			machineType:  MachineSet,
+			wantErr:      `no kind "AzureMachineProviderSpec" is registered for version "invalid/v1beta1" in scheme`,
+		},
+		{
+			name:         "fail - invalid type",
+			providerSpec: []byte(`{"apiVersion": "v1","kind": "Node"}`),
+			machineType:  MachineSet,
+			wantErr:      `failed to read provider spec:`,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			actualProviderSpec, err := UnmarshalAzureProviderSpec(machineName, tt.machineType, tt.providerSpec)
+			if err == nil {
+				if tt.wantErr != "" {
+					t.Error(err)
+				}
+
+				// Ignoring check to support both azureproviderconfig and machine providerSpec APIVersions
+				actualProviderSpec.TypeMeta = metav1.TypeMeta{}
+
+				if !reflect.DeepEqual(actualProviderSpec, machineAzureProviderSpec) {
+					t.Errorf("got '%v' wanted '%v'", spew.Sdump(actualProviderSpec), spew.Sdump(machineAzureProviderSpec))
+				}
+			} else {
+				// Use contains as one test case contains line numbers which may change
+				if !strings.Contains(err.Error(), tt.wantErr) {
+					t.Error(err)
+				}
+			}
+		})
+	}
+}
diff --git a/test/e2e/adminapi_cluster_getlogs.go b/test/e2e/adminapi_cluster_getlogs.go
index 390432005..fdff21402 100644
--- a/test/e2e/adminapi_cluster_getlogs.go
+++ b/test/e2e/adminapi_cluster_getlogs.go
@@ -5,6 +5,7 @@ package e2e
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"net/url"
 	"time"
@@ -52,13 +53,21 @@ func testGetPodLogsOK(ctx context.Context, containerName, podName, namespace str
 		err = clients.Kubernetes.CoreV1().Pods(namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{})
 		Expect(err).NotTo(HaveOccurred())
 	}()
-	err = wait.Poll(time.Second, time.Second*60, func() (done bool, err error) {
+	err = wait.PollInfinite(time.Second*5, func() (done bool, err error) {
 		pod, err = clients.Kubernetes.CoreV1().Pods(namespace).Get(ctx, pod.Name, metav1.GetOptions{})
 		Expect(err).NotTo(HaveOccurred())
-		if pod.Status.Phase == corev1.PodSucceeded {
+
+		switch pod.Status.Phase {
+		case corev1.PodSucceeded:
 			return true, nil
+		case corev1.PodPending:
+			if pod.CreationTimestamp.Time.Add(5*time.Minute).Unix() < time.Now().Unix() {
+				return false, errors.New("pod was pending for more than 5min")
+			}
+			return false, nil
+		case corev1.PodFailed:
+			return true, errors.New(pod.Status.Message)
 		}
-
 		return false, nil
 	})
 	Expect(err).NotTo(HaveOccurred())