Repository: Azure/ARO-RP
Pull request #3590 (2024-05-21 20:09:14)

Retry failed ARM template deployments during cluster installation when FPSP is missing roleassignments
### Which issue this PR addresses:

Fixes [ARO-7598](https://issues.redhat.com/browse/ARO-7598)

### What this PR does / why we need it:

Handle ARM deployment preflight errors containing `"Authorization failed for template resource"` as authorization errors within our `AuthorizationRetryingAction` step. 

These errors occur in local development environments, where we fake the behavior of the first-party service principal automatically being granted Owner permissions over a cluster's managed resource group, by explicitly and manually granting this role assignment after MRG creation. This role assignment can take time to propagate, and may not be recognized by the time we apply ARM templates to the managed resource group.

ARM returns an InvalidTemplateDeployment error (HTTP 400), which was not previously considered to be an authorization error and we would thus fail the install when this occurred. 

### Test plan for issue:

- [X] Unit tests were added for the error matching functionality
- [x] E2E tests (incl. cluster creation) continue to pass
- [x] Manual test of the new functionality in a full-service RP dev environment confirmed to be experiencing the issue

### Is there any documentation that needs to be updated for this PR?

No

### How do you know this will function as expected in production? 

We have not observed this specific error in production, so functionality should stay the same. 


Patch:
diff --git a/.sha256sum b/.sha256sum
index 8fae07dc7..cca696c9d 100644
--- a/.sha256sum
+++ b/.sha256sum
@@ -6,4 +6,4 @@ b1f1de0fe40d05de90742b17928968923b936adc294000f58974f50a297581dd  swagger/redhat
 c023515341196746454c0ae7af077d40d3ec13f6b88b33cb558f0a7ab17a5a24  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2023-07-01-preview/redhatopenshift.json
 440748951dd1c3b34b5ccbdcb7cd966e3b89490887a1f1d64429561fad789515  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/stable/2023-09-04/redhatopenshift.json
 74a46fdde6ceb0121fe1515c7e11e902dd921b54cffe693307fb02b3dc88f26e  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/stable/2023-11-22/redhatopenshift.json
-a27184734436629e24b344c3b5c015437f144e18e7eddce7e252a1ed4cda7bca  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
+f1e9d42d7c4c0081282e065e7845455db28ed6924687f1acecafb5fbc43ae0c3  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
diff --git a/Dockerfile.ci-portal b/Dockerfile.ci-portal
deleted file mode 100644
index d54d1af07..000000000
--- a/Dockerfile.ci-portal
+++ /dev/null
@@ -1,13 +0,0 @@
-ARG REGISTRY
-
-FROM ${REGISTRY}/ubi8/nodejs-16 AS portal-base
-WORKDIR /build/portal/v2
-USER root
-
-FROM portal-base AS portal-build
-COPY /portal/v2/package*.json ./
-RUN npm ci
-RUN npm audit --omit=dev
-COPY /portal/v2/ ./
-RUN npm run lint
-RUN npm run build
\ No newline at end of file
diff --git a/Dockerfile.ci-rp b/Dockerfile.ci-rp
index 4750d88bb..0c8cfe587 100644
--- a/Dockerfile.ci-rp
+++ b/Dockerfile.ci-rp
@@ -1,21 +1,40 @@
 ARG REGISTRY
-ARG VERSION
+ARG ARO_VERSION
 
 ###############################################################################
+# Stage 1: Build the SRE Portal Assets
 # builder is responsible for all compilation and validation of the RP
 ###############################################################################
-FROM ${REGISTRY}/ubi8/go-toolset:1.20.12-5 as builder
+FROM ${REGISTRY}/ubi8/nodejs-16  as portal-build
+WORKDIR /build/portal/v2
+USER root
+
+# Copying package files and installing dependencies
+COPY portal/v2/package*.json ./
+RUN npm ci
+RUN npm audit --omit=dev # Run audit without dev dependencies
+
+# Copying the rest of the source and build
+COPY --chown=root:root portal/v2/ ./
+RUN npm run lint && npm run build
+
+###############################################################################
+# Stage 2: Compile the Golang RP code
+###############################################################################
+FROM ${REGISTRY}/ubi8/go-toolset:1.20.12-5 AS builder
+ARG ARO_VERSION
 USER root
 WORKDIR /app
 
-# golang config
+# golang config and build steps
 ENV GOPATH=/root/go
 ENV GOFLAGS="-tags=containers_image_openpgp,exclude_graphdriver_btrfs,exclude_graphdriver_devicemapper"
 
-# install general build tools
-RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b /usr/local/bin v1.56.2
+# Install golangci-lint and verify
+RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b /usr/local/bin v1.56.2 && \
+    golangci-lint --version || (echo "golangci-lint not found" && exit 1)
 
-# copy from least changed -> most changed to improve cache hits
+# Copy dependencies and source files
 COPY go.mod go.sum ./
 COPY vendor vendor
 COPY swagger swagger
@@ -25,20 +44,21 @@ COPY cmd cmd
 COPY pkg pkg
 COPY test test
 
-# lint
-RUN golangci-lint run --verbose
+# Ensure JS assets are available before generating Go code
+COPY --from=portal-build /build/pkg/portal/assets/v2/build /app/pkg/portal/assets/v2/build
 
-# build
+# Lint, generate, build, and test
+RUN golangci-lint run --verbose
 RUN go generate ./...
-RUN go build -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${VERSION}" ./cmd/aro
-RUN go test ./test/e2e/... -tags e2e,codec.safe -c -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${VERSION}" -o e2e.test
+RUN go build -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${ARO_VERSION}" ./cmd/aro
+RUN go test ./test/e2e/... -tags e2e,codec.safe -c -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${ARO_VERSION}" -o e2e.test
 
-# test
+# Additional tests
 RUN ARO_RUN_PKI_TESTS=nope go run gotest.tools/gotestsum@v1.11.0 --format pkgname --junitfile report.xml -- -coverprofile=cover.out ./...
 RUN hack/fips/validate-fips.sh ./aro
 
 ###############################################################################
-# final is our slim image with minimal layers and tools
+# Stage 3: final is our slim image with minimal layers and tools
 ###############################################################################
 FROM ${REGISTRY}/ubi8/ubi-minimal AS final
 RUN microdnf update && microdnf clean all
diff --git a/Dockerfile.ci-rp.dockerignore b/Dockerfile.ci-rp.dockerignore
index a984a1402..052c75c86 100644
--- a/Dockerfile.ci-rp.dockerignore
+++ b/Dockerfile.ci-rp.dockerignore
@@ -10,4 +10,17 @@
 !pkg
 !swagger
 !test
-!vendor
\ No newline at end of file
+!vendor
+
+# Ignore locally built NPM artifacts - using new automated process
+pkg/portal/assets/v2/build
+
+# Allow files and directories for JavaScript build
+!portal/v2/package*.json
+!portal/v2/src
+!portal/v2/public
+!portal/v2/config-overrides.js
+!portal/v2/tsconfig.json
+!portal/v2/.eslintrc.yml
+!portal/v2/.npmrc
+
diff --git a/Dockerfile.proxy b/Dockerfile.proxy
index 87643f947..006172cc3 100644
--- a/Dockerfile.proxy
+++ b/Dockerfile.proxy
@@ -7,11 +7,13 @@ ENV GOPATH=/root/go
 RUN mkdir -p /app
 WORKDIR /app
 
+COPY . /app
 RUN make proxy
 
 FROM ${REGISTRY}/ubi8/ubi-minimal
 RUN microdnf update && microdnf clean all
-COPY --from=builder /go/src/github.com/Azure/ARO-RP/proxy /usr/local/bin/
+COPY --from=builder /app/proxy /usr/local/bin/
 ENTRYPOINT ["proxy"]
 EXPOSE 8443/tcp
+EXPOSE 8080/tcp
 USER 1000
diff --git a/Makefile b/Makefile
index a0953075d..d263c3e8d 100644
--- a/Makefile
+++ b/Makefile
@@ -77,10 +77,7 @@ client: generate
 	hack/build-client.sh "${AUTOREST_IMAGE}" 2020-04-30 2021-09-01-preview 2022-04-01 2022-09-04 2023-04-01 2023-07-01-preview 2023-09-04 2023-11-22 2024-08-12-preview
 
 ci-rp: fix-macos-vendor
-	docker build . -f Dockerfile.ci-rp --ulimit=nofile=4096:4096 --build-arg REGISTRY=$(REGISTRY) --build-arg VERSION=$(VERSION) --no-cache=$(NO_CACHE)
-
-ci-portal:
-	docker build . -f Dockerfile.ci-portal --build-arg REGISTRY=$(REGISTRY) --no-cache=$(NO_CACHE)
+	docker build . -f Dockerfile.ci-rp --ulimit=nofile=4096:4096 --build-arg REGISTRY=$(REGISTRY) --build-arg ARO_VERSION=$(VERSION) --no-cache=$(NO_CACHE)
 
 # TODO: hard coding dev-config.yaml is clunky; it is also probably convenient to
 # override COMMIT.
@@ -123,7 +120,6 @@ generate-kiota:
 
 init-contrib:
 	install -v hack/git/hooks/* .git/hooks/
-	
 
 image-aro-multistage:
 	docker build --platform=linux/amd64 --network=host --no-cache -f Dockerfile.aro-multistage -t $(ARO_IMAGE) --build-arg REGISTRY=$(REGISTRY) .
diff --git a/docs/prepare-your-dev-environment.md b/docs/prepare-your-dev-environment.md
index 159acc766..f5b67a9e8 100644
--- a/docs/prepare-your-dev-environment.md
+++ b/docs/prepare-your-dev-environment.md
@@ -34,7 +34,7 @@ This document goes through the development dependencies one requires in order to
         If you're using podman-machine, you will need to export the socket, for example::
 
             ```bash
-            export ARO_PODMAN_SOCKET=unix:///$HOME/.local/share/containers/podman/machine/qemu/podman.sock
+            export ARO_PODMAN_SOCKET=unix://$HOME/.local/share/containers/podman/machine/qemu/podman.sock
             ```
         
         You will also need to ensure that podman machine has enough resources::
diff --git a/pkg/api/admin/openshiftcluster.go b/pkg/api/admin/openshiftcluster.go
index d1bab4bb6..ed328772e 100644
--- a/pkg/api/admin/openshiftcluster.go
+++ b/pkg/api/admin/openshiftcluster.go
@@ -418,9 +418,13 @@ type IngressProfile struct {
 
 // PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.
 type PlatformWorkloadIdentityProfile struct {
+	UpgradeableTo              *UpgradeableTo             `json:"upgradeableTo,omitempty"`
 	PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"`
 }
 
+// UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to
+type UpgradeableTo string
+
 // PlatformWorkloadIdentity stores information representing a single workload identity.
 type PlatformWorkloadIdentity struct {
 	OperatorName string `json:"operatorName,omitempty"`
diff --git a/pkg/api/admin/openshiftcluster_convert.go b/pkg/api/admin/openshiftcluster_convert.go
index b0a563bcf..e91a2a46c 100644
--- a/pkg/api/admin/openshiftcluster_convert.go
+++ b/pkg/api/admin/openshiftcluster_convert.go
@@ -305,6 +305,7 @@ func (c openShiftClusterConverter) ToInternal(_oc interface{}, out *api.OpenShif
 	out.Properties.NetworkProfile.APIServerPrivateEndpointIP = oc.Properties.NetworkProfile.APIServerPrivateEndpointIP
 	out.Properties.NetworkProfile.GatewayPrivateEndpointIP = oc.Properties.NetworkProfile.GatewayPrivateEndpointIP
 	out.Properties.NetworkProfile.GatewayPrivateLinkID = oc.Properties.NetworkProfile.GatewayPrivateLinkID
+	out.Properties.NetworkProfile.PreconfiguredNSG = api.PreconfiguredNSG(oc.Properties.NetworkProfile.PreconfiguredNSG)
 	if oc.Properties.NetworkProfile.LoadBalancerProfile != nil {
 		loadBalancerProfile := api.LoadBalancerProfile{}
 
diff --git a/pkg/api/openshiftcluster.go b/pkg/api/openshiftcluster.go
index 0826520d0..caa157d71 100644
--- a/pkg/api/openshiftcluster.go
+++ b/pkg/api/openshiftcluster.go
@@ -773,9 +773,13 @@ type HiveProfile struct {
 type PlatformWorkloadIdentityProfile struct {
 	MissingFields
 
+	UpgradeableTo              *UpgradeableTo             `json:"upgradeableTo,omitempty"`
 	PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"`
 }
 
+// UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to
+type UpgradeableTo string
+
 // PlatformWorkloadIdentity stores information representing a single workload identity.
 type PlatformWorkloadIdentity struct {
 	MissingFields
@@ -803,4 +807,5 @@ type Identity struct {
 
 	Type                   string                 `json:"type,omitempty"`
 	UserAssignedIdentities UserAssignedIdentities `json:"userAssignedIdentities,omitempty"`
+	IdentityURL            string                 `json:"identityURL,omitempty" mutable:"true"`
 }
diff --git a/pkg/api/v20230904/openshiftcluster_convert.go b/pkg/api/v20230904/openshiftcluster_convert.go
index fd8069d55..5bfbf0bbd 100644
--- a/pkg/api/v20230904/openshiftcluster_convert.go
+++ b/pkg/api/v20230904/openshiftcluster_convert.go
@@ -169,6 +169,7 @@ func (c openShiftClusterConverter) ToInternal(_oc interface{}, out *api.OpenShif
 	out.Properties.NetworkProfile.PodCIDR = oc.Properties.NetworkProfile.PodCIDR
 	out.Properties.NetworkProfile.ServiceCIDR = oc.Properties.NetworkProfile.ServiceCIDR
 	out.Properties.NetworkProfile.OutboundType = api.OutboundType(oc.Properties.NetworkProfile.OutboundType)
+	out.Properties.NetworkProfile.PreconfiguredNSG = api.PreconfiguredNSG(oc.Properties.NetworkProfile.PreconfiguredNSG)
 	out.Properties.MasterProfile.VMSize = api.VMSize(oc.Properties.MasterProfile.VMSize)
 	out.Properties.MasterProfile.SubnetID = oc.Properties.MasterProfile.SubnetID
 	out.Properties.MasterProfile.EncryptionAtHost = api.EncryptionAtHost(oc.Properties.MasterProfile.EncryptionAtHost)
diff --git a/pkg/api/v20231122/openshiftcluster_convert.go b/pkg/api/v20231122/openshiftcluster_convert.go
index 53c89d1da..59a2037c2 100644
--- a/pkg/api/v20231122/openshiftcluster_convert.go
+++ b/pkg/api/v20231122/openshiftcluster_convert.go
@@ -190,6 +190,7 @@ func (c openShiftClusterConverter) ToInternal(_oc interface{}, out *api.OpenShif
 	out.Properties.NetworkProfile.PodCIDR = oc.Properties.NetworkProfile.PodCIDR
 	out.Properties.NetworkProfile.ServiceCIDR = oc.Properties.NetworkProfile.ServiceCIDR
 	out.Properties.NetworkProfile.OutboundType = api.OutboundType(oc.Properties.NetworkProfile.OutboundType)
+	out.Properties.NetworkProfile.PreconfiguredNSG = api.PreconfiguredNSG(oc.Properties.NetworkProfile.PreconfiguredNSG)
 
 	if oc.Properties.NetworkProfile.LoadBalancerProfile != nil {
 		loadBalancerProfile := api.LoadBalancerProfile{}
diff --git a/pkg/api/v20240812preview/openshiftcluster.go b/pkg/api/v20240812preview/openshiftcluster.go
index 09a5a632d..28e7d4bd1 100644
--- a/pkg/api/v20240812preview/openshiftcluster.go
+++ b/pkg/api/v20240812preview/openshiftcluster.go
@@ -290,15 +290,19 @@ type IngressProfile struct {
 
 // PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.
 type PlatformWorkloadIdentityProfile struct {
-	PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"`
+	UpgradeableTo              *UpgradeableTo             `json:"upgradeableTo,omitempty" mutable:"true"`
+	PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty" mutable:"true"`
 }
 
+// UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to
+type UpgradeableTo string
+
 // PlatformWorkloadIdentity stores information representing a single workload identity.
 type PlatformWorkloadIdentity struct {
-	OperatorName string `json:"operatorName,omitempty"`
-	ResourceID   string `json:"resourceId,omitempty"`
-	ClientID     string `json:"clientId,omitempty" swagger:"readOnly"`
-	ObjectID     string `json:"objectId,omitempty" swagger:"readOnly"`
+	OperatorName string `json:"operatorName,omitempty" mutable:"true"`
+	ResourceID   string `json:"resourceId,omitempty" mutable:"true"`
+	ClientID     string `json:"clientId,omitempty" swagger:"readOnly" mutable:"true"`
+	ObjectID     string `json:"objectId,omitempty" swagger:"readOnly" mutable:"true"`
 }
 
 // ClusterUserAssignedIdentity stores information about a user-assigned managed identity in a predefined format required by Microsoft's Managed Identity team.
diff --git a/pkg/api/v20240812preview/openshiftcluster_convert.go b/pkg/api/v20240812preview/openshiftcluster_convert.go
index ad51f4406..80d0793b6 100644
--- a/pkg/api/v20240812preview/openshiftcluster_convert.go
+++ b/pkg/api/v20240812preview/openshiftcluster_convert.go
@@ -141,6 +141,12 @@ func (c openShiftClusterConverter) ToExternal(oc *api.OpenShiftCluster) interfac
 
 	if oc.Properties.PlatformWorkloadIdentityProfile != nil && oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities != nil {
 		out.Properties.PlatformWorkloadIdentityProfile = &PlatformWorkloadIdentityProfile{}
+
+		if oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo != nil {
+			temp := UpgradeableTo(*oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo)
+			out.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo = &temp
+		}
+
 		out.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities = make([]PlatformWorkloadIdentity, len(oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities))
 
 		for i := range oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities {
@@ -225,6 +231,12 @@ func (c openShiftClusterConverter) ToInternal(_oc interface{}, out *api.OpenShif
 	}
 	if oc.Properties.PlatformWorkloadIdentityProfile != nil && oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities != nil {
 		out.Properties.PlatformWorkloadIdentityProfile = &api.PlatformWorkloadIdentityProfile{}
+
+		if oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo != nil {
+			temp := api.UpgradeableTo(*oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo)
+			out.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo = &temp
+		}
+
 		out.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities = make([]api.PlatformWorkloadIdentity, len(oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities))
 
 		for i := range oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities {
@@ -238,6 +250,7 @@ func (c openShiftClusterConverter) ToInternal(_oc interface{}, out *api.OpenShif
 	out.Properties.NetworkProfile.PodCIDR = oc.Properties.NetworkProfile.PodCIDR
 	out.Properties.NetworkProfile.ServiceCIDR = oc.Properties.NetworkProfile.ServiceCIDR
 	out.Properties.NetworkProfile.OutboundType = api.OutboundType(oc.Properties.NetworkProfile.OutboundType)
+	out.Properties.NetworkProfile.PreconfiguredNSG = api.PreconfiguredNSG(oc.Properties.NetworkProfile.PreconfiguredNSG)
 
 	if oc.Properties.NetworkProfile.LoadBalancerProfile != nil {
 		loadBalancerProfile := api.LoadBalancerProfile{}
diff --git a/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go b/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go
index e438d5fc1..2535b27f7 100644
--- a/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go
+++ b/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go
@@ -1525,6 +1525,7 @@ func (pwi PlatformWorkloadIdentity) MarshalJSON() ([]byte, error) {
 // PlatformWorkloadIdentityProfile platformWorkloadIdentityProfile encapsulates all information that is
 // specific to workload identity clusters.
 type PlatformWorkloadIdentityProfile struct {
+	UpgradeableTo              *string                     `json:"upgradeableTo,omitempty"`
 	PlatformWorkloadIdentities *[]PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"`
 }
 
diff --git a/pkg/cluster/arooperator.go b/pkg/cluster/arooperator.go
index 7afbb5b3c..ca187e23f 100644
--- a/pkg/cluster/arooperator.go
+++ b/pkg/cluster/arooperator.go
@@ -65,6 +65,10 @@ func (m *manager) ensureCredentialsRequest(ctx context.Context) error {
 	return m.aroOperatorDeployer.CreateOrUpdateCredentialsRequest(ctx)
 }
 
+func (m *manager) ensureUpgradeAnnotation(ctx context.Context) error {
+	return m.aroOperatorDeployer.EnsureUpgradeAnnotation(ctx)
+}
+
 func (m *manager) renewMDSDCertificate(ctx context.Context) error {
 	return m.aroOperatorDeployer.RenewMDSDCertificate(ctx)
 }
diff --git a/pkg/cluster/install.go b/pkg/cluster/install.go
index 5e14d57ef..072b9cb29 100644
--- a/pkg/cluster/install.go
+++ b/pkg/cluster/install.go
@@ -213,6 +213,7 @@ func (m *manager) Update(ctx context.Context) error {
 		steps.Action(m.updateAROSecret),
 		steps.Action(m.restartAROOperatorMaster), // depends on m.updateOpenShiftSecret; the point of restarting is to pick up any changes made to the secret
 		steps.Condition(m.aroDeploymentReady, 5*time.Minute, true),
+		steps.Action(m.ensureUpgradeAnnotation),
 		steps.Action(m.reconcileLoadBalancerProfile),
 	}
 
@@ -524,7 +525,7 @@ func (m *manager) initializeKubernetesClients(ctx context.Context) error {
 // initializeKubernetesClients initializes clients which are used
 // once the cluster is up later on in the install process.
 func (m *manager) initializeOperatorDeployer(ctx context.Context) (err error) {
-	m.aroOperatorDeployer, err = deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.client, m.extensionscli, m.kubernetescli)
+	m.aroOperatorDeployer, err = deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.client, m.extensionscli, m.kubernetescli, m.operatorcli)
 	return
 }
 
diff --git a/pkg/deploy/assets/env-development.json b/pkg/deploy/assets/env-development.json
index 4328b7592..1c738e4fc 100644
--- a/pkg/deploy/assets/env-development.json
+++ b/pkg/deploy/assets/env-development.json
@@ -143,6 +143,62 @@
                 "[resourceId('Microsoft.Network/virtualNetworks', 'dev-vpn-vnet')]"
             ]
         },
+        {
+            "sku": {
+                "name": "Basic"
+            },
+            "properties": {
+                "frontendIPConfigurations": [
+                    {
+                        "properties": {
+                            "subnet": {
+                                "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'rp-vnet', 'rp-subnet')]"
+                            }
+                        },
+                        "name": "not-used"
+                    }
+                ],
+                "backendAddressPools": [
+                    {
+                        "name": "dev-backend"
+                    }
+                ],
+                "loadBalancingRules": [
+                    {
+                        "properties": {
+                            "frontendIPConfiguration": {
+                                "id": "[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', 'dev-lb-internal', 'not-used')]"
+                            },
+                            "backendAddressPool": {
+                                "id": "[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'dev-lb-internal', 'dev-backend')]"
+                            },
+                            "probe": {
+                                "id": "[resourceId('Microsoft.Network/loadBalancers/probes', 'dev-lb-internal', 'dev-probe')]"
+                            },
+                            "protocol": "Tcp",
+                            "loadDistribution": "Default",
+                            "frontendPort": 443,
+                            "backendPort": 443
+                        },
+                        "name": "dev-lbrule"
+                    }
+                ],
+                "probes": [
+                    {
+                        "properties": {
+                            "protocol": "Tcp",
+                            "port": 443,
+                            "numberOfProbes": 3
+                        },
+                        "name": "dev-probe"
+                    }
+                ]
+            },
+            "name": "dev-lb-internal",
+            "type": "Microsoft.Network/loadBalancers",
+            "location": "[resourceGroup().location]",
+            "apiVersion": "2020-08-01"
+        },
         {
             "name": "[concat(take(resourceGroup().name,10), '-dev-sharedKV')]",
             "type": "Microsoft.KeyVault/vaults",
@@ -244,9 +300,9 @@
                     },
                     "storageProfile": {
                         "imageReference": {
-                            "publisher": "RedHat",
-                            "offer": "RHEL",
-                            "sku": "8-LVM",
+                            "publisher": "MicrosoftCBLMariner",
+                            "offer": "cbl-mariner",
+                            "sku": "cbl-mariner-2-gen2",
                             "version": "latest"
                         },
                         "osDisk": {
@@ -257,6 +313,9 @@
                         }
                     },
                     "networkProfile": {
+                        "healthProbe": {
+                            "id": "[resourceId('Microsoft.Network/loadBalancers/probes', 'dev-lb-internal', 'dev-probe')]"
+                        },
                         "networkInterfaceConfigurations": [
                             {
                                 "name": "dev-proxy-vmss-nic",
@@ -277,7 +336,12 @@
                                                             "domainNameLabel": "[parameters('proxyDomainNameLabel')]"
                                                         }
                                                     }
-                                                }
+                                                },
+                                                "loadBalancerBackendAddressPools": [
+                                                    {
+                                                        "id": "[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'dev-lb-internal', 'dev-backend')]"
+                                                    }
+                                                ]
                                             }
                                         }
                                     ]
@@ -296,7 +360,7 @@
                                     "autoUpgradeMinorVersion": true,
                                     "settings": {},
                                     "protectedSettings": {
-                                        "script": "[base64(concat(base64ToString('c2V0IC1leAoK'),'PROXYIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImage')),''')\n','PROXYIMAGEAUTH=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImageAuth')),''')\n','PROXYCERT=''',parameters('proxyCert'),'''\n','PROXYCLIENTCERT=''',parameters('proxyClientCert'),'''\n','PROXYKEY=''',parameters('proxyKey'),'''\n','\n',base64ToString('I0FkZGluZyByZXRyeSBsb2dpYyB0byB5dW0gY29tbWFuZHMgaW4gb3JkZXIgdG8gYXZvaWQgc3RhbGxpbmcgb3V0IG9uIHJlc291cmNlIGxvY2tzCmVjaG8gInJ1bm5pbmcgUkhVSSBmaXgiCmZvciBhdHRlbXB0IGluIHsxLi42MH07IGRvCiAgeXVtIHVwZGF0ZSAteSAtLWRpc2FibGVyZXBvPScqJyAtLWVuYWJsZXJlcG89J3JodWktbWljcm9zb2Z0LWF6dXJlKicgJiYgYnJlYWsKICBpZiBbWyAke2F0dGVtcHR9IC1sdCA2MCBdXTsgdGhlbiBzbGVlcCAzMDsgZWxzZSBleGl0IDE7IGZpCmRvbmUKCmVjaG8gInJ1bm5pbmcgeXVtIHVwZGF0ZSIKZm9yIGF0dGVtcHQgaW4gezEuLjYwfTsgZG8KICB5dW0gLXkgLXggV0FMaW51eEFnZW50IC14IFdBTGludXhBZ2VudC11ZGV2IHVwZGF0ZSAtLWFsbG93ZXJhc2luZyAmJiBicmVhawogIGlmIFtbICR7YXR0ZW1wdH0gLWx0IDYwIF1dOyB0aGVuIHNsZWVwIDMwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKZWNobyAiaW5zdGFsbGluZyBwb2RtYW4tZG9ja2VyIgpmb3IgYXR0ZW1wdCBpbiB7MS4uNjB9OyBkbwogIHl1bSAteSBpbnN0YWxsIHBvZG1hbi1kb2NrZXIgJiYgYnJlYWsKICBpZiBbWyAke2F0dGVtcHR9IC1sdCA2MCBdXTsgdGhlbiBzbGVlcCAzMDsgZWxzZSBleGl0IDE7IGZpCmRvbmUKCmZpcmV3YWxsLWNtZCAtLWFkZC1wb3J0PTQ0My90Y3AgLS1wZXJtYW5lbnQKCm1rZGlyIC9yb290Ly5kb2NrZXIKY2F0ID4vcm9vdC8uZG9ja2VyL2NvbmZpZy5qc29uIDw8RU9GCnsKCSJhdXRocyI6IHsKCQkiJHtQUk9YWUlNQUdFJSUvKn0iOiB7CgkJCSJhdXRoIjogIiRQUk9YWUlNQUdFQVVUSCIKCQl9Cgl9Cn0KRU9GCgpta2RpciAtcCAvZXRjL2NvbnRhaW5lcnMvCnRvdWNoIC9ldGMvY29udGFpbmVycy9ub2RvY2tlcgoKZG9ja2VyIHB1bGwgIiRQUk9YWUlNQUdFIgoKbWtkaXIgL2V0Yy9wcm94eQpiYXNlNjQgLWQgPDw8IiRQUk9YWUNFUlQiID4vZXRjL3Byb3h5L3Byb3h5LmNydApiYXNlNjQgLWQgPDw8IiRQUk9YWUtFWSIgPi9ldGMvcHJveHkvcHJveHkua2V5CmJhc2U2NCAtZCA8PDwiJFBST1hZQ0xJRU5UQ0VSVCIgPi9ldGMvcHJveHkvcHJveHktY2xpZW50LmNydApjaG93biAtUiAxMDAwOjEwMDAgL2V0Yy9wcm94eQpjaG1vZCAwNjAwIC9ldGMvcHJveHkvcHJveHkua2V5CgpjYXQgPi9ldGMvc3lzY29uZmlnL3Byb3h5IDw8RU9GClBST1hZX0lNQUdFPSckUFJPWFlJTUFHRScKRU9GCgpjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vcHJveHkuc2VydmljZSA8PCdFT0YnCltVbml0XQpBZnRlcj1uZXR3b3JrLW9ubGluZS50YXJnZXQKV2FudHM9bmV0d29yay1vbmxpbmUudGFyZ2V0CgpbU2VydmljZV0KRW52aXJvbm1lbnRGaWxlPS9ldGMvc3lzY29uZmlnL3Byb3h5CkV4ZWNTdGFydFByZT0tL3Vzci9iaW4vZG9ja2VyIHJtIC1mICVuCkV4ZWNTdGFydD0vdXNyL2Jpbi9kb2NrZXIgcnVuIC0tcm0gLS1uYW1lICVuIC1wIDQ0Mzo4NDQzIC12IC9ldGMvcHJveHk6L3NlY3JldHMgJFBST1hZX0lNQUdFCkV4ZWNTdG9wPS91c3IvYmluL2RvY2tlciBzdG9wICVuClJlc3RhcnQ9YWx3YXlzClJlc3RhcnRTZWM9MQpTdGFydExpbWl0SW50ZXJ2YWw9MAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIudGFyZ2V0CkVPRgoKc3lzdGVtY3RsIGVuYWJsZSBwcm94eS5zZXJ2aWNlCgpjYXQgPi9ldGMvY3Jvbi53ZWVrbHkvcHVsbC1pbWFnZSA8PCdFT0YnCiMhL2Jpbi9iYXNoCgpkb2NrZXIgcHVsbCAkUFJPWFlJTUFHRQpzeXN0ZW1jdGwgcmVzdGFydCBwcm94eS5zZXJ2aWNlCkVPRgpjaG1vZCAreCAvZXRjL2Nyb24ud2Vla2x5L3B1bGwtaW1hZ2UKCmNhdCA+L2V0Yy9jcm9uLndlZWtseS95dW11cGRhdGUgPDwnRU9GJwojIS9iaW4vYmFzaAoKeXVtIHVwZGF0ZSAteQpFT0YKY2htb2QgK3ggL2V0Yy9jcm9uLndlZWtseS95dW11cGRhdGUKCmNhdCA+L2V0Yy9jcm9uLmRhaWx5L3Jlc3RhcnQtcHJveHkgPDwnRU9GJwojIS9iaW4vYmFzaAoKc3lzdGVtY3RsIHJlc3RhcnQgcHJveHkuc2VydmljZQpFT0YKY2htb2QgK3ggL2V0Yy9jcm9uLmRhaWx5L3Jlc3RhcnQtcHJveHkKCigKCXNsZWVwIDMwCglyZWJvb3QKKSAmCg==')))]"
+                                        "script": "[base64(concat(base64ToString('c2V0IC1leAoK'),'PROXYIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImage')),''')\n','PROXYIMAGEAUTH=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImageAuth')),''')\n','PROXYCERT=''',parameters('proxyCert'),'''\n','PROXYCLIENTCERT=''',parameters('proxyClientCert'),'''\n','PROXYKEY=''',parameters('proxyKey'),'''\n','\n',base64ToString('I0FkZGluZyByZXRyeSBsb2dpYyB0byB5dW0gY29tbWFuZHMgaW4gb3JkZXIgdG8gYXZvaWQgc3RhbGxpbmcgb3V0IG9uIHJlc291cmNlIGxvY2tzCmVjaG8gImluc3RhbGxpbmcgbW9ieS1lbmdpbmUgKGRvY2tlcikiCmZvciBhdHRlbXB0IGluIHsxLi42MH07IGRvCgl0ZG5mIGluc3RhbGwgLXkgbW9ieS1lbmdpbmUgbW9ieS1jbGkgJiYgYnJlYWsKCWlmIFtbICR7YXR0ZW1wdH0gLWx0IDYwIF1dOyB0aGVuIHNsZWVwIDMwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKc3lzdGVtY3RsIGVuYWJsZSBkb2NrZXIKc3lzdGVtY3RsIHN0YXJ0IGRvY2tlcgoKbWtkaXIgL3Jvb3QvLmRvY2tlcgpjYXQgPi9yb290Ly5kb2NrZXIvY29uZmlnLmpzb24gPDxFT0YKewoJImF1dGhzIjogewoJCSIke1BST1hZSU1BR0UlJS8qfSI6IHsKCQkJImF1dGgiOiAiJFBST1hZSU1BR0VBVVRIIgoJCX0KCX0KfQpFT0YKCmRvY2tlciBwdWxsICIkUFJPWFlJTUFHRSIKCm1rZGlyIC9ldGMvcHJveHkKYmFzZTY0IC1kIDw8PCIkUFJPWFlDRVJUIiA+L2V0Yy9wcm94eS9wcm94eS5jcnQKYmFzZTY0IC1kIDw8PCIkUFJPWFlLRVkiID4vZXRjL3Byb3h5L3Byb3h5LmtleQpiYXNlNjQgLWQgPDw8IiRQUk9YWUNMSUVOVENFUlQiID4vZXRjL3Byb3h5L3Byb3h5LWNsaWVudC5jcnQKY2hvd24gLVIgMTAwMDoxMDAwIC9ldGMvcHJveHkKY2htb2QgMDYwMCAvZXRjL3Byb3h5L3Byb3h5LmtleQoKY2F0ID4vZXRjL3N5c2NvbmZpZy9wcm94eSA8PEVPRgpQUk9YWV9JTUFHRT0nJFBST1hZSU1BR0UnCkVPRgoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL3Byb3h5LnNlcnZpY2UgPDwnRU9GJwpbVW5pdF0KQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CldhbnRzPW5ldHdvcmstb25saW5lLnRhcmdldAoKW1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9wcm94eQpFeGVjU3RhcnRQcmU9LS91c3IvYmluL2RvY2tlciBybSAtZiAlbgpFeGVjU3RhcnQ9L3Vzci9iaW4vZG9ja2VyIHJ1biAtLXJtIC0tbmFtZSAlbiAtcCA0NDM6ODQ0MyAtdiAvZXRjL3Byb3h5Oi9zZWNyZXRzICRQUk9YWV9JTUFHRQpFeGVjU3RvcD0vdXNyL2Jpbi9kb2NrZXIgc3RvcCAlbgpSZXN0YXJ0PWFsd2F5cwpSZXN0YXJ0U2VjPTEKU3RhcnRMaW1pdEludGVydmFsPTAKCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCnN5c3RlbWN0bCBlbmFibGUgcHJveHkuc2VydmljZQoKY2F0ID4vZXRjL2Nyb24ud2Vla2x5L3B1bGwtaW1hZ2UgPDwnRU9GJwojIS9iaW4vYmFzaAoKZG9ja2VyIHB1bGwgJFBST1hZSU1BR0UKc3lzdGVtY3RsIHJlc3RhcnQgcHJveHkuc2VydmljZQpFT0YKY2htb2QgK3ggL2V0Yy9jcm9uLndlZWtseS9wdWxsLWltYWdlCgpjYXQgPi9ldGMvY3Jvbi53ZWVrbHkveXVtdXBkYXRlIDw8J0VPRicKIyEvYmluL2Jhc2gKCnl1bSB1cGRhdGUgLXkKRU9GCmNobW9kICt4IC9ldGMvY3Jvbi53ZWVrbHkveXVtdXBkYXRlCgpjYXQgPi9ldGMvY3Jvbi5kYWlseS9yZXN0YXJ0LXByb3h5IDw8J0VPRicKIyEvYmluL2Jhc2gKCnN5c3RlbWN0bCByZXN0YXJ0IHByb3h5LnNlcnZpY2UKRU9GCmNobW9kICt4IC9ldGMvY3Jvbi5kYWlseS9yZXN0YXJ0LXByb3h5CgooCglzbGVlcCAzMAoJcmVib290CikgJgo=')))]"
                                     },
                                     "provisionAfterExtensions": [
                                         "Microsoft.Azure.Monitor.AzureMonitorLinuxAgent",
@@ -337,10 +401,22 @@
                 },
                 "overprovision": false
             },
+            "identity": {
+                "type": "UserAssigned",
+                "userAssignedIdentities": {
+                    "[resourceId('AzSecPackAutoConfigRG', 'Microsoft.ManagedIdentity/userAssignedIdentities', concat('AzSecPackAutoConfigUA-', resourceGroup().location))]": {}
+                }
+            },
             "name": "dev-proxy-vmss",
             "type": "Microsoft.Compute/virtualMachineScaleSets",
             "location": "[resourceGroup().location]",
-            "apiVersion": "2020-12-01"
+            "tags": {
+                "azsecpack": "nonprod"
+            },
+            "apiVersion": "2020-12-01",
+            "dependsOn": [
+                "[resourceId('Microsoft.Network/loadBalancers', 'dev-lb-internal')]"
+            ]
         },
         {
             "properties": {
diff --git a/pkg/deploy/generator/resources_dev.go b/pkg/deploy/generator/resources_dev.go
index 668b7fee3..a1788a804 100644
--- a/pkg/deploy/generator/resources_dev.go
+++ b/pkg/deploy/generator/resources_dev.go
@@ -17,6 +17,69 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/azureclient"
 )
 
+// devLBInternal is needed for defining a healthprobe.
+// VMSS with auto upgrademode requires a healthprobe from an LB.
+func (g *generator) devLBInternal() *arm.Resource {
+	return &arm.Resource{
+		Resource: &mgmtnetwork.LoadBalancer{
+			Sku: &mgmtnetwork.LoadBalancerSku{
+				Name: mgmtnetwork.LoadBalancerSkuNameBasic,
+			},
+			LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{
+				FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{
+					{
+						FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{
+							Subnet: &mgmtnetwork.Subnet{
+								ID: to.StringPtr("[resourceId('Microsoft.Network/virtualNetworks/subnets', 'rp-vnet', 'rp-subnet')]"),
+							},
+						},
+						Name: to.StringPtr("not-used"),
+					},
+				},
+				BackendAddressPools: &[]mgmtnetwork.BackendAddressPool{
+					{
+						Name: to.StringPtr("dev-backend"),
+					},
+				},
+				LoadBalancingRules: &[]mgmtnetwork.LoadBalancingRule{
+					{
+						LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{
+							FrontendIPConfiguration: &mgmtnetwork.SubResource{
+								ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', 'dev-lb-internal', 'not-used')]"),
+							},
+							BackendAddressPool: &mgmtnetwork.SubResource{
+								ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'dev-lb-internal', 'dev-backend')]"),
+							},
+							Probe: &mgmtnetwork.SubResource{
+								ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/probes', 'dev-lb-internal', 'dev-probe')]"),
+							},
+							Protocol:         mgmtnetwork.TransportProtocolTCP,
+							LoadDistribution: mgmtnetwork.LoadDistributionDefault,
+							FrontendPort:     to.Int32Ptr(443),
+							BackendPort:      to.Int32Ptr(443),
+						},
+						Name: to.StringPtr("dev-lbrule"),
+					},
+				},
+				Probes: &[]mgmtnetwork.Probe{
+					{
+						ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{
+							Protocol:       mgmtnetwork.ProbeProtocolTCP,
+							Port:           to.Int32Ptr(443),
+							NumberOfProbes: to.Int32Ptr(3),
+						},
+						Name: to.StringPtr("dev-probe"),
+					},
+				},
+			},
+			Name:     to.StringPtr("dev-lb-internal"),
+			Type:     to.StringPtr("Microsoft.Network/loadBalancers"),
+			Location: to.StringPtr("[resourceGroup().location]"),
+		},
+		APIVersion: azureclient.APIVersion("Microsoft.Network"),
+	}
+}
+
 func (g *generator) devProxyVMSS() *arm.Resource {
 	parts := []string{
 		fmt.Sprintf("base64ToString('%s')", base64.StdEncoding.EncodeToString([]byte("set -ex\n\n"))),
@@ -51,6 +114,12 @@ func (g *generator) devProxyVMSS() *arm.Resource {
 				Tier:     to.StringPtr("Standard"),
 				Capacity: to.Int64Ptr(1),
 			},
+			Identity: &mgmtcompute.VirtualMachineScaleSetIdentity{
+				Type: mgmtcompute.ResourceIdentityTypeUserAssigned,
+				UserAssignedIdentities: map[string]*mgmtcompute.VirtualMachineScaleSetIdentityUserAssignedIdentitiesValue{
+					"[resourceId('AzSecPackAutoConfigRG', 'Microsoft.ManagedIdentity/userAssignedIdentities', concat('AzSecPackAutoConfigUA-', resourceGroup().location))]": {},
+				},
+			},
 			VirtualMachineScaleSetProperties: &mgmtcompute.VirtualMachineScaleSetProperties{
 				UpgradePolicy: &mgmtcompute.UpgradePolicy{
 					Mode: mgmtcompute.UpgradeModeRolling,
@@ -73,9 +142,9 @@ func (g *generator) devProxyVMSS() *arm.Resource {
 					},
 					StorageProfile: &mgmtcompute.VirtualMachineScaleSetStorageProfile{
 						ImageReference: &mgmtcompute.ImageReference{
-							Publisher: to.StringPtr("RedHat"),
-							Offer:     to.StringPtr("RHEL"),
-							Sku:       to.StringPtr("8-LVM"),
+							Publisher: to.StringPtr("MicrosoftCBLMariner"),
+							Offer:     to.StringPtr("cbl-mariner"),
+							Sku:       to.StringPtr("cbl-mariner-2-gen2"),
 							Version:   to.StringPtr("latest"),
 						},
 						OsDisk: &mgmtcompute.VirtualMachineScaleSetOSDisk{
@@ -86,6 +155,9 @@ func (g *generator) devProxyVMSS() *arm.Resource {
 						},
 					},
 					NetworkProfile: &mgmtcompute.VirtualMachineScaleSetNetworkProfile{
+						HealthProbe: &mgmtcompute.APIEntityReference{
+							ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/probes', 'dev-lb-internal', 'dev-probe')]"),
+						},
 						NetworkInterfaceConfigurations: &[]mgmtcompute.VirtualMachineScaleSetNetworkConfiguration{
 							{
 								Name: to.StringPtr("dev-proxy-vmss-nic"),
@@ -107,6 +179,11 @@ func (g *generator) devProxyVMSS() *arm.Resource {
 														},
 													},
 												},
+												LoadBalancerBackendAddressPools: &[]mgmtcompute.SubResource{
+													{
+														ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'dev-lb-internal', 'dev-backend')]"),
+													},
+												},
 											},
 										},
 									},
@@ -171,6 +248,12 @@ func (g *generator) devProxyVMSS() *arm.Resource {
 			Location: to.StringPtr("[resourceGroup().location]"),
 		},
 		APIVersion: azureclient.APIVersion("Microsoft.Compute"),
+		Tags: map[string]any{
+			"azsecpack": "nonprod",
+		},
+		DependsOn: []string{
+			"[resourceId('Microsoft.Network/loadBalancers', 'dev-lb-internal')]",
+		},
 	}
 }
 
diff --git a/pkg/deploy/generator/scripts/devProxyVMSS.sh b/pkg/deploy/generator/scripts/devProxyVMSS.sh
index cb8e15bf8..f09c9f57f 100644
--- a/pkg/deploy/generator/scripts/devProxyVMSS.sh
+++ b/pkg/deploy/generator/scripts/devProxyVMSS.sh
@@ -1,23 +1,12 @@
 #Adding retry logic to yum commands in order to avoid stalling out on resource locks
-echo "running RHUI fix"
+echo "installing moby-engine (docker)"
 for attempt in {1..60}; do
-  yum update -y --disablerepo='*' --enablerepo='rhui-microsoft-azure*' && break
-  if [[ ${attempt} -lt 60 ]]; then sleep 30; else exit 1; fi
+	tdnf install -y moby-engine moby-cli && break
+	if [[ ${attempt} -lt 60 ]]; then sleep 30; else exit 1; fi
 done
 
-echo "running yum update"
-for attempt in {1..60}; do
-  yum -y -x WALinuxAgent -x WALinuxAgent-udev update --allowerasing && break
-  if [[ ${attempt} -lt 60 ]]; then sleep 30; else exit 1; fi
-done
-
-echo "installing podman-docker"
-for attempt in {1..60}; do
-  yum -y install podman-docker && break
-  if [[ ${attempt} -lt 60 ]]; then sleep 30; else exit 1; fi
-done
-
-firewall-cmd --add-port=443/tcp --permanent
+systemctl enable docker
+systemctl start docker
 
 mkdir /root/.docker
 cat >/root/.docker/config.json <<EOF
@@ -30,9 +19,6 @@ cat >/root/.docker/config.json <<EOF
 }
 EOF
 
-mkdir -p /etc/containers/
-touch /etc/containers/nodocker
-
 docker pull "$PROXYIMAGE"
 
 mkdir /etc/proxy
diff --git a/pkg/deploy/generator/templates_dev.go b/pkg/deploy/generator/templates_dev.go
index 7f3bd4b00..6eca6e892 100644
--- a/pkg/deploy/generator/templates_dev.go
+++ b/pkg/deploy/generator/templates_dev.go
@@ -33,6 +33,7 @@ func (g *generator) devSharedTemplate() *arm.Template {
 		g.devVnet(),
 		g.devVPNVnet(),
 		g.devVPN(),
+		g.devLBInternal(),
 		g.devDiskEncryptionKeyvault(),
 		g.devDiskEncryptionKey(),
 		g.devDiskEncryptionKeyVaultAccessPolicy(),
diff --git a/pkg/frontend/openshiftcluster_putorpatch.go b/pkg/frontend/openshiftcluster_putorpatch.go
index 1fa6f26fd..5c6308978 100644
--- a/pkg/frontend/openshiftcluster_putorpatch.go
+++ b/pkg/frontend/openshiftcluster_putorpatch.go
@@ -25,6 +25,8 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/version"
 )
 
+var errMissingIdentityURL error = fmt.Errorf("identityURL not provided but required for workload identity cluster")
+
 func (f *frontend) putOrPatchOpenShiftCluster(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 	log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry)
@@ -41,10 +43,12 @@ func (f *frontend) putOrPatchOpenShiftCluster(w http.ResponseWriter, r *http.Req
 	subId := chi.URLParam(r, "subscriptionId")
 	resourceProviderNamespace := chi.URLParam(r, "resourceProviderNamespace")
 
+	identityURL := r.Header.Get("x-ms-identity-url")
+
 	apiVersion := r.URL.Query().Get(api.APIVersionKey)
 	err := cosmosdb.RetryOnPreconditionFailed(func() error {
 		var err error
-		b, err = f._putOrPatchOpenShiftCluster(ctx, log, body, correlationData, systemData, r.URL.Path, originalPath, r.Method, referer, &header, f.apis[apiVersion].OpenShiftClusterConverter, f.apis[apiVersion].OpenShiftClusterStaticValidator, subId, resourceProviderNamespace, apiVersion)
+		b, err = f._putOrPatchOpenShiftCluster(ctx, log, body, correlationData, systemData, r.URL.Path, originalPath, r.Method, referer, &header, f.apis[apiVersion].OpenShiftClusterConverter, f.apis[apiVersion].OpenShiftClusterStaticValidator, subId, resourceProviderNamespace, apiVersion, identityURL)
 		return err
 	})
 
@@ -52,7 +56,7 @@ func (f *frontend) putOrPatchOpenShiftCluster(w http.ResponseWriter, r *http.Req
 	reply(log, w, header, b, err)
 }
 
-func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus.Entry, body []byte, correlationData *api.CorrelationData, systemData *api.SystemData, path, originalPath, method, referer string, header *http.Header, converter api.OpenShiftClusterConverter, staticValidator api.OpenShiftClusterStaticValidator, subId, resourceProviderNamespace string, apiVersion string) ([]byte, error) {
+func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus.Entry, body []byte, correlationData *api.CorrelationData, systemData *api.SystemData, path, originalPath, method, referer string, header *http.Header, converter api.OpenShiftClusterConverter, staticValidator api.OpenShiftClusterStaticValidator, subId, resourceProviderNamespace string, apiVersion string, identityURL string) ([]byte, error) {
 	subscription, err := f.validateSubscriptionState(ctx, path, api.SubscriptionStateRegistered)
 	if err != nil {
 		return nil, err
@@ -86,11 +90,17 @@ func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus.
 				},
 			},
 		}
+
 		if !f.env.IsLocalDevelopmentMode() /* not local dev or CI */ {
 			doc.OpenShiftCluster.Properties.FeatureProfile.GatewayEnabled = true
 		}
 	}
 
+	err = validateIdentityUrl(doc.OpenShiftCluster, identityURL, isCreate)
+	if err != nil {
+		return nil, err
+	}
+
 	doc.CorrelationData = correlationData
 
 	err = validateTerminalProvisioningState(doc.OpenShiftCluster.Properties.ProvisioningState)
@@ -288,6 +298,24 @@ func enrichClusterSystemData(doc *api.OpenShiftClusterDocument, systemData *api.
 	}
 }
 
+func validateIdentityUrl(cluster *api.OpenShiftCluster, identityURL string, isCreate bool) error {
+	// Don't persist identity URL in non-wimi clusters
+	if cluster.Properties.ServicePrincipalProfile != nil || cluster.Identity == nil {
+		return nil
+	}
+
+	if identityURL == "" {
+		if isCreate {
+			return errMissingIdentityURL
+		}
+		return nil
+	}
+
+	cluster.Identity.IdentityURL = identityURL
+
+	return nil
+}
+
 func (f *frontend) ValidateNewCluster(ctx context.Context, subscription *api.SubscriptionDocument, cluster *api.OpenShiftCluster, staticValidator api.OpenShiftClusterStaticValidator, ext interface{}, path string) error {
 	err := staticValidator.Static(ext, nil, f.env.Location(), f.env.Domain(), f.env.FeatureIsSet(env.FeatureRequireD2sV3Workers), path)
 	if err != nil {
diff --git a/pkg/operator/deploy/deploy.go b/pkg/operator/deploy/deploy.go
index ab18e2505..1fbc93a7c 100644
--- a/pkg/operator/deploy/deploy.go
+++ b/pkg/operator/deploy/deploy.go
@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/go-multierror"
+	operatorclient "github.com/openshift/client-go/operator/clientset/versioned"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	extensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -56,6 +57,7 @@ type Operator interface {
 	Restart(context.Context, []string) error
 	IsRunningDesiredVersion(context.Context) (bool, error)
 	RenewMDSDCertificate(context.Context) error
+	EnsureUpgradeAnnotation(context.Context) error
 }
 
 type operator struct {
@@ -67,10 +69,11 @@ type operator struct {
 	client        client.Client
 	extensionscli extensionsclient.Interface
 	kubernetescli kubernetes.Interface
+	operatorcli   operatorclient.Interface
 	dh            dynamichelper.Interface
 }
 
-func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli aroclient.Interface, client client.Client, extensionscli extensionsclient.Interface, kubernetescli kubernetes.Interface) (Operator, error) {
+func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli aroclient.Interface, client client.Client, extensionscli extensionsclient.Interface, kubernetescli kubernetes.Interface, operatorcli operatorclient.Interface) (Operator, error) {
 	restConfig, err := restconfig.RestConfig(env, oc)
 	if err != nil {
 		return nil, err
@@ -89,6 +92,7 @@ func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli
 		client:        client,
 		extensionscli: extensionscli,
 		kubernetescli: kubernetescli,
+		operatorcli:   operatorcli,
 		dh:            dh,
 	}, nil
 }
@@ -432,6 +436,34 @@ func (o *operator) RenewMDSDCertificate(ctx context.Context) error {
 	return nil
 }
 
+func (o *operator) EnsureUpgradeAnnotation(ctx context.Context) error {
+	if o.oc.Properties.PlatformWorkloadIdentityProfile == nil ||
+		o.oc.Properties.ServicePrincipalProfile != nil {
+		return nil
+	}
+
+	upgradeableTo := string(*o.oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo)
+	upgradeableAnnotation := "cloudcredential.openshift.io/upgradeable-to"
+
+	cloudcredentialobject, err := o.operatorcli.OperatorV1().CloudCredentials().Get(ctx, "cluster", metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	if cloudcredentialobject.Annotations == nil {
+		cloudcredentialobject.Annotations = map[string]string{}
+	}
+
+	cloudcredentialobject.Annotations[upgradeableAnnotation] = upgradeableTo
+
+	_, err = o.operatorcli.OperatorV1().CloudCredentials().Update(ctx, cloudcredentialobject, metav1.UpdateOptions{})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (o *operator) IsReady(ctx context.Context) (bool, error) {
 	ok, err := ready.CheckDeploymentIsReady(ctx, o.kubernetescli.AppsV1().Deployments(pkgoperator.Namespace), "aro-operator-master")()
 	o.log.Infof("deployment %q ok status is: %v, err is: %v", "aro-operator-master", ok, err)
diff --git a/pkg/proxy/proxy.go b/pkg/proxy/proxy.go
index 746a7d1c9..3bcd1e858 100644
--- a/pkg/proxy/proxy.go
+++ b/pkg/proxy/proxy.go
@@ -7,6 +7,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"errors"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -29,7 +30,16 @@ type Server struct {
 	subnet         *net.IPNet
 }
 
+func health(w http.ResponseWriter, _ *http.Request) {
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprintf(w, "Running.")
+}
+
 func (s *Server) Run() error {
+	healthMux := http.NewServeMux()
+	healthMux.HandleFunc("/", health)
+	go http.ListenAndServe(":8080", healthMux)
+
 	_, subnet, err := net.ParseCIDR(s.Subnet)
 	if err != nil {
 		return err
diff --git a/pkg/util/arm/deploy.go b/pkg/util/arm/deploy.go
index 3bad254fc..6a34d51a6 100644
--- a/pkg/util/arm/deploy.go
+++ b/pkg/util/arm/deploy.go
@@ -34,7 +34,8 @@ func DeployTemplate(ctx context.Context, log *logrus.Entry, deployments features
 	}
 
 	if azureerrors.HasAuthorizationFailedError(err) ||
-		azureerrors.HasLinkedAuthorizationFailedError(err) {
+		azureerrors.HasLinkedAuthorizationFailedError(err) ||
+		azureerrors.IsDeploymentMissingPermissionsError(err) {
 		return err
 	}
 
diff --git a/pkg/util/azureerrors/error.go b/pkg/util/azureerrors/error.go
index 9211ef695..1262558c0 100644
--- a/pkg/util/azureerrors/error.go
+++ b/pkg/util/azureerrors/error.go
@@ -14,6 +14,10 @@ import (
 	"github.com/Azure/ARO-RP/pkg/api"
 )
 
+const (
+	CodeInvalidTemplateDeployment = "InvalidTemplateDeployment"
+)
+
 // HasAuthorizationFailedError returns true it the error is, or contains, an
 // AuthorizationFailed error
 func HasAuthorizationFailedError(err error) bool {
@@ -60,6 +64,22 @@ func deploymentFailedDueToAuthError(err error, authCode string) bool {
 	return false
 }
 
+// IsDeploymentMissingPermissionsError returns true if the error indicates that
+// ARM rejected a template deployment pre-flight due to missing role
+// assignments.
+// This can be an indicator of role assignment propagation delay.
+func IsDeploymentMissingPermissionsError(err error) bool {
+	if detailedErr, ok := err.(autorest.DetailedError); ok {
+		if serviceErr, ok := detailedErr.Original.(*azure.ServiceError); ok {
+			if serviceErr.Code == CodeInvalidTemplateDeployment && strings.Contains(serviceErr.Message, "Authorization failed for template resource") {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
 // IsDeploymentActiveError returns true it the error is a DeploymentActive error
 func IsDeploymentActiveError(err error) bool {
 	if detailedErr, ok := err.(autorest.DetailedError); ok {
diff --git a/pkg/util/mocks/operator/deploy/deploy.go b/pkg/util/mocks/operator/deploy/deploy.go
index 99ad4f224..30f4efc25 100644
--- a/pkg/util/mocks/operator/deploy/deploy.go
+++ b/pkg/util/mocks/operator/deploy/deploy.go
@@ -62,6 +62,20 @@ func (mr *MockOperatorMockRecorder) CreateOrUpdateCredentialsRequest(arg0 interf
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateCredentialsRequest", reflect.TypeOf((*MockOperator)(nil).CreateOrUpdateCredentialsRequest), arg0)
 }
 
+// EnsureUpgradeAnnotation mocks base method.
+func (m *MockOperator) EnsureUpgradeAnnotation(arg0 context.Context) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "EnsureUpgradeAnnotation", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// EnsureUpgradeAnnotation indicates an expected call of EnsureUpgradeAnnotation.
+func (mr *MockOperatorMockRecorder) EnsureUpgradeAnnotation(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureUpgradeAnnotation", reflect.TypeOf((*MockOperator)(nil).EnsureUpgradeAnnotation), arg0)
+}
+
 // IsReady mocks base method.
 func (m *MockOperator) IsReady(arg0 context.Context) (bool, error) {
 	m.ctrl.T.Helper()
diff --git a/pkg/util/steps/refreshing.go b/pkg/util/steps/refreshing.go
index 96fbdf71f..8138ecda6 100644
--- a/pkg/util/steps/refreshing.go
+++ b/pkg/util/steps/refreshing.go
@@ -77,6 +77,7 @@ func (s *authorizationRefreshingActionStep) run(ctx context.Context, log *logrus
 			(azureerrors.IsUnauthorizedClientError(err) ||
 				azureerrors.HasAuthorizationFailedError(err) ||
 				azureerrors.IsInvalidSecretError(err) ||
+				azureerrors.IsDeploymentMissingPermissionsError(err) ||
 				err == ErrWantRefresh) {
 			log.Printf("auth error, refreshing and retrying: %v", err)
 			// Try refreshing auth.
diff --git a/pkg/util/version/const.go b/pkg/util/version/const.go
index e1b96a1b3..01f05a4f1 100644
--- a/pkg/util/version/const.go
+++ b/pkg/util/version/const.go
@@ -34,8 +34,8 @@ type Stream struct {
 // This default is left here ONLY for use by local development mode,
 // until we can come up with a better solution.
 var DefaultInstallStream = Stream{
-	Version:  NewVersion(4, 13, 23),
-	PullSpec: "quay.io/openshift-release-dev/ocp-release@sha256:ca556d3494d08765c90481f15dd965995371168ea7ee7a551000bed4481931c8",
+	Version:  NewVersion(4, 13, 40),
+	PullSpec: "quay.io/openshift-release-dev/ocp-release@sha256:c1f69e6137bc9cda2c6da56bafbc7ea969900acb5e5c349b1ebb2103b10b424f",
 }
 
 // FluentbitImage contains the location of the Fluentbit container image
diff --git a/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models.py b/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models.py
index 3e77e2302..b7e1fab63 100644
--- a/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models.py
+++ b/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models.py
@@ -1303,12 +1303,16 @@ class PlatformWorkloadIdentity(msrest.serialization.Model):
 class PlatformWorkloadIdentityProfile(msrest.serialization.Model):
     """PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.
 
+    :ivar upgradeable_to: UpgradeableTo stores a single OpenShift version a workload identity
+     cluster can be upgraded to.
+    :vartype upgradeable_to: str
     :ivar platform_workload_identities:
     :vartype platform_workload_identities:
      list[~azure.mgmt.redhatopenshift.v2024_08_12_preview.models.PlatformWorkloadIdentity]
     """
 
     _attribute_map = {
+        'upgradeable_to': {'key': 'upgradeableTo', 'type': 'str'},
         'platform_workload_identities': {'key': 'platformWorkloadIdentities', 'type': '[PlatformWorkloadIdentity]'},
     }
 
@@ -1317,11 +1321,15 @@ class PlatformWorkloadIdentityProfile(msrest.serialization.Model):
         **kwargs
     ):
         """
+        :keyword upgradeable_to: UpgradeableTo stores a single OpenShift version a workload identity
+         cluster can be upgraded to.
+        :paramtype upgradeable_to: str
         :keyword platform_workload_identities:
         :paramtype platform_workload_identities:
          list[~azure.mgmt.redhatopenshift.v2024_08_12_preview.models.PlatformWorkloadIdentity]
         """
         super(PlatformWorkloadIdentityProfile, self).__init__(**kwargs)
+        self.upgradeable_to = kwargs.get('upgradeable_to', None)
         self.platform_workload_identities = kwargs.get('platform_workload_identities', None)
 
 
diff --git a/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models_py3.py b/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models_py3.py
index 9828713ff..b8c2d9a1d 100644
--- a/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models_py3.py
+++ b/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models_py3.py
@@ -1412,27 +1412,36 @@ class PlatformWorkloadIdentity(msrest.serialization.Model):
 class PlatformWorkloadIdentityProfile(msrest.serialization.Model):
     """PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.
 
+    :ivar upgradeable_to: UpgradeableTo stores a single OpenShift version a workload identity
+     cluster can be upgraded to.
+    :vartype upgradeable_to: str
     :ivar platform_workload_identities:
     :vartype platform_workload_identities:
      list[~azure.mgmt.redhatopenshift.v2024_08_12_preview.models.PlatformWorkloadIdentity]
     """
 
     _attribute_map = {
+        'upgradeable_to': {'key': 'upgradeableTo', 'type': 'str'},
         'platform_workload_identities': {'key': 'platformWorkloadIdentities', 'type': '[PlatformWorkloadIdentity]'},
     }
 
     def __init__(
         self,
         *,
+        upgradeable_to: Optional[str] = None,
         platform_workload_identities: Optional[List["PlatformWorkloadIdentity"]] = None,
         **kwargs
     ):
         """
+        :keyword upgradeable_to: UpgradeableTo stores a single OpenShift version a workload identity
+         cluster can be upgraded to.
+        :paramtype upgradeable_to: str
         :keyword platform_workload_identities:
         :paramtype platform_workload_identities:
          list[~azure.mgmt.redhatopenshift.v2024_08_12_preview.models.PlatformWorkloadIdentity]
         """
         super(PlatformWorkloadIdentityProfile, self).__init__(**kwargs)
+        self.upgradeable_to = upgradeable_to
         self.platform_workload_identities = platform_workload_identities
 
 
diff --git a/swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json b/swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
index c65beac00..95e0131f5 100644
--- a/swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
+++ b/swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
@@ -2349,6 +2349,9 @@
       "description": "PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.",
       "type": "object",
       "properties": {
+        "upgradeableTo": {
+          "$ref": "#/definitions/UpgradeableTo"
+        },
         "platformWorkloadIdentities": {
           "type": "array",
           "items": {
@@ -2603,6 +2606,10 @@
         "type": "string"
       }
     },
+    "UpgradeableTo": {
+      "description": "UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to",
+      "type": "string"
+    },
     "UserAssignedIdentities": {
       "description": "UserAssignedIdentities stores a mapping from resource IDs of managed identities to their client/principal IDs.",
       "type": "object",


Test patch:
diff --git a/pkg/api/v20240812preview/openshiftcluster_validatestatic.go b/pkg/api/v20240812preview/openshiftcluster_validatestatic.go
index c027b1cd7..b5011fdd2 100644
--- a/pkg/api/v20240812preview/openshiftcluster_validatestatic.go
+++ b/pkg/api/v20240812preview/openshiftcluster_validatestatic.go
@@ -12,6 +12,7 @@ import (
 
 	azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
 	"github.com/Azure/go-autorest/autorest/azure"
+	"github.com/coreos/go-semver/semver"
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/api/util/immutable"
@@ -442,6 +443,14 @@ func (sv openShiftClusterStaticValidator) validatePlatformWorkloadIdentityProfil
 			return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, fmt.Sprintf("%s.PlatformWorkloadIdentities[%d].resourceID", path, n), "Resource must be a user assigned identity.")
 		}
 	}
+
+	if pwip.UpgradeableTo != nil {
+		_, err := semver.NewVersion(string(*pwip.UpgradeableTo))
+		if err != nil {
+			return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, fmt.Sprintf("%s.UpgradeableTo[%v]", path, *pwip.UpgradeableTo), "UpgradeableTo must be a valid OpenShift version in the format 'x.y.z'.")
+		}
+	}
+
 	return nil
 }
 
diff --git a/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go b/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go
index 97e0b802f..884df39ea 100644
--- a/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go
+++ b/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go
@@ -1163,6 +1163,9 @@ func TestOpenShiftClusterStaticValidateDelta(t *testing.T) {
 }
 
 func TestOpenShiftClusterStaticValidatePlatformWorkloadIdentityProfile(t *testing.T) {
+	validUpgradeableToValue := UpgradeableTo("4.14.29")
+	invalidUpgradeableToValue := UpgradeableTo("16.107.invalid")
+
 	createTests := []*validateTest{
 		{
 			name: "valid empty workloadIdentityProfile",
@@ -1323,6 +1326,27 @@ func TestOpenShiftClusterStaticValidatePlatformWorkloadIdentityProfile(t *testin
 			},
 			wantErr: "400: InvalidParameter: properties.servicePrincipalProfile: Must provide either an identity or service principal credentials.",
 		},
+		{
+			name: "valid UpgradeableTo value",
+			modify: func(oc *OpenShiftCluster) {
+				oc.Identity = &Identity{}
+				oc.Properties.ServicePrincipalProfile = nil
+				oc.Properties.PlatformWorkloadIdentityProfile = &PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &validUpgradeableToValue,
+				}
+			},
+		},
+		{
+			name: "invalid UpgradeableTo value",
+			modify: func(oc *OpenShiftCluster) {
+				oc.Identity = &Identity{}
+				oc.Properties.ServicePrincipalProfile = nil
+				oc.Properties.PlatformWorkloadIdentityProfile = &PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &invalidUpgradeableToValue,
+				}
+			},
+			wantErr: `400: InvalidParameter: properties.platformWorkloadIdentityProfile.UpgradeableTo[16.107.invalid]: UpgradeableTo must be a valid OpenShift version in the format 'x.y.z'.`,
+		},
 	}
 
 	runTests(t, testModeCreate, createTests)
diff --git a/pkg/frontend/openshiftcluster_putorpatch_test.go b/pkg/frontend/openshiftcluster_putorpatch_test.go
index f7c10ade9..540616ccf 100644
--- a/pkg/frontend/openshiftcluster_putorpatch_test.go
+++ b/pkg/frontend/openshiftcluster_putorpatch_test.go
@@ -3305,3 +3305,84 @@ func TestEnrichClusterSystemData(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateIdentityUrl(t *testing.T) {
+	for _, tt := range []struct {
+		name        string
+		identityURL string
+		cluster     *api.OpenShiftCluster
+		expected    *api.OpenShiftCluster
+		isCreate    bool
+		wantError   error
+	}{
+		{
+			name:        "identity URL is empty, is not wi/mi cluster create",
+			identityURL: "",
+			cluster:     &api.OpenShiftCluster{},
+			expected:    &api.OpenShiftCluster{},
+			isCreate:    false,
+		},
+		{
+			name:        "identity URL is empty, is wi/mi cluster create",
+			identityURL: "",
+			cluster:     &api.OpenShiftCluster{},
+			expected:    &api.OpenShiftCluster{},
+			isCreate:    true,
+			wantError:   errMissingIdentityURL,
+		},
+		{
+			name:        "cluster is not wi/mi, identityURL passed",
+			identityURL: "http://foo.bar",
+			cluster: &api.OpenShiftCluster{
+				Properties: api.OpenShiftClusterProperties{
+					ServicePrincipalProfile: &api.ServicePrincipalProfile{},
+				},
+			},
+			expected: &api.OpenShiftCluster{
+				Properties: api.OpenShiftClusterProperties{
+					ServicePrincipalProfile: &api.ServicePrincipalProfile{},
+				},
+			},
+			isCreate: true,
+		},
+		{
+			name:        "cluster is not wi/mi, identityURL not passed",
+			identityURL: "",
+			cluster: &api.OpenShiftCluster{
+				Properties: api.OpenShiftClusterProperties{
+					ServicePrincipalProfile: &api.ServicePrincipalProfile{},
+				},
+			},
+			expected: &api.OpenShiftCluster{
+				Properties: api.OpenShiftClusterProperties{
+					ServicePrincipalProfile: &api.ServicePrincipalProfile{},
+				},
+			},
+			isCreate: true,
+		},
+		{
+			name: "pass - identity URL passed on wi/mi cluster",
+			cluster: &api.OpenShiftCluster{
+				Identity: &api.Identity{},
+			},
+			identityURL: "http://foo.bar",
+			expected: &api.OpenShiftCluster{
+				Identity: &api.Identity{
+					IdentityURL: "http://foo.bar",
+				},
+			},
+			isCreate: true,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateIdentityUrl(tt.cluster, tt.identityURL, tt.isCreate)
+			if err != nil && err != tt.wantError {
+				t.Error(cmp.Diff(err, tt.wantError))
+			}
+
+			if !reflect.DeepEqual(tt.cluster, tt.expected) {
+				t.Error(cmp.Diff(tt.cluster, tt.expected))
+			}
+		})
+	}
+}
diff --git a/pkg/operator/deploy/deploy_test.go b/pkg/operator/deploy/deploy_test.go
index 6002e65dd..44f5e257a 100644
--- a/pkg/operator/deploy/deploy_test.go
+++ b/pkg/operator/deploy/deploy_test.go
@@ -10,6 +10,8 @@ import (
 
 	"github.com/golang/mock/gomock"
 	configv1 "github.com/openshift/api/config/v1"
+	operatorv1 "github.com/openshift/api/operator/v1"
+	operatorfake "github.com/openshift/client-go/operator/clientset/versioned/fake"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -461,3 +463,122 @@ func TestCheckPodImageVersion(t *testing.T) {
 		})
 	}
 }
+
+func TestTestEnsureUpgradeAnnotation(t *testing.T) {
+	UpgradeableTo1 := api.UpgradeableTo("4.14.59")
+
+	for _, tt := range []struct {
+		name                 string
+		cluster              api.OpenShiftClusterProperties
+		annotation           map[string]string
+		wantAnnotation       map[string]string
+		wantErr              string
+		cloudCredentialsName string
+	}{
+		{
+			name: "nil PlatformWorkloadIdentityProfile, no version persisted in cluster document",
+		},
+		{
+			name: "non-nil ServicePrincipalProfile, no version persisted in cluster document",
+			cluster: api.OpenShiftClusterProperties{
+				ServicePrincipalProfile: &api.ServicePrincipalProfile{
+					ClientID:     "",
+					ClientSecret: "",
+				},
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{},
+			},
+		},
+		{
+			name: "no version persisted in cluster document, persist it",
+			cluster: api.OpenShiftClusterProperties{
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &UpgradeableTo1,
+				},
+			},
+			annotation: nil,
+			wantAnnotation: map[string]string{
+				"cloudcredential.openshift.io/upgradeable-to": "4.14.59",
+			},
+		},
+		{
+			name: "cloud credential 'cluster' doesn't exist",
+			cluster: api.OpenShiftClusterProperties{
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &UpgradeableTo1,
+				},
+			},
+			cloudCredentialsName: "oh_no",
+			annotation:           nil,
+			wantAnnotation:       nil,
+			wantErr:              `cloudcredentials.operator.openshift.io "cluster" not found`,
+		},
+		{
+			name: "version persisted in cluster document, replace it",
+			cluster: api.OpenShiftClusterProperties{
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &UpgradeableTo1,
+				},
+			},
+			annotation: map[string]string{
+				"cloudcredential.openshift.io/upgradeable-to": "4.14.02",
+			},
+			wantAnnotation: map[string]string{
+				"cloudcredential.openshift.io/upgradeable-to": "4.14.59",
+			},
+		},
+		{
+			name: "annotations exist, append the upgradeable annotation",
+			cluster: api.OpenShiftClusterProperties{
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &UpgradeableTo1,
+				},
+			},
+			annotation: map[string]string{
+				"foo": "bar",
+			},
+			wantAnnotation: map[string]string{
+				"foo": "bar",
+				"cloudcredential.openshift.io/upgradeable-to": "4.14.59",
+			},
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := context.Background()
+			controller := gomock.NewController(t)
+			defer controller.Finish()
+
+			env := mock_env.NewMockInterface(controller)
+
+			oc := &api.OpenShiftCluster{
+				Properties: tt.cluster,
+			}
+
+			if tt.cloudCredentialsName == "" {
+				tt.cloudCredentialsName = "cluster"
+			}
+
+			cloudcredentialobject := &operatorv1.CloudCredential{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        tt.cloudCredentialsName,
+					Annotations: tt.annotation,
+				},
+			}
+
+			o := operator{
+				oc:          oc,
+				env:         env,
+				operatorcli: operatorfake.NewSimpleClientset(cloudcredentialobject),
+			}
+
+			err := o.EnsureUpgradeAnnotation(ctx)
+			utilerror.AssertErrorMessage(t, err, tt.wantErr)
+			result, _ := o.operatorcli.OperatorV1().CloudCredentials().List(ctx, metav1.ListOptions{})
+			for _, v := range result.Items {
+				actualAnnotations := v.ObjectMeta.Annotations
+				if !reflect.DeepEqual(actualAnnotations, tt.wantAnnotation) {
+					t.Errorf("actual annotation: %v, wanted %v", tt.annotation, tt.wantAnnotation)
+				}
+			}
+		})
+	}
+}
diff --git a/pkg/util/azureerrors/error_test.go b/pkg/util/azureerrors/error_test.go
index 77c9c333e..ee925060d 100644
--- a/pkg/util/azureerrors/error_test.go
+++ b/pkg/util/azureerrors/error_test.go
@@ -120,3 +120,36 @@ func TestIsDeploymentActiveError(t *testing.T) {
 		})
 	}
 }
+
+func TestIsDeploymentMissingPermissionsError(t *testing.T) {
+	for _, tt := range []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{
+			name: "Another error",
+			err:  errors.New("something happened"),
+		},
+		{
+			name: "Missing RoleAssignment",
+			err: autorest.DetailedError{
+				PackageType: "features.DeploymentsClient",
+				Method:      "CreateOrUpdate",
+				Message:     "Failure sending request",
+				Original: &azure.ServiceError{
+					Code:    CodeInvalidTemplateDeployment,
+					Message: "The template deployment failed with error: 'Authorization failed for template resource '$RESOURCE' of type 'Microsoft.Authorization/roleAssignments'. The client '$CLIENT' with object id '$CLIENT' does not have permission to perform action '$ACTION' at scope '$SCOPE'.'.",
+				},
+			},
+			want: true,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			got := IsDeploymentMissingPermissionsError(autorest.NewErrorWithError(tt.err, "", "", nil, ""))
+			if got != tt.want {
+				t.Errorf("got %v, want %v", got, tt.want)
+			}
+		})
+	}
+}