Repository: Azure/ARO-RP
Pull request #3575 (2024-05-10 18:06:47)

Allow customers to annotate CloudCredential resource on update
### Which issue this PR addresses:

<!--
Please include a link to the ADO work item as well as any GitHub issues.

Usage: `Fixes #<GitHub issue number>`, or `Fixes (paste link of issue)`.
-->
Fixes ARO-7348

### What this PR does / why we need it:

Clusters with manual credentials mode need to be annotated in order to be upgraded: https://docs.openshift.com/container-platform/4.14/updating/preparing_for_updates/preparing-manual-creds-update.html#cco-manual-upgrade-annotation_preparing-manual-creds-update

This PR adds a new API field that the user can patch on update, which will then annotate the cluster itself with the version.

Requires https://github.com/Azure/ARO-RP/pull/3563 as this will also require some validation to ensure that the user is passing in a version and not some other string.
<!--
Include a brief summary of what the PR is intended to accomplish and how the PR
does it. (2-3 sentences)
-->

### Test plan for issue:

<!--
How did you test that this PR works?

- Are there unit tests?
- Are there integration/e2e tests?
- If it is not possible to write automated tests, explain why and document how
  to manually test and verify the feature.
-->

### Is there any documentation that needs to be updated for this PR?

<!--
- If yes and the docs are in GitHub, include doc updates in the PR.
- If yes and the docs are not in GitHub (i.e. ADO wiki), include a link to the
  docs.
- If no, explain why (e.g. "tech debt cleanup, N/A").
-->

### How do you know this will function as expected in production? 

<!--
- Does adequate telemetry, monitoring and documentation exist to effectively operate your change?
- Have failure modes been identified and mitigated? 
-->



Patch:
diff --git a/.sha256sum b/.sha256sum
index 8fae07dc7..cca696c9d 100644
--- a/.sha256sum
+++ b/.sha256sum
@@ -6,4 +6,4 @@ b1f1de0fe40d05de90742b17928968923b936adc294000f58974f50a297581dd  swagger/redhat
 c023515341196746454c0ae7af077d40d3ec13f6b88b33cb558f0a7ab17a5a24  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2023-07-01-preview/redhatopenshift.json
 440748951dd1c3b34b5ccbdcb7cd966e3b89490887a1f1d64429561fad789515  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/stable/2023-09-04/redhatopenshift.json
 74a46fdde6ceb0121fe1515c7e11e902dd921b54cffe693307fb02b3dc88f26e  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/stable/2023-11-22/redhatopenshift.json
-a27184734436629e24b344c3b5c015437f144e18e7eddce7e252a1ed4cda7bca  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
+f1e9d42d7c4c0081282e065e7845455db28ed6924687f1acecafb5fbc43ae0c3  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
diff --git a/Dockerfile.ci-portal b/Dockerfile.ci-portal
new file mode 100644
index 000000000..d54d1af07
--- /dev/null
+++ b/Dockerfile.ci-portal
@@ -0,0 +1,13 @@
+ARG REGISTRY
+
+FROM ${REGISTRY}/ubi8/nodejs-16 AS portal-base
+WORKDIR /build/portal/v2
+USER root
+
+FROM portal-base AS portal-build
+COPY /portal/v2/package*.json ./
+RUN npm ci
+RUN npm audit --omit=dev
+COPY /portal/v2/ ./
+RUN npm run lint
+RUN npm run build
\ No newline at end of file
diff --git a/Dockerfile.ci-rp b/Dockerfile.ci-rp
index 23ee37330..4750d88bb 100644
--- a/Dockerfile.ci-rp
+++ b/Dockerfile.ci-rp
@@ -2,38 +2,20 @@ ARG REGISTRY
 ARG VERSION
 
 ###############################################################################
-# Stage 1: Build the SRE Portal Assets
 # builder is responsible for all compilation and validation of the RP
 ###############################################################################
-FROM ${REGISTRY}/ubi8/nodejs-16  as portal-build
-WORKDIR /build/portal/v2
-USER root
-
-# Copying package files and installing dependencies
-COPY portal/v2/package*.json ./
-RUN npm ci
-RUN npm audit --omit=dev # Run audit without dev dependencies
-
-# Copying the rest of the source and build
-COPY --chown=root:root portal/v2/ ./
-RUN npm run lint && npm run build
-
-###############################################################################
-# Stage 2: Compile the Golang RP code
-###############################################################################
-FROM ${REGISTRY}/ubi8/go-toolset:1.20.12-5 AS builder
+FROM ${REGISTRY}/ubi8/go-toolset:1.20.12-5 as builder
 USER root
 WORKDIR /app
 
-# golang config and build steps
+# golang config
 ENV GOPATH=/root/go
 ENV GOFLAGS="-tags=containers_image_openpgp,exclude_graphdriver_btrfs,exclude_graphdriver_devicemapper"
 
-# Install golangci-lint and verify
-RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b /usr/local/bin v1.56.2 && \
-    golangci-lint --version || (echo "golangci-lint not found" && exit 1)
+# install general build tools
+RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b /usr/local/bin v1.56.2
 
-# Copy dependencies and source files
+# copy from least changed -> most changed to improve cache hits
 COPY go.mod go.sum ./
 COPY vendor vendor
 COPY swagger swagger
@@ -43,21 +25,20 @@ COPY cmd cmd
 COPY pkg pkg
 COPY test test
 
-# Ensure JS assets are available before generating Go code
-COPY --from=portal-build /build/pkg/portal/assets/v2/build /app/pkg/portal/assets/v2/build
-
-# Lint, generate, build, and test
+# lint
 RUN golangci-lint run --verbose
+
+# build
 RUN go generate ./...
 RUN go build -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${VERSION}" ./cmd/aro
 RUN go test ./test/e2e/... -tags e2e,codec.safe -c -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${VERSION}" -o e2e.test
 
-# Additional tests
+# test
 RUN ARO_RUN_PKI_TESTS=nope go run gotest.tools/gotestsum@v1.11.0 --format pkgname --junitfile report.xml -- -coverprofile=cover.out ./...
 RUN hack/fips/validate-fips.sh ./aro
 
 ###############################################################################
-# Stage 3: final is our slim image with minimal layers and tools
+# final is our slim image with minimal layers and tools
 ###############################################################################
 FROM ${REGISTRY}/ubi8/ubi-minimal AS final
 RUN microdnf update && microdnf clean all
diff --git a/Dockerfile.ci-rp.dockerignore b/Dockerfile.ci-rp.dockerignore
index 052c75c86..a984a1402 100644
--- a/Dockerfile.ci-rp.dockerignore
+++ b/Dockerfile.ci-rp.dockerignore
@@ -10,17 +10,4 @@
 !pkg
 !swagger
 !test
-!vendor
-
-# Ignore locally built NPM artifacts - using new automated process
-pkg/portal/assets/v2/build
-
-# Allow files and directories for JavaScript build
-!portal/v2/package*.json
-!portal/v2/src
-!portal/v2/public
-!portal/v2/config-overrides.js
-!portal/v2/tsconfig.json
-!portal/v2/.eslintrc.yml
-!portal/v2/.npmrc
-
+!vendor
\ No newline at end of file
diff --git a/Dockerfile.proxy b/Dockerfile.proxy
index 006172cc3..87643f947 100644
--- a/Dockerfile.proxy
+++ b/Dockerfile.proxy
@@ -7,13 +7,11 @@ ENV GOPATH=/root/go
 RUN mkdir -p /app
 WORKDIR /app
 
-COPY . /app
 RUN make proxy
 
 FROM ${REGISTRY}/ubi8/ubi-minimal
 RUN microdnf update && microdnf clean all
-COPY --from=builder /app/proxy /usr/local/bin/
+COPY --from=builder /go/src/github.com/Azure/ARO-RP/proxy /usr/local/bin/
 ENTRYPOINT ["proxy"]
 EXPOSE 8443/tcp
-EXPOSE 8080/tcp
 USER 1000
diff --git a/Makefile b/Makefile
index e5b53a309..a0953075d 100644
--- a/Makefile
+++ b/Makefile
@@ -79,6 +79,9 @@ client: generate
 ci-rp: fix-macos-vendor
 	docker build . -f Dockerfile.ci-rp --ulimit=nofile=4096:4096 --build-arg REGISTRY=$(REGISTRY) --build-arg VERSION=$(VERSION) --no-cache=$(NO_CACHE)
 
+ci-portal:
+	docker build . -f Dockerfile.ci-portal --build-arg REGISTRY=$(REGISTRY) --no-cache=$(NO_CACHE)
+
 # TODO: hard coding dev-config.yaml is clunky; it is also probably convenient to
 # override COMMIT.
 deploy:
@@ -120,6 +123,7 @@ generate-kiota:
 
 init-contrib:
 	install -v hack/git/hooks/* .git/hooks/
+	
 
 image-aro-multistage:
 	docker build --platform=linux/amd64 --network=host --no-cache -f Dockerfile.aro-multistage -t $(ARO_IMAGE) --build-arg REGISTRY=$(REGISTRY) .
diff --git a/pkg/api/admin/openshiftcluster.go b/pkg/api/admin/openshiftcluster.go
index d1bab4bb6..ed328772e 100644
--- a/pkg/api/admin/openshiftcluster.go
+++ b/pkg/api/admin/openshiftcluster.go
@@ -418,9 +418,13 @@ type IngressProfile struct {
 
 // PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.
 type PlatformWorkloadIdentityProfile struct {
+	UpgradeableTo              *UpgradeableTo             `json:"upgradeableTo,omitempty"`
 	PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"`
 }
 
+// UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to
+type UpgradeableTo string
+
 // PlatformWorkloadIdentity stores information representing a single workload identity.
 type PlatformWorkloadIdentity struct {
 	OperatorName string `json:"operatorName,omitempty"`
diff --git a/pkg/api/openshiftcluster.go b/pkg/api/openshiftcluster.go
index 0826520d0..e0517ad10 100644
--- a/pkg/api/openshiftcluster.go
+++ b/pkg/api/openshiftcluster.go
@@ -773,9 +773,13 @@ type HiveProfile struct {
 type PlatformWorkloadIdentityProfile struct {
 	MissingFields
 
+	UpgradeableTo              *UpgradeableTo             `json:"upgradeableTo,omitempty"`
 	PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"`
 }
 
+// UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to
+type UpgradeableTo string
+
 // PlatformWorkloadIdentity stores information representing a single workload identity.
 type PlatformWorkloadIdentity struct {
 	MissingFields
diff --git a/pkg/api/v20240812preview/openshiftcluster.go b/pkg/api/v20240812preview/openshiftcluster.go
index 09a5a632d..28e7d4bd1 100644
--- a/pkg/api/v20240812preview/openshiftcluster.go
+++ b/pkg/api/v20240812preview/openshiftcluster.go
@@ -290,15 +290,19 @@ type IngressProfile struct {
 
 // PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.
 type PlatformWorkloadIdentityProfile struct {
-	PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"`
+	UpgradeableTo              *UpgradeableTo             `json:"upgradeableTo,omitempty" mutable:"true"`
+	PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty" mutable:"true"`
 }
 
+// UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to
+type UpgradeableTo string
+
 // PlatformWorkloadIdentity stores information representing a single workload identity.
 type PlatformWorkloadIdentity struct {
-	OperatorName string `json:"operatorName,omitempty"`
-	ResourceID   string `json:"resourceId,omitempty"`
-	ClientID     string `json:"clientId,omitempty" swagger:"readOnly"`
-	ObjectID     string `json:"objectId,omitempty" swagger:"readOnly"`
+	OperatorName string `json:"operatorName,omitempty" mutable:"true"`
+	ResourceID   string `json:"resourceId,omitempty" mutable:"true"`
+	ClientID     string `json:"clientId,omitempty" swagger:"readOnly" mutable:"true"`
+	ObjectID     string `json:"objectId,omitempty" swagger:"readOnly" mutable:"true"`
 }
 
 // ClusterUserAssignedIdentity stores information about a user-assigned managed identity in a predefined format required by Microsoft's Managed Identity team.
diff --git a/pkg/api/v20240812preview/openshiftcluster_convert.go b/pkg/api/v20240812preview/openshiftcluster_convert.go
index ad51f4406..078d0ad7f 100644
--- a/pkg/api/v20240812preview/openshiftcluster_convert.go
+++ b/pkg/api/v20240812preview/openshiftcluster_convert.go
@@ -141,6 +141,12 @@ func (c openShiftClusterConverter) ToExternal(oc *api.OpenShiftCluster) interfac
 
 	if oc.Properties.PlatformWorkloadIdentityProfile != nil && oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities != nil {
 		out.Properties.PlatformWorkloadIdentityProfile = &PlatformWorkloadIdentityProfile{}
+
+		if oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo != nil {
+			temp := UpgradeableTo(*oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo)
+			out.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo = &temp
+		}
+
 		out.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities = make([]PlatformWorkloadIdentity, len(oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities))
 
 		for i := range oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities {
@@ -225,6 +231,12 @@ func (c openShiftClusterConverter) ToInternal(_oc interface{}, out *api.OpenShif
 	}
 	if oc.Properties.PlatformWorkloadIdentityProfile != nil && oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities != nil {
 		out.Properties.PlatformWorkloadIdentityProfile = &api.PlatformWorkloadIdentityProfile{}
+
+		if oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo != nil {
+			temp := api.UpgradeableTo(*oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo)
+			out.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo = &temp
+		}
+
 		out.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities = make([]api.PlatformWorkloadIdentity, len(oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities))
 
 		for i := range oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities {
diff --git a/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go b/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go
index e438d5fc1..2535b27f7 100644
--- a/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go
+++ b/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go
@@ -1525,6 +1525,7 @@ func (pwi PlatformWorkloadIdentity) MarshalJSON() ([]byte, error) {
 // PlatformWorkloadIdentityProfile platformWorkloadIdentityProfile encapsulates all information that is
 // specific to workload identity clusters.
 type PlatformWorkloadIdentityProfile struct {
+	UpgradeableTo              *string                     `json:"upgradeableTo,omitempty"`
 	PlatformWorkloadIdentities *[]PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"`
 }
 
diff --git a/pkg/cluster/arooperator.go b/pkg/cluster/arooperator.go
index 7afbb5b3c..ca187e23f 100644
--- a/pkg/cluster/arooperator.go
+++ b/pkg/cluster/arooperator.go
@@ -65,6 +65,10 @@ func (m *manager) ensureCredentialsRequest(ctx context.Context) error {
 	return m.aroOperatorDeployer.CreateOrUpdateCredentialsRequest(ctx)
 }
 
+func (m *manager) ensureUpgradeAnnotation(ctx context.Context) error {
+	return m.aroOperatorDeployer.EnsureUpgradeAnnotation(ctx)
+}
+
 func (m *manager) renewMDSDCertificate(ctx context.Context) error {
 	return m.aroOperatorDeployer.RenewMDSDCertificate(ctx)
 }
diff --git a/pkg/cluster/install.go b/pkg/cluster/install.go
index 5e14d57ef..072b9cb29 100644
--- a/pkg/cluster/install.go
+++ b/pkg/cluster/install.go
@@ -213,6 +213,7 @@ func (m *manager) Update(ctx context.Context) error {
 		steps.Action(m.updateAROSecret),
 		steps.Action(m.restartAROOperatorMaster), // depends on m.updateOpenShiftSecret; the point of restarting is to pick up any changes made to the secret
 		steps.Condition(m.aroDeploymentReady, 5*time.Minute, true),
+		steps.Action(m.ensureUpgradeAnnotation),
 		steps.Action(m.reconcileLoadBalancerProfile),
 	}
 
@@ -524,7 +525,7 @@ func (m *manager) initializeKubernetesClients(ctx context.Context) error {
 // initializeKubernetesClients initializes clients which are used
 // once the cluster is up later on in the install process.
 func (m *manager) initializeOperatorDeployer(ctx context.Context) (err error) {
-	m.aroOperatorDeployer, err = deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.client, m.extensionscli, m.kubernetescli)
+	m.aroOperatorDeployer, err = deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.client, m.extensionscli, m.kubernetescli, m.operatorcli)
 	return
 }
 
diff --git a/pkg/deploy/assets/env-development.json b/pkg/deploy/assets/env-development.json
index 1c738e4fc..4328b7592 100644
--- a/pkg/deploy/assets/env-development.json
+++ b/pkg/deploy/assets/env-development.json
@@ -143,62 +143,6 @@
                 "[resourceId('Microsoft.Network/virtualNetworks', 'dev-vpn-vnet')]"
             ]
         },
-        {
-            "sku": {
-                "name": "Basic"
-            },
-            "properties": {
-                "frontendIPConfigurations": [
-                    {
-                        "properties": {
-                            "subnet": {
-                                "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'rp-vnet', 'rp-subnet')]"
-                            }
-                        },
-                        "name": "not-used"
-                    }
-                ],
-                "backendAddressPools": [
-                    {
-                        "name": "dev-backend"
-                    }
-                ],
-                "loadBalancingRules": [
-                    {
-                        "properties": {
-                            "frontendIPConfiguration": {
-                                "id": "[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', 'dev-lb-internal', 'not-used')]"
-                            },
-                            "backendAddressPool": {
-                                "id": "[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'dev-lb-internal', 'dev-backend')]"
-                            },
-                            "probe": {
-                                "id": "[resourceId('Microsoft.Network/loadBalancers/probes', 'dev-lb-internal', 'dev-probe')]"
-                            },
-                            "protocol": "Tcp",
-                            "loadDistribution": "Default",
-                            "frontendPort": 443,
-                            "backendPort": 443
-                        },
-                        "name": "dev-lbrule"
-                    }
-                ],
-                "probes": [
-                    {
-                        "properties": {
-                            "protocol": "Tcp",
-                            "port": 443,
-                            "numberOfProbes": 3
-                        },
-                        "name": "dev-probe"
-                    }
-                ]
-            },
-            "name": "dev-lb-internal",
-            "type": "Microsoft.Network/loadBalancers",
-            "location": "[resourceGroup().location]",
-            "apiVersion": "2020-08-01"
-        },
         {
             "name": "[concat(take(resourceGroup().name,10), '-dev-sharedKV')]",
             "type": "Microsoft.KeyVault/vaults",
@@ -300,9 +244,9 @@
                     },
                     "storageProfile": {
                         "imageReference": {
-                            "publisher": "MicrosoftCBLMariner",
-                            "offer": "cbl-mariner",
-                            "sku": "cbl-mariner-2-gen2",
+                            "publisher": "RedHat",
+                            "offer": "RHEL",
+                            "sku": "8-LVM",
                             "version": "latest"
                         },
                         "osDisk": {
@@ -313,9 +257,6 @@
                         }
                     },
                     "networkProfile": {
-                        "healthProbe": {
-                            "id": "[resourceId('Microsoft.Network/loadBalancers/probes', 'dev-lb-internal', 'dev-probe')]"
-                        },
                         "networkInterfaceConfigurations": [
                             {
                                 "name": "dev-proxy-vmss-nic",
@@ -336,12 +277,7 @@
                                                             "domainNameLabel": "[parameters('proxyDomainNameLabel')]"
                                                         }
                                                     }
-                                                },
-                                                "loadBalancerBackendAddressPools": [
-                                                    {
-                                                        "id": "[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'dev-lb-internal', 'dev-backend')]"
-                                                    }
-                                                ]
+                                                }
                                             }
                                         }
                                     ]
@@ -360,7 +296,7 @@
                                     "autoUpgradeMinorVersion": true,
                                     "settings": {},
                                     "protectedSettings": {
-                                        "script": "[base64(concat(base64ToString('c2V0IC1leAoK'),'PROXYIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImage')),''')\n','PROXYIMAGEAUTH=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImageAuth')),''')\n','PROXYCERT=''',parameters('proxyCert'),'''\n','PROXYCLIENTCERT=''',parameters('proxyClientCert'),'''\n','PROXYKEY=''',parameters('proxyKey'),'''\n','\n',base64ToString('I0FkZGluZyByZXRyeSBsb2dpYyB0byB5dW0gY29tbWFuZHMgaW4gb3JkZXIgdG8gYXZvaWQgc3RhbGxpbmcgb3V0IG9uIHJlc291cmNlIGxvY2tzCmVjaG8gImluc3RhbGxpbmcgbW9ieS1lbmdpbmUgKGRvY2tlcikiCmZvciBhdHRlbXB0IGluIHsxLi42MH07IGRvCgl0ZG5mIGluc3RhbGwgLXkgbW9ieS1lbmdpbmUgbW9ieS1jbGkgJiYgYnJlYWsKCWlmIFtbICR7YXR0ZW1wdH0gLWx0IDYwIF1dOyB0aGVuIHNsZWVwIDMwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKc3lzdGVtY3RsIGVuYWJsZSBkb2NrZXIKc3lzdGVtY3RsIHN0YXJ0IGRvY2tlcgoKbWtkaXIgL3Jvb3QvLmRvY2tlcgpjYXQgPi9yb290Ly5kb2NrZXIvY29uZmlnLmpzb24gPDxFT0YKewoJImF1dGhzIjogewoJCSIke1BST1hZSU1BR0UlJS8qfSI6IHsKCQkJImF1dGgiOiAiJFBST1hZSU1BR0VBVVRIIgoJCX0KCX0KfQpFT0YKCmRvY2tlciBwdWxsICIkUFJPWFlJTUFHRSIKCm1rZGlyIC9ldGMvcHJveHkKYmFzZTY0IC1kIDw8PCIkUFJPWFlDRVJUIiA+L2V0Yy9wcm94eS9wcm94eS5jcnQKYmFzZTY0IC1kIDw8PCIkUFJPWFlLRVkiID4vZXRjL3Byb3h5L3Byb3h5LmtleQpiYXNlNjQgLWQgPDw8IiRQUk9YWUNMSUVOVENFUlQiID4vZXRjL3Byb3h5L3Byb3h5LWNsaWVudC5jcnQKY2hvd24gLVIgMTAwMDoxMDAwIC9ldGMvcHJveHkKY2htb2QgMDYwMCAvZXRjL3Byb3h5L3Byb3h5LmtleQoKY2F0ID4vZXRjL3N5c2NvbmZpZy9wcm94eSA8PEVPRgpQUk9YWV9JTUFHRT0nJFBST1hZSU1BR0UnCkVPRgoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL3Byb3h5LnNlcnZpY2UgPDwnRU9GJwpbVW5pdF0KQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CldhbnRzPW5ldHdvcmstb25saW5lLnRhcmdldAoKW1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9wcm94eQpFeGVjU3RhcnRQcmU9LS91c3IvYmluL2RvY2tlciBybSAtZiAlbgpFeGVjU3RhcnQ9L3Vzci9iaW4vZG9ja2VyIHJ1biAtLXJtIC0tbmFtZSAlbiAtcCA0NDM6ODQ0MyAtdiAvZXRjL3Byb3h5Oi9zZWNyZXRzICRQUk9YWV9JTUFHRQpFeGVjU3RvcD0vdXNyL2Jpbi9kb2NrZXIgc3RvcCAlbgpSZXN0YXJ0PWFsd2F5cwpSZXN0YXJ0U2VjPTEKU3RhcnRMaW1pdEludGVydmFsPTAKCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCnN5c3RlbWN0bCBlbmFibGUgcHJveHkuc2VydmljZQoKY2F0ID4vZXRjL2Nyb24ud2Vla2x5L3B1bGwtaW1hZ2UgPDwnRU9GJwojIS9iaW4vYmFzaAoKZG9ja2VyIHB1bGwgJFBST1hZSU1BR0UKc3lzdGVtY3RsIHJlc3RhcnQgcHJveHkuc2VydmljZQpFT0YKY2htb2QgK3ggL2V0Yy9jcm9uLndlZWtseS9wdWxsLWltYWdlCgpjYXQgPi9ldGMvY3Jvbi53ZWVrbHkveXVtdXBkYXRlIDw8J0VPRicKIyEvYmluL2Jhc2gKCnl1bSB1cGRhdGUgLXkKRU9GCmNobW9kICt4IC9ldGMvY3Jvbi53ZWVrbHkveXVtdXBkYXRlCgpjYXQgPi9ldGMvY3Jvbi5kYWlseS9yZXN0YXJ0LXByb3h5IDw8J0VPRicKIyEvYmluL2Jhc2gKCnN5c3RlbWN0bCByZXN0YXJ0IHByb3h5LnNlcnZpY2UKRU9GCmNobW9kICt4IC9ldGMvY3Jvbi5kYWlseS9yZXN0YXJ0LXByb3h5CgooCglzbGVlcCAzMAoJcmVib290CikgJgo=')))]"
+                                        "script": "[base64(concat(base64ToString('c2V0IC1leAoK'),'PROXYIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImage')),''')\n','PROXYIMAGEAUTH=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImageAuth')),''')\n','PROXYCERT=''',parameters('proxyCert'),'''\n','PROXYCLIENTCERT=''',parameters('proxyClientCert'),'''\n','PROXYKEY=''',parameters('proxyKey'),'''\n','\n',base64ToString('I0FkZGluZyByZXRyeSBsb2dpYyB0byB5dW0gY29tbWFuZHMgaW4gb3JkZXIgdG8gYXZvaWQgc3RhbGxpbmcgb3V0IG9uIHJlc291cmNlIGxvY2tzCmVjaG8gInJ1bm5pbmcgUkhVSSBmaXgiCmZvciBhdHRlbXB0IGluIHsxLi42MH07IGRvCiAgeXVtIHVwZGF0ZSAteSAtLWRpc2FibGVyZXBvPScqJyAtLWVuYWJsZXJlcG89J3JodWktbWljcm9zb2Z0LWF6dXJlKicgJiYgYnJlYWsKICBpZiBbWyAke2F0dGVtcHR9IC1sdCA2MCBdXTsgdGhlbiBzbGVlcCAzMDsgZWxzZSBleGl0IDE7IGZpCmRvbmUKCmVjaG8gInJ1bm5pbmcgeXVtIHVwZGF0ZSIKZm9yIGF0dGVtcHQgaW4gezEuLjYwfTsgZG8KICB5dW0gLXkgLXggV0FMaW51eEFnZW50IC14IFdBTGludXhBZ2VudC11ZGV2IHVwZGF0ZSAtLWFsbG93ZXJhc2luZyAmJiBicmVhawogIGlmIFtbICR7YXR0ZW1wdH0gLWx0IDYwIF1dOyB0aGVuIHNsZWVwIDMwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKZWNobyAiaW5zdGFsbGluZyBwb2RtYW4tZG9ja2VyIgpmb3IgYXR0ZW1wdCBpbiB7MS4uNjB9OyBkbwogIHl1bSAteSBpbnN0YWxsIHBvZG1hbi1kb2NrZXIgJiYgYnJlYWsKICBpZiBbWyAke2F0dGVtcHR9IC1sdCA2MCBdXTsgdGhlbiBzbGVlcCAzMDsgZWxzZSBleGl0IDE7IGZpCmRvbmUKCmZpcmV3YWxsLWNtZCAtLWFkZC1wb3J0PTQ0My90Y3AgLS1wZXJtYW5lbnQKCm1rZGlyIC9yb290Ly5kb2NrZXIKY2F0ID4vcm9vdC8uZG9ja2VyL2NvbmZpZy5qc29uIDw8RU9GCnsKCSJhdXRocyI6IHsKCQkiJHtQUk9YWUlNQUdFJSUvKn0iOiB7CgkJCSJhdXRoIjogIiRQUk9YWUlNQUdFQVVUSCIKCQl9Cgl9Cn0KRU9GCgpta2RpciAtcCAvZXRjL2NvbnRhaW5lcnMvCnRvdWNoIC9ldGMvY29udGFpbmVycy9ub2RvY2tlcgoKZG9ja2VyIHB1bGwgIiRQUk9YWUlNQUdFIgoKbWtkaXIgL2V0Yy9wcm94eQpiYXNlNjQgLWQgPDw8IiRQUk9YWUNFUlQiID4vZXRjL3Byb3h5L3Byb3h5LmNydApiYXNlNjQgLWQgPDw8IiRQUk9YWUtFWSIgPi9ldGMvcHJveHkvcHJveHkua2V5CmJhc2U2NCAtZCA8PDwiJFBST1hZQ0xJRU5UQ0VSVCIgPi9ldGMvcHJveHkvcHJveHktY2xpZW50LmNydApjaG93biAtUiAxMDAwOjEwMDAgL2V0Yy9wcm94eQpjaG1vZCAwNjAwIC9ldGMvcHJveHkvcHJveHkua2V5CgpjYXQgPi9ldGMvc3lzY29uZmlnL3Byb3h5IDw8RU9GClBST1hZX0lNQUdFPSckUFJPWFlJTUFHRScKRU9GCgpjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vcHJveHkuc2VydmljZSA8PCdFT0YnCltVbml0XQpBZnRlcj1uZXR3b3JrLW9ubGluZS50YXJnZXQKV2FudHM9bmV0d29yay1vbmxpbmUudGFyZ2V0CgpbU2VydmljZV0KRW52aXJvbm1lbnRGaWxlPS9ldGMvc3lzY29uZmlnL3Byb3h5CkV4ZWNTdGFydFByZT0tL3Vzci9iaW4vZG9ja2VyIHJtIC1mICVuCkV4ZWNTdGFydD0vdXNyL2Jpbi9kb2NrZXIgcnVuIC0tcm0gLS1uYW1lICVuIC1wIDQ0Mzo4NDQzIC12IC9ldGMvcHJveHk6L3NlY3JldHMgJFBST1hZX0lNQUdFCkV4ZWNTdG9wPS91c3IvYmluL2RvY2tlciBzdG9wICVuClJlc3RhcnQ9YWx3YXlzClJlc3RhcnRTZWM9MQpTdGFydExpbWl0SW50ZXJ2YWw9MAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIudGFyZ2V0CkVPRgoKc3lzdGVtY3RsIGVuYWJsZSBwcm94eS5zZXJ2aWNlCgpjYXQgPi9ldGMvY3Jvbi53ZWVrbHkvcHVsbC1pbWFnZSA8PCdFT0YnCiMhL2Jpbi9iYXNoCgpkb2NrZXIgcHVsbCAkUFJPWFlJTUFHRQpzeXN0ZW1jdGwgcmVzdGFydCBwcm94eS5zZXJ2aWNlCkVPRgpjaG1vZCAreCAvZXRjL2Nyb24ud2Vla2x5L3B1bGwtaW1hZ2UKCmNhdCA+L2V0Yy9jcm9uLndlZWtseS95dW11cGRhdGUgPDwnRU9GJwojIS9iaW4vYmFzaAoKeXVtIHVwZGF0ZSAteQpFT0YKY2htb2QgK3ggL2V0Yy9jcm9uLndlZWtseS95dW11cGRhdGUKCmNhdCA+L2V0Yy9jcm9uLmRhaWx5L3Jlc3RhcnQtcHJveHkgPDwnRU9GJwojIS9iaW4vYmFzaAoKc3lzdGVtY3RsIHJlc3RhcnQgcHJveHkuc2VydmljZQpFT0YKY2htb2QgK3ggL2V0Yy9jcm9uLmRhaWx5L3Jlc3RhcnQtcHJveHkKCigKCXNsZWVwIDMwCglyZWJvb3QKKSAmCg==')))]"
                                     },
                                     "provisionAfterExtensions": [
                                         "Microsoft.Azure.Monitor.AzureMonitorLinuxAgent",
@@ -401,22 +337,10 @@
                 },
                 "overprovision": false
             },
-            "identity": {
-                "type": "UserAssigned",
-                "userAssignedIdentities": {
-                    "[resourceId('AzSecPackAutoConfigRG', 'Microsoft.ManagedIdentity/userAssignedIdentities', concat('AzSecPackAutoConfigUA-', resourceGroup().location))]": {}
-                }
-            },
             "name": "dev-proxy-vmss",
             "type": "Microsoft.Compute/virtualMachineScaleSets",
             "location": "[resourceGroup().location]",
-            "tags": {
-                "azsecpack": "nonprod"
-            },
-            "apiVersion": "2020-12-01",
-            "dependsOn": [
-                "[resourceId('Microsoft.Network/loadBalancers', 'dev-lb-internal')]"
-            ]
+            "apiVersion": "2020-12-01"
         },
         {
             "properties": {
diff --git a/pkg/deploy/generator/resources_dev.go b/pkg/deploy/generator/resources_dev.go
index a1788a804..668b7fee3 100644
--- a/pkg/deploy/generator/resources_dev.go
+++ b/pkg/deploy/generator/resources_dev.go
@@ -17,69 +17,6 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/azureclient"
 )
 
-// devLBInternal is needed for defining a healthprobe.
-// VMSS with auto upgrademode requires a healthprobe from an LB.
-func (g *generator) devLBInternal() *arm.Resource {
-	return &arm.Resource{
-		Resource: &mgmtnetwork.LoadBalancer{
-			Sku: &mgmtnetwork.LoadBalancerSku{
-				Name: mgmtnetwork.LoadBalancerSkuNameBasic,
-			},
-			LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{
-				FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{
-					{
-						FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{
-							Subnet: &mgmtnetwork.Subnet{
-								ID: to.StringPtr("[resourceId('Microsoft.Network/virtualNetworks/subnets', 'rp-vnet', 'rp-subnet')]"),
-							},
-						},
-						Name: to.StringPtr("not-used"),
-					},
-				},
-				BackendAddressPools: &[]mgmtnetwork.BackendAddressPool{
-					{
-						Name: to.StringPtr("dev-backend"),
-					},
-				},
-				LoadBalancingRules: &[]mgmtnetwork.LoadBalancingRule{
-					{
-						LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{
-							FrontendIPConfiguration: &mgmtnetwork.SubResource{
-								ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', 'dev-lb-internal', 'not-used')]"),
-							},
-							BackendAddressPool: &mgmtnetwork.SubResource{
-								ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'dev-lb-internal', 'dev-backend')]"),
-							},
-							Probe: &mgmtnetwork.SubResource{
-								ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/probes', 'dev-lb-internal', 'dev-probe')]"),
-							},
-							Protocol:         mgmtnetwork.TransportProtocolTCP,
-							LoadDistribution: mgmtnetwork.LoadDistributionDefault,
-							FrontendPort:     to.Int32Ptr(443),
-							BackendPort:      to.Int32Ptr(443),
-						},
-						Name: to.StringPtr("dev-lbrule"),
-					},
-				},
-				Probes: &[]mgmtnetwork.Probe{
-					{
-						ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{
-							Protocol:       mgmtnetwork.ProbeProtocolTCP,
-							Port:           to.Int32Ptr(443),
-							NumberOfProbes: to.Int32Ptr(3),
-						},
-						Name: to.StringPtr("dev-probe"),
-					},
-				},
-			},
-			Name:     to.StringPtr("dev-lb-internal"),
-			Type:     to.StringPtr("Microsoft.Network/loadBalancers"),
-			Location: to.StringPtr("[resourceGroup().location]"),
-		},
-		APIVersion: azureclient.APIVersion("Microsoft.Network"),
-	}
-}
-
 func (g *generator) devProxyVMSS() *arm.Resource {
 	parts := []string{
 		fmt.Sprintf("base64ToString('%s')", base64.StdEncoding.EncodeToString([]byte("set -ex\n\n"))),
@@ -114,12 +51,6 @@ func (g *generator) devProxyVMSS() *arm.Resource {
 				Tier:     to.StringPtr("Standard"),
 				Capacity: to.Int64Ptr(1),
 			},
-			Identity: &mgmtcompute.VirtualMachineScaleSetIdentity{
-				Type: mgmtcompute.ResourceIdentityTypeUserAssigned,
-				UserAssignedIdentities: map[string]*mgmtcompute.VirtualMachineScaleSetIdentityUserAssignedIdentitiesValue{
-					"[resourceId('AzSecPackAutoConfigRG', 'Microsoft.ManagedIdentity/userAssignedIdentities', concat('AzSecPackAutoConfigUA-', resourceGroup().location))]": {},
-				},
-			},
 			VirtualMachineScaleSetProperties: &mgmtcompute.VirtualMachineScaleSetProperties{
 				UpgradePolicy: &mgmtcompute.UpgradePolicy{
 					Mode: mgmtcompute.UpgradeModeRolling,
@@ -142,9 +73,9 @@ func (g *generator) devProxyVMSS() *arm.Resource {
 					},
 					StorageProfile: &mgmtcompute.VirtualMachineScaleSetStorageProfile{
 						ImageReference: &mgmtcompute.ImageReference{
-							Publisher: to.StringPtr("MicrosoftCBLMariner"),
-							Offer:     to.StringPtr("cbl-mariner"),
-							Sku:       to.StringPtr("cbl-mariner-2-gen2"),
+							Publisher: to.StringPtr("RedHat"),
+							Offer:     to.StringPtr("RHEL"),
+							Sku:       to.StringPtr("8-LVM"),
 							Version:   to.StringPtr("latest"),
 						},
 						OsDisk: &mgmtcompute.VirtualMachineScaleSetOSDisk{
@@ -155,9 +86,6 @@ func (g *generator) devProxyVMSS() *arm.Resource {
 						},
 					},
 					NetworkProfile: &mgmtcompute.VirtualMachineScaleSetNetworkProfile{
-						HealthProbe: &mgmtcompute.APIEntityReference{
-							ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/probes', 'dev-lb-internal', 'dev-probe')]"),
-						},
 						NetworkInterfaceConfigurations: &[]mgmtcompute.VirtualMachineScaleSetNetworkConfiguration{
 							{
 								Name: to.StringPtr("dev-proxy-vmss-nic"),
@@ -179,11 +107,6 @@ func (g *generator) devProxyVMSS() *arm.Resource {
 														},
 													},
 												},
-												LoadBalancerBackendAddressPools: &[]mgmtcompute.SubResource{
-													{
-														ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'dev-lb-internal', 'dev-backend')]"),
-													},
-												},
 											},
 										},
 									},
@@ -248,12 +171,6 @@ func (g *generator) devProxyVMSS() *arm.Resource {
 			Location: to.StringPtr("[resourceGroup().location]"),
 		},
 		APIVersion: azureclient.APIVersion("Microsoft.Compute"),
-		Tags: map[string]any{
-			"azsecpack": "nonprod",
-		},
-		DependsOn: []string{
-			"[resourceId('Microsoft.Network/loadBalancers', 'dev-lb-internal')]",
-		},
 	}
 }
 
diff --git a/pkg/deploy/generator/scripts/devProxyVMSS.sh b/pkg/deploy/generator/scripts/devProxyVMSS.sh
index f09c9f57f..cb8e15bf8 100644
--- a/pkg/deploy/generator/scripts/devProxyVMSS.sh
+++ b/pkg/deploy/generator/scripts/devProxyVMSS.sh
@@ -1,12 +1,23 @@
 #Adding retry logic to yum commands in order to avoid stalling out on resource locks
-echo "installing moby-engine (docker)"
+echo "running RHUI fix"
 for attempt in {1..60}; do
-	tdnf install -y moby-engine moby-cli && break
-	if [[ ${attempt} -lt 60 ]]; then sleep 30; else exit 1; fi
+  yum update -y --disablerepo='*' --enablerepo='rhui-microsoft-azure*' && break
+  if [[ ${attempt} -lt 60 ]]; then sleep 30; else exit 1; fi
 done
 
-systemctl enable docker
-systemctl start docker
+echo "running yum update"
+for attempt in {1..60}; do
+  yum -y -x WALinuxAgent -x WALinuxAgent-udev update --allowerasing && break
+  if [[ ${attempt} -lt 60 ]]; then sleep 30; else exit 1; fi
+done
+
+echo "installing podman-docker"
+for attempt in {1..60}; do
+  yum -y install podman-docker && break
+  if [[ ${attempt} -lt 60 ]]; then sleep 30; else exit 1; fi
+done
+
+firewall-cmd --add-port=443/tcp --permanent
 
 mkdir /root/.docker
 cat >/root/.docker/config.json <<EOF
@@ -19,6 +30,9 @@ cat >/root/.docker/config.json <<EOF
 }
 EOF
 
+mkdir -p /etc/containers/
+touch /etc/containers/nodocker
+
 docker pull "$PROXYIMAGE"
 
 mkdir /etc/proxy
diff --git a/pkg/deploy/generator/templates_dev.go b/pkg/deploy/generator/templates_dev.go
index 6eca6e892..7f3bd4b00 100644
--- a/pkg/deploy/generator/templates_dev.go
+++ b/pkg/deploy/generator/templates_dev.go
@@ -33,7 +33,6 @@ func (g *generator) devSharedTemplate() *arm.Template {
 		g.devVnet(),
 		g.devVPNVnet(),
 		g.devVPN(),
-		g.devLBInternal(),
 		g.devDiskEncryptionKeyvault(),
 		g.devDiskEncryptionKey(),
 		g.devDiskEncryptionKeyVaultAccessPolicy(),
diff --git a/pkg/operator/deploy/deploy.go b/pkg/operator/deploy/deploy.go
index ab18e2505..1fbc93a7c 100644
--- a/pkg/operator/deploy/deploy.go
+++ b/pkg/operator/deploy/deploy.go
@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/go-multierror"
+	operatorclient "github.com/openshift/client-go/operator/clientset/versioned"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	extensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -56,6 +57,7 @@ type Operator interface {
 	Restart(context.Context, []string) error
 	IsRunningDesiredVersion(context.Context) (bool, error)
 	RenewMDSDCertificate(context.Context) error
+	EnsureUpgradeAnnotation(context.Context) error
 }
 
 type operator struct {
@@ -67,10 +69,11 @@ type operator struct {
 	client        client.Client
 	extensionscli extensionsclient.Interface
 	kubernetescli kubernetes.Interface
+	operatorcli   operatorclient.Interface
 	dh            dynamichelper.Interface
 }
 
-func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli aroclient.Interface, client client.Client, extensionscli extensionsclient.Interface, kubernetescli kubernetes.Interface) (Operator, error) {
+func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli aroclient.Interface, client client.Client, extensionscli extensionsclient.Interface, kubernetescli kubernetes.Interface, operatorcli operatorclient.Interface) (Operator, error) {
 	restConfig, err := restconfig.RestConfig(env, oc)
 	if err != nil {
 		return nil, err
@@ -89,6 +92,7 @@ func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli
 		client:        client,
 		extensionscli: extensionscli,
 		kubernetescli: kubernetescli,
+		operatorcli:   operatorcli,
 		dh:            dh,
 	}, nil
 }
@@ -432,6 +436,34 @@ func (o *operator) RenewMDSDCertificate(ctx context.Context) error {
 	return nil
 }
 
+func (o *operator) EnsureUpgradeAnnotation(ctx context.Context) error {
+	if o.oc.Properties.PlatformWorkloadIdentityProfile == nil ||
+		o.oc.Properties.ServicePrincipalProfile != nil {
+		return nil
+	}
+
+	upgradeableTo := string(*o.oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo)
+	upgradeableAnnotation := "cloudcredential.openshift.io/upgradeable-to"
+
+	cloudcredentialobject, err := o.operatorcli.OperatorV1().CloudCredentials().Get(ctx, "cluster", metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	if cloudcredentialobject.Annotations == nil {
+		cloudcredentialobject.Annotations = map[string]string{}
+	}
+
+	cloudcredentialobject.Annotations[upgradeableAnnotation] = upgradeableTo
+
+	_, err = o.operatorcli.OperatorV1().CloudCredentials().Update(ctx, cloudcredentialobject, metav1.UpdateOptions{})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (o *operator) IsReady(ctx context.Context) (bool, error) {
 	ok, err := ready.CheckDeploymentIsReady(ctx, o.kubernetescli.AppsV1().Deployments(pkgoperator.Namespace), "aro-operator-master")()
 	o.log.Infof("deployment %q ok status is: %v, err is: %v", "aro-operator-master", ok, err)
diff --git a/pkg/proxy/proxy.go b/pkg/proxy/proxy.go
index 3bcd1e858..746a7d1c9 100644
--- a/pkg/proxy/proxy.go
+++ b/pkg/proxy/proxy.go
@@ -7,7 +7,6 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"errors"
-	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -30,16 +29,7 @@ type Server struct {
 	subnet         *net.IPNet
 }
 
-func health(w http.ResponseWriter, _ *http.Request) {
-	w.WriteHeader(http.StatusOK)
-	fmt.Fprintf(w, "Running.")
-}
-
 func (s *Server) Run() error {
-	healthMux := http.NewServeMux()
-	healthMux.HandleFunc("/", health)
-	go http.ListenAndServe(":8080", healthMux)
-
 	_, subnet, err := net.ParseCIDR(s.Subnet)
 	if err != nil {
 		return err
diff --git a/pkg/util/mocks/operator/deploy/deploy.go b/pkg/util/mocks/operator/deploy/deploy.go
index 99ad4f224..30f4efc25 100644
--- a/pkg/util/mocks/operator/deploy/deploy.go
+++ b/pkg/util/mocks/operator/deploy/deploy.go
@@ -62,6 +62,20 @@ func (mr *MockOperatorMockRecorder) CreateOrUpdateCredentialsRequest(arg0 interf
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateCredentialsRequest", reflect.TypeOf((*MockOperator)(nil).CreateOrUpdateCredentialsRequest), arg0)
 }
 
+// EnsureUpgradeAnnotation mocks base method.
+func (m *MockOperator) EnsureUpgradeAnnotation(arg0 context.Context) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "EnsureUpgradeAnnotation", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// EnsureUpgradeAnnotation indicates an expected call of EnsureUpgradeAnnotation.
+func (mr *MockOperatorMockRecorder) EnsureUpgradeAnnotation(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureUpgradeAnnotation", reflect.TypeOf((*MockOperator)(nil).EnsureUpgradeAnnotation), arg0)
+}
+
 // IsReady mocks base method.
 func (m *MockOperator) IsReady(arg0 context.Context) (bool, error) {
 	m.ctrl.T.Helper()
diff --git a/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models.py b/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models.py
index 3e77e2302..b7e1fab63 100644
--- a/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models.py
+++ b/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models.py
@@ -1303,12 +1303,16 @@ class PlatformWorkloadIdentity(msrest.serialization.Model):
 class PlatformWorkloadIdentityProfile(msrest.serialization.Model):
     """PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.
 
+    :ivar upgradeable_to: UpgradeableTo stores a single OpenShift version a workload identity
+     cluster can be upgraded to.
+    :vartype upgradeable_to: str
     :ivar platform_workload_identities:
     :vartype platform_workload_identities:
      list[~azure.mgmt.redhatopenshift.v2024_08_12_preview.models.PlatformWorkloadIdentity]
     """
 
     _attribute_map = {
+        'upgradeable_to': {'key': 'upgradeableTo', 'type': 'str'},
         'platform_workload_identities': {'key': 'platformWorkloadIdentities', 'type': '[PlatformWorkloadIdentity]'},
     }
 
@@ -1317,11 +1321,15 @@ class PlatformWorkloadIdentityProfile(msrest.serialization.Model):
         **kwargs
     ):
         """
+        :keyword upgradeable_to: UpgradeableTo stores a single OpenShift version a workload identity
+         cluster can be upgraded to.
+        :paramtype upgradeable_to: str
         :keyword platform_workload_identities:
         :paramtype platform_workload_identities:
          list[~azure.mgmt.redhatopenshift.v2024_08_12_preview.models.PlatformWorkloadIdentity]
         """
         super(PlatformWorkloadIdentityProfile, self).__init__(**kwargs)
+        self.upgradeable_to = kwargs.get('upgradeable_to', None)
         self.platform_workload_identities = kwargs.get('platform_workload_identities', None)
 
 
diff --git a/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models_py3.py b/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models_py3.py
index 9828713ff..b8c2d9a1d 100644
--- a/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models_py3.py
+++ b/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models_py3.py
@@ -1412,27 +1412,36 @@ class PlatformWorkloadIdentity(msrest.serialization.Model):
 class PlatformWorkloadIdentityProfile(msrest.serialization.Model):
     """PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.
 
+    :ivar upgradeable_to: UpgradeableTo stores a single OpenShift version a workload identity
+     cluster can be upgraded to.
+    :vartype upgradeable_to: str
     :ivar platform_workload_identities:
     :vartype platform_workload_identities:
      list[~azure.mgmt.redhatopenshift.v2024_08_12_preview.models.PlatformWorkloadIdentity]
     """
 
     _attribute_map = {
+        'upgradeable_to': {'key': 'upgradeableTo', 'type': 'str'},
         'platform_workload_identities': {'key': 'platformWorkloadIdentities', 'type': '[PlatformWorkloadIdentity]'},
     }
 
     def __init__(
         self,
         *,
+        upgradeable_to: Optional[str] = None,
         platform_workload_identities: Optional[List["PlatformWorkloadIdentity"]] = None,
         **kwargs
     ):
         """
+        :keyword upgradeable_to: UpgradeableTo stores a single OpenShift version a workload identity
+         cluster can be upgraded to.
+        :paramtype upgradeable_to: str
         :keyword platform_workload_identities:
         :paramtype platform_workload_identities:
          list[~azure.mgmt.redhatopenshift.v2024_08_12_preview.models.PlatformWorkloadIdentity]
         """
         super(PlatformWorkloadIdentityProfile, self).__init__(**kwargs)
+        self.upgradeable_to = upgradeable_to
         self.platform_workload_identities = platform_workload_identities
 
 
diff --git a/swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json b/swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
index c65beac00..95e0131f5 100644
--- a/swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
+++ b/swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
@@ -2349,6 +2349,9 @@
       "description": "PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.",
       "type": "object",
       "properties": {
+        "upgradeableTo": {
+          "$ref": "#/definitions/UpgradeableTo"
+        },
         "platformWorkloadIdentities": {
           "type": "array",
           "items": {
@@ -2603,6 +2606,10 @@
         "type": "string"
       }
     },
+    "UpgradeableTo": {
+      "description": "UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to",
+      "type": "string"
+    },
     "UserAssignedIdentities": {
       "description": "UserAssignedIdentities stores a mapping from resource IDs of managed identities to their client/principal IDs.",
       "type": "object",


Test patch:
diff --git a/pkg/api/v20240812preview/openshiftcluster_validatestatic.go b/pkg/api/v20240812preview/openshiftcluster_validatestatic.go
index c027b1cd7..b5011fdd2 100644
--- a/pkg/api/v20240812preview/openshiftcluster_validatestatic.go
+++ b/pkg/api/v20240812preview/openshiftcluster_validatestatic.go
@@ -12,6 +12,7 @@ import (
 
 	azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
 	"github.com/Azure/go-autorest/autorest/azure"
+	"github.com/coreos/go-semver/semver"
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/api/util/immutable"
@@ -442,6 +443,14 @@ func (sv openShiftClusterStaticValidator) validatePlatformWorkloadIdentityProfil
 			return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, fmt.Sprintf("%s.PlatformWorkloadIdentities[%d].resourceID", path, n), "Resource must be a user assigned identity.")
 		}
 	}
+
+	if pwip.UpgradeableTo != nil {
+		_, err := semver.NewVersion(string(*pwip.UpgradeableTo))
+		if err != nil {
+			return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, fmt.Sprintf("%s.UpgradeableTo[%v]", path, *pwip.UpgradeableTo), "UpgradeableTo must be a valid OpenShift version in the format 'x.y.z'.")
+		}
+	}
+
 	return nil
 }
 
diff --git a/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go b/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go
index 97e0b802f..884df39ea 100644
--- a/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go
+++ b/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go
@@ -1163,6 +1163,9 @@ func TestOpenShiftClusterStaticValidateDelta(t *testing.T) {
 }
 
 func TestOpenShiftClusterStaticValidatePlatformWorkloadIdentityProfile(t *testing.T) {
+	validUpgradeableToValue := UpgradeableTo("4.14.29")
+	invalidUpgradeableToValue := UpgradeableTo("16.107.invalid")
+
 	createTests := []*validateTest{
 		{
 			name: "valid empty workloadIdentityProfile",
@@ -1323,6 +1326,27 @@ func TestOpenShiftClusterStaticValidatePlatformWorkloadIdentityProfile(t *testin
 			},
 			wantErr: "400: InvalidParameter: properties.servicePrincipalProfile: Must provide either an identity or service principal credentials.",
 		},
+		{
+			name: "valid UpgradeableTo value",
+			modify: func(oc *OpenShiftCluster) {
+				oc.Identity = &Identity{}
+				oc.Properties.ServicePrincipalProfile = nil
+				oc.Properties.PlatformWorkloadIdentityProfile = &PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &validUpgradeableToValue,
+				}
+			},
+		},
+		{
+			name: "invalid UpgradeableTo value",
+			modify: func(oc *OpenShiftCluster) {
+				oc.Identity = &Identity{}
+				oc.Properties.ServicePrincipalProfile = nil
+				oc.Properties.PlatformWorkloadIdentityProfile = &PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &invalidUpgradeableToValue,
+				}
+			},
+			wantErr: `400: InvalidParameter: properties.platformWorkloadIdentityProfile.UpgradeableTo[16.107.invalid]: UpgradeableTo must be a valid OpenShift version in the format 'x.y.z'.`,
+		},
 	}
 
 	runTests(t, testModeCreate, createTests)
diff --git a/pkg/operator/deploy/deploy_test.go b/pkg/operator/deploy/deploy_test.go
index 6002e65dd..44f5e257a 100644
--- a/pkg/operator/deploy/deploy_test.go
+++ b/pkg/operator/deploy/deploy_test.go
@@ -10,6 +10,8 @@ import (
 
 	"github.com/golang/mock/gomock"
 	configv1 "github.com/openshift/api/config/v1"
+	operatorv1 "github.com/openshift/api/operator/v1"
+	operatorfake "github.com/openshift/client-go/operator/clientset/versioned/fake"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -461,3 +463,122 @@ func TestCheckPodImageVersion(t *testing.T) {
 		})
 	}
 }
+
+func TestTestEnsureUpgradeAnnotation(t *testing.T) {
+	UpgradeableTo1 := api.UpgradeableTo("4.14.59")
+
+	for _, tt := range []struct {
+		name                 string
+		cluster              api.OpenShiftClusterProperties
+		annotation           map[string]string
+		wantAnnotation       map[string]string
+		wantErr              string
+		cloudCredentialsName string
+	}{
+		{
+			name: "nil PlatformWorkloadIdentityProfile, no version persisted in cluster document",
+		},
+		{
+			name: "non-nil ServicePrincipalProfile, no version persisted in cluster document",
+			cluster: api.OpenShiftClusterProperties{
+				ServicePrincipalProfile: &api.ServicePrincipalProfile{
+					ClientID:     "",
+					ClientSecret: "",
+				},
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{},
+			},
+		},
+		{
+			name: "no version persisted in cluster document, persist it",
+			cluster: api.OpenShiftClusterProperties{
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &UpgradeableTo1,
+				},
+			},
+			annotation: nil,
+			wantAnnotation: map[string]string{
+				"cloudcredential.openshift.io/upgradeable-to": "4.14.59",
+			},
+		},
+		{
+			name: "cloud credential 'cluster' doesn't exist",
+			cluster: api.OpenShiftClusterProperties{
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &UpgradeableTo1,
+				},
+			},
+			cloudCredentialsName: "oh_no",
+			annotation:           nil,
+			wantAnnotation:       nil,
+			wantErr:              `cloudcredentials.operator.openshift.io "cluster" not found`,
+		},
+		{
+			name: "version persisted in cluster document, replace it",
+			cluster: api.OpenShiftClusterProperties{
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &UpgradeableTo1,
+				},
+			},
+			annotation: map[string]string{
+				"cloudcredential.openshift.io/upgradeable-to": "4.14.02",
+			},
+			wantAnnotation: map[string]string{
+				"cloudcredential.openshift.io/upgradeable-to": "4.14.59",
+			},
+		},
+		{
+			name: "annotations exist, append the upgradeable annotation",
+			cluster: api.OpenShiftClusterProperties{
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &UpgradeableTo1,
+				},
+			},
+			annotation: map[string]string{
+				"foo": "bar",
+			},
+			wantAnnotation: map[string]string{
+				"foo": "bar",
+				"cloudcredential.openshift.io/upgradeable-to": "4.14.59",
+			},
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := context.Background()
+			controller := gomock.NewController(t)
+			defer controller.Finish()
+
+			env := mock_env.NewMockInterface(controller)
+
+			oc := &api.OpenShiftCluster{
+				Properties: tt.cluster,
+			}
+
+			if tt.cloudCredentialsName == "" {
+				tt.cloudCredentialsName = "cluster"
+			}
+
+			cloudcredentialobject := &operatorv1.CloudCredential{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        tt.cloudCredentialsName,
+					Annotations: tt.annotation,
+				},
+			}
+
+			o := operator{
+				oc:          oc,
+				env:         env,
+				operatorcli: operatorfake.NewSimpleClientset(cloudcredentialobject),
+			}
+
+			err := o.EnsureUpgradeAnnotation(ctx)
+			utilerror.AssertErrorMessage(t, err, tt.wantErr)
+			result, _ := o.operatorcli.OperatorV1().CloudCredentials().List(ctx, metav1.ListOptions{})
+			for _, v := range result.Items {
+				actualAnnotations := v.ObjectMeta.Annotations
+				if !reflect.DeepEqual(actualAnnotations, tt.wantAnnotation) {
+					t.Errorf("actual annotation: %v, wanted %v", tt.annotation, tt.wantAnnotation)
+				}
+			}
+		})
+	}
+}