Repository: Azure/ARO-RP
Pull request #3609 (2024-05-31 17:17:19)

Revender hive to commit d7ead609f495785360aeea7c318f28fe82f9bcbf
### Which issue this PR addresses:

[ARO-5062](https://issues.redhat.com/browse/ARO-5062)

Fixes

### What this PR does / why we need it:

Keep Hive updated to mitigate vulnerabilities.

### Test plan for issue:

@slawande2 has tested this by creating a cluster with this hive commit in a full service dev rp environment.

### Is there any documentation that needs to be updated for this PR?

Updated [ARO Wiki](https://msazure.visualstudio.com/AzureRedHatOpenShift/_wiki/wikis/ARO.wiki/635070/Hive-Deploy-ARO-Pipelines) with `make ci-rp` step.

### How do you know this will function as expected in production? 

Cluster creation during E2E of Hive deployment will ensure hive functions.



Patch:
diff --git a/.sha256sum b/.sha256sum
index 8fae07dc7..cca696c9d 100644
--- a/.sha256sum
+++ b/.sha256sum
@@ -6,4 +6,4 @@ b1f1de0fe40d05de90742b17928968923b936adc294000f58974f50a297581dd  swagger/redhat
 c023515341196746454c0ae7af077d40d3ec13f6b88b33cb558f0a7ab17a5a24  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2023-07-01-preview/redhatopenshift.json
 440748951dd1c3b34b5ccbdcb7cd966e3b89490887a1f1d64429561fad789515  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/stable/2023-09-04/redhatopenshift.json
 74a46fdde6ceb0121fe1515c7e11e902dd921b54cffe693307fb02b3dc88f26e  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/stable/2023-11-22/redhatopenshift.json
-a27184734436629e24b344c3b5c015437f144e18e7eddce7e252a1ed4cda7bca  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
+f1e9d42d7c4c0081282e065e7845455db28ed6924687f1acecafb5fbc43ae0c3  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
diff --git a/Dockerfile.ci-rp b/Dockerfile.ci-rp
index 23ee37330..c1d608541 100644
--- a/Dockerfile.ci-rp
+++ b/Dockerfile.ci-rp
@@ -1,5 +1,5 @@
 ARG REGISTRY
-ARG VERSION
+ARG ARO_VERSION
 
 ###############################################################################
 # Stage 1: Build the SRE Portal Assets
@@ -22,6 +22,7 @@ RUN npm run lint && npm run build
 # Stage 2: Compile the Golang RP code
 ###############################################################################
 FROM ${REGISTRY}/ubi8/go-toolset:1.20.12-5 AS builder
+ARG ARO_VERSION
 USER root
 WORKDIR /app
 
@@ -49,11 +50,11 @@ COPY --from=portal-build /build/pkg/portal/assets/v2/build /app/pkg/portal/asset
 # Lint, generate, build, and test
 RUN golangci-lint run --verbose
 RUN go generate ./...
-RUN go build -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${VERSION}" ./cmd/aro
-RUN go test ./test/e2e/... -tags e2e,codec.safe -c -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${VERSION}" -o e2e.test
+RUN go build -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${ARO_VERSION}" ./cmd/aro
+RUN go test ./test/e2e/... -tags e2e,codec.safe -c -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${ARO_VERSION}" -o e2e.test
 
 # Additional tests
-RUN ARO_RUN_PKI_TESTS=nope go run gotest.tools/gotestsum@v1.11.0 --format pkgname --junitfile report.xml -- -coverprofile=cover.out ./...
+RUN ARO_SKIP_PKI_TESTS=true go run gotest.tools/gotestsum@v1.11.0 --format pkgname --junitfile report.xml -- -coverprofile=cover.out ./...
 RUN hack/fips/validate-fips.sh ./aro
 
 ###############################################################################
diff --git a/Makefile b/Makefile
index e5b53a309..d263c3e8d 100644
--- a/Makefile
+++ b/Makefile
@@ -77,7 +77,7 @@ client: generate
 	hack/build-client.sh "${AUTOREST_IMAGE}" 2020-04-30 2021-09-01-preview 2022-04-01 2022-09-04 2023-04-01 2023-07-01-preview 2023-09-04 2023-11-22 2024-08-12-preview
 
 ci-rp: fix-macos-vendor
-	docker build . -f Dockerfile.ci-rp --ulimit=nofile=4096:4096 --build-arg REGISTRY=$(REGISTRY) --build-arg VERSION=$(VERSION) --no-cache=$(NO_CACHE)
+	docker build . -f Dockerfile.ci-rp --ulimit=nofile=4096:4096 --build-arg REGISTRY=$(REGISTRY) --build-arg ARO_VERSION=$(VERSION) --no-cache=$(NO_CACHE)
 
 # TODO: hard coding dev-config.yaml is clunky; it is also probably convenient to
 # override COMMIT.
diff --git a/cmd/aro/mirror.go b/cmd/aro/mirror.go
index db0c7ec92..6e7e95bb7 100644
--- a/cmd/aro/mirror.go
+++ b/cmd/aro/mirror.go
@@ -134,7 +134,7 @@ func mirror(ctx context.Context, log *logrus.Entry) error {
 		"quay.io/app-sre/managed-upgrade-operator:v0.1.952-44b631a",
 
 		// https://quay.io/repository/app-sre/hive?tab=tags
-		"quay.io/app-sre/hive:83aedb9f6e",
+		"quay.io/app-sre/hive:d7ead609f4",
 	} {
 		log.Printf("mirroring %s -> %s", ref, pkgmirror.Dest(dstAcr+acrDomainSuffix, ref))
 
diff --git a/docs/prepare-your-dev-environment.md b/docs/prepare-your-dev-environment.md
index 159acc766..f5b67a9e8 100644
--- a/docs/prepare-your-dev-environment.md
+++ b/docs/prepare-your-dev-environment.md
@@ -34,7 +34,7 @@ This document goes through the development dependencies one requires in order to
         If you're using podman-machine, you will need to export the socket, for example::
 
             ```bash
-            export ARO_PODMAN_SOCKET=unix:///$HOME/.local/share/containers/podman/machine/qemu/podman.sock
+            export ARO_PODMAN_SOCKET=unix://$HOME/.local/share/containers/podman/machine/qemu/podman.sock
             ```
         
         You will also need to ensure that podman machine has enough resources::
diff --git a/docs/upstream-differences.md b/docs/upstream-differences.md
deleted file mode 100644
index f94f7ee71..000000000
--- a/docs/upstream-differences.md
+++ /dev/null
@@ -1,204 +0,0 @@
-# Upstream differences
-
-This file catalogues the differences of install approach between ARO and
-upstream OCP.
-
-## Installer carry patches
-
-1. Clone ARO and upstream repos:
-    ```sh
-    # clone our forked installer
-    git clone https://github.com/openshift/installer-aro.git
-    cd installer-aro
-
-    # add the upstream as a remote source
-    git remote add upstream https://github.com/openshift/installer.git
-    git fetch upstream -a
-    ```
-1. See carry patches from previous release:
-    ```sh
-    # list patches
-    git log upstream/release-X.Y-1..origin/release-X.Y-1-azure
-
-    # see diff of patches
-    git show upstream/release-X.Y-1..origin/release-X.Y-1-azure
-    ```
-
-## Installation differences
-
-* ARO does not use Terraform to create clusters, and instead uses ARM templates directly
-
-* ARO persists the install graph in the cluster storage account in a new "aro"
-  container / "graph" blob.
-
-* No managed identity (for now).
-
-* No IPv6 support (for now).
-
-* Upstream installer closely binds the installConfig (cluster) name, cluster
-  domain name, infra ID and Azure resource name prefix.  ARO separates these out
-  a little.  The installConfig (cluster) name and the domain name remain bound;
-  the infra ID and Azure resource name prefix are taken from the ARO resource
-  name.
-
-* API server public IP domain name label is not set.
-
-* ARO uses first party RHCOS OS images published by Microsoft.
-
-* ARO never creates xxxxx-bootstrap-pip-* for bootstrap VM, or the corresponding
-  NSG rule.
-
-* ARO does not create a outbound-provider Service on port 27627.
-
-* ARO deploys a private link service in order for the RP to be able to
-  communicate with the cluster.
-
-* ARO runs a dnsmasq service on the nodes through the use of a machineconfig to resolve api-int and *.apps domains on the node locally allowing for custom DNS configured on the VNET.
-
-# Introducing new OCP release into ARO RP
-
-To support a new version of OpenShift on ARO, you will need to reconcile [upstream changes](https://github.com/openshift/installer) with our [forked installer](https://github.com/openshift/installer-aro). This will not be a merge, but a cherry-pick of patches we've implemented.
-
-## Update installer fork
-
-To bring new OCP release branch into ARO installer fork:
-
-1. If not done already, fetch our fork and upstream repos:
-    ```sh
-    # clone our forked installer
-    # Alternatively, fork openshift/installer-aro and clone your fork
-    git clone https://github.com/openshift/installer-aro.git
-    cd installer-aro
-    
-    # add the upstream as a remote source
-    git remote add upstream https://github.com/openshift/installer.git
-    git fetch upstream -a
-    ```
-1. Assess and document differences in X.Y and X.Y-1 in upstream
-    ```sh
-    # diff the upstream X.Y with X.Y-1 and search for architecture changes
-    git diff upstream/release-X.Y-1 upstream/release-X.Y
-    
-    # pay particular attention to Terraform files, which may need to be moved into ARO's ARM templates
-    git diff upstream/release-X.Y-1 upstream/release-X.Y */azure/*.tf
-    ```
-2. Create a new X.Y release branch in our forked installer
-    ```sh
-    # create a new release branch in the fork based on the upstream
-    git checkout upstream/release-X.Y
-    git checkout -b release-X.Y-azure
-    ```
-3. If there is a golang version bump in this release, modify `./hack/build.sh` and `./hack/go-test.sh` with the new version, then verify these scripts still work and commit them
-4. Determine the patches you need to cherry-pick, based on the last (Y-1) release
-    ```sh
-    # find commit shas to cherry-pick from last time
-    git checkout release-X.Y-1-azure
-    git log
-    ```
-5. For every commit you need to cherry-pick (in-order), do:
-    ```sh
-    # WARNING: when you reach the commit for `commit data/assets_vfsdata.go`, look ahead
-    git cherry-pick abc123 # may require manually fixing a merge
-    ./hack/build.sh   # fix any failures
-    ./hack/go-test.sh # fix any failures
-    # if you had to manually merge, you can now `git cherry-pick --continue`
-    ```
-    - When cherry-picking the specific patch `commit data/assets_vfsdata.go`, instead run:
-        ```sh
-        git cherry-pick abc123 # may require manually fixing a merge
-        ./hack/build.sh   # fix any failures
-        ./hack/go-test.sh # fix any failures
-        # if you had to manually merge, you can now `git cherry-pick --continue`
-        pushd ./hack/assets && go run ./assets.go && popd
-        ./hack/build.sh   # fix any failures
-        ./hack/go-test.sh # fix any failures
-        git add data/assets_vfsdata.go
-        git commit --amend
-        ```
-
-**Note:** If any changes are required during the process, make sure to amend the relevant patch or create a new one.
-Each commit should be atomic/complete - you should be able to cherry-pick it into the upstream installer and bring
-the fix or feature it carries in full, without a need to cherry-pick additional commits.
-This makes it easier to understand the nature of the patch as well as contribute our carry patches
-back to the upstream installer.
-
-## Update Installer Fork Again
-
-In the far or near future after you have [initially patched the installer](#update-installer-fork), you may need to pull in additional upstream changes that have happened since you patched the installer (e.g. upstream added a bugfix since your cherry-picking). The easiest way to pull in these changes safely is:
-
-```sh
-git fetch upstream -a
-git checkout release-X.Y-azure
-git pull upstream/release-X.Y --rebase=interactive
-```
-
-When you get to the editor mode to set up your rebase, you should do a few things:
-
-1. after each `pick` line, add the verification commands with:
-    ```text
-    exec ./hack/build.sh
-    exec ./hack/go-test.sh
-    ```
-1. change the line for commit `data/assets_vfsdata.go` from `pick` to `edit` so you can regenerate assets as outlined [above](#update-installer-fork), and verify it manually.
-
-By the end of this editing process, you should have a rebase file that looks something like:
-
-```text
-pick abc123 patch A
-exec ./hack/build.sh
-exec ./hack/go-test.sh
-pick def234 patch B
-exec ./hack/build.sh
-exec ./hack/go-test.sh
-edit ghi345 patch data/assets_vfsdata.go
-exec ./hack/build.sh
-exec ./hack/go-test.sh
-pick jkl456 patch C
-exec ./hack/build.sh
-exec ./hack/go-test.sh
-{...}
-```
-
-When you are finished, you can write/close the file editor to perform the rebase on the upstream differences while verifying that every patch still works along the way.
-
-Get a repo admin to create `release-X.Y-azure` in [openshift/installer-aro](https://github.com/openshift/installer-aro). You can now push your local branch `release-X.Y-azure` to your own fork of [openshift/installer-aro](https://github.com/openshift/installer-aro) and send a PR.
-
-# Update ARO Installer Wrapper and ARO-RP
-
-Once installer fork is ready, perform the following changes in the [ARO Installer Wrapper](https://github.com/openshift/installer-aro-wrapper):
-
-1. Update `go mod edit -replace` calls in `hack/update-go-module-dependencies.sh` to use a new release-X.Y branch.
-    * Make sure to read comments in the script.
-1. `make vendor`.
-    * You most likely will have to make changes to the codebase at this point to adjust it to new versions of dependencies.
-    * Also you likely will have to repeat this step several time until you resolve all conflicting dependencies.
-      Follow `go mod` failures, which will tell you what module requires what other module.
-      You will probably need to look at the `go.mod` files of these modules and see whether they set own replace directives,
-      as the script is likely to fail with something like this:
-
-      ```
-      go: github.com/openshift/installer@v0.16.1 requires
-          github.com/openshift/cluster-api-provider-kubevirt@v0.0.0-20201214114543-e5aed9c73f1f requires
-          kubevirt.io/client-go@v0.0.0-00010101000000-000000000000: invalid version: unknown revision 000000000000
-      ```
-
-      In the example above you need to:
-        * Checkout `github.com/openshift/cluster-api-provider-kubevirt` at commit `e5aed9c73f1f`.
-        * In go.mod find a replace directive for `kubevirt.io/client-go`.
-        * Add/update relevant replace directive in `go.mod`.
-1. `make generate`.
-1. A local image can be built for testing purposes and pushed to a dev ACR repo. The process for [publishing the final image uses ADO](https://msazure.visualstudio.com/AzureRedHatOpenShift/_wiki/wikis/ARO.wiki/452838/ARO-Installer-Image-Deployment-Process)
-
-In the ARO-RP codebase:
-
-1. Update `pkg/util/version/const.go` to point to the new release.
-    * You should be able to find latest published release and image hash [on quay.io](https://quay.io/repository/openshift-release-dev/ocp-release?tab=tags).
-1. (Optional) `make discoverycache`.
-    * This command requires a running cluster with the new version.
-1. The list of the hard-coded namespaces in `pkg/util/namespace/namespace.go` needs to be updated regularly as every
-   minor version of upstream OCP introduces a new namespace or two.
-
-
-Publish RHCOS image to the Azure Cloud Partner Portal:
-1. Publish RHCOS image. See [this document](https://github.com/openshift/installer-aro-wrapper/blob/main/docs/publish-rhcos-image.md).
-1. After this point, you should be able to create a dev cluster using the RP and it should use the new release.
diff --git a/env.example b/env.example
index a265550d1..2f420e764 100644
--- a/env.example
+++ b/env.example
@@ -3,4 +3,6 @@ export ARO_IMAGE=arointsvc.azurecr.io/aro:latest
 export NO_CACHE=false
 export AZURE_EXTENSION_DEV_SOURCES="$(pwd)/python"
 
+export ARO_SKIP_PKI_TESTS=true
+
 . secrets/env
diff --git a/go.mod b/go.mod
index 3c3c3f0d4..6bcf71d8d 100644
--- a/go.mod
+++ b/go.mod
@@ -29,7 +29,7 @@ require (
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
 	github.com/go-chi/chi/v5 v5.0.8
-	github.com/go-logr/logr v1.4.1
+	github.com/go-logr/logr v1.4.2
 	github.com/go-test/deep v1.1.0
 	github.com/gofrs/uuid v4.2.0+incompatible
 	github.com/golang-jwt/jwt/v4 v4.5.0
@@ -59,7 +59,7 @@ require (
 	github.com/openshift/api v3.9.1-0.20191111211345-a27ff30ebf09+incompatible
 	github.com/openshift/client-go v0.0.0-20220525160904-9e1acff93e4a
 	github.com/openshift/cloud-credential-operator v0.0.0-00010101000000-000000000000
-	github.com/openshift/hive/apis v0.0.0-20240510150258-83aedb9f6e73
+	github.com/openshift/hive/apis v0.0.0-20240529172037-d7ead609f495
 	github.com/openshift/library-go v0.0.0-20220525173854-9b950a41acdc
 	github.com/openshift/machine-config-operator v0.0.1-0.20230519222939-1abc13efbb0d
 	github.com/pires/go-proxyproto v0.6.2
@@ -80,9 +80,9 @@ require (
 	golang.org/x/sync v0.6.0
 	golang.org/x/text v0.15.0
 	golang.org/x/tools v0.19.0
-	k8s.io/api v0.30.0
+	k8s.io/api v0.30.1
 	k8s.io/apiextensions-apiserver v0.25.0
-	k8s.io/apimachinery v0.30.0
+	k8s.io/apimachinery v0.30.1
 	k8s.io/cli-runtime v0.25.16
 	k8s.io/client-go v0.26.2
 	k8s.io/code-generator v0.25.16
diff --git a/go.sum b/go.sum
index cb7db633d..5132972fd 100644
--- a/go.sum
+++ b/go.sum
@@ -233,8 +233,8 @@ github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQr
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
-github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A=
diff --git a/pkg/api/admin/openshiftcluster.go b/pkg/api/admin/openshiftcluster.go
index d1bab4bb6..ed328772e 100644
--- a/pkg/api/admin/openshiftcluster.go
+++ b/pkg/api/admin/openshiftcluster.go
@@ -418,9 +418,13 @@ type IngressProfile struct {
 
 // PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.
 type PlatformWorkloadIdentityProfile struct {
+	UpgradeableTo              *UpgradeableTo             `json:"upgradeableTo,omitempty"`
 	PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"`
 }
 
+// UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to
+type UpgradeableTo string
+
 // PlatformWorkloadIdentity stores information representing a single workload identity.
 type PlatformWorkloadIdentity struct {
 	OperatorName string `json:"operatorName,omitempty"`
diff --git a/pkg/api/openshiftcluster.go b/pkg/api/openshiftcluster.go
index 0826520d0..caa157d71 100644
--- a/pkg/api/openshiftcluster.go
+++ b/pkg/api/openshiftcluster.go
@@ -773,9 +773,13 @@ type HiveProfile struct {
 type PlatformWorkloadIdentityProfile struct {
 	MissingFields
 
+	UpgradeableTo              *UpgradeableTo             `json:"upgradeableTo,omitempty"`
 	PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"`
 }
 
+// UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to
+type UpgradeableTo string
+
 // PlatformWorkloadIdentity stores information representing a single workload identity.
 type PlatformWorkloadIdentity struct {
 	MissingFields
@@ -803,4 +807,5 @@ type Identity struct {
 
 	Type                   string                 `json:"type,omitempty"`
 	UserAssignedIdentities UserAssignedIdentities `json:"userAssignedIdentities,omitempty"`
+	IdentityURL            string                 `json:"identityURL,omitempty" mutable:"true"`
 }
diff --git a/pkg/api/v20240812preview/openshiftcluster.go b/pkg/api/v20240812preview/openshiftcluster.go
index 09a5a632d..28e7d4bd1 100644
--- a/pkg/api/v20240812preview/openshiftcluster.go
+++ b/pkg/api/v20240812preview/openshiftcluster.go
@@ -290,15 +290,19 @@ type IngressProfile struct {
 
 // PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.
 type PlatformWorkloadIdentityProfile struct {
-	PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"`
+	UpgradeableTo              *UpgradeableTo             `json:"upgradeableTo,omitempty" mutable:"true"`
+	PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty" mutable:"true"`
 }
 
+// UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to
+type UpgradeableTo string
+
 // PlatformWorkloadIdentity stores information representing a single workload identity.
 type PlatformWorkloadIdentity struct {
-	OperatorName string `json:"operatorName,omitempty"`
-	ResourceID   string `json:"resourceId,omitempty"`
-	ClientID     string `json:"clientId,omitempty" swagger:"readOnly"`
-	ObjectID     string `json:"objectId,omitempty" swagger:"readOnly"`
+	OperatorName string `json:"operatorName,omitempty" mutable:"true"`
+	ResourceID   string `json:"resourceId,omitempty" mutable:"true"`
+	ClientID     string `json:"clientId,omitempty" swagger:"readOnly" mutable:"true"`
+	ObjectID     string `json:"objectId,omitempty" swagger:"readOnly" mutable:"true"`
 }
 
 // ClusterUserAssignedIdentity stores information about a user-assigned managed identity in a predefined format required by Microsoft's Managed Identity team.
diff --git a/pkg/api/v20240812preview/openshiftcluster_convert.go b/pkg/api/v20240812preview/openshiftcluster_convert.go
index 1d6ab240a..80d0793b6 100644
--- a/pkg/api/v20240812preview/openshiftcluster_convert.go
+++ b/pkg/api/v20240812preview/openshiftcluster_convert.go
@@ -141,6 +141,12 @@ func (c openShiftClusterConverter) ToExternal(oc *api.OpenShiftCluster) interfac
 
 	if oc.Properties.PlatformWorkloadIdentityProfile != nil && oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities != nil {
 		out.Properties.PlatformWorkloadIdentityProfile = &PlatformWorkloadIdentityProfile{}
+
+		if oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo != nil {
+			temp := UpgradeableTo(*oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo)
+			out.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo = &temp
+		}
+
 		out.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities = make([]PlatformWorkloadIdentity, len(oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities))
 
 		for i := range oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities {
@@ -225,6 +231,12 @@ func (c openShiftClusterConverter) ToInternal(_oc interface{}, out *api.OpenShif
 	}
 	if oc.Properties.PlatformWorkloadIdentityProfile != nil && oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities != nil {
 		out.Properties.PlatformWorkloadIdentityProfile = &api.PlatformWorkloadIdentityProfile{}
+
+		if oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo != nil {
+			temp := api.UpgradeableTo(*oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo)
+			out.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo = &temp
+		}
+
 		out.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities = make([]api.PlatformWorkloadIdentity, len(oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities))
 
 		for i := range oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities {
diff --git a/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go b/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go
index e438d5fc1..2535b27f7 100644
--- a/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go
+++ b/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go
@@ -1525,6 +1525,7 @@ func (pwi PlatformWorkloadIdentity) MarshalJSON() ([]byte, error) {
 // PlatformWorkloadIdentityProfile platformWorkloadIdentityProfile encapsulates all information that is
 // specific to workload identity clusters.
 type PlatformWorkloadIdentityProfile struct {
+	UpgradeableTo              *string                     `json:"upgradeableTo,omitempty"`
 	PlatformWorkloadIdentities *[]PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"`
 }
 
diff --git a/pkg/cluster/arooperator.go b/pkg/cluster/arooperator.go
index 7afbb5b3c..ca187e23f 100644
--- a/pkg/cluster/arooperator.go
+++ b/pkg/cluster/arooperator.go
@@ -65,6 +65,10 @@ func (m *manager) ensureCredentialsRequest(ctx context.Context) error {
 	return m.aroOperatorDeployer.CreateOrUpdateCredentialsRequest(ctx)
 }
 
+func (m *manager) ensureUpgradeAnnotation(ctx context.Context) error {
+	return m.aroOperatorDeployer.EnsureUpgradeAnnotation(ctx)
+}
+
 func (m *manager) renewMDSDCertificate(ctx context.Context) error {
 	return m.aroOperatorDeployer.RenewMDSDCertificate(ctx)
 }
diff --git a/pkg/cluster/install.go b/pkg/cluster/install.go
index 5e14d57ef..072b9cb29 100644
--- a/pkg/cluster/install.go
+++ b/pkg/cluster/install.go
@@ -213,6 +213,7 @@ func (m *manager) Update(ctx context.Context) error {
 		steps.Action(m.updateAROSecret),
 		steps.Action(m.restartAROOperatorMaster), // depends on m.updateOpenShiftSecret; the point of restarting is to pick up any changes made to the secret
 		steps.Condition(m.aroDeploymentReady, 5*time.Minute, true),
+		steps.Action(m.ensureUpgradeAnnotation),
 		steps.Action(m.reconcileLoadBalancerProfile),
 	}
 
@@ -524,7 +525,7 @@ func (m *manager) initializeKubernetesClients(ctx context.Context) error {
 // initializeKubernetesClients initializes clients which are used
 // once the cluster is up later on in the install process.
 func (m *manager) initializeOperatorDeployer(ctx context.Context) (err error) {
-	m.aroOperatorDeployer, err = deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.client, m.extensionscli, m.kubernetescli)
+	m.aroOperatorDeployer, err = deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.client, m.extensionscli, m.kubernetescli, m.operatorcli)
 	return
 }
 
diff --git a/pkg/frontend/openshiftcluster_putorpatch.go b/pkg/frontend/openshiftcluster_putorpatch.go
index 1fa6f26fd..5c6308978 100644
--- a/pkg/frontend/openshiftcluster_putorpatch.go
+++ b/pkg/frontend/openshiftcluster_putorpatch.go
@@ -25,6 +25,8 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/version"
 )
 
+var errMissingIdentityURL error = fmt.Errorf("identityURL not provided but required for workload identity cluster")
+
 func (f *frontend) putOrPatchOpenShiftCluster(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 	log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry)
@@ -41,10 +43,12 @@ func (f *frontend) putOrPatchOpenShiftCluster(w http.ResponseWriter, r *http.Req
 	subId := chi.URLParam(r, "subscriptionId")
 	resourceProviderNamespace := chi.URLParam(r, "resourceProviderNamespace")
 
+	identityURL := r.Header.Get("x-ms-identity-url")
+
 	apiVersion := r.URL.Query().Get(api.APIVersionKey)
 	err := cosmosdb.RetryOnPreconditionFailed(func() error {
 		var err error
-		b, err = f._putOrPatchOpenShiftCluster(ctx, log, body, correlationData, systemData, r.URL.Path, originalPath, r.Method, referer, &header, f.apis[apiVersion].OpenShiftClusterConverter, f.apis[apiVersion].OpenShiftClusterStaticValidator, subId, resourceProviderNamespace, apiVersion)
+		b, err = f._putOrPatchOpenShiftCluster(ctx, log, body, correlationData, systemData, r.URL.Path, originalPath, r.Method, referer, &header, f.apis[apiVersion].OpenShiftClusterConverter, f.apis[apiVersion].OpenShiftClusterStaticValidator, subId, resourceProviderNamespace, apiVersion, identityURL)
 		return err
 	})
 
@@ -52,7 +56,7 @@ func (f *frontend) putOrPatchOpenShiftCluster(w http.ResponseWriter, r *http.Req
 	reply(log, w, header, b, err)
 }
 
-func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus.Entry, body []byte, correlationData *api.CorrelationData, systemData *api.SystemData, path, originalPath, method, referer string, header *http.Header, converter api.OpenShiftClusterConverter, staticValidator api.OpenShiftClusterStaticValidator, subId, resourceProviderNamespace string, apiVersion string) ([]byte, error) {
+func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus.Entry, body []byte, correlationData *api.CorrelationData, systemData *api.SystemData, path, originalPath, method, referer string, header *http.Header, converter api.OpenShiftClusterConverter, staticValidator api.OpenShiftClusterStaticValidator, subId, resourceProviderNamespace string, apiVersion string, identityURL string) ([]byte, error) {
 	subscription, err := f.validateSubscriptionState(ctx, path, api.SubscriptionStateRegistered)
 	if err != nil {
 		return nil, err
@@ -86,11 +90,17 @@ func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus.
 				},
 			},
 		}
+
 		if !f.env.IsLocalDevelopmentMode() /* not local dev or CI */ {
 			doc.OpenShiftCluster.Properties.FeatureProfile.GatewayEnabled = true
 		}
 	}
 
+	err = validateIdentityUrl(doc.OpenShiftCluster, identityURL, isCreate)
+	if err != nil {
+		return nil, err
+	}
+
 	doc.CorrelationData = correlationData
 
 	err = validateTerminalProvisioningState(doc.OpenShiftCluster.Properties.ProvisioningState)
@@ -288,6 +298,24 @@ func enrichClusterSystemData(doc *api.OpenShiftClusterDocument, systemData *api.
 	}
 }
 
+func validateIdentityUrl(cluster *api.OpenShiftCluster, identityURL string, isCreate bool) error {
+	// Don't persist identity URL in non-wimi clusters
+	if cluster.Properties.ServicePrincipalProfile != nil || cluster.Identity == nil {
+		return nil
+	}
+
+	if identityURL == "" {
+		if isCreate {
+			return errMissingIdentityURL
+		}
+		return nil
+	}
+
+	cluster.Identity.IdentityURL = identityURL
+
+	return nil
+}
+
 func (f *frontend) ValidateNewCluster(ctx context.Context, subscription *api.SubscriptionDocument, cluster *api.OpenShiftCluster, staticValidator api.OpenShiftClusterStaticValidator, ext interface{}, path string) error {
 	err := staticValidator.Static(ext, nil, f.env.Location(), f.env.Domain(), f.env.FeatureIsSet(env.FeatureRequireD2sV3Workers), path)
 	if err != nil {
diff --git a/pkg/operator/deploy/deploy.go b/pkg/operator/deploy/deploy.go
index ab18e2505..1fbc93a7c 100644
--- a/pkg/operator/deploy/deploy.go
+++ b/pkg/operator/deploy/deploy.go
@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/go-multierror"
+	operatorclient "github.com/openshift/client-go/operator/clientset/versioned"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	extensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -56,6 +57,7 @@ type Operator interface {
 	Restart(context.Context, []string) error
 	IsRunningDesiredVersion(context.Context) (bool, error)
 	RenewMDSDCertificate(context.Context) error
+	EnsureUpgradeAnnotation(context.Context) error
 }
 
 type operator struct {
@@ -67,10 +69,11 @@ type operator struct {
 	client        client.Client
 	extensionscli extensionsclient.Interface
 	kubernetescli kubernetes.Interface
+	operatorcli   operatorclient.Interface
 	dh            dynamichelper.Interface
 }
 
-func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli aroclient.Interface, client client.Client, extensionscli extensionsclient.Interface, kubernetescli kubernetes.Interface) (Operator, error) {
+func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli aroclient.Interface, client client.Client, extensionscli extensionsclient.Interface, kubernetescli kubernetes.Interface, operatorcli operatorclient.Interface) (Operator, error) {
 	restConfig, err := restconfig.RestConfig(env, oc)
 	if err != nil {
 		return nil, err
@@ -89,6 +92,7 @@ func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli
 		client:        client,
 		extensionscli: extensionscli,
 		kubernetescli: kubernetescli,
+		operatorcli:   operatorcli,
 		dh:            dh,
 	}, nil
 }
@@ -432,6 +436,34 @@ func (o *operator) RenewMDSDCertificate(ctx context.Context) error {
 	return nil
 }
 
+func (o *operator) EnsureUpgradeAnnotation(ctx context.Context) error {
+	if o.oc.Properties.PlatformWorkloadIdentityProfile == nil ||
+		o.oc.Properties.ServicePrincipalProfile != nil {
+		return nil
+	}
+
+	upgradeableTo := string(*o.oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo)
+	upgradeableAnnotation := "cloudcredential.openshift.io/upgradeable-to"
+
+	cloudcredentialobject, err := o.operatorcli.OperatorV1().CloudCredentials().Get(ctx, "cluster", metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	if cloudcredentialobject.Annotations == nil {
+		cloudcredentialobject.Annotations = map[string]string{}
+	}
+
+	cloudcredentialobject.Annotations[upgradeableAnnotation] = upgradeableTo
+
+	_, err = o.operatorcli.OperatorV1().CloudCredentials().Update(ctx, cloudcredentialobject, metav1.UpdateOptions{})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (o *operator) IsReady(ctx context.Context) (bool, error) {
 	ok, err := ready.CheckDeploymentIsReady(ctx, o.kubernetescli.AppsV1().Deployments(pkgoperator.Namespace), "aro-operator-master")()
 	o.log.Infof("deployment %q ok status is: %v, err is: %v", "aro-operator-master", ok, err)
diff --git a/pkg/util/arm/deploy.go b/pkg/util/arm/deploy.go
index 3bad254fc..6a34d51a6 100644
--- a/pkg/util/arm/deploy.go
+++ b/pkg/util/arm/deploy.go
@@ -34,7 +34,8 @@ func DeployTemplate(ctx context.Context, log *logrus.Entry, deployments features
 	}
 
 	if azureerrors.HasAuthorizationFailedError(err) ||
-		azureerrors.HasLinkedAuthorizationFailedError(err) {
+		azureerrors.HasLinkedAuthorizationFailedError(err) ||
+		azureerrors.IsDeploymentMissingPermissionsError(err) {
 		return err
 	}
 
diff --git a/pkg/util/azureerrors/error.go b/pkg/util/azureerrors/error.go
index 9211ef695..1262558c0 100644
--- a/pkg/util/azureerrors/error.go
+++ b/pkg/util/azureerrors/error.go
@@ -14,6 +14,10 @@ import (
 	"github.com/Azure/ARO-RP/pkg/api"
 )
 
+const (
+	CodeInvalidTemplateDeployment = "InvalidTemplateDeployment"
+)
+
 // HasAuthorizationFailedError returns true it the error is, or contains, an
 // AuthorizationFailed error
 func HasAuthorizationFailedError(err error) bool {
@@ -60,6 +64,22 @@ func deploymentFailedDueToAuthError(err error, authCode string) bool {
 	return false
 }
 
+// IsDeploymentMissingPermissionsError returns true if the error indicates that
+// ARM rejected a template deployment pre-flight due to missing role
+// assignments.
+// This can be an indicator of role assignment propagation delay.
+func IsDeploymentMissingPermissionsError(err error) bool {
+	if detailedErr, ok := err.(autorest.DetailedError); ok {
+		if serviceErr, ok := detailedErr.Original.(*azure.ServiceError); ok {
+			if serviceErr.Code == CodeInvalidTemplateDeployment && strings.Contains(serviceErr.Message, "Authorization failed for template resource") {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
 // IsDeploymentActiveError returns true it the error is a DeploymentActive error
 func IsDeploymentActiveError(err error) bool {
 	if detailedErr, ok := err.(autorest.DetailedError); ok {
diff --git a/pkg/util/mocks/operator/deploy/deploy.go b/pkg/util/mocks/operator/deploy/deploy.go
index 99ad4f224..30f4efc25 100644
--- a/pkg/util/mocks/operator/deploy/deploy.go
+++ b/pkg/util/mocks/operator/deploy/deploy.go
@@ -62,6 +62,20 @@ func (mr *MockOperatorMockRecorder) CreateOrUpdateCredentialsRequest(arg0 interf
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateCredentialsRequest", reflect.TypeOf((*MockOperator)(nil).CreateOrUpdateCredentialsRequest), arg0)
 }
 
+// EnsureUpgradeAnnotation mocks base method.
+func (m *MockOperator) EnsureUpgradeAnnotation(arg0 context.Context) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "EnsureUpgradeAnnotation", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// EnsureUpgradeAnnotation indicates an expected call of EnsureUpgradeAnnotation.
+func (mr *MockOperatorMockRecorder) EnsureUpgradeAnnotation(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureUpgradeAnnotation", reflect.TypeOf((*MockOperator)(nil).EnsureUpgradeAnnotation), arg0)
+}
+
 // IsReady mocks base method.
 func (m *MockOperator) IsReady(arg0 context.Context) (bool, error) {
 	m.ctrl.T.Helper()
diff --git a/pkg/util/steps/refreshing.go b/pkg/util/steps/refreshing.go
index 96fbdf71f..8138ecda6 100644
--- a/pkg/util/steps/refreshing.go
+++ b/pkg/util/steps/refreshing.go
@@ -77,6 +77,7 @@ func (s *authorizationRefreshingActionStep) run(ctx context.Context, log *logrus
 			(azureerrors.IsUnauthorizedClientError(err) ||
 				azureerrors.HasAuthorizationFailedError(err) ||
 				azureerrors.IsInvalidSecretError(err) ||
+				azureerrors.IsDeploymentMissingPermissionsError(err) ||
 				err == ErrWantRefresh) {
 			log.Printf("auth error, refreshing and retrying: %v", err)
 			// Try refreshing auth.
diff --git a/pkg/util/version/const.go b/pkg/util/version/const.go
index e1b96a1b3..01f05a4f1 100644
--- a/pkg/util/version/const.go
+++ b/pkg/util/version/const.go
@@ -34,8 +34,8 @@ type Stream struct {
 // This default is left here ONLY for use by local development mode,
 // until we can come up with a better solution.
 var DefaultInstallStream = Stream{
-	Version:  NewVersion(4, 13, 23),
-	PullSpec: "quay.io/openshift-release-dev/ocp-release@sha256:ca556d3494d08765c90481f15dd965995371168ea7ee7a551000bed4481931c8",
+	Version:  NewVersion(4, 13, 40),
+	PullSpec: "quay.io/openshift-release-dev/ocp-release@sha256:c1f69e6137bc9cda2c6da56bafbc7ea969900acb5e5c349b1ebb2103b10b424f",
 }
 
 // FluentbitImage contains the location of the Fluentbit container image
diff --git a/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models.py b/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models.py
index 3e77e2302..b7e1fab63 100644
--- a/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models.py
+++ b/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models.py
@@ -1303,12 +1303,16 @@ class PlatformWorkloadIdentity(msrest.serialization.Model):
 class PlatformWorkloadIdentityProfile(msrest.serialization.Model):
     """PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.
 
+    :ivar upgradeable_to: UpgradeableTo stores a single OpenShift version a workload identity
+     cluster can be upgraded to.
+    :vartype upgradeable_to: str
     :ivar platform_workload_identities:
     :vartype platform_workload_identities:
      list[~azure.mgmt.redhatopenshift.v2024_08_12_preview.models.PlatformWorkloadIdentity]
     """
 
     _attribute_map = {
+        'upgradeable_to': {'key': 'upgradeableTo', 'type': 'str'},
         'platform_workload_identities': {'key': 'platformWorkloadIdentities', 'type': '[PlatformWorkloadIdentity]'},
     }
 
@@ -1317,11 +1321,15 @@ class PlatformWorkloadIdentityProfile(msrest.serialization.Model):
         **kwargs
     ):
         """
+        :keyword upgradeable_to: UpgradeableTo stores a single OpenShift version a workload identity
+         cluster can be upgraded to.
+        :paramtype upgradeable_to: str
         :keyword platform_workload_identities:
         :paramtype platform_workload_identities:
          list[~azure.mgmt.redhatopenshift.v2024_08_12_preview.models.PlatformWorkloadIdentity]
         """
         super(PlatformWorkloadIdentityProfile, self).__init__(**kwargs)
+        self.upgradeable_to = kwargs.get('upgradeable_to', None)
         self.platform_workload_identities = kwargs.get('platform_workload_identities', None)
 
 
diff --git a/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models_py3.py b/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models_py3.py
index 9828713ff..b8c2d9a1d 100644
--- a/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models_py3.py
+++ b/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models_py3.py
@@ -1412,27 +1412,36 @@ class PlatformWorkloadIdentity(msrest.serialization.Model):
 class PlatformWorkloadIdentityProfile(msrest.serialization.Model):
     """PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.
 
+    :ivar upgradeable_to: UpgradeableTo stores a single OpenShift version a workload identity
+     cluster can be upgraded to.
+    :vartype upgradeable_to: str
     :ivar platform_workload_identities:
     :vartype platform_workload_identities:
      list[~azure.mgmt.redhatopenshift.v2024_08_12_preview.models.PlatformWorkloadIdentity]
     """
 
     _attribute_map = {
+        'upgradeable_to': {'key': 'upgradeableTo', 'type': 'str'},
         'platform_workload_identities': {'key': 'platformWorkloadIdentities', 'type': '[PlatformWorkloadIdentity]'},
     }
 
     def __init__(
         self,
         *,
+        upgradeable_to: Optional[str] = None,
         platform_workload_identities: Optional[List["PlatformWorkloadIdentity"]] = None,
         **kwargs
     ):
         """
+        :keyword upgradeable_to: UpgradeableTo stores a single OpenShift version a workload identity
+         cluster can be upgraded to.
+        :paramtype upgradeable_to: str
         :keyword platform_workload_identities:
         :paramtype platform_workload_identities:
          list[~azure.mgmt.redhatopenshift.v2024_08_12_preview.models.PlatformWorkloadIdentity]
         """
         super(PlatformWorkloadIdentityProfile, self).__init__(**kwargs)
+        self.upgradeable_to = upgradeable_to
         self.platform_workload_identities = platform_workload_identities
 
 
diff --git a/swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json b/swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
index c65beac00..95e0131f5 100644
--- a/swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
+++ b/swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
@@ -2349,6 +2349,9 @@
       "description": "PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.",
       "type": "object",
       "properties": {
+        "upgradeableTo": {
+          "$ref": "#/definitions/UpgradeableTo"
+        },
         "platformWorkloadIdentities": {
           "type": "array",
           "items": {
@@ -2603,6 +2606,10 @@
         "type": "string"
       }
     },
+    "UpgradeableTo": {
+      "description": "UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to",
+      "type": "string"
+    },
     "UserAssignedIdentities": {
       "description": "UserAssignedIdentities stores a mapping from resource IDs of managed identities to their client/principal IDs.",
       "type": "object",
diff --git a/vendor/github.com/go-logr/logr/README.md b/vendor/github.com/go-logr/logr/README.md
index 8969526a6..7c7f0c69c 100644
--- a/vendor/github.com/go-logr/logr/README.md
+++ b/vendor/github.com/go-logr/logr/README.md
@@ -1,6 +1,7 @@
 # A minimal logging API for Go
 
 [![Go Reference](https://pkg.go.dev/badge/github.com/go-logr/logr.svg)](https://pkg.go.dev/github.com/go-logr/logr)
+[![Go Report Card](https://goreportcard.com/badge/github.com/go-logr/logr)](https://goreportcard.com/report/github.com/go-logr/logr)
 [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/go-logr/logr/badge)](https://securityscorecards.dev/viewer/?platform=github.com&org=go-logr&repo=logr)
 
 logr offers an(other) opinion on how Go programs and libraries can do logging
diff --git a/vendor/github.com/go-logr/logr/funcr/funcr.go b/vendor/github.com/go-logr/logr/funcr/funcr.go
index fb2f866f4..30568e768 100644
--- a/vendor/github.com/go-logr/logr/funcr/funcr.go
+++ b/vendor/github.com/go-logr/logr/funcr/funcr.go
@@ -236,15 +236,14 @@ func newFormatter(opts Options, outfmt outputFormat) Formatter {
 // implementation. It should be constructed with NewFormatter. Some of
 // its methods directly implement logr.LogSink.
 type Formatter struct {
-	outputFormat    outputFormat
-	prefix          string
-	values          []any
-	valuesStr       string
-	parentValuesStr string
-	depth           int
-	opts            *Options
-	group           string // for slog groups
-	groupDepth      int
+	outputFormat outputFormat
+	prefix       string
+	values       []any
+	valuesStr    string
+	depth        int
+	opts         *Options
+	groupName    string // for slog groups
+	groups       []groupDef
 }
 
 // outputFormat indicates which outputFormat to use.
@@ -257,6 +256,13 @@ const (
 	outputJSON
 )
 
+// groupDef represents a saved group.  The values may be empty, but we don't
+// know if we need to render the group until the final record is rendered.
+type groupDef struct {
+	name   string
+	values string
+}
+
 // PseudoStruct is a list of key-value pairs that gets logged as a struct.
 type PseudoStruct []any
 
@@ -264,76 +270,102 @@ type PseudoStruct []any
 func (f Formatter) render(builtins, args []any) string {
 	// Empirically bytes.Buffer is faster than strings.Builder for this.
 	buf := bytes.NewBuffer(make([]byte, 0, 1024))
+
 	if f.outputFormat == outputJSON {
-		buf.WriteByte('{') // for the whole line
+		buf.WriteByte('{') // for the whole record
 	}
 
+	// Render builtins
 	vals := builtins
 	if hook := f.opts.RenderBuiltinsHook; hook != nil {
 		vals = hook(f.sanitize(vals))
 	}
-	f.flatten(buf, vals, false, false) // keys are ours, no need to escape
+	f.flatten(buf, vals, false) // keys are ours, no need to escape
 	continuing := len(builtins) > 0
 
-	if f.parentValuesStr != "" {
-		if continuing {
-			buf.WriteByte(f.comma())
+	// Turn the inner-most group into a string
+	argsStr := func() string {
+		buf := bytes.NewBuffer(make([]byte, 0, 1024))
+
+		vals = args
+		if hook := f.opts.RenderArgsHook; hook != nil {
+			vals = hook(f.sanitize(vals))
 		}
-		buf.WriteString(f.parentValuesStr)
-		continuing = true
-	}
+		f.flatten(buf, vals, true) // escape user-provided keys
 
-	groupDepth := f.groupDepth
-	if f.group != "" {
-		if f.valuesStr != "" || len(args) != 0 {
-			if continuing {
-				buf.WriteByte(f.comma())
-			}
-			buf.WriteString(f.quoted(f.group, true)) // escape user-provided keys
-			buf.WriteByte(f.colon())
-			buf.WriteByte('{') // for the group
-			continuing = false
-		} else {
-			// The group was empty
-			groupDepth--
+		return buf.String()
+	}()
+
+	// Render the stack of groups from the inside out.
+	bodyStr := f.renderGroup(f.groupName, f.valuesStr, argsStr)
+	for i := len(f.groups) - 1; i >= 0; i-- {
+		grp := &f.groups[i]
+		if grp.values == "" && bodyStr == "" {
+			// no contents, so we must elide the whole group
+			continue
 		}
+		bodyStr = f.renderGroup(grp.name, grp.values, bodyStr)
 	}
 
-	if f.valuesStr != "" {
+	if bodyStr != "" {
 		if continuing {
 			buf.WriteByte(f.comma())
 		}
-		buf.WriteString(f.valuesStr)
-		continuing = true
+		buf.WriteString(bodyStr)
 	}
 
-	vals = args
-	if hook := f.opts.RenderArgsHook; hook != nil {
-		vals = hook(f.sanitize(vals))
+	if f.outputFormat == outputJSON {
+		buf.WriteByte('}') // for the whole record
 	}
-	f.flatten(buf, vals, continuing, true) // escape user-provided keys
 
-	for i := 0; i < groupDepth; i++ {
-		buf.WriteByte('}') // for the groups
+	return buf.String()
+}
+
+// renderGroup returns a string representation of the named group with rendered
+// values and args.  If the name is empty, this will return the values and args,
+// joined.  If the name is not empty, this will return a single key-value pair,
+// where the value is a grouping of the values and args.  If the values and
+// args are both empty, this will return an empty string, even if the name was
+// specified.
+func (f Formatter) renderGroup(name string, values string, args string) string {
+	buf := bytes.NewBuffer(make([]byte, 0, 1024))
+
+	needClosingBrace := false
+	if name != "" && (values != "" || args != "") {
+		buf.WriteString(f.quoted(name, true)) // escape user-provided keys
+		buf.WriteByte(f.colon())
+		buf.WriteByte('{')
+		needClosingBrace = true
 	}
 
-	if f.outputFormat == outputJSON {
-		buf.WriteByte('}') // for the whole line
+	continuing := false
+	if values != "" {
+		buf.WriteString(values)
+		continuing = true
+	}
+
+	if args != "" {
+		if continuing {
+			buf.WriteByte(f.comma())
+		}
+		buf.WriteString(args)
+	}
+
+	if needClosingBrace {
+		buf.WriteByte('}')
 	}
 
 	return buf.String()
 }
 
-// flatten renders a list of key-value pairs into a buffer.  If continuing is
-// true, it assumes that the buffer has previous values and will emit a
-// separator (which depends on the output format) before the first pair it
-// writes.  If escapeKeys is true, the keys are assumed to have
-// non-JSON-compatible characters in them and must be evaluated for escapes.
+// flatten renders a list of key-value pairs into a buffer.  If escapeKeys is
+// true, the keys are assumed to have non-JSON-compatible characters in them
+// and must be evaluated for escapes.
 //
 // This function returns a potentially modified version of kvList, which
 // ensures that there is a value for every key (adding a value if needed) and
 // that each key is a string (substituting a key if needed).
-func (f Formatter) flatten(buf *bytes.Buffer, kvList []any, continuing bool, escapeKeys bool) []any {
+func (f Formatter) flatten(buf *bytes.Buffer, kvList []any, escapeKeys bool) []any {
 	// This logic overlaps with sanitize() but saves one type-cast per key,
 	// which can be measurable.
 	if len(kvList)%2 != 0 {
@@ -354,7 +386,7 @@ func (f Formatter) flatten(buf *bytes.Buffer, kvList []any, continuing bool, esc
 		}
 		v := kvList[i+1]
 
-		if i > 0 || continuing {
+		if i > 0 {
 			if f.outputFormat == outputJSON {
 				buf.WriteByte(f.comma())
 			} else {
@@ -766,46 +798,17 @@ func (f Formatter) sanitize(kvList []any) []any {
 // startGroup opens a new group scope (basically a sub-struct), which locks all
 // the current saved values and starts them anew.  This is needed to satisfy
 // slog.
-func (f *Formatter) startGroup(group string) {
+func (f *Formatter) startGroup(name string) {
 	// Unnamed groups are just inlined.
-	if group == "" {
+	if name == "" {
 		return
 	}
 
-	// Any saved values can no longer be changed.
-	buf := bytes.NewBuffer(make([]byte, 0, 1024))
-	continuing := false
-
-	if f.parentValuesStr != "" {
-		buf.WriteString(f.parentValuesStr)
-		continuing = true
-	}
-
-	if f.group != "" && f.valuesStr != "" {
-		if continuing {
-			buf.WriteByte(f.comma())
-		}
-		buf.WriteString(f.quoted(f.group, true)) // escape user-provided keys
-		buf.WriteByte(f.colon())
-		buf.WriteByte('{') // for the group
-		continuing = false
-	}
-
-	if f.valuesStr != "" {
-		if continuing {
-			buf.WriteByte(f.comma())
-		}
-		buf.WriteString(f.valuesStr)
-	}
-
-	// NOTE: We don't close the scope here - that's done later, when a log line
-	// is actually rendered (because we have N scopes to close).
-
-	f.parentValuesStr = buf.String()
+	n := len(f.groups)
+	f.groups = append(f.groups[:n:n], groupDef{f.groupName, f.valuesStr})
 
 	// Start collecting new values.
-	f.group = group
-	f.groupDepth++
+	f.groupName = name
 	f.valuesStr = ""
 	f.values = nil
 }
@@ -900,7 +903,7 @@ func (f *Formatter) AddValues(kvList []any) {
 
 	// Pre-render values, so we don't have to do it on each Info/Error call.
 	buf := bytes.NewBuffer(make([]byte, 0, 1024))
-	f.flatten(buf, vals, false, true) // escape user-provided keys
+	f.flatten(buf, vals, true) // escape user-provided keys
 	f.valuesStr = buf.String()
 }
 
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 94eb75e3f..3bc9ff1a2 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -550,7 +550,7 @@ github.com/go-errors/errors
 github.com/go-jose/go-jose/v3
 github.com/go-jose/go-jose/v3/cipher
 github.com/go-jose/go-jose/v3/json
-# github.com/go-logr/logr v1.4.1
+# github.com/go-logr/logr v1.4.2
 ## explicit; go 1.18
 github.com/go-logr/logr
 github.com/go-logr/logr/funcr
@@ -1076,7 +1076,7 @@ github.com/openshift/cloud-credential-operator/pkg/apis/cloudcredential/v1
 # github.com/openshift/custom-resource-status v1.1.3-0.20220503160415-f2fdb4999d87
 ## explicit; go 1.12
 github.com/openshift/custom-resource-status/conditions/v1
-# github.com/openshift/hive/apis v0.0.0-20240510150258-83aedb9f6e73 => github.com/openshift/hive/apis v0.0.0-20231116161336-9dd47f8bfa1f
+# github.com/openshift/hive/apis v0.0.0-20240529172037-d7ead609f495 => github.com/openshift/hive/apis v0.0.0-20231116161336-9dd47f8bfa1f
 ## explicit; go 1.20
 github.com/openshift/hive/apis/hive/v1
 github.com/openshift/hive/apis/hive/v1/agent
@@ -1577,7 +1577,7 @@ gopkg.in/yaml.v2
 # gopkg.in/yaml.v3 v3.0.1
 ## explicit
 gopkg.in/yaml.v3
-# k8s.io/api v0.30.0 => k8s.io/api v0.25.16
+# k8s.io/api v0.30.1 => k8s.io/api v0.25.16
 ## explicit; go 1.19
 k8s.io/api/admission/v1
 k8s.io/api/admission/v1beta1
@@ -1646,7 +1646,7 @@ k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme
 k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1
 k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1
 k8s.io/apiextensions-apiserver/third_party/forked/celopenapi/model
-# k8s.io/apimachinery v0.30.0 => k8s.io/apimachinery v0.25.16
+# k8s.io/apimachinery v0.30.1 => k8s.io/apimachinery v0.25.16
 ## explicit; go 1.19
 k8s.io/apimachinery/pkg/api/equality
 k8s.io/apimachinery/pkg/api/errors


Test patch:
diff --git a/pkg/api/v20240812preview/openshiftcluster_validatestatic.go b/pkg/api/v20240812preview/openshiftcluster_validatestatic.go
index c027b1cd7..b5011fdd2 100644
--- a/pkg/api/v20240812preview/openshiftcluster_validatestatic.go
+++ b/pkg/api/v20240812preview/openshiftcluster_validatestatic.go
@@ -12,6 +12,7 @@ import (
 
 	azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
 	"github.com/Azure/go-autorest/autorest/azure"
+	"github.com/coreos/go-semver/semver"
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/api/util/immutable"
@@ -442,6 +443,14 @@ func (sv openShiftClusterStaticValidator) validatePlatformWorkloadIdentityProfil
 			return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, fmt.Sprintf("%s.PlatformWorkloadIdentities[%d].resourceID", path, n), "Resource must be a user assigned identity.")
 		}
 	}
+
+	if pwip.UpgradeableTo != nil {
+		_, err := semver.NewVersion(string(*pwip.UpgradeableTo))
+		if err != nil {
+			return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, fmt.Sprintf("%s.UpgradeableTo[%v]", path, *pwip.UpgradeableTo), "UpgradeableTo must be a valid OpenShift version in the format 'x.y.z'.")
+		}
+	}
+
 	return nil
 }
 
diff --git a/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go b/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go
index 97e0b802f..884df39ea 100644
--- a/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go
+++ b/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go
@@ -1163,6 +1163,9 @@ func TestOpenShiftClusterStaticValidateDelta(t *testing.T) {
 }
 
 func TestOpenShiftClusterStaticValidatePlatformWorkloadIdentityProfile(t *testing.T) {
+	validUpgradeableToValue := UpgradeableTo("4.14.29")
+	invalidUpgradeableToValue := UpgradeableTo("16.107.invalid")
+
 	createTests := []*validateTest{
 		{
 			name: "valid empty workloadIdentityProfile",
@@ -1323,6 +1326,27 @@ func TestOpenShiftClusterStaticValidatePlatformWorkloadIdentityProfile(t *testin
 			},
 			wantErr: "400: InvalidParameter: properties.servicePrincipalProfile: Must provide either an identity or service principal credentials.",
 		},
+		{
+			name: "valid UpgradeableTo value",
+			modify: func(oc *OpenShiftCluster) {
+				oc.Identity = &Identity{}
+				oc.Properties.ServicePrincipalProfile = nil
+				oc.Properties.PlatformWorkloadIdentityProfile = &PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &validUpgradeableToValue,
+				}
+			},
+		},
+		{
+			name: "invalid UpgradeableTo value",
+			modify: func(oc *OpenShiftCluster) {
+				oc.Identity = &Identity{}
+				oc.Properties.ServicePrincipalProfile = nil
+				oc.Properties.PlatformWorkloadIdentityProfile = &PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &invalidUpgradeableToValue,
+				}
+			},
+			wantErr: `400: InvalidParameter: properties.platformWorkloadIdentityProfile.UpgradeableTo[16.107.invalid]: UpgradeableTo must be a valid OpenShift version in the format 'x.y.z'.`,
+		},
 	}
 
 	runTests(t, testModeCreate, createTests)
diff --git a/pkg/frontend/openshiftcluster_putorpatch_test.go b/pkg/frontend/openshiftcluster_putorpatch_test.go
index f7c10ade9..540616ccf 100644
--- a/pkg/frontend/openshiftcluster_putorpatch_test.go
+++ b/pkg/frontend/openshiftcluster_putorpatch_test.go
@@ -3305,3 +3305,84 @@ func TestEnrichClusterSystemData(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateIdentityUrl(t *testing.T) {
+	for _, tt := range []struct {
+		name        string
+		identityURL string
+		cluster     *api.OpenShiftCluster
+		expected    *api.OpenShiftCluster
+		isCreate    bool
+		wantError   error
+	}{
+		{
+			name:        "identity URL is empty, is not wi/mi cluster create",
+			identityURL: "",
+			cluster:     &api.OpenShiftCluster{},
+			expected:    &api.OpenShiftCluster{},
+			isCreate:    false,
+		},
+		{
+			name:        "identity URL is empty, is wi/mi cluster create",
+			identityURL: "",
+			cluster:     &api.OpenShiftCluster{},
+			expected:    &api.OpenShiftCluster{},
+			isCreate:    true,
+			wantError:   errMissingIdentityURL,
+		},
+		{
+			name:        "cluster is not wi/mi, identityURL passed",
+			identityURL: "http://foo.bar",
+			cluster: &api.OpenShiftCluster{
+				Properties: api.OpenShiftClusterProperties{
+					ServicePrincipalProfile: &api.ServicePrincipalProfile{},
+				},
+			},
+			expected: &api.OpenShiftCluster{
+				Properties: api.OpenShiftClusterProperties{
+					ServicePrincipalProfile: &api.ServicePrincipalProfile{},
+				},
+			},
+			isCreate: true,
+		},
+		{
+			name:        "cluster is not wi/mi, identityURL not passed",
+			identityURL: "",
+			cluster: &api.OpenShiftCluster{
+				Properties: api.OpenShiftClusterProperties{
+					ServicePrincipalProfile: &api.ServicePrincipalProfile{},
+				},
+			},
+			expected: &api.OpenShiftCluster{
+				Properties: api.OpenShiftClusterProperties{
+					ServicePrincipalProfile: &api.ServicePrincipalProfile{},
+				},
+			},
+			isCreate: true,
+		},
+		{
+			name: "pass - identity URL passed on wi/mi cluster",
+			cluster: &api.OpenShiftCluster{
+				Identity: &api.Identity{},
+			},
+			identityURL: "http://foo.bar",
+			expected: &api.OpenShiftCluster{
+				Identity: &api.Identity{
+					IdentityURL: "http://foo.bar",
+				},
+			},
+			isCreate: true,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateIdentityUrl(tt.cluster, tt.identityURL, tt.isCreate)
+			if err != nil && err != tt.wantError {
+				t.Error(cmp.Diff(err, tt.wantError))
+			}
+
+			if !reflect.DeepEqual(tt.cluster, tt.expected) {
+				t.Error(cmp.Diff(tt.cluster, tt.expected))
+			}
+		})
+	}
+}
diff --git a/pkg/operator/deploy/deploy_test.go b/pkg/operator/deploy/deploy_test.go
index 6002e65dd..44f5e257a 100644
--- a/pkg/operator/deploy/deploy_test.go
+++ b/pkg/operator/deploy/deploy_test.go
@@ -10,6 +10,8 @@ import (
 
 	"github.com/golang/mock/gomock"
 	configv1 "github.com/openshift/api/config/v1"
+	operatorv1 "github.com/openshift/api/operator/v1"
+	operatorfake "github.com/openshift/client-go/operator/clientset/versioned/fake"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -461,3 +463,122 @@ func TestCheckPodImageVersion(t *testing.T) {
 		})
 	}
 }
+
+func TestTestEnsureUpgradeAnnotation(t *testing.T) {
+	UpgradeableTo1 := api.UpgradeableTo("4.14.59")
+
+	for _, tt := range []struct {
+		name                 string
+		cluster              api.OpenShiftClusterProperties
+		annotation           map[string]string
+		wantAnnotation       map[string]string
+		wantErr              string
+		cloudCredentialsName string
+	}{
+		{
+			name: "nil PlatformWorkloadIdentityProfile, no version persisted in cluster document",
+		},
+		{
+			name: "non-nil ServicePrincipalProfile, no version persisted in cluster document",
+			cluster: api.OpenShiftClusterProperties{
+				ServicePrincipalProfile: &api.ServicePrincipalProfile{
+					ClientID:     "",
+					ClientSecret: "",
+				},
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{},
+			},
+		},
+		{
+			name: "no version persisted in cluster document, persist it",
+			cluster: api.OpenShiftClusterProperties{
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &UpgradeableTo1,
+				},
+			},
+			annotation: nil,
+			wantAnnotation: map[string]string{
+				"cloudcredential.openshift.io/upgradeable-to": "4.14.59",
+			},
+		},
+		{
+			name: "cloud credential 'cluster' doesn't exist",
+			cluster: api.OpenShiftClusterProperties{
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &UpgradeableTo1,
+				},
+			},
+			cloudCredentialsName: "oh_no",
+			annotation:           nil,
+			wantAnnotation:       nil,
+			wantErr:              `cloudcredentials.operator.openshift.io "cluster" not found`,
+		},
+		{
+			name: "version persisted in cluster document, replace it",
+			cluster: api.OpenShiftClusterProperties{
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &UpgradeableTo1,
+				},
+			},
+			annotation: map[string]string{
+				"cloudcredential.openshift.io/upgradeable-to": "4.14.02",
+			},
+			wantAnnotation: map[string]string{
+				"cloudcredential.openshift.io/upgradeable-to": "4.14.59",
+			},
+		},
+		{
+			name: "annotations exist, append the upgradeable annotation",
+			cluster: api.OpenShiftClusterProperties{
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &UpgradeableTo1,
+				},
+			},
+			annotation: map[string]string{
+				"foo": "bar",
+			},
+			wantAnnotation: map[string]string{
+				"foo": "bar",
+				"cloudcredential.openshift.io/upgradeable-to": "4.14.59",
+			},
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := context.Background()
+			controller := gomock.NewController(t)
+			defer controller.Finish()
+
+			env := mock_env.NewMockInterface(controller)
+
+			oc := &api.OpenShiftCluster{
+				Properties: tt.cluster,
+			}
+
+			if tt.cloudCredentialsName == "" {
+				tt.cloudCredentialsName = "cluster"
+			}
+
+			cloudcredentialobject := &operatorv1.CloudCredential{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        tt.cloudCredentialsName,
+					Annotations: tt.annotation,
+				},
+			}
+
+			o := operator{
+				oc:          oc,
+				env:         env,
+				operatorcli: operatorfake.NewSimpleClientset(cloudcredentialobject),
+			}
+
+			err := o.EnsureUpgradeAnnotation(ctx)
+			utilerror.AssertErrorMessage(t, err, tt.wantErr)
+			result, _ := o.operatorcli.OperatorV1().CloudCredentials().List(ctx, metav1.ListOptions{})
+			for _, v := range result.Items {
+				actualAnnotations := v.ObjectMeta.Annotations
+				if !reflect.DeepEqual(actualAnnotations, tt.wantAnnotation) {
+					t.Errorf("actual annotation: %v, wanted %v", tt.annotation, tt.wantAnnotation)
+				}
+			}
+		})
+	}
+}
diff --git a/pkg/util/azureerrors/error_test.go b/pkg/util/azureerrors/error_test.go
index 77c9c333e..ee925060d 100644
--- a/pkg/util/azureerrors/error_test.go
+++ b/pkg/util/azureerrors/error_test.go
@@ -120,3 +120,36 @@ func TestIsDeploymentActiveError(t *testing.T) {
 		})
 	}
 }
+
+func TestIsDeploymentMissingPermissionsError(t *testing.T) {
+	for _, tt := range []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{
+			name: "Another error",
+			err:  errors.New("something happened"),
+		},
+		{
+			name: "Missing RoleAssignment",
+			err: autorest.DetailedError{
+				PackageType: "features.DeploymentsClient",
+				Method:      "CreateOrUpdate",
+				Message:     "Failure sending request",
+				Original: &azure.ServiceError{
+					Code:    CodeInvalidTemplateDeployment,
+					Message: "The template deployment failed with error: 'Authorization failed for template resource '$RESOURCE' of type 'Microsoft.Authorization/roleAssignments'. The client '$CLIENT' with object id '$CLIENT' does not have permission to perform action '$ACTION' at scope '$SCOPE'.'.",
+				},
+			},
+			want: true,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			got := IsDeploymentMissingPermissionsError(autorest.NewErrorWithError(tt.err, "", "", nil, ""))
+			if got != tt.want {
+				t.Errorf("got %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
diff --git a/pkg/util/pki/pki_test.go b/pkg/util/pki/pki_test.go
index 25676c561..c7a9d30bb 100644
--- a/pkg/util/pki/pki_test.go
+++ b/pkg/util/pki/pki_test.go
@@ -13,7 +13,7 @@ import (
 )
 
 func TestGetTlsConfig(t *testing.T) {
-	if os.Getenv("ARO_RUN_PKI_TESTS") != "" {
+	if os.Getenv("ARO_SKIP_PKI_TESTS") != "" {
 		t.Skip("")
 	}
 	kpiUrl := "https://issuer.pki.azure.com/dsms/issuercertificates?getissuersv3&caName=%s"