Repository: Azure/ARO-RP
Pull request #3583 (2024-05-15 21:24:02)

Use single ARO API/client version in pkg/util/cluster
### Which issue this PR addresses:

Fixes no Jira

### What this PR does / why we need it:

Updates pkg/util/cluster/cluster.go to use a single ARO API version and client, replacing the 4 different API versions previously in use. 

This version is currently set to the v20230904 API - the latest public-facing API as of now. This allows the functionality in pkg/util/cluster to continue to be used against production RP instances, e.g. in production E2E contexts. 

### Test plan for issue:

- [x] Local cluster creation kicked off successfully
- [x] Local cluster deletion kicked off successfully
- [x] PR E2E cluster creation/deletion works as expected

### Is there any documentation that needs to be updated for this PR?

No. 

### How do you know this will function as expected in production? 

N/A (this functionality is only used during local development and when running E2E)


Patch:
diff --git a/.sha256sum b/.sha256sum
index 8fae07dc7..cca696c9d 100644
--- a/.sha256sum
+++ b/.sha256sum
@@ -6,4 +6,4 @@ b1f1de0fe40d05de90742b17928968923b936adc294000f58974f50a297581dd  swagger/redhat
 c023515341196746454c0ae7af077d40d3ec13f6b88b33cb558f0a7ab17a5a24  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2023-07-01-preview/redhatopenshift.json
 440748951dd1c3b34b5ccbdcb7cd966e3b89490887a1f1d64429561fad789515  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/stable/2023-09-04/redhatopenshift.json
 74a46fdde6ceb0121fe1515c7e11e902dd921b54cffe693307fb02b3dc88f26e  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/stable/2023-11-22/redhatopenshift.json
-a27184734436629e24b344c3b5c015437f144e18e7eddce7e252a1ed4cda7bca  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
+f1e9d42d7c4c0081282e065e7845455db28ed6924687f1acecafb5fbc43ae0c3  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
diff --git a/Dockerfile.ci-portal b/Dockerfile.ci-portal
deleted file mode 100644
index d54d1af07..000000000
--- a/Dockerfile.ci-portal
+++ /dev/null
@@ -1,13 +0,0 @@
-ARG REGISTRY
-
-FROM ${REGISTRY}/ubi8/nodejs-16 AS portal-base
-WORKDIR /build/portal/v2
-USER root
-
-FROM portal-base AS portal-build
-COPY /portal/v2/package*.json ./
-RUN npm ci
-RUN npm audit --omit=dev
-COPY /portal/v2/ ./
-RUN npm run lint
-RUN npm run build
\ No newline at end of file
diff --git a/Dockerfile.ci-rp b/Dockerfile.ci-rp
index 4750d88bb..c1d608541 100644
--- a/Dockerfile.ci-rp
+++ b/Dockerfile.ci-rp
@@ -1,21 +1,40 @@
 ARG REGISTRY
-ARG VERSION
+ARG ARO_VERSION
 
 ###############################################################################
+# Stage 1: Build the SRE Portal Assets
 # builder is responsible for all compilation and validation of the RP
 ###############################################################################
-FROM ${REGISTRY}/ubi8/go-toolset:1.20.12-5 as builder
+FROM ${REGISTRY}/ubi8/nodejs-16  as portal-build
+WORKDIR /build/portal/v2
+USER root
+
+# Copying package files and installing dependencies
+COPY portal/v2/package*.json ./
+RUN npm ci
+RUN npm audit --omit=dev # Run audit without dev dependencies
+
+# Copying the rest of the source and build
+COPY --chown=root:root portal/v2/ ./
+RUN npm run lint && npm run build
+
+###############################################################################
+# Stage 2: Compile the Golang RP code
+###############################################################################
+FROM ${REGISTRY}/ubi8/go-toolset:1.20.12-5 AS builder
+ARG ARO_VERSION
 USER root
 WORKDIR /app
 
-# golang config
+# golang config and build steps
 ENV GOPATH=/root/go
 ENV GOFLAGS="-tags=containers_image_openpgp,exclude_graphdriver_btrfs,exclude_graphdriver_devicemapper"
 
-# install general build tools
-RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b /usr/local/bin v1.56.2
+# Install golangci-lint and verify
+RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b /usr/local/bin v1.56.2 && \
+    golangci-lint --version || (echo "golangci-lint not found" && exit 1)
 
-# copy from least changed -> most changed to improve cache hits
+# Copy dependencies and source files
 COPY go.mod go.sum ./
 COPY vendor vendor
 COPY swagger swagger
@@ -25,20 +44,21 @@ COPY cmd cmd
 COPY pkg pkg
 COPY test test
 
-# lint
-RUN golangci-lint run --verbose
+# Ensure JS assets are available before generating Go code
+COPY --from=portal-build /build/pkg/portal/assets/v2/build /app/pkg/portal/assets/v2/build
 
-# build
+# Lint, generate, build, and test
+RUN golangci-lint run --verbose
 RUN go generate ./...
-RUN go build -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${VERSION}" ./cmd/aro
-RUN go test ./test/e2e/... -tags e2e,codec.safe -c -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${VERSION}" -o e2e.test
+RUN go build -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${ARO_VERSION}" ./cmd/aro
+RUN go test ./test/e2e/... -tags e2e,codec.safe -c -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${ARO_VERSION}" -o e2e.test
 
-# test
-RUN ARO_RUN_PKI_TESTS=nope go run gotest.tools/gotestsum@v1.11.0 --format pkgname --junitfile report.xml -- -coverprofile=cover.out ./...
+# Additional tests
+RUN ARO_SKIP_PKI_TESTS=true go run gotest.tools/gotestsum@v1.11.0 --format pkgname --junitfile report.xml -- -coverprofile=cover.out ./...
 RUN hack/fips/validate-fips.sh ./aro
 
 ###############################################################################
-# final is our slim image with minimal layers and tools
+# Stage 3: final is our slim image with minimal layers and tools
 ###############################################################################
 FROM ${REGISTRY}/ubi8/ubi-minimal AS final
 RUN microdnf update && microdnf clean all
diff --git a/Dockerfile.ci-rp.dockerignore b/Dockerfile.ci-rp.dockerignore
index a984a1402..052c75c86 100644
--- a/Dockerfile.ci-rp.dockerignore
+++ b/Dockerfile.ci-rp.dockerignore
@@ -10,4 +10,17 @@
 !pkg
 !swagger
 !test
-!vendor
\ No newline at end of file
+!vendor
+
+# Ignore locally built NPM artifacts - using new automated process
+pkg/portal/assets/v2/build
+
+# Allow files and directories for JavaScript build
+!portal/v2/package*.json
+!portal/v2/src
+!portal/v2/public
+!portal/v2/config-overrides.js
+!portal/v2/tsconfig.json
+!portal/v2/.eslintrc.yml
+!portal/v2/.npmrc
+
diff --git a/Dockerfile.proxy b/Dockerfile.proxy
index 87643f947..006172cc3 100644
--- a/Dockerfile.proxy
+++ b/Dockerfile.proxy
@@ -7,11 +7,13 @@ ENV GOPATH=/root/go
 RUN mkdir -p /app
 WORKDIR /app
 
+COPY . /app
 RUN make proxy
 
 FROM ${REGISTRY}/ubi8/ubi-minimal
 RUN microdnf update && microdnf clean all
-COPY --from=builder /go/src/github.com/Azure/ARO-RP/proxy /usr/local/bin/
+COPY --from=builder /app/proxy /usr/local/bin/
 ENTRYPOINT ["proxy"]
 EXPOSE 8443/tcp
+EXPOSE 8080/tcp
 USER 1000
diff --git a/Makefile b/Makefile
index a0953075d..d263c3e8d 100644
--- a/Makefile
+++ b/Makefile
@@ -77,10 +77,7 @@ client: generate
 	hack/build-client.sh "${AUTOREST_IMAGE}" 2020-04-30 2021-09-01-preview 2022-04-01 2022-09-04 2023-04-01 2023-07-01-preview 2023-09-04 2023-11-22 2024-08-12-preview
 
 ci-rp: fix-macos-vendor
-	docker build . -f Dockerfile.ci-rp --ulimit=nofile=4096:4096 --build-arg REGISTRY=$(REGISTRY) --build-arg VERSION=$(VERSION) --no-cache=$(NO_CACHE)
-
-ci-portal:
-	docker build . -f Dockerfile.ci-portal --build-arg REGISTRY=$(REGISTRY) --no-cache=$(NO_CACHE)
+	docker build . -f Dockerfile.ci-rp --ulimit=nofile=4096:4096 --build-arg REGISTRY=$(REGISTRY) --build-arg ARO_VERSION=$(VERSION) --no-cache=$(NO_CACHE)
 
 # TODO: hard coding dev-config.yaml is clunky; it is also probably convenient to
 # override COMMIT.
@@ -123,7 +120,6 @@ generate-kiota:
 
 init-contrib:
 	install -v hack/git/hooks/* .git/hooks/
-	
 
 image-aro-multistage:
 	docker build --platform=linux/amd64 --network=host --no-cache -f Dockerfile.aro-multistage -t $(ARO_IMAGE) --build-arg REGISTRY=$(REGISTRY) .
diff --git a/cmd/aro/mirror.go b/cmd/aro/mirror.go
index db0c7ec92..6e7e95bb7 100644
--- a/cmd/aro/mirror.go
+++ b/cmd/aro/mirror.go
@@ -134,7 +134,7 @@ func mirror(ctx context.Context, log *logrus.Entry) error {
 		"quay.io/app-sre/managed-upgrade-operator:v0.1.952-44b631a",
 
 		// https://quay.io/repository/app-sre/hive?tab=tags
-		"quay.io/app-sre/hive:83aedb9f6e",
+		"quay.io/app-sre/hive:d7ead609f4",
 	} {
 		log.Printf("mirroring %s -> %s", ref, pkgmirror.Dest(dstAcr+acrDomainSuffix, ref))
 
diff --git a/docs/deploy-development-rp.md b/docs/deploy-development-rp.md
index 62c1f38ac..d475ed86a 100644
--- a/docs/deploy-development-rp.md
+++ b/docs/deploy-development-rp.md
@@ -386,9 +386,9 @@ To access the cluster for oc / kubectl or SSH'ing into the cluster you need to c
 
   * Run the ssh-aks.sh script, specifying the cluster name and the node number of the VM you are trying to ssh to.
   ```bash
-  hack/ssk-aks.sh aro-aks-cluster 0 # The first VM node in 'aro-aks-cluster'
-  hack/ssk-aks.sh aro-aks-cluster 1 # The second VM node in 'aro-aks-cluster'
-  hack/ssk-aks.sh aro-aks-cluster 2 # The third VM node in 'aro-aks-cluster'
+  hack/ssh-aks.sh aro-aks-cluster 0 # The first VM node in 'aro-aks-cluster'
+  hack/ssh-aks.sh aro-aks-cluster 1 # The second VM node in 'aro-aks-cluster'
+  hack/ssh-aks.sh aro-aks-cluster 2 # The third VM node in 'aro-aks-cluster'
   ```
 
 * Access via Azure Portal
diff --git a/docs/prepare-your-dev-environment.md b/docs/prepare-your-dev-environment.md
index 159acc766..f5b67a9e8 100644
--- a/docs/prepare-your-dev-environment.md
+++ b/docs/prepare-your-dev-environment.md
@@ -34,7 +34,7 @@ This document goes through the development dependencies one requires in order to
         If you're using podman-machine, you will need to export the socket, for example::
 
             ```bash
-            export ARO_PODMAN_SOCKET=unix:///$HOME/.local/share/containers/podman/machine/qemu/podman.sock
+            export ARO_PODMAN_SOCKET=unix://$HOME/.local/share/containers/podman/machine/qemu/podman.sock
             ```
         
         You will also need to ensure that podman machine has enough resources::
diff --git a/docs/upstream-differences.md b/docs/upstream-differences.md
deleted file mode 100644
index f94f7ee71..000000000
--- a/docs/upstream-differences.md
+++ /dev/null
@@ -1,204 +0,0 @@
-# Upstream differences
-
-This file catalogues the differences of install approach between ARO and
-upstream OCP.
-
-## Installer carry patches
-
-1. Clone ARO and upstream repos:
-    ```sh
-    # clone our forked installer
-    git clone https://github.com/openshift/installer-aro.git
-    cd installer-aro
-
-    # add the upstream as a remote source
-    git remote add upstream https://github.com/openshift/installer.git
-    git fetch upstream -a
-    ```
-1. See carry patches from previous release:
-    ```sh
-    # list patches
-    git log upstream/release-X.Y-1..origin/release-X.Y-1-azure
-
-    # see diff of patches
-    git show upstream/release-X.Y-1..origin/release-X.Y-1-azure
-    ```
-
-## Installation differences
-
-* ARO does not use Terraform to create clusters, and instead uses ARM templates directly
-
-* ARO persists the install graph in the cluster storage account in a new "aro"
-  container / "graph" blob.
-
-* No managed identity (for now).
-
-* No IPv6 support (for now).
-
-* Upstream installer closely binds the installConfig (cluster) name, cluster
-  domain name, infra ID and Azure resource name prefix.  ARO separates these out
-  a little.  The installConfig (cluster) name and the domain name remain bound;
-  the infra ID and Azure resource name prefix are taken from the ARO resource
-  name.
-
-* API server public IP domain name label is not set.
-
-* ARO uses first party RHCOS OS images published by Microsoft.
-
-* ARO never creates xxxxx-bootstrap-pip-* for bootstrap VM, or the corresponding
-  NSG rule.
-
-* ARO does not create a outbound-provider Service on port 27627.
-
-* ARO deploys a private link service in order for the RP to be able to
-  communicate with the cluster.
-
-* ARO runs a dnsmasq service on the nodes through the use of a machineconfig to resolve api-int and *.apps domains on the node locally allowing for custom DNS configured on the VNET.
-
-# Introducing new OCP release into ARO RP
-
-To support a new version of OpenShift on ARO, you will need to reconcile [upstream changes](https://github.com/openshift/installer) with our [forked installer](https://github.com/openshift/installer-aro). This will not be a merge, but a cherry-pick of patches we've implemented.
-
-## Update installer fork
-
-To bring new OCP release branch into ARO installer fork:
-
-1. If not done already, fetch our fork and upstream repos:
-    ```sh
-    # clone our forked installer
-    # Alternatively, fork openshift/installer-aro and clone your fork
-    git clone https://github.com/openshift/installer-aro.git
-    cd installer-aro
-    
-    # add the upstream as a remote source
-    git remote add upstream https://github.com/openshift/installer.git
-    git fetch upstream -a
-    ```
-1. Assess and document differences in X.Y and X.Y-1 in upstream
-    ```sh
-    # diff the upstream X.Y with X.Y-1 and search for architecture changes
-    git diff upstream/release-X.Y-1 upstream/release-X.Y
-    
-    # pay particular attention to Terraform files, which may need to be moved into ARO's ARM templates
-    git diff upstream/release-X.Y-1 upstream/release-X.Y */azure/*.tf
-    ```
-2. Create a new X.Y release branch in our forked installer
-    ```sh
-    # create a new release branch in the fork based on the upstream
-    git checkout upstream/release-X.Y
-    git checkout -b release-X.Y-azure
-    ```
-3. If there is a golang version bump in this release, modify `./hack/build.sh` and `./hack/go-test.sh` with the new version, then verify these scripts still work and commit them
-4. Determine the patches you need to cherry-pick, based on the last (Y-1) release
-    ```sh
-    # find commit shas to cherry-pick from last time
-    git checkout release-X.Y-1-azure
-    git log
-    ```
-5. For every commit you need to cherry-pick (in-order), do:
-    ```sh
-    # WARNING: when you reach the commit for `commit data/assets_vfsdata.go`, look ahead
-    git cherry-pick abc123 # may require manually fixing a merge
-    ./hack/build.sh   # fix any failures
-    ./hack/go-test.sh # fix any failures
-    # if you had to manually merge, you can now `git cherry-pick --continue`
-    ```
-    - When cherry-picking the specific patch `commit data/assets_vfsdata.go`, instead run:
-        ```sh
-        git cherry-pick abc123 # may require manually fixing a merge
-        ./hack/build.sh   # fix any failures
-        ./hack/go-test.sh # fix any failures
-        # if you had to manually merge, you can now `git cherry-pick --continue`
-        pushd ./hack/assets && go run ./assets.go && popd
-        ./hack/build.sh   # fix any failures
-        ./hack/go-test.sh # fix any failures
-        git add data/assets_vfsdata.go
-        git commit --amend
-        ```
-
-**Note:** If any changes are required during the process, make sure to amend the relevant patch or create a new one.
-Each commit should be atomic/complete - you should be able to cherry-pick it into the upstream installer and bring
-the fix or feature it carries in full, without a need to cherry-pick additional commits.
-This makes it easier to understand the nature of the patch as well as contribute our carry patches
-back to the upstream installer.
-
-## Update Installer Fork Again
-
-In the far or near future after you have [initially patched the installer](#update-installer-fork), you may need to pull in additional upstream changes that have happened since you patched the installer (e.g. upstream added a bugfix since your cherry-picking). The easiest way to pull in these changes safely is:
-
-```sh
-git fetch upstream -a
-git checkout release-X.Y-azure
-git pull upstream/release-X.Y --rebase=interactive
-```
-
-When you get to the editor mode to set up your rebase, you should do a few things:
-
-1. after each `pick` line, add the verification commands with:
-    ```text
-    exec ./hack/build.sh
-    exec ./hack/go-test.sh
-    ```
-1. change the line for commit `data/assets_vfsdata.go` from `pick` to `edit` so you can regenerate assets as outlined [above](#update-installer-fork), and verify it manually.
-
-By the end of this editing process, you should have a rebase file that looks something like:
-
-```text
-pick abc123 patch A
-exec ./hack/build.sh
-exec ./hack/go-test.sh
-pick def234 patch B
-exec ./hack/build.sh
-exec ./hack/go-test.sh
-edit ghi345 patch data/assets_vfsdata.go
-exec ./hack/build.sh
-exec ./hack/go-test.sh
-pick jkl456 patch C
-exec ./hack/build.sh
-exec ./hack/go-test.sh
-{...}
-```
-
-When you are finished, you can write/close the file editor to perform the rebase on the upstream differences while verifying that every patch still works along the way.
-
-Get a repo admin to create `release-X.Y-azure` in [openshift/installer-aro](https://github.com/openshift/installer-aro). You can now push your local branch `release-X.Y-azure` to your own fork of [openshift/installer-aro](https://github.com/openshift/installer-aro) and send a PR.
-
-# Update ARO Installer Wrapper and ARO-RP
-
-Once installer fork is ready, perform the following changes in the [ARO Installer Wrapper](https://github.com/openshift/installer-aro-wrapper):
-
-1. Update `go mod edit -replace` calls in `hack/update-go-module-dependencies.sh` to use a new release-X.Y branch.
-    * Make sure to read comments in the script.
-1. `make vendor`.
-    * You most likely will have to make changes to the codebase at this point to adjust it to new versions of dependencies.
-    * Also you likely will have to repeat this step several time until you resolve all conflicting dependencies.
-      Follow `go mod` failures, which will tell you what module requires what other module.
-      You will probably need to look at the `go.mod` files of these modules and see whether they set own replace directives,
-      as the script is likely to fail with something like this:
-
-      ```
-      go: github.com/openshift/installer@v0.16.1 requires
-          github.com/openshift/cluster-api-provider-kubevirt@v0.0.0-20201214114543-e5aed9c73f1f requires
-          kubevirt.io/client-go@v0.0.0-00010101000000-000000000000: invalid version: unknown revision 000000000000
-      ```
-
-      In the example above you need to:
-        * Checkout `github.com/openshift/cluster-api-provider-kubevirt` at commit `e5aed9c73f1f`.
-        * In go.mod find a replace directive for `kubevirt.io/client-go`.
-        * Add/update relevant replace directive in `go.mod`.
-1. `make generate`.
-1. A local image can be built for testing purposes and pushed to a dev ACR repo. The process for [publishing the final image uses ADO](https://msazure.visualstudio.com/AzureRedHatOpenShift/_wiki/wikis/ARO.wiki/452838/ARO-Installer-Image-Deployment-Process)
-
-In the ARO-RP codebase:
-
-1. Update `pkg/util/version/const.go` to point to the new release.
-    * You should be able to find latest published release and image hash [on quay.io](https://quay.io/repository/openshift-release-dev/ocp-release?tab=tags).
-1. (Optional) `make discoverycache`.
-    * This command requires a running cluster with the new version.
-1. The list of the hard-coded namespaces in `pkg/util/namespace/namespace.go` needs to be updated regularly as every
-   minor version of upstream OCP introduces a new namespace or two.
-
-
-Publish RHCOS image to the Azure Cloud Partner Portal:
-1. Publish RHCOS image. See [this document](https://github.com/openshift/installer-aro-wrapper/blob/main/docs/publish-rhcos-image.md).
-1. After this point, you should be able to create a dev cluster using the RP and it should use the new release.
diff --git a/env.example b/env.example
index a265550d1..2f420e764 100644
--- a/env.example
+++ b/env.example
@@ -3,4 +3,6 @@ export ARO_IMAGE=arointsvc.azurecr.io/aro:latest
 export NO_CACHE=false
 export AZURE_EXTENSION_DEV_SOURCES="$(pwd)/python"
 
+export ARO_SKIP_PKI_TESTS=true
+
 . secrets/env
diff --git a/go.mod b/go.mod
index 3c3c3f0d4..6bcf71d8d 100644
--- a/go.mod
+++ b/go.mod
@@ -29,7 +29,7 @@ require (
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
 	github.com/go-chi/chi/v5 v5.0.8
-	github.com/go-logr/logr v1.4.1
+	github.com/go-logr/logr v1.4.2
 	github.com/go-test/deep v1.1.0
 	github.com/gofrs/uuid v4.2.0+incompatible
 	github.com/golang-jwt/jwt/v4 v4.5.0
@@ -59,7 +59,7 @@ require (
 	github.com/openshift/api v3.9.1-0.20191111211345-a27ff30ebf09+incompatible
 	github.com/openshift/client-go v0.0.0-20220525160904-9e1acff93e4a
 	github.com/openshift/cloud-credential-operator v0.0.0-00010101000000-000000000000
-	github.com/openshift/hive/apis v0.0.0-20240510150258-83aedb9f6e73
+	github.com/openshift/hive/apis v0.0.0-20240529172037-d7ead609f495
 	github.com/openshift/library-go v0.0.0-20220525173854-9b950a41acdc
 	github.com/openshift/machine-config-operator v0.0.1-0.20230519222939-1abc13efbb0d
 	github.com/pires/go-proxyproto v0.6.2
@@ -80,9 +80,9 @@ require (
 	golang.org/x/sync v0.6.0
 	golang.org/x/text v0.15.0
 	golang.org/x/tools v0.19.0
-	k8s.io/api v0.30.0
+	k8s.io/api v0.30.1
 	k8s.io/apiextensions-apiserver v0.25.0
-	k8s.io/apimachinery v0.30.0
+	k8s.io/apimachinery v0.30.1
 	k8s.io/cli-runtime v0.25.16
 	k8s.io/client-go v0.26.2
 	k8s.io/code-generator v0.25.16
diff --git a/go.sum b/go.sum
index cb7db633d..5132972fd 100644
--- a/go.sum
+++ b/go.sum
@@ -233,8 +233,8 @@ github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQr
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
-github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A=
diff --git a/pkg/api/admin/openshiftcluster.go b/pkg/api/admin/openshiftcluster.go
index d1bab4bb6..ed328772e 100644
--- a/pkg/api/admin/openshiftcluster.go
+++ b/pkg/api/admin/openshiftcluster.go
@@ -418,9 +418,13 @@ type IngressProfile struct {
 
 // PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.
 type PlatformWorkloadIdentityProfile struct {
+	UpgradeableTo              *UpgradeableTo             `json:"upgradeableTo,omitempty"`
 	PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"`
 }
 
+// UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to
+type UpgradeableTo string
+
 // PlatformWorkloadIdentity stores information representing a single workload identity.
 type PlatformWorkloadIdentity struct {
 	OperatorName string `json:"operatorName,omitempty"`
diff --git a/pkg/api/admin/openshiftcluster_convert.go b/pkg/api/admin/openshiftcluster_convert.go
index b0a563bcf..e91a2a46c 100644
--- a/pkg/api/admin/openshiftcluster_convert.go
+++ b/pkg/api/admin/openshiftcluster_convert.go
@@ -305,6 +305,7 @@ func (c openShiftClusterConverter) ToInternal(_oc interface{}, out *api.OpenShif
 	out.Properties.NetworkProfile.APIServerPrivateEndpointIP = oc.Properties.NetworkProfile.APIServerPrivateEndpointIP
 	out.Properties.NetworkProfile.GatewayPrivateEndpointIP = oc.Properties.NetworkProfile.GatewayPrivateEndpointIP
 	out.Properties.NetworkProfile.GatewayPrivateLinkID = oc.Properties.NetworkProfile.GatewayPrivateLinkID
+	out.Properties.NetworkProfile.PreconfiguredNSG = api.PreconfiguredNSG(oc.Properties.NetworkProfile.PreconfiguredNSG)
 	if oc.Properties.NetworkProfile.LoadBalancerProfile != nil {
 		loadBalancerProfile := api.LoadBalancerProfile{}
 
diff --git a/pkg/api/openshiftcluster.go b/pkg/api/openshiftcluster.go
index 0826520d0..caa157d71 100644
--- a/pkg/api/openshiftcluster.go
+++ b/pkg/api/openshiftcluster.go
@@ -773,9 +773,13 @@ type HiveProfile struct {
 type PlatformWorkloadIdentityProfile struct {
 	MissingFields
 
+	UpgradeableTo              *UpgradeableTo             `json:"upgradeableTo,omitempty"`
 	PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"`
 }
 
+// UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to
+type UpgradeableTo string
+
 // PlatformWorkloadIdentity stores information representing a single workload identity.
 type PlatformWorkloadIdentity struct {
 	MissingFields
@@ -803,4 +807,5 @@ type Identity struct {
 
 	Type                   string                 `json:"type,omitempty"`
 	UserAssignedIdentities UserAssignedIdentities `json:"userAssignedIdentities,omitempty"`
+	IdentityURL            string                 `json:"identityURL,omitempty" mutable:"true"`
 }
diff --git a/pkg/api/v20230904/openshiftcluster_convert.go b/pkg/api/v20230904/openshiftcluster_convert.go
index fd8069d55..5bfbf0bbd 100644
--- a/pkg/api/v20230904/openshiftcluster_convert.go
+++ b/pkg/api/v20230904/openshiftcluster_convert.go
@@ -169,6 +169,7 @@ func (c openShiftClusterConverter) ToInternal(_oc interface{}, out *api.OpenShif
 	out.Properties.NetworkProfile.PodCIDR = oc.Properties.NetworkProfile.PodCIDR
 	out.Properties.NetworkProfile.ServiceCIDR = oc.Properties.NetworkProfile.ServiceCIDR
 	out.Properties.NetworkProfile.OutboundType = api.OutboundType(oc.Properties.NetworkProfile.OutboundType)
+	out.Properties.NetworkProfile.PreconfiguredNSG = api.PreconfiguredNSG(oc.Properties.NetworkProfile.PreconfiguredNSG)
 	out.Properties.MasterProfile.VMSize = api.VMSize(oc.Properties.MasterProfile.VMSize)
 	out.Properties.MasterProfile.SubnetID = oc.Properties.MasterProfile.SubnetID
 	out.Properties.MasterProfile.EncryptionAtHost = api.EncryptionAtHost(oc.Properties.MasterProfile.EncryptionAtHost)
diff --git a/pkg/api/v20231122/openshiftcluster_convert.go b/pkg/api/v20231122/openshiftcluster_convert.go
index 53c89d1da..59a2037c2 100644
--- a/pkg/api/v20231122/openshiftcluster_convert.go
+++ b/pkg/api/v20231122/openshiftcluster_convert.go
@@ -190,6 +190,7 @@ func (c openShiftClusterConverter) ToInternal(_oc interface{}, out *api.OpenShif
 	out.Properties.NetworkProfile.PodCIDR = oc.Properties.NetworkProfile.PodCIDR
 	out.Properties.NetworkProfile.ServiceCIDR = oc.Properties.NetworkProfile.ServiceCIDR
 	out.Properties.NetworkProfile.OutboundType = api.OutboundType(oc.Properties.NetworkProfile.OutboundType)
+	out.Properties.NetworkProfile.PreconfiguredNSG = api.PreconfiguredNSG(oc.Properties.NetworkProfile.PreconfiguredNSG)
 
 	if oc.Properties.NetworkProfile.LoadBalancerProfile != nil {
 		loadBalancerProfile := api.LoadBalancerProfile{}
diff --git a/pkg/api/v20240812preview/openshiftcluster.go b/pkg/api/v20240812preview/openshiftcluster.go
index 09a5a632d..28e7d4bd1 100644
--- a/pkg/api/v20240812preview/openshiftcluster.go
+++ b/pkg/api/v20240812preview/openshiftcluster.go
@@ -290,15 +290,19 @@ type IngressProfile struct {
 
 // PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.
 type PlatformWorkloadIdentityProfile struct {
-	PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"`
+	UpgradeableTo              *UpgradeableTo             `json:"upgradeableTo,omitempty" mutable:"true"`
+	PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty" mutable:"true"`
 }
 
+// UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to
+type UpgradeableTo string
+
 // PlatformWorkloadIdentity stores information representing a single workload identity.
 type PlatformWorkloadIdentity struct {
-	OperatorName string `json:"operatorName,omitempty"`
-	ResourceID   string `json:"resourceId,omitempty"`
-	ClientID     string `json:"clientId,omitempty" swagger:"readOnly"`
-	ObjectID     string `json:"objectId,omitempty" swagger:"readOnly"`
+	OperatorName string `json:"operatorName,omitempty" mutable:"true"`
+	ResourceID   string `json:"resourceId,omitempty" mutable:"true"`
+	ClientID     string `json:"clientId,omitempty" swagger:"readOnly" mutable:"true"`
+	ObjectID     string `json:"objectId,omitempty" swagger:"readOnly" mutable:"true"`
 }
 
 // ClusterUserAssignedIdentity stores information about a user-assigned managed identity in a predefined format required by Microsoft's Managed Identity team.
diff --git a/pkg/api/v20240812preview/openshiftcluster_convert.go b/pkg/api/v20240812preview/openshiftcluster_convert.go
index ad51f4406..80d0793b6 100644
--- a/pkg/api/v20240812preview/openshiftcluster_convert.go
+++ b/pkg/api/v20240812preview/openshiftcluster_convert.go
@@ -141,6 +141,12 @@ func (c openShiftClusterConverter) ToExternal(oc *api.OpenShiftCluster) interfac
 
 	if oc.Properties.PlatformWorkloadIdentityProfile != nil && oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities != nil {
 		out.Properties.PlatformWorkloadIdentityProfile = &PlatformWorkloadIdentityProfile{}
+
+		if oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo != nil {
+			temp := UpgradeableTo(*oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo)
+			out.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo = &temp
+		}
+
 		out.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities = make([]PlatformWorkloadIdentity, len(oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities))
 
 		for i := range oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities {
@@ -225,6 +231,12 @@ func (c openShiftClusterConverter) ToInternal(_oc interface{}, out *api.OpenShif
 	}
 	if oc.Properties.PlatformWorkloadIdentityProfile != nil && oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities != nil {
 		out.Properties.PlatformWorkloadIdentityProfile = &api.PlatformWorkloadIdentityProfile{}
+
+		if oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo != nil {
+			temp := api.UpgradeableTo(*oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo)
+			out.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo = &temp
+		}
+
 		out.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities = make([]api.PlatformWorkloadIdentity, len(oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities))
 
 		for i := range oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities {
@@ -238,6 +250,7 @@ func (c openShiftClusterConverter) ToInternal(_oc interface{}, out *api.OpenShif
 	out.Properties.NetworkProfile.PodCIDR = oc.Properties.NetworkProfile.PodCIDR
 	out.Properties.NetworkProfile.ServiceCIDR = oc.Properties.NetworkProfile.ServiceCIDR
 	out.Properties.NetworkProfile.OutboundType = api.OutboundType(oc.Properties.NetworkProfile.OutboundType)
+	out.Properties.NetworkProfile.PreconfiguredNSG = api.PreconfiguredNSG(oc.Properties.NetworkProfile.PreconfiguredNSG)
 
 	if oc.Properties.NetworkProfile.LoadBalancerProfile != nil {
 		loadBalancerProfile := api.LoadBalancerProfile{}
diff --git a/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go b/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go
index e438d5fc1..2535b27f7 100644
--- a/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go
+++ b/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go
@@ -1525,6 +1525,7 @@ func (pwi PlatformWorkloadIdentity) MarshalJSON() ([]byte, error) {
 // PlatformWorkloadIdentityProfile platformWorkloadIdentityProfile encapsulates all information that is
 // specific to workload identity clusters.
 type PlatformWorkloadIdentityProfile struct {
+	UpgradeableTo              *string                     `json:"upgradeableTo,omitempty"`
 	PlatformWorkloadIdentities *[]PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"`
 }
 
diff --git a/pkg/cluster/arooperator.go b/pkg/cluster/arooperator.go
index 7afbb5b3c..ca187e23f 100644
--- a/pkg/cluster/arooperator.go
+++ b/pkg/cluster/arooperator.go
@@ -65,6 +65,10 @@ func (m *manager) ensureCredentialsRequest(ctx context.Context) error {
 	return m.aroOperatorDeployer.CreateOrUpdateCredentialsRequest(ctx)
 }
 
+func (m *manager) ensureUpgradeAnnotation(ctx context.Context) error {
+	return m.aroOperatorDeployer.EnsureUpgradeAnnotation(ctx)
+}
+
 func (m *manager) renewMDSDCertificate(ctx context.Context) error {
 	return m.aroOperatorDeployer.RenewMDSDCertificate(ctx)
 }
diff --git a/pkg/cluster/install.go b/pkg/cluster/install.go
index 5e14d57ef..072b9cb29 100644
--- a/pkg/cluster/install.go
+++ b/pkg/cluster/install.go
@@ -213,6 +213,7 @@ func (m *manager) Update(ctx context.Context) error {
 		steps.Action(m.updateAROSecret),
 		steps.Action(m.restartAROOperatorMaster), // depends on m.updateOpenShiftSecret; the point of restarting is to pick up any changes made to the secret
 		steps.Condition(m.aroDeploymentReady, 5*time.Minute, true),
+		steps.Action(m.ensureUpgradeAnnotation),
 		steps.Action(m.reconcileLoadBalancerProfile),
 	}
 
@@ -524,7 +525,7 @@ func (m *manager) initializeKubernetesClients(ctx context.Context) error {
 // initializeKubernetesClients initializes clients which are used
 // once the cluster is up later on in the install process.
 func (m *manager) initializeOperatorDeployer(ctx context.Context) (err error) {
-	m.aroOperatorDeployer, err = deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.client, m.extensionscli, m.kubernetescli)
+	m.aroOperatorDeployer, err = deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.client, m.extensionscli, m.kubernetescli, m.operatorcli)
 	return
 }
 
diff --git a/pkg/database/cosmosdb/generate.go b/pkg/database/cosmosdb/generate.go
index d82fa1a5d..b89f19f2b 100644
--- a/pkg/database/cosmosdb/generate.go
+++ b/pkg/database/cosmosdb/generate.go
@@ -3,7 +3,7 @@ package cosmosdb
 // Copyright (c) Microsoft Corporation.
 // Licensed under the Apache License 2.0.
 
-//go:generate go run ../../../vendor/github.com/jewzaam/go-cosmosdb/cmd/gencosmosdb github.com/Azure/ARO-RP/pkg/api,AsyncOperationDocument github.com/Azure/ARO-RP/pkg/api,BillingDocument github.com/Azure/ARO-RP/pkg/api,GatewayDocument github.com/Azure/ARO-RP/pkg/api,MonitorDocument github.com/Azure/ARO-RP/pkg/api,OpenShiftClusterDocument github.com/Azure/ARO-RP/pkg/api,SubscriptionDocument github.com/Azure/ARO-RP/pkg/api,OpenShiftVersionDocument github.com/Azure/ARO-RP/pkg/api,ClusterManagerConfigurationDocument
+//go:generate go run ../../../vendor/github.com/jewzaam/go-cosmosdb/cmd/gencosmosdb github.com/Azure/ARO-RP/pkg/api,AsyncOperationDocument github.com/Azure/ARO-RP/pkg/api,BillingDocument github.com/Azure/ARO-RP/pkg/api,GatewayDocument github.com/Azure/ARO-RP/pkg/api,MonitorDocument github.com/Azure/ARO-RP/pkg/api,OpenShiftClusterDocument github.com/Azure/ARO-RP/pkg/api,SubscriptionDocument github.com/Azure/ARO-RP/pkg/api,OpenShiftVersionDocument github.com/Azure/ARO-RP/pkg/api,ClusterManagerConfigurationDocument github.com/Azure/ARO-RP/pkg/api,PlatformWorkloadIdentityRoleSetDocument
 //go:generate go run ../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ./
 //go:generate go run ../../../vendor/github.com/golang/mock/mockgen -destination=../../util/mocks/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/database/$GOPACKAGE PermissionClient
 //go:generate go run ../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../util/mocks/$GOPACKAGE/$GOPACKAGE.go
diff --git a/pkg/database/cosmosdb/zz_generated_platformworkloadidentityrolesetdocument.go b/pkg/database/cosmosdb/zz_generated_platformworkloadidentityrolesetdocument.go
new file mode 100644
index 000000000..5b7696418
--- /dev/null
+++ b/pkg/database/cosmosdb/zz_generated_platformworkloadidentityrolesetdocument.go
@@ -0,0 +1,313 @@
+// Code generated by github.com/jewzaam/go-cosmosdb, DO NOT EDIT.
+
+package cosmosdb
+
+import (
+	"context"
+	"net/http"
+	"strconv"
+	"strings"
+
+	pkg "github.com/Azure/ARO-RP/pkg/api"
+)
+
+type platformWorkloadIdentityRoleSetDocumentClient struct {
+	*databaseClient
+	path string
+}
+
+// PlatformWorkloadIdentityRoleSetDocumentClient is a platformWorkloadIdentityRoleSetDocument client
+type PlatformWorkloadIdentityRoleSetDocumentClient interface {
+	Create(context.Context, string, *pkg.PlatformWorkloadIdentityRoleSetDocument, *Options) (*pkg.PlatformWorkloadIdentityRoleSetDocument, error)
+	List(*Options) PlatformWorkloadIdentityRoleSetDocumentIterator
+	ListAll(context.Context, *Options) (*pkg.PlatformWorkloadIdentityRoleSetDocuments, error)
+	Get(context.Context, string, string, *Options) (*pkg.PlatformWorkloadIdentityRoleSetDocument, error)
+	Replace(context.Context, string, *pkg.PlatformWorkloadIdentityRoleSetDocument, *Options) (*pkg.PlatformWorkloadIdentityRoleSetDocument, error)
+	Delete(context.Context, string, *pkg.PlatformWorkloadIdentityRoleSetDocument, *Options) error
+	Query(string, *Query, *Options) PlatformWorkloadIdentityRoleSetDocumentRawIterator
+	QueryAll(context.Context, string, *Query, *Options) (*pkg.PlatformWorkloadIdentityRoleSetDocuments, error)
+	ChangeFeed(*Options) PlatformWorkloadIdentityRoleSetDocumentIterator
+}
+
+type platformWorkloadIdentityRoleSetDocumentChangeFeedIterator struct {
+	*platformWorkloadIdentityRoleSetDocumentClient
+	continuation string
+	options      *Options
+}
+
+type platformWorkloadIdentityRoleSetDocumentListIterator struct {
+	*platformWorkloadIdentityRoleSetDocumentClient
+	continuation string
+	done         bool
+	options      *Options
+}
+
+type platformWorkloadIdentityRoleSetDocumentQueryIterator struct {
+	*platformWorkloadIdentityRoleSetDocumentClient
+	partitionkey string
+	query        *Query
+	continuation string
+	done         bool
+	options      *Options
+}
+
+// PlatformWorkloadIdentityRoleSetDocumentIterator is a platformWorkloadIdentityRoleSetDocument iterator
+type PlatformWorkloadIdentityRoleSetDocumentIterator interface {
+	Next(context.Context, int) (*pkg.PlatformWorkloadIdentityRoleSetDocuments, error)
+	Continuation() string
+}
+
+// PlatformWorkloadIdentityRoleSetDocumentRawIterator is a platformWorkloadIdentityRoleSetDocument raw iterator
+type PlatformWorkloadIdentityRoleSetDocumentRawIterator interface {
+	PlatformWorkloadIdentityRoleSetDocumentIterator
+	NextRaw(context.Context, int, interface{}) error
+}
+
+// NewPlatformWorkloadIdentityRoleSetDocumentClient returns a new platformWorkloadIdentityRoleSetDocument client
+func NewPlatformWorkloadIdentityRoleSetDocumentClient(collc CollectionClient, collid string) PlatformWorkloadIdentityRoleSetDocumentClient {
+	return &platformWorkloadIdentityRoleSetDocumentClient{
+		databaseClient: collc.(*collectionClient).databaseClient,
+		path:           collc.(*collectionClient).path + "/colls/" + collid,
+	}
+}
+
+func (c *platformWorkloadIdentityRoleSetDocumentClient) all(ctx context.Context, i PlatformWorkloadIdentityRoleSetDocumentIterator) (*pkg.PlatformWorkloadIdentityRoleSetDocuments, error) {
+	allplatformWorkloadIdentityRoleSetDocuments := &pkg.PlatformWorkloadIdentityRoleSetDocuments{}
+
+	for {
+		platformWorkloadIdentityRoleSetDocuments, err := i.Next(ctx, -1)
+		if err != nil {
+			return nil, err
+		}
+		if platformWorkloadIdentityRoleSetDocuments == nil {
+			break
+		}
+
+		allplatformWorkloadIdentityRoleSetDocuments.Count += platformWorkloadIdentityRoleSetDocuments.Count
+		allplatformWorkloadIdentityRoleSetDocuments.ResourceID = platformWorkloadIdentityRoleSetDocuments.ResourceID
+		allplatformWorkloadIdentityRoleSetDocuments.PlatformWorkloadIdentityRoleSetDocuments = append(allplatformWorkloadIdentityRoleSetDocuments.PlatformWorkloadIdentityRoleSetDocuments, platformWorkloadIdentityRoleSetDocuments.PlatformWorkloadIdentityRoleSetDocuments...)
+	}
+
+	return allplatformWorkloadIdentityRoleSetDocuments, nil
+}
+
+func (c *platformWorkloadIdentityRoleSetDocumentClient) Create(ctx context.Context, partitionkey string, newplatformWorkloadIdentityRoleSetDocument *pkg.PlatformWorkloadIdentityRoleSetDocument, options *Options) (platformWorkloadIdentityRoleSetDocument *pkg.PlatformWorkloadIdentityRoleSetDocument, err error) {
+	headers := http.Header{}
+	headers.Set("X-Ms-Documentdb-Partitionkey", `["`+partitionkey+`"]`)
+
+	if options == nil {
+		options = &Options{}
+	}
+	options.NoETag = true
+
+	err = c.setOptions(options, newplatformWorkloadIdentityRoleSetDocument, headers)
+	if err != nil {
+		return
+	}
+
+	err = c.do(ctx, http.MethodPost, c.path+"/docs", "docs", c.path, http.StatusCreated, &newplatformWorkloadIdentityRoleSetDocument, &platformWorkloadIdentityRoleSetDocument, headers)
+	return
+}
+
+func (c *platformWorkloadIdentityRoleSetDocumentClient) List(options *Options) PlatformWorkloadIdentityRoleSetDocumentIterator {
+	continuation := ""
+	if options != nil {
+		continuation = options.Continuation
+	}
+
+	return &platformWorkloadIdentityRoleSetDocumentListIterator{platformWorkloadIdentityRoleSetDocumentClient: c, options: options, continuation: continuation}
+}
+
+func (c *platformWorkloadIdentityRoleSetDocumentClient) ListAll(ctx context.Context, options *Options) (*pkg.PlatformWorkloadIdentityRoleSetDocuments, error) {
+	return c.all(ctx, c.List(options))
+}
+
+func (c *platformWorkloadIdentityRoleSetDocumentClient) Get(ctx context.Context, partitionkey, platformWorkloadIdentityRoleSetDocumentid string, options *Options) (platformWorkloadIdentityRoleSetDocument *pkg.PlatformWorkloadIdentityRoleSetDocument, err error) {
+	headers := http.Header{}
+	headers.Set("X-Ms-Documentdb-Partitionkey", `["`+partitionkey+`"]`)
+
+	err = c.setOptions(options, nil, headers)
+	if err != nil {
+		return
+	}
+
+	err = c.do(ctx, http.MethodGet, c.path+"/docs/"+platformWorkloadIdentityRoleSetDocumentid, "docs", c.path+"/docs/"+platformWorkloadIdentityRoleSetDocumentid, http.StatusOK, nil, &platformWorkloadIdentityRoleSetDocument, headers)
+	return
+}
+
+func (c *platformWorkloadIdentityRoleSetDocumentClient) Replace(ctx context.Context, partitionkey string, newplatformWorkloadIdentityRoleSetDocument *pkg.PlatformWorkloadIdentityRoleSetDocument, options *Options) (platformWorkloadIdentityRoleSetDocument *pkg.PlatformWorkloadIdentityRoleSetDocument, err error) {
+	headers := http.Header{}
+	headers.Set("X-Ms-Documentdb-Partitionkey", `["`+partitionkey+`"]`)
+
+	err = c.setOptions(options, newplatformWorkloadIdentityRoleSetDocument, headers)
+	if err != nil {
+		return
+	}
+
+	err = c.do(ctx, http.MethodPut, c.path+"/docs/"+newplatformWorkloadIdentityRoleSetDocument.ID, "docs", c.path+"/docs/"+newplatformWorkloadIdentityRoleSetDocument.ID, http.StatusOK, &newplatformWorkloadIdentityRoleSetDocument, &platformWorkloadIdentityRoleSetDocument, headers)
+	return
+}
+
+func (c *platformWorkloadIdentityRoleSetDocumentClient) Delete(ctx context.Context, partitionkey string, platformWorkloadIdentityRoleSetDocument *pkg.PlatformWorkloadIdentityRoleSetDocument, options *Options) (err error) {
+	headers := http.Header{}
+	headers.Set("X-Ms-Documentdb-Partitionkey", `["`+partitionkey+`"]`)
+
+	err = c.setOptions(options, platformWorkloadIdentityRoleSetDocument, headers)
+	if err != nil {
+		return
+	}
+
+	err = c.do(ctx, http.MethodDelete, c.path+"/docs/"+platformWorkloadIdentityRoleSetDocument.ID, "docs", c.path+"/docs/"+platformWorkloadIdentityRoleSetDocument.ID, http.StatusNoContent, nil, nil, headers)
+	return
+}
+
+func (c *platformWorkloadIdentityRoleSetDocumentClient) Query(partitionkey string, query *Query, options *Options) PlatformWorkloadIdentityRoleSetDocumentRawIterator {
+	continuation := ""
+	if options != nil {
+		continuation = options.Continuation
+	}
+
+	return &platformWorkloadIdentityRoleSetDocumentQueryIterator{platformWorkloadIdentityRoleSetDocumentClient: c, partitionkey: partitionkey, query: query, options: options, continuation: continuation}
+}
+
+func (c *platformWorkloadIdentityRoleSetDocumentClient) QueryAll(ctx context.Context, partitionkey string, query *Query, options *Options) (*pkg.PlatformWorkloadIdentityRoleSetDocuments, error) {
+	return c.all(ctx, c.Query(partitionkey, query, options))
+}
+
+func (c *platformWorkloadIdentityRoleSetDocumentClient) ChangeFeed(options *Options) PlatformWorkloadIdentityRoleSetDocumentIterator {
+	continuation := ""
+	if options != nil {
+		continuation = options.Continuation
+	}
+
+	return &platformWorkloadIdentityRoleSetDocumentChangeFeedIterator{platformWorkloadIdentityRoleSetDocumentClient: c, options: options, continuation: continuation}
+}
+
+func (c *platformWorkloadIdentityRoleSetDocumentClient) setOptions(options *Options, platformWorkloadIdentityRoleSetDocument *pkg.PlatformWorkloadIdentityRoleSetDocument, headers http.Header) error {
+	if options == nil {
+		return nil
+	}
+
+	if platformWorkloadIdentityRoleSetDocument != nil && !options.NoETag {
+		if platformWorkloadIdentityRoleSetDocument.ETag == "" {
+			return ErrETagRequired
+		}
+		headers.Set("If-Match", platformWorkloadIdentityRoleSetDocument.ETag)
+	}
+	if len(options.PreTriggers) > 0 {
+		headers.Set("X-Ms-Documentdb-Pre-Trigger-Include", strings.Join(options.PreTriggers, ","))
+	}
+	if len(options.PostTriggers) > 0 {
+		headers.Set("X-Ms-Documentdb-Post-Trigger-Include", strings.Join(options.PostTriggers, ","))
+	}
+	if len(options.PartitionKeyRangeID) > 0 {
+		headers.Set("X-Ms-Documentdb-PartitionKeyRangeID", options.PartitionKeyRangeID)
+	}
+
+	return nil
+}
+
+func (i *platformWorkloadIdentityRoleSetDocumentChangeFeedIterator) Next(ctx context.Context, maxItemCount int) (platformWorkloadIdentityRoleSetDocuments *pkg.PlatformWorkloadIdentityRoleSetDocuments, err error) {
+	headers := http.Header{}
+	headers.Set("A-IM", "Incremental feed")
+
+	headers.Set("X-Ms-Max-Item-Count", strconv.Itoa(maxItemCount))
+	if i.continuation != "" {
+		headers.Set("If-None-Match", i.continuation)
+	}
+
+	err = i.setOptions(i.options, nil, headers)
+	if err != nil {
+		return
+	}
+
+	err = i.do(ctx, http.MethodGet, i.path+"/docs", "docs", i.path, http.StatusOK, nil, &platformWorkloadIdentityRoleSetDocuments, headers)
+	if IsErrorStatusCode(err, http.StatusNotModified) {
+		err = nil
+	}
+	if err != nil {
+		return
+	}
+
+	i.continuation = headers.Get("Etag")
+
+	return
+}
+
+func (i *platformWorkloadIdentityRoleSetDocumentChangeFeedIterator) Continuation() string {
+	return i.continuation
+}
+
+func (i *platformWorkloadIdentityRoleSetDocumentListIterator) Next(ctx context.Context, maxItemCount int) (platformWorkloadIdentityRoleSetDocuments *pkg.PlatformWorkloadIdentityRoleSetDocuments, err error) {
+	if i.done {
+		return
+	}
+
+	headers := http.Header{}
+	headers.Set("X-Ms-Max-Item-Count", strconv.Itoa(maxItemCount))
+	if i.continuation != "" {
+		headers.Set("X-Ms-Continuation", i.continuation)
+	}
+
+	err = i.setOptions(i.options, nil, headers)
+	if err != nil {
+		return
+	}
+
+	err = i.do(ctx, http.MethodGet, i.path+"/docs", "docs", i.path, http.StatusOK, nil, &platformWorkloadIdentityRoleSetDocuments, headers)
+	if err != nil {
+		return
+	}
+
+	i.continuation = headers.Get("X-Ms-Continuation")
+	i.done = i.continuation == ""
+
+	return
+}
+
+func (i *platformWorkloadIdentityRoleSetDocumentListIterator) Continuation() string {
+	return i.continuation
+}
+
+func (i *platformWorkloadIdentityRoleSetDocumentQueryIterator) Next(ctx context.Context, maxItemCount int) (platformWorkloadIdentityRoleSetDocuments *pkg.PlatformWorkloadIdentityRoleSetDocuments, err error) {
+	err = i.NextRaw(ctx, maxItemCount, &platformWorkloadIdentityRoleSetDocuments)
+	return
+}
+
+func (i *platformWorkloadIdentityRoleSetDocumentQueryIterator) NextRaw(ctx context.Context, maxItemCount int, raw interface{}) (err error) {
+	if i.done {
+		return
+	}
+
+	headers := http.Header{}
+	headers.Set("X-Ms-Max-Item-Count", strconv.Itoa(maxItemCount))
+	headers.Set("X-Ms-Documentdb-Isquery", "True")
+	headers.Set("Content-Type", "application/query+json")
+	if i.partitionkey != "" {
+		headers.Set("X-Ms-Documentdb-Partitionkey", `["`+i.partitionkey+`"]`)
+	} else {
+		headers.Set("X-Ms-Documentdb-Query-Enablecrosspartition", "True")
+	}
+	if i.continuation != "" {
+		headers.Set("X-Ms-Continuation", i.continuation)
+	}
+
+	err = i.setOptions(i.options, nil, headers)
+	if err != nil {
+		return
+	}
+
+	err = i.do(ctx, http.MethodPost, i.path+"/docs", "docs", i.path, http.StatusOK, &i.query, &raw, headers)
+	if err != nil {
+		return
+	}
+
+	i.continuation = headers.Get("X-Ms-Continuation")
+	i.done = i.continuation == ""
+
+	return
+}
+
+func (i *platformWorkloadIdentityRoleSetDocumentQueryIterator) Continuation() string {
+	return i.continuation
+}
diff --git a/pkg/database/cosmosdb/zz_generated_platformworkloadidentityrolesetdocument_fake.go b/pkg/database/cosmosdb/zz_generated_platformworkloadidentityrolesetdocument_fake.go
new file mode 100644
index 000000000..78a2f8109
--- /dev/null
+++ b/pkg/database/cosmosdb/zz_generated_platformworkloadidentityrolesetdocument_fake.go
@@ -0,0 +1,361 @@
+// Code generated by github.com/jewzaam/go-cosmosdb, DO NOT EDIT.
+
+package cosmosdb
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"sync"
+
+	"github.com/ugorji/go/codec"
+
+	pkg "github.com/Azure/ARO-RP/pkg/api"
+)
+
+type fakePlatformWorkloadIdentityRoleSetDocumentTriggerHandler func(context.Context, *pkg.PlatformWorkloadIdentityRoleSetDocument) error
+type fakePlatformWorkloadIdentityRoleSetDocumentQueryHandler func(PlatformWorkloadIdentityRoleSetDocumentClient, *Query, *Options) PlatformWorkloadIdentityRoleSetDocumentRawIterator
+
+var _ PlatformWorkloadIdentityRoleSetDocumentClient = &FakePlatformWorkloadIdentityRoleSetDocumentClient{}
+
+// NewFakePlatformWorkloadIdentityRoleSetDocumentClient returns a FakePlatformWorkloadIdentityRoleSetDocumentClient
+func NewFakePlatformWorkloadIdentityRoleSetDocumentClient(h *codec.JsonHandle) *FakePlatformWorkloadIdentityRoleSetDocumentClient {
+	return &FakePlatformWorkloadIdentityRoleSetDocumentClient{
+		jsonHandle:                               h,
+		platformWorkloadIdentityRoleSetDocuments: make(map[string]*pkg.PlatformWorkloadIdentityRoleSetDocument),
+		triggerHandlers:                          make(map[string]fakePlatformWorkloadIdentityRoleSetDocumentTriggerHandler),
+		queryHandlers:                            make(map[string]fakePlatformWorkloadIdentityRoleSetDocumentQueryHandler),
+	}
+}
+
+// FakePlatformWorkloadIdentityRoleSetDocumentClient is a FakePlatformWorkloadIdentityRoleSetDocumentClient
+type FakePlatformWorkloadIdentityRoleSetDocumentClient struct {
+	lock                                     sync.RWMutex
+	jsonHandle                               *codec.JsonHandle
+	platformWorkloadIdentityRoleSetDocuments map[string]*pkg.PlatformWorkloadIdentityRoleSetDocument
+	triggerHandlers                          map[string]fakePlatformWorkloadIdentityRoleSetDocumentTriggerHandler
+	queryHandlers                            map[string]fakePlatformWorkloadIdentityRoleSetDocumentQueryHandler
+	sorter                                   func([]*pkg.PlatformWorkloadIdentityRoleSetDocument)
+	etag                                     int
+
+	// returns true if documents conflict
+	conflictChecker func(*pkg.PlatformWorkloadIdentityRoleSetDocument, *pkg.PlatformWorkloadIdentityRoleSetDocument) bool
+
+	// err, if not nil, is an error to return when attempting to communicate
+	// with this Client
+	err error
+}
+
+// SetError sets or unsets an error that will be returned on any
+// FakePlatformWorkloadIdentityRoleSetDocumentClient method invocation
+func (c *FakePlatformWorkloadIdentityRoleSetDocumentClient) SetError(err error) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	c.err = err
+}
+
+// SetSorter sets or unsets a sorter function which will be used to sort values
+// returned by List() for test stability
+func (c *FakePlatformWorkloadIdentityRoleSetDocumentClient) SetSorter(sorter func([]*pkg.PlatformWorkloadIdentityRoleSetDocument)) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	c.sorter = sorter
+}
+
+// SetConflictChecker sets or unsets a function which can be used to validate
+// additional unique keys in a PlatformWorkloadIdentityRoleSetDocument
+func (c *FakePlatformWorkloadIdentityRoleSetDocumentClient) SetConflictChecker(conflictChecker func(*pkg.PlatformWorkloadIdentityRoleSetDocument, *pkg.PlatformWorkloadIdentityRoleSetDocument) bool) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	c.conflictChecker = conflictChecker
+}
+
+// SetTriggerHandler sets or unsets a trigger handler
+func (c *FakePlatformWorkloadIdentityRoleSetDocumentClient) SetTriggerHandler(triggerName string, trigger fakePlatformWorkloadIdentityRoleSetDocumentTriggerHandler) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	c.triggerHandlers[triggerName] = trigger
+}
+
+// SetQueryHandler sets or unsets a query handler
+func (c *FakePlatformWorkloadIdentityRoleSetDocumentClient) SetQueryHandler(queryName string, query fakePlatformWorkloadIdentityRoleSetDocumentQueryHandler) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	c.queryHandlers[queryName] = query
+}
+
+func (c *FakePlatformWorkloadIdentityRoleSetDocumentClient) deepCopy(platformWorkloadIdentityRoleSetDocument *pkg.PlatformWorkloadIdentityRoleSetDocument) (*pkg.PlatformWorkloadIdentityRoleSetDocument, error) {
+	var b []byte
+	err := codec.NewEncoderBytes(&b, c.jsonHandle).Encode(platformWorkloadIdentityRoleSetDocument)
+	if err != nil {
+		return nil, err
+	}
+
+	platformWorkloadIdentityRoleSetDocument = nil
+	err = codec.NewDecoderBytes(b, c.jsonHandle).Decode(&platformWorkloadIdentityRoleSetDocument)
+	if err != nil {
+		return nil, err
+	}
+
+	return platformWorkloadIdentityRoleSetDocument, nil
+}
+
+func (c *FakePlatformWorkloadIdentityRoleSetDocumentClient) apply(ctx context.Context, partitionkey string, platformWorkloadIdentityRoleSetDocument *pkg.PlatformWorkloadIdentityRoleSetDocument, options *Options, isCreate bool) (*pkg.PlatformWorkloadIdentityRoleSetDocument, error) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	if c.err != nil {
+		return nil, c.err
+	}
+
+	platformWorkloadIdentityRoleSetDocument, err := c.deepCopy(platformWorkloadIdentityRoleSetDocument) // copy now because pretriggers can mutate platformWorkloadIdentityRoleSetDocument
+	if err != nil {
+		return nil, err
+	}
+
+	if options != nil {
+		err := c.processPreTriggers(ctx, platformWorkloadIdentityRoleSetDocument, options)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	existingPlatformWorkloadIdentityRoleSetDocument, exists := c.platformWorkloadIdentityRoleSetDocuments[platformWorkloadIdentityRoleSetDocument.ID]
+	if isCreate && exists {
+		return nil, &Error{
+			StatusCode: http.StatusConflict,
+			Message:    "Entity with the specified id already exists in the system",
+		}
+	}
+	if !isCreate {
+		if !exists {
+			return nil, &Error{StatusCode: http.StatusNotFound}
+		}
+
+		if platformWorkloadIdentityRoleSetDocument.ETag != existingPlatformWorkloadIdentityRoleSetDocument.ETag {
+			return nil, &Error{StatusCode: http.StatusPreconditionFailed}
+		}
+	}
+
+	if c.conflictChecker != nil {
+		for _, platformWorkloadIdentityRoleSetDocumentToCheck := range c.platformWorkloadIdentityRoleSetDocuments {
+			if c.conflictChecker(platformWorkloadIdentityRoleSetDocumentToCheck, platformWorkloadIdentityRoleSetDocument) {
+				return nil, &Error{
+					StatusCode: http.StatusConflict,
+					Message:    "Entity with the specified id already exists in the system",
+				}
+			}
+		}
+	}
+
+	platformWorkloadIdentityRoleSetDocument.ETag = fmt.Sprint(c.etag)
+	c.etag++
+
+	c.platformWorkloadIdentityRoleSetDocuments[platformWorkloadIdentityRoleSetDocument.ID] = platformWorkloadIdentityRoleSetDocument
+
+	return c.deepCopy(platformWorkloadIdentityRoleSetDocument)
+}
+
+// Create creates a PlatformWorkloadIdentityRoleSetDocument in the database
+func (c *FakePlatformWorkloadIdentityRoleSetDocumentClient) Create(ctx context.Context, partitionkey string, platformWorkloadIdentityRoleSetDocument *pkg.PlatformWorkloadIdentityRoleSetDocument, options *Options) (*pkg.PlatformWorkloadIdentityRoleSetDocument, error) {
+	return c.apply(ctx, partitionkey, platformWorkloadIdentityRoleSetDocument, options, true)
+}
+
+// Replace replaces a PlatformWorkloadIdentityRoleSetDocument in the database
+func (c *FakePlatformWorkloadIdentityRoleSetDocumentClient) Replace(ctx context.Context, partitionkey string, platformWorkloadIdentityRoleSetDocument *pkg.PlatformWorkloadIdentityRoleSetDocument, options *Options) (*pkg.PlatformWorkloadIdentityRoleSetDocument, error) {
+	return c.apply(ctx, partitionkey, platformWorkloadIdentityRoleSetDocument, options, false)
+}
+
+// List returns a PlatformWorkloadIdentityRoleSetDocumentIterator to list all PlatformWorkloadIdentityRoleSetDocuments in the database
+func (c *FakePlatformWorkloadIdentityRoleSetDocumentClient) List(*Options) PlatformWorkloadIdentityRoleSetDocumentIterator {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+
+	if c.err != nil {
+		return NewFakePlatformWorkloadIdentityRoleSetDocumentErroringRawIterator(c.err)
+	}
+
+	platformWorkloadIdentityRoleSetDocuments := make([]*pkg.PlatformWorkloadIdentityRoleSetDocument, 0, len(c.platformWorkloadIdentityRoleSetDocuments))
+	for _, platformWorkloadIdentityRoleSetDocument := range c.platformWorkloadIdentityRoleSetDocuments {
+		platformWorkloadIdentityRoleSetDocument, err := c.deepCopy(platformWorkloadIdentityRoleSetDocument)
+		if err != nil {
+			return NewFakePlatformWorkloadIdentityRoleSetDocumentErroringRawIterator(err)
+		}
+		platformWorkloadIdentityRoleSetDocuments = append(platformWorkloadIdentityRoleSetDocuments, platformWorkloadIdentityRoleSetDocument)
+	}
+
+	if c.sorter != nil {
+		c.sorter(platformWorkloadIdentityRoleSetDocuments)
+	}
+
+	return NewFakePlatformWorkloadIdentityRoleSetDocumentIterator(platformWorkloadIdentityRoleSetDocuments, 0)
+}
+
+// ListAll lists all PlatformWorkloadIdentityRoleSetDocuments in the database
+func (c *FakePlatformWorkloadIdentityRoleSetDocumentClient) ListAll(ctx context.Context, options *Options) (*pkg.PlatformWorkloadIdentityRoleSetDocuments, error) {
+	iter := c.List(options)
+	return iter.Next(ctx, -1)
+}
+
+// Get gets a PlatformWorkloadIdentityRoleSetDocument from the database
+func (c *FakePlatformWorkloadIdentityRoleSetDocumentClient) Get(ctx context.Context, partitionkey string, id string, options *Options) (*pkg.PlatformWorkloadIdentityRoleSetDocument, error) {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+
+	if c.err != nil {
+		return nil, c.err
+	}
+
+	platformWorkloadIdentityRoleSetDocument, exists := c.platformWorkloadIdentityRoleSetDocuments[id]
+	if !exists {
+		return nil, &Error{StatusCode: http.StatusNotFound}
+	}
+
+	return c.deepCopy(platformWorkloadIdentityRoleSetDocument)
+}
+
+// Delete deletes a PlatformWorkloadIdentityRoleSetDocument from the database
+func (c *FakePlatformWorkloadIdentityRoleSetDocumentClient) Delete(ctx context.Context, partitionKey string, platformWorkloadIdentityRoleSetDocument *pkg.PlatformWorkloadIdentityRoleSetDocument, options *Options) error {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	if c.err != nil {
+		return c.err
+	}
+
+	_, exists := c.platformWorkloadIdentityRoleSetDocuments[platformWorkloadIdentityRoleSetDocument.ID]
+	if !exists {
+		return &Error{StatusCode: http.StatusNotFound}
+	}
+
+	delete(c.platformWorkloadIdentityRoleSetDocuments, platformWorkloadIdentityRoleSetDocument.ID)
+	return nil
+}
+
+// ChangeFeed is unimplemented
+func (c *FakePlatformWorkloadIdentityRoleSetDocumentClient) ChangeFeed(*Options) PlatformWorkloadIdentityRoleSetDocumentIterator {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+
+	if c.err != nil {
+		return NewFakePlatformWorkloadIdentityRoleSetDocumentErroringRawIterator(c.err)
+	}
+
+	return NewFakePlatformWorkloadIdentityRoleSetDocumentErroringRawIterator(ErrNotImplemented)
+}
+
+func (c *FakePlatformWorkloadIdentityRoleSetDocumentClient) processPreTriggers(ctx context.Context, platformWorkloadIdentityRoleSetDocument *pkg.PlatformWorkloadIdentityRoleSetDocument, options *Options) error {
+	for _, triggerName := range options.PreTriggers {
+		if triggerHandler := c.triggerHandlers[triggerName]; triggerHandler != nil {
+			c.lock.Unlock()
+			err := triggerHandler(ctx, platformWorkloadIdentityRoleSetDocument)
+			c.lock.Lock()
+			if err != nil {
+				return err
+			}
+		} else {
+			return ErrNotImplemented
+		}
+	}
+
+	return nil
+}
+
+// Query calls a query handler to implement database querying
+func (c *FakePlatformWorkloadIdentityRoleSetDocumentClient) Query(name string, query *Query, options *Options) PlatformWorkloadIdentityRoleSetDocumentRawIterator {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+
+	if c.err != nil {
+		return NewFakePlatformWorkloadIdentityRoleSetDocumentErroringRawIterator(c.err)
+	}
+
+	if queryHandler := c.queryHandlers[query.Query]; queryHandler != nil {
+		c.lock.RUnlock()
+		i := queryHandler(c, query, options)
+		c.lock.RLock()
+		return i
+	}
+
+	return NewFakePlatformWorkloadIdentityRoleSetDocumentErroringRawIterator(ErrNotImplemented)
+}
+
+// QueryAll calls a query handler to implement database querying
+func (c *FakePlatformWorkloadIdentityRoleSetDocumentClient) QueryAll(ctx context.Context, partitionkey string, query *Query, options *Options) (*pkg.PlatformWorkloadIdentityRoleSetDocuments, error) {
+	iter := c.Query("", query, options)
+	return iter.Next(ctx, -1)
+}
+
+func NewFakePlatformWorkloadIdentityRoleSetDocumentIterator(platformWorkloadIdentityRoleSetDocuments []*pkg.PlatformWorkloadIdentityRoleSetDocument, continuation int) PlatformWorkloadIdentityRoleSetDocumentRawIterator {
+	return &fakePlatformWorkloadIdentityRoleSetDocumentIterator{platformWorkloadIdentityRoleSetDocuments: platformWorkloadIdentityRoleSetDocuments, continuation: continuation}
+}
+
+type fakePlatformWorkloadIdentityRoleSetDocumentIterator struct {
+	platformWorkloadIdentityRoleSetDocuments []*pkg.PlatformWorkloadIdentityRoleSetDocument
+	continuation                             int
+	done                                     bool
+}
+
+func (i *fakePlatformWorkloadIdentityRoleSetDocumentIterator) NextRaw(ctx context.Context, maxItemCount int, out interface{}) error {
+	return ErrNotImplemented
+}
+
+func (i *fakePlatformWorkloadIdentityRoleSetDocumentIterator) Next(ctx context.Context, maxItemCount int) (*pkg.PlatformWorkloadIdentityRoleSetDocuments, error) {
+	if i.done {
+		return nil, nil
+	}
+
+	var platformWorkloadIdentityRoleSetDocuments []*pkg.PlatformWorkloadIdentityRoleSetDocument
+	if maxItemCount == -1 {
+		platformWorkloadIdentityRoleSetDocuments = i.platformWorkloadIdentityRoleSetDocuments[i.continuation:]
+		i.continuation = len(i.platformWorkloadIdentityRoleSetDocuments)
+		i.done = true
+	} else {
+		max := i.continuation + maxItemCount
+		if max > len(i.platformWorkloadIdentityRoleSetDocuments) {
+			max = len(i.platformWorkloadIdentityRoleSetDocuments)
+		}
+		platformWorkloadIdentityRoleSetDocuments = i.platformWorkloadIdentityRoleSetDocuments[i.continuation:max]
+		i.continuation += max
+		i.done = i.Continuation() == ""
+	}
+
+	return &pkg.PlatformWorkloadIdentityRoleSetDocuments{
+		PlatformWorkloadIdentityRoleSetDocuments: platformWorkloadIdentityRoleSetDocuments,
+		Count:                                    len(platformWorkloadIdentityRoleSetDocuments),
+	}, nil
+}
+
+func (i *fakePlatformWorkloadIdentityRoleSetDocumentIterator) Continuation() string {
+	if i.continuation >= len(i.platformWorkloadIdentityRoleSetDocuments) {
+		return ""
+	}
+	return fmt.Sprintf("%d", i.continuation)
+}
+
+// NewFakePlatformWorkloadIdentityRoleSetDocumentErroringRawIterator returns a PlatformWorkloadIdentityRoleSetDocumentRawIterator which
+// whose methods return the given error
+func NewFakePlatformWorkloadIdentityRoleSetDocumentErroringRawIterator(err error) PlatformWorkloadIdentityRoleSetDocumentRawIterator {
+	return &fakePlatformWorkloadIdentityRoleSetDocumentErroringRawIterator{err: err}
+}
+
+type fakePlatformWorkloadIdentityRoleSetDocumentErroringRawIterator struct {
+	err error
+}
+
+func (i *fakePlatformWorkloadIdentityRoleSetDocumentErroringRawIterator) Next(ctx context.Context, maxItemCount int) (*pkg.PlatformWorkloadIdentityRoleSetDocuments, error) {
+	return nil, i.err
+}
+
+func (i *fakePlatformWorkloadIdentityRoleSetDocumentErroringRawIterator) NextRaw(context.Context, int, interface{}) error {
+	return i.err
+}
+
+func (i *fakePlatformWorkloadIdentityRoleSetDocumentErroringRawIterator) Continuation() string {
+	return ""
+}
diff --git a/pkg/database/database.go b/pkg/database/database.go
index 4e6ad73d7..550e93cb8 100644
--- a/pkg/database/database.go
+++ b/pkg/database/database.go
@@ -26,15 +26,16 @@ import (
 )
 
 const (
-	collAsyncOperations   = "AsyncOperations"
-	collBilling           = "Billing"
-	collClusterManager    = "ClusterManagerConfigurations"
-	collGateway           = "Gateway"
-	collMonitors          = "Monitors"
-	collOpenShiftClusters = "OpenShiftClusters"
-	collOpenShiftVersion  = "OpenShiftVersions"
-	collPortal            = "Portal"
-	collSubscriptions     = "Subscriptions"
+	collAsyncOperations                 = "AsyncOperations"
+	collBilling                         = "Billing"
+	collClusterManager                  = "ClusterManagerConfigurations"
+	collGateway                         = "Gateway"
+	collMonitors                        = "Monitors"
+	collOpenShiftClusters               = "OpenShiftClusters"
+	collOpenShiftVersion                = "OpenShiftVersions"
+	collPlatformWorkloadIdentityRoleSet = "PlatformWorkloadIdentityRoleSets"
+	collPortal                          = "Portal"
+	collSubscriptions                   = "Subscriptions"
 )
 
 func NewDatabaseClient(log *logrus.Entry, _env env.Core, authorizer cosmosdb.Authorizer, m metrics.Emitter, aead encryption.AEAD, databaseAccountName string) (cosmosdb.DatabaseClient, error) {
diff --git a/pkg/database/platformworkloadidentityrolesets.go b/pkg/database/platformworkloadidentityrolesets.go
new file mode 100644
index 000000000..d8832fec2
--- /dev/null
+++ b/pkg/database/platformworkloadidentityrolesets.go
@@ -0,0 +1,113 @@
+package database
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	"github.com/Azure/ARO-RP/pkg/database/cosmosdb"
+	"github.com/Azure/ARO-RP/pkg/util/uuid"
+)
+
+type platformWorkloadIdentityRoleSets struct {
+	c    cosmosdb.PlatformWorkloadIdentityRoleSetDocumentClient
+	uuid uuid.Generator
+}
+
+type PlatformWorkloadIdentityRoleSets interface {
+	ChangeFeed() cosmosdb.PlatformWorkloadIdentityRoleSetDocumentIterator
+	Create(context.Context, *api.PlatformWorkloadIdentityRoleSetDocument) (*api.PlatformWorkloadIdentityRoleSetDocument, error)
+	Delete(context.Context, *api.PlatformWorkloadIdentityRoleSetDocument) error
+	Get(context.Context, string) (*api.PlatformWorkloadIdentityRoleSetDocument, error)
+	Update(context.Context, *api.PlatformWorkloadIdentityRoleSetDocument) (*api.PlatformWorkloadIdentityRoleSetDocument, error)
+	Patch(context.Context, string, func(*api.PlatformWorkloadIdentityRoleSetDocument) error) (*api.PlatformWorkloadIdentityRoleSetDocument, error)
+	ListAll(context.Context) (*api.PlatformWorkloadIdentityRoleSetDocuments, error)
+	NewUUID() string
+}
+
+func NewPlatformWorkloadIdentityRoleSets(ctx context.Context, dbc cosmosdb.DatabaseClient, dbName string) (PlatformWorkloadIdentityRoleSets, error) {
+	collc := cosmosdb.NewCollectionClient(dbc, dbName)
+
+	documentClient := cosmosdb.NewPlatformWorkloadIdentityRoleSetDocumentClient(collc, collPlatformWorkloadIdentityRoleSet)
+	return NewPlatformWorkloadIdentityRoleSetsWithProvidedClient(documentClient, uuid.DefaultGenerator), nil
+}
+
+func NewPlatformWorkloadIdentityRoleSetsWithProvidedClient(client cosmosdb.PlatformWorkloadIdentityRoleSetDocumentClient, uuid uuid.Generator) PlatformWorkloadIdentityRoleSets {
+	return &platformWorkloadIdentityRoleSets{
+		c:    client,
+		uuid: uuid,
+	}
+}
+
+func (c *platformWorkloadIdentityRoleSets) ChangeFeed() cosmosdb.PlatformWorkloadIdentityRoleSetDocumentIterator {
+	return c.c.ChangeFeed(nil)
+}
+
+func (c *platformWorkloadIdentityRoleSets) Create(ctx context.Context, doc *api.PlatformWorkloadIdentityRoleSetDocument) (*api.PlatformWorkloadIdentityRoleSetDocument, error) {
+	if doc.ID != strings.ToLower(doc.ID) {
+		return nil, fmt.Errorf("id %q is not lower case", doc.ID)
+	}
+
+	return c.c.Create(ctx, doc.ID, doc, nil)
+}
+
+func (c *platformWorkloadIdentityRoleSets) Delete(ctx context.Context, doc *api.PlatformWorkloadIdentityRoleSetDocument) error {
+	if doc.ID != strings.ToLower(doc.ID) {
+		return fmt.Errorf("id %q is not lower case", doc.ID)
+	}
+
+	return c.c.Delete(ctx, doc.ID, doc, &cosmosdb.Options{NoETag: true})
+}
+
+func (c *platformWorkloadIdentityRoleSets) Get(ctx context.Context, id string) (*api.PlatformWorkloadIdentityRoleSetDocument, error) {
+	if id != strings.ToLower(id) {
+		return nil, fmt.Errorf("id %q is not lower case", id)
+	}
+
+	return c.c.Get(ctx, id, id, nil)
+}
+
+func (c *platformWorkloadIdentityRoleSets) Patch(ctx context.Context, id string, f func(*api.PlatformWorkloadIdentityRoleSetDocument) error) (*api.PlatformWorkloadIdentityRoleSetDocument, error) {
+	var doc *api.PlatformWorkloadIdentityRoleSetDocument
+
+	err := cosmosdb.RetryOnPreconditionFailed(func() (err error) {
+		doc, err = c.Get(ctx, id)
+		if err != nil {
+			return
+		}
+
+		err = f(doc)
+		if err != nil {
+			return
+		}
+
+		doc, err = c.update(ctx, doc)
+		return
+	})
+
+	return doc, err
+}
+
+func (c *platformWorkloadIdentityRoleSets) Update(ctx context.Context, doc *api.PlatformWorkloadIdentityRoleSetDocument) (*api.PlatformWorkloadIdentityRoleSetDocument, error) {
+	return c.update(ctx, doc)
+}
+
+func (c *platformWorkloadIdentityRoleSets) update(ctx context.Context, doc *api.PlatformWorkloadIdentityRoleSetDocument) (*api.PlatformWorkloadIdentityRoleSetDocument, error) {
+	if doc.ID != strings.ToLower(doc.ID) {
+		return nil, fmt.Errorf("id %q is not lower case", doc.ID)
+	}
+
+	return c.c.Replace(ctx, doc.ID, doc, nil)
+}
+
+func (c *platformWorkloadIdentityRoleSets) ListAll(ctx context.Context) (*api.PlatformWorkloadIdentityRoleSetDocuments, error) {
+	return c.c.ListAll(ctx, nil)
+}
+
+func (c *platformWorkloadIdentityRoleSets) NewUUID() string {
+	return c.uuid.Generate()
+}
diff --git a/pkg/deploy/assets/databases-development.json b/pkg/deploy/assets/databases-development.json
index d0c7aabfa..04ae16974 100644
--- a/pkg/deploy/assets/databases-development.json
+++ b/pkg/deploy/assets/databases-development.json
@@ -70,6 +70,28 @@
             },
             "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers"
         },
+        {
+            "apiVersion": "2023-04-15",
+            "dependsOn": [
+                "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), parameters('databaseName'))]"
+            ],
+            "location": "[resourceGroup().location]",
+            "name": "[concat(parameters('databaseAccountName'), '/', parameters('databaseName'), '/PlatformWorkloadIdentityRoleSets')]",
+            "properties": {
+                "options": {},
+                "resource": {
+                    "defaultTtl": -1,
+                    "id": "PlatformWorkloadIdentityRoleSets",
+                    "partitionKey": {
+                        "kind": "Hash",
+                        "paths": [
+                            "/id"
+                        ]
+                    }
+                }
+            },
+            "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers"
+        },
         {
             "apiVersion": "2023-04-15",
             "dependsOn": [
diff --git a/pkg/deploy/assets/env-development.json b/pkg/deploy/assets/env-development.json
index 4328b7592..1c738e4fc 100644
--- a/pkg/deploy/assets/env-development.json
+++ b/pkg/deploy/assets/env-development.json
@@ -143,6 +143,62 @@
                 "[resourceId('Microsoft.Network/virtualNetworks', 'dev-vpn-vnet')]"
             ]
         },
+        {
+            "sku": {
+                "name": "Basic"
+            },
+            "properties": {
+                "frontendIPConfigurations": [
+                    {
+                        "properties": {
+                            "subnet": {
+                                "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'rp-vnet', 'rp-subnet')]"
+                            }
+                        },
+                        "name": "not-used"
+                    }
+                ],
+                "backendAddressPools": [
+                    {
+                        "name": "dev-backend"
+                    }
+                ],
+                "loadBalancingRules": [
+                    {
+                        "properties": {
+                            "frontendIPConfiguration": {
+                                "id": "[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', 'dev-lb-internal', 'not-used')]"
+                            },
+                            "backendAddressPool": {
+                                "id": "[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'dev-lb-internal', 'dev-backend')]"
+                            },
+                            "probe": {
+                                "id": "[resourceId('Microsoft.Network/loadBalancers/probes', 'dev-lb-internal', 'dev-probe')]"
+                            },
+                            "protocol": "Tcp",
+                            "loadDistribution": "Default",
+                            "frontendPort": 443,
+                            "backendPort": 443
+                        },
+                        "name": "dev-lbrule"
+                    }
+                ],
+                "probes": [
+                    {
+                        "properties": {
+                            "protocol": "Tcp",
+                            "port": 443,
+                            "numberOfProbes": 3
+                        },
+                        "name": "dev-probe"
+                    }
+                ]
+            },
+            "name": "dev-lb-internal",
+            "type": "Microsoft.Network/loadBalancers",
+            "location": "[resourceGroup().location]",
+            "apiVersion": "2020-08-01"
+        },
         {
             "name": "[concat(take(resourceGroup().name,10), '-dev-sharedKV')]",
             "type": "Microsoft.KeyVault/vaults",
@@ -244,9 +300,9 @@
                     },
                     "storageProfile": {
                         "imageReference": {
-                            "publisher": "RedHat",
-                            "offer": "RHEL",
-                            "sku": "8-LVM",
+                            "publisher": "MicrosoftCBLMariner",
+                            "offer": "cbl-mariner",
+                            "sku": "cbl-mariner-2-gen2",
                             "version": "latest"
                         },
                         "osDisk": {
@@ -257,6 +313,9 @@
                         }
                     },
                     "networkProfile": {
+                        "healthProbe": {
+                            "id": "[resourceId('Microsoft.Network/loadBalancers/probes', 'dev-lb-internal', 'dev-probe')]"
+                        },
                         "networkInterfaceConfigurations": [
                             {
                                 "name": "dev-proxy-vmss-nic",
@@ -277,7 +336,12 @@
                                                             "domainNameLabel": "[parameters('proxyDomainNameLabel')]"
                                                         }
                                                     }
-                                                }
+                                                },
+                                                "loadBalancerBackendAddressPools": [
+                                                    {
+                                                        "id": "[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'dev-lb-internal', 'dev-backend')]"
+                                                    }
+                                                ]
                                             }
                                         }
                                     ]
@@ -296,7 +360,7 @@
                                     "autoUpgradeMinorVersion": true,
                                     "settings": {},
                                     "protectedSettings": {
-                                        "script": "[base64(concat(base64ToString('c2V0IC1leAoK'),'PROXYIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImage')),''')\n','PROXYIMAGEAUTH=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImageAuth')),''')\n','PROXYCERT=''',parameters('proxyCert'),'''\n','PROXYCLIENTCERT=''',parameters('proxyClientCert'),'''\n','PROXYKEY=''',parameters('proxyKey'),'''\n','\n',base64ToString('I0FkZGluZyByZXRyeSBsb2dpYyB0byB5dW0gY29tbWFuZHMgaW4gb3JkZXIgdG8gYXZvaWQgc3RhbGxpbmcgb3V0IG9uIHJlc291cmNlIGxvY2tzCmVjaG8gInJ1bm5pbmcgUkhVSSBmaXgiCmZvciBhdHRlbXB0IGluIHsxLi42MH07IGRvCiAgeXVtIHVwZGF0ZSAteSAtLWRpc2FibGVyZXBvPScqJyAtLWVuYWJsZXJlcG89J3JodWktbWljcm9zb2Z0LWF6dXJlKicgJiYgYnJlYWsKICBpZiBbWyAke2F0dGVtcHR9IC1sdCA2MCBdXTsgdGhlbiBzbGVlcCAzMDsgZWxzZSBleGl0IDE7IGZpCmRvbmUKCmVjaG8gInJ1bm5pbmcgeXVtIHVwZGF0ZSIKZm9yIGF0dGVtcHQgaW4gezEuLjYwfTsgZG8KICB5dW0gLXkgLXggV0FMaW51eEFnZW50IC14IFdBTGludXhBZ2VudC11ZGV2IHVwZGF0ZSAtLWFsbG93ZXJhc2luZyAmJiBicmVhawogIGlmIFtbICR7YXR0ZW1wdH0gLWx0IDYwIF1dOyB0aGVuIHNsZWVwIDMwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKZWNobyAiaW5zdGFsbGluZyBwb2RtYW4tZG9ja2VyIgpmb3IgYXR0ZW1wdCBpbiB7MS4uNjB9OyBkbwogIHl1bSAteSBpbnN0YWxsIHBvZG1hbi1kb2NrZXIgJiYgYnJlYWsKICBpZiBbWyAke2F0dGVtcHR9IC1sdCA2MCBdXTsgdGhlbiBzbGVlcCAzMDsgZWxzZSBleGl0IDE7IGZpCmRvbmUKCmZpcmV3YWxsLWNtZCAtLWFkZC1wb3J0PTQ0My90Y3AgLS1wZXJtYW5lbnQKCm1rZGlyIC9yb290Ly5kb2NrZXIKY2F0ID4vcm9vdC8uZG9ja2VyL2NvbmZpZy5qc29uIDw8RU9GCnsKCSJhdXRocyI6IHsKCQkiJHtQUk9YWUlNQUdFJSUvKn0iOiB7CgkJCSJhdXRoIjogIiRQUk9YWUlNQUdFQVVUSCIKCQl9Cgl9Cn0KRU9GCgpta2RpciAtcCAvZXRjL2NvbnRhaW5lcnMvCnRvdWNoIC9ldGMvY29udGFpbmVycy9ub2RvY2tlcgoKZG9ja2VyIHB1bGwgIiRQUk9YWUlNQUdFIgoKbWtkaXIgL2V0Yy9wcm94eQpiYXNlNjQgLWQgPDw8IiRQUk9YWUNFUlQiID4vZXRjL3Byb3h5L3Byb3h5LmNydApiYXNlNjQgLWQgPDw8IiRQUk9YWUtFWSIgPi9ldGMvcHJveHkvcHJveHkua2V5CmJhc2U2NCAtZCA8PDwiJFBST1hZQ0xJRU5UQ0VSVCIgPi9ldGMvcHJveHkvcHJveHktY2xpZW50LmNydApjaG93biAtUiAxMDAwOjEwMDAgL2V0Yy9wcm94eQpjaG1vZCAwNjAwIC9ldGMvcHJveHkvcHJveHkua2V5CgpjYXQgPi9ldGMvc3lzY29uZmlnL3Byb3h5IDw8RU9GClBST1hZX0lNQUdFPSckUFJPWFlJTUFHRScKRU9GCgpjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vcHJveHkuc2VydmljZSA8PCdFT0YnCltVbml0XQpBZnRlcj1uZXR3b3JrLW9ubGluZS50YXJnZXQKV2FudHM9bmV0d29yay1vbmxpbmUudGFyZ2V0CgpbU2VydmljZV0KRW52aXJvbm1lbnRGaWxlPS9ldGMvc3lzY29uZmlnL3Byb3h5CkV4ZWNTdGFydFByZT0tL3Vzci9iaW4vZG9ja2VyIHJtIC1mICVuCkV4ZWNTdGFydD0vdXNyL2Jpbi9kb2NrZXIgcnVuIC0tcm0gLS1uYW1lICVuIC1wIDQ0Mzo4NDQzIC12IC9ldGMvcHJveHk6L3NlY3JldHMgJFBST1hZX0lNQUdFCkV4ZWNTdG9wPS91c3IvYmluL2RvY2tlciBzdG9wICVuClJlc3RhcnQ9YWx3YXlzClJlc3RhcnRTZWM9MQpTdGFydExpbWl0SW50ZXJ2YWw9MAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIudGFyZ2V0CkVPRgoKc3lzdGVtY3RsIGVuYWJsZSBwcm94eS5zZXJ2aWNlCgpjYXQgPi9ldGMvY3Jvbi53ZWVrbHkvcHVsbC1pbWFnZSA8PCdFT0YnCiMhL2Jpbi9iYXNoCgpkb2NrZXIgcHVsbCAkUFJPWFlJTUFHRQpzeXN0ZW1jdGwgcmVzdGFydCBwcm94eS5zZXJ2aWNlCkVPRgpjaG1vZCAreCAvZXRjL2Nyb24ud2Vla2x5L3B1bGwtaW1hZ2UKCmNhdCA+L2V0Yy9jcm9uLndlZWtseS95dW11cGRhdGUgPDwnRU9GJwojIS9iaW4vYmFzaAoKeXVtIHVwZGF0ZSAteQpFT0YKY2htb2QgK3ggL2V0Yy9jcm9uLndlZWtseS95dW11cGRhdGUKCmNhdCA+L2V0Yy9jcm9uLmRhaWx5L3Jlc3RhcnQtcHJveHkgPDwnRU9GJwojIS9iaW4vYmFzaAoKc3lzdGVtY3RsIHJlc3RhcnQgcHJveHkuc2VydmljZQpFT0YKY2htb2QgK3ggL2V0Yy9jcm9uLmRhaWx5L3Jlc3RhcnQtcHJveHkKCigKCXNsZWVwIDMwCglyZWJvb3QKKSAmCg==')))]"
+                                        "script": "[base64(concat(base64ToString('c2V0IC1leAoK'),'PROXYIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImage')),''')\n','PROXYIMAGEAUTH=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImageAuth')),''')\n','PROXYCERT=''',parameters('proxyCert'),'''\n','PROXYCLIENTCERT=''',parameters('proxyClientCert'),'''\n','PROXYKEY=''',parameters('proxyKey'),'''\n','\n',base64ToString('I0FkZGluZyByZXRyeSBsb2dpYyB0byB5dW0gY29tbWFuZHMgaW4gb3JkZXIgdG8gYXZvaWQgc3RhbGxpbmcgb3V0IG9uIHJlc291cmNlIGxvY2tzCmVjaG8gImluc3RhbGxpbmcgbW9ieS1lbmdpbmUgKGRvY2tlcikiCmZvciBhdHRlbXB0IGluIHsxLi42MH07IGRvCgl0ZG5mIGluc3RhbGwgLXkgbW9ieS1lbmdpbmUgbW9ieS1jbGkgJiYgYnJlYWsKCWlmIFtbICR7YXR0ZW1wdH0gLWx0IDYwIF1dOyB0aGVuIHNsZWVwIDMwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKc3lzdGVtY3RsIGVuYWJsZSBkb2NrZXIKc3lzdGVtY3RsIHN0YXJ0IGRvY2tlcgoKbWtkaXIgL3Jvb3QvLmRvY2tlcgpjYXQgPi9yb290Ly5kb2NrZXIvY29uZmlnLmpzb24gPDxFT0YKewoJImF1dGhzIjogewoJCSIke1BST1hZSU1BR0UlJS8qfSI6IHsKCQkJImF1dGgiOiAiJFBST1hZSU1BR0VBVVRIIgoJCX0KCX0KfQpFT0YKCmRvY2tlciBwdWxsICIkUFJPWFlJTUFHRSIKCm1rZGlyIC9ldGMvcHJveHkKYmFzZTY0IC1kIDw8PCIkUFJPWFlDRVJUIiA+L2V0Yy9wcm94eS9wcm94eS5jcnQKYmFzZTY0IC1kIDw8PCIkUFJPWFlLRVkiID4vZXRjL3Byb3h5L3Byb3h5LmtleQpiYXNlNjQgLWQgPDw8IiRQUk9YWUNMSUVOVENFUlQiID4vZXRjL3Byb3h5L3Byb3h5LWNsaWVudC5jcnQKY2hvd24gLVIgMTAwMDoxMDAwIC9ldGMvcHJveHkKY2htb2QgMDYwMCAvZXRjL3Byb3h5L3Byb3h5LmtleQoKY2F0ID4vZXRjL3N5c2NvbmZpZy9wcm94eSA8PEVPRgpQUk9YWV9JTUFHRT0nJFBST1hZSU1BR0UnCkVPRgoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL3Byb3h5LnNlcnZpY2UgPDwnRU9GJwpbVW5pdF0KQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CldhbnRzPW5ldHdvcmstb25saW5lLnRhcmdldAoKW1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9wcm94eQpFeGVjU3RhcnRQcmU9LS91c3IvYmluL2RvY2tlciBybSAtZiAlbgpFeGVjU3RhcnQ9L3Vzci9iaW4vZG9ja2VyIHJ1biAtLXJtIC0tbmFtZSAlbiAtcCA0NDM6ODQ0MyAtdiAvZXRjL3Byb3h5Oi9zZWNyZXRzICRQUk9YWV9JTUFHRQpFeGVjU3RvcD0vdXNyL2Jpbi9kb2NrZXIgc3RvcCAlbgpSZXN0YXJ0PWFsd2F5cwpSZXN0YXJ0U2VjPTEKU3RhcnRMaW1pdEludGVydmFsPTAKCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCnN5c3RlbWN0bCBlbmFibGUgcHJveHkuc2VydmljZQoKY2F0ID4vZXRjL2Nyb24ud2Vla2x5L3B1bGwtaW1hZ2UgPDwnRU9GJwojIS9iaW4vYmFzaAoKZG9ja2VyIHB1bGwgJFBST1hZSU1BR0UKc3lzdGVtY3RsIHJlc3RhcnQgcHJveHkuc2VydmljZQpFT0YKY2htb2QgK3ggL2V0Yy9jcm9uLndlZWtseS9wdWxsLWltYWdlCgpjYXQgPi9ldGMvY3Jvbi53ZWVrbHkveXVtdXBkYXRlIDw8J0VPRicKIyEvYmluL2Jhc2gKCnl1bSB1cGRhdGUgLXkKRU9GCmNobW9kICt4IC9ldGMvY3Jvbi53ZWVrbHkveXVtdXBkYXRlCgpjYXQgPi9ldGMvY3Jvbi5kYWlseS9yZXN0YXJ0LXByb3h5IDw8J0VPRicKIyEvYmluL2Jhc2gKCnN5c3RlbWN0bCByZXN0YXJ0IHByb3h5LnNlcnZpY2UKRU9GCmNobW9kICt4IC9ldGMvY3Jvbi5kYWlseS9yZXN0YXJ0LXByb3h5CgooCglzbGVlcCAzMAoJcmVib290CikgJgo=')))]"
                                     },
                                     "provisionAfterExtensions": [
                                         "Microsoft.Azure.Monitor.AzureMonitorLinuxAgent",
@@ -337,10 +401,22 @@
                 },
                 "overprovision": false
             },
+            "identity": {
+                "type": "UserAssigned",
+                "userAssignedIdentities": {
+                    "[resourceId('AzSecPackAutoConfigRG', 'Microsoft.ManagedIdentity/userAssignedIdentities', concat('AzSecPackAutoConfigUA-', resourceGroup().location))]": {}
+                }
+            },
             "name": "dev-proxy-vmss",
             "type": "Microsoft.Compute/virtualMachineScaleSets",
             "location": "[resourceGroup().location]",
-            "apiVersion": "2020-12-01"
+            "tags": {
+                "azsecpack": "nonprod"
+            },
+            "apiVersion": "2020-12-01",
+            "dependsOn": [
+                "[resourceId('Microsoft.Network/loadBalancers', 'dev-lb-internal')]"
+            ]
         },
         {
             "properties": {
diff --git a/pkg/deploy/assets/rp-production.json b/pkg/deploy/assets/rp-production.json
index 2fd7a5ade..26282772f 100644
--- a/pkg/deploy/assets/rp-production.json
+++ b/pkg/deploy/assets/rp-production.json
@@ -818,6 +818,29 @@
             },
             "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers"
         },
+        {
+            "apiVersion": "2023-04-15",
+            "dependsOn": [
+                "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), 'ARO')]",
+                "[resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName'))]"
+            ],
+            "location": "[resourceGroup().location]",
+            "name": "[concat(parameters('databaseAccountName'), '/', 'ARO', '/PlatformWorkloadIdentityRoleSets')]",
+            "properties": {
+                "options": {},
+                "resource": {
+                    "defaultTtl": -1,
+                    "id": "PlatformWorkloadIdentityRoleSets",
+                    "partitionKey": {
+                        "kind": "Hash",
+                        "paths": [
+                            "/id"
+                        ]
+                    }
+                }
+            },
+            "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers"
+        },
         {
             "apiVersion": "2023-04-15",
             "dependsOn": [
diff --git a/pkg/deploy/generator/resources_dev.go b/pkg/deploy/generator/resources_dev.go
index 668b7fee3..a1788a804 100644
--- a/pkg/deploy/generator/resources_dev.go
+++ b/pkg/deploy/generator/resources_dev.go
@@ -17,6 +17,69 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/azureclient"
 )
 
+// devLBInternal is needed for defining a healthprobe.
+// VMSS with auto upgrademode requires a healthprobe from an LB.
+func (g *generator) devLBInternal() *arm.Resource {
+	return &arm.Resource{
+		Resource: &mgmtnetwork.LoadBalancer{
+			Sku: &mgmtnetwork.LoadBalancerSku{
+				Name: mgmtnetwork.LoadBalancerSkuNameBasic,
+			},
+			LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{
+				FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{
+					{
+						FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{
+							Subnet: &mgmtnetwork.Subnet{
+								ID: to.StringPtr("[resourceId('Microsoft.Network/virtualNetworks/subnets', 'rp-vnet', 'rp-subnet')]"),
+							},
+						},
+						Name: to.StringPtr("not-used"),
+					},
+				},
+				BackendAddressPools: &[]mgmtnetwork.BackendAddressPool{
+					{
+						Name: to.StringPtr("dev-backend"),
+					},
+				},
+				LoadBalancingRules: &[]mgmtnetwork.LoadBalancingRule{
+					{
+						LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{
+							FrontendIPConfiguration: &mgmtnetwork.SubResource{
+								ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', 'dev-lb-internal', 'not-used')]"),
+							},
+							BackendAddressPool: &mgmtnetwork.SubResource{
+								ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'dev-lb-internal', 'dev-backend')]"),
+							},
+							Probe: &mgmtnetwork.SubResource{
+								ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/probes', 'dev-lb-internal', 'dev-probe')]"),
+							},
+							Protocol:         mgmtnetwork.TransportProtocolTCP,
+							LoadDistribution: mgmtnetwork.LoadDistributionDefault,
+							FrontendPort:     to.Int32Ptr(443),
+							BackendPort:      to.Int32Ptr(443),
+						},
+						Name: to.StringPtr("dev-lbrule"),
+					},
+				},
+				Probes: &[]mgmtnetwork.Probe{
+					{
+						ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{
+							Protocol:       mgmtnetwork.ProbeProtocolTCP,
+							Port:           to.Int32Ptr(443),
+							NumberOfProbes: to.Int32Ptr(3),
+						},
+						Name: to.StringPtr("dev-probe"),
+					},
+				},
+			},
+			Name:     to.StringPtr("dev-lb-internal"),
+			Type:     to.StringPtr("Microsoft.Network/loadBalancers"),
+			Location: to.StringPtr("[resourceGroup().location]"),
+		},
+		APIVersion: azureclient.APIVersion("Microsoft.Network"),
+	}
+}
+
 func (g *generator) devProxyVMSS() *arm.Resource {
 	parts := []string{
 		fmt.Sprintf("base64ToString('%s')", base64.StdEncoding.EncodeToString([]byte("set -ex\n\n"))),
@@ -51,6 +114,12 @@ func (g *generator) devProxyVMSS() *arm.Resource {
 				Tier:     to.StringPtr("Standard"),
 				Capacity: to.Int64Ptr(1),
 			},
+			Identity: &mgmtcompute.VirtualMachineScaleSetIdentity{
+				Type: mgmtcompute.ResourceIdentityTypeUserAssigned,
+				UserAssignedIdentities: map[string]*mgmtcompute.VirtualMachineScaleSetIdentityUserAssignedIdentitiesValue{
+					"[resourceId('AzSecPackAutoConfigRG', 'Microsoft.ManagedIdentity/userAssignedIdentities', concat('AzSecPackAutoConfigUA-', resourceGroup().location))]": {},
+				},
+			},
 			VirtualMachineScaleSetProperties: &mgmtcompute.VirtualMachineScaleSetProperties{
 				UpgradePolicy: &mgmtcompute.UpgradePolicy{
 					Mode: mgmtcompute.UpgradeModeRolling,
@@ -73,9 +142,9 @@ func (g *generator) devProxyVMSS() *arm.Resource {
 					},
 					StorageProfile: &mgmtcompute.VirtualMachineScaleSetStorageProfile{
 						ImageReference: &mgmtcompute.ImageReference{
-							Publisher: to.StringPtr("RedHat"),
-							Offer:     to.StringPtr("RHEL"),
-							Sku:       to.StringPtr("8-LVM"),
+							Publisher: to.StringPtr("MicrosoftCBLMariner"),
+							Offer:     to.StringPtr("cbl-mariner"),
+							Sku:       to.StringPtr("cbl-mariner-2-gen2"),
 							Version:   to.StringPtr("latest"),
 						},
 						OsDisk: &mgmtcompute.VirtualMachineScaleSetOSDisk{
@@ -86,6 +155,9 @@ func (g *generator) devProxyVMSS() *arm.Resource {
 						},
 					},
 					NetworkProfile: &mgmtcompute.VirtualMachineScaleSetNetworkProfile{
+						HealthProbe: &mgmtcompute.APIEntityReference{
+							ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/probes', 'dev-lb-internal', 'dev-probe')]"),
+						},
 						NetworkInterfaceConfigurations: &[]mgmtcompute.VirtualMachineScaleSetNetworkConfiguration{
 							{
 								Name: to.StringPtr("dev-proxy-vmss-nic"),
@@ -107,6 +179,11 @@ func (g *generator) devProxyVMSS() *arm.Resource {
 														},
 													},
 												},
+												LoadBalancerBackendAddressPools: &[]mgmtcompute.SubResource{
+													{
+														ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'dev-lb-internal', 'dev-backend')]"),
+													},
+												},
 											},
 										},
 									},
@@ -171,6 +248,12 @@ func (g *generator) devProxyVMSS() *arm.Resource {
 			Location: to.StringPtr("[resourceGroup().location]"),
 		},
 		APIVersion: azureclient.APIVersion("Microsoft.Compute"),
+		Tags: map[string]any{
+			"azsecpack": "nonprod",
+		},
+		DependsOn: []string{
+			"[resourceId('Microsoft.Network/loadBalancers', 'dev-lb-internal')]",
+		},
 	}
 }
 
diff --git a/pkg/deploy/generator/resources_rp.go b/pkg/deploy/generator/resources_rp.go
index 907d9fcd8..3d961e70d 100644
--- a/pkg/deploy/generator/resources_rp.go
+++ b/pkg/deploy/generator/resources_rp.go
@@ -1172,6 +1172,31 @@ func (g *generator) database(databaseName string, addDependsOn bool) []*arm.Reso
 			},
 			Type: "Microsoft.DocumentDB/databaseAccounts/sqlDatabases",
 		},
+		{
+			Resource: &sdkcosmos.SQLContainerCreateUpdateParameters{
+				Properties: &sdkcosmos.SQLContainerCreateUpdateProperties{
+					Resource: &sdkcosmos.SQLContainerResource{
+						ID: to.StringPtr("PlatformWorkloadIdentityRoleSets"),
+						PartitionKey: &sdkcosmos.ContainerPartitionKey{
+							Paths: []*string{
+								to.StringPtr("/id"),
+							},
+							Kind: &hashPartitionKey,
+						},
+						DefaultTTL: to.Int32Ptr(-1),
+					},
+					Options: &sdkcosmos.CreateUpdateOptions{},
+				},
+				Name:     to.StringPtr("[concat(parameters('databaseAccountName'), '/', " + databaseName + ", '/PlatformWorkloadIdentityRoleSets')]"),
+				Type:     to.StringPtr("Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers"),
+				Location: to.StringPtr("[resourceGroup().location]"),
+			},
+			APIVersion: azureclient.APIVersion("Microsoft.DocumentDB"),
+			DependsOn: []string{
+				"[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), " + databaseName + ")]",
+			},
+			Type: "Microsoft.DocumentDB/databaseAccounts/sqlDatabases",
+		},
 		{
 			Resource: &sdkcosmos.SQLContainerCreateUpdateParameters{
 				Properties: &sdkcosmos.SQLContainerCreateUpdateProperties{
diff --git a/pkg/deploy/generator/scripts/devProxyVMSS.sh b/pkg/deploy/generator/scripts/devProxyVMSS.sh
index cb8e15bf8..f09c9f57f 100644
--- a/pkg/deploy/generator/scripts/devProxyVMSS.sh
+++ b/pkg/deploy/generator/scripts/devProxyVMSS.sh
@@ -1,23 +1,12 @@
 #Adding retry logic to yum commands in order to avoid stalling out on resource locks
-echo "running RHUI fix"
+echo "installing moby-engine (docker)"
 for attempt in {1..60}; do
-  yum update -y --disablerepo='*' --enablerepo='rhui-microsoft-azure*' && break
-  if [[ ${attempt} -lt 60 ]]; then sleep 30; else exit 1; fi
+	tdnf install -y moby-engine moby-cli && break
+	if [[ ${attempt} -lt 60 ]]; then sleep 30; else exit 1; fi
 done
 
-echo "running yum update"
-for attempt in {1..60}; do
-  yum -y -x WALinuxAgent -x WALinuxAgent-udev update --allowerasing && break
-  if [[ ${attempt} -lt 60 ]]; then sleep 30; else exit 1; fi
-done
-
-echo "installing podman-docker"
-for attempt in {1..60}; do
-  yum -y install podman-docker && break
-  if [[ ${attempt} -lt 60 ]]; then sleep 30; else exit 1; fi
-done
-
-firewall-cmd --add-port=443/tcp --permanent
+systemctl enable docker
+systemctl start docker
 
 mkdir /root/.docker
 cat >/root/.docker/config.json <<EOF
@@ -30,9 +19,6 @@ cat >/root/.docker/config.json <<EOF
 }
 EOF
 
-mkdir -p /etc/containers/
-touch /etc/containers/nodocker
-
 docker pull "$PROXYIMAGE"
 
 mkdir /etc/proxy
diff --git a/pkg/deploy/generator/templates_dev.go b/pkg/deploy/generator/templates_dev.go
index 7f3bd4b00..6eca6e892 100644
--- a/pkg/deploy/generator/templates_dev.go
+++ b/pkg/deploy/generator/templates_dev.go
@@ -33,6 +33,7 @@ func (g *generator) devSharedTemplate() *arm.Template {
 		g.devVnet(),
 		g.devVPNVnet(),
 		g.devVPN(),
+		g.devLBInternal(),
 		g.devDiskEncryptionKeyvault(),
 		g.devDiskEncryptionKey(),
 		g.devDiskEncryptionKeyVaultAccessPolicy(),
diff --git a/pkg/frontend/openshiftcluster_putorpatch.go b/pkg/frontend/openshiftcluster_putorpatch.go
index 1fa6f26fd..5c6308978 100644
--- a/pkg/frontend/openshiftcluster_putorpatch.go
+++ b/pkg/frontend/openshiftcluster_putorpatch.go
@@ -25,6 +25,8 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/version"
 )
 
+var errMissingIdentityURL error = fmt.Errorf("identityURL not provided but required for workload identity cluster")
+
 func (f *frontend) putOrPatchOpenShiftCluster(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 	log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry)
@@ -41,10 +43,12 @@ func (f *frontend) putOrPatchOpenShiftCluster(w http.ResponseWriter, r *http.Req
 	subId := chi.URLParam(r, "subscriptionId")
 	resourceProviderNamespace := chi.URLParam(r, "resourceProviderNamespace")
 
+	identityURL := r.Header.Get("x-ms-identity-url")
+
 	apiVersion := r.URL.Query().Get(api.APIVersionKey)
 	err := cosmosdb.RetryOnPreconditionFailed(func() error {
 		var err error
-		b, err = f._putOrPatchOpenShiftCluster(ctx, log, body, correlationData, systemData, r.URL.Path, originalPath, r.Method, referer, &header, f.apis[apiVersion].OpenShiftClusterConverter, f.apis[apiVersion].OpenShiftClusterStaticValidator, subId, resourceProviderNamespace, apiVersion)
+		b, err = f._putOrPatchOpenShiftCluster(ctx, log, body, correlationData, systemData, r.URL.Path, originalPath, r.Method, referer, &header, f.apis[apiVersion].OpenShiftClusterConverter, f.apis[apiVersion].OpenShiftClusterStaticValidator, subId, resourceProviderNamespace, apiVersion, identityURL)
 		return err
 	})
 
@@ -52,7 +56,7 @@ func (f *frontend) putOrPatchOpenShiftCluster(w http.ResponseWriter, r *http.Req
 	reply(log, w, header, b, err)
 }
 
-func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus.Entry, body []byte, correlationData *api.CorrelationData, systemData *api.SystemData, path, originalPath, method, referer string, header *http.Header, converter api.OpenShiftClusterConverter, staticValidator api.OpenShiftClusterStaticValidator, subId, resourceProviderNamespace string, apiVersion string) ([]byte, error) {
+func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus.Entry, body []byte, correlationData *api.CorrelationData, systemData *api.SystemData, path, originalPath, method, referer string, header *http.Header, converter api.OpenShiftClusterConverter, staticValidator api.OpenShiftClusterStaticValidator, subId, resourceProviderNamespace string, apiVersion string, identityURL string) ([]byte, error) {
 	subscription, err := f.validateSubscriptionState(ctx, path, api.SubscriptionStateRegistered)
 	if err != nil {
 		return nil, err
@@ -86,11 +90,17 @@ func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus.
 				},
 			},
 		}
+
 		if !f.env.IsLocalDevelopmentMode() /* not local dev or CI */ {
 			doc.OpenShiftCluster.Properties.FeatureProfile.GatewayEnabled = true
 		}
 	}
 
+	err = validateIdentityUrl(doc.OpenShiftCluster, identityURL, isCreate)
+	if err != nil {
+		return nil, err
+	}
+
 	doc.CorrelationData = correlationData
 
 	err = validateTerminalProvisioningState(doc.OpenShiftCluster.Properties.ProvisioningState)
@@ -288,6 +298,24 @@ func enrichClusterSystemData(doc *api.OpenShiftClusterDocument, systemData *api.
 	}
 }
 
+func validateIdentityUrl(cluster *api.OpenShiftCluster, identityURL string, isCreate bool) error {
+	// Don't persist identity URL in non-wimi clusters
+	if cluster.Properties.ServicePrincipalProfile != nil || cluster.Identity == nil {
+		return nil
+	}
+
+	if identityURL == "" {
+		if isCreate {
+			return errMissingIdentityURL
+		}
+		return nil
+	}
+
+	cluster.Identity.IdentityURL = identityURL
+
+	return nil
+}
+
 func (f *frontend) ValidateNewCluster(ctx context.Context, subscription *api.SubscriptionDocument, cluster *api.OpenShiftCluster, staticValidator api.OpenShiftClusterStaticValidator, ext interface{}, path string) error {
 	err := staticValidator.Static(ext, nil, f.env.Location(), f.env.Domain(), f.env.FeatureIsSet(env.FeatureRequireD2sV3Workers), path)
 	if err != nil {
diff --git a/pkg/operator/deploy/deploy.go b/pkg/operator/deploy/deploy.go
index ab18e2505..1fbc93a7c 100644
--- a/pkg/operator/deploy/deploy.go
+++ b/pkg/operator/deploy/deploy.go
@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/go-multierror"
+	operatorclient "github.com/openshift/client-go/operator/clientset/versioned"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	extensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -56,6 +57,7 @@ type Operator interface {
 	Restart(context.Context, []string) error
 	IsRunningDesiredVersion(context.Context) (bool, error)
 	RenewMDSDCertificate(context.Context) error
+	EnsureUpgradeAnnotation(context.Context) error
 }
 
 type operator struct {
@@ -67,10 +69,11 @@ type operator struct {
 	client        client.Client
 	extensionscli extensionsclient.Interface
 	kubernetescli kubernetes.Interface
+	operatorcli   operatorclient.Interface
 	dh            dynamichelper.Interface
 }
 
-func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli aroclient.Interface, client client.Client, extensionscli extensionsclient.Interface, kubernetescli kubernetes.Interface) (Operator, error) {
+func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli aroclient.Interface, client client.Client, extensionscli extensionsclient.Interface, kubernetescli kubernetes.Interface, operatorcli operatorclient.Interface) (Operator, error) {
 	restConfig, err := restconfig.RestConfig(env, oc)
 	if err != nil {
 		return nil, err
@@ -89,6 +92,7 @@ func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli
 		client:        client,
 		extensionscli: extensionscli,
 		kubernetescli: kubernetescli,
+		operatorcli:   operatorcli,
 		dh:            dh,
 	}, nil
 }
@@ -432,6 +436,34 @@ func (o *operator) RenewMDSDCertificate(ctx context.Context) error {
 	return nil
 }
 
+func (o *operator) EnsureUpgradeAnnotation(ctx context.Context) error {
+	if o.oc.Properties.PlatformWorkloadIdentityProfile == nil ||
+		o.oc.Properties.ServicePrincipalProfile != nil {
+		return nil
+	}
+
+	upgradeableTo := string(*o.oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo)
+	upgradeableAnnotation := "cloudcredential.openshift.io/upgradeable-to"
+
+	cloudcredentialobject, err := o.operatorcli.OperatorV1().CloudCredentials().Get(ctx, "cluster", metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	if cloudcredentialobject.Annotations == nil {
+		cloudcredentialobject.Annotations = map[string]string{}
+	}
+
+	cloudcredentialobject.Annotations[upgradeableAnnotation] = upgradeableTo
+
+	_, err = o.operatorcli.OperatorV1().CloudCredentials().Update(ctx, cloudcredentialobject, metav1.UpdateOptions{})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (o *operator) IsReady(ctx context.Context) (bool, error) {
 	ok, err := ready.CheckDeploymentIsReady(ctx, o.kubernetescli.AppsV1().Deployments(pkgoperator.Namespace), "aro-operator-master")()
 	o.log.Infof("deployment %q ok status is: %v, err is: %v", "aro-operator-master", ok, err)
diff --git a/pkg/proxy/proxy.go b/pkg/proxy/proxy.go
index 746a7d1c9..3bcd1e858 100644
--- a/pkg/proxy/proxy.go
+++ b/pkg/proxy/proxy.go
@@ -7,6 +7,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"errors"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -29,7 +30,16 @@ type Server struct {
 	subnet         *net.IPNet
 }
 
+func health(w http.ResponseWriter, _ *http.Request) {
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprintf(w, "Running.")
+}
+
 func (s *Server) Run() error {
+	healthMux := http.NewServeMux()
+	healthMux.HandleFunc("/", health)
+	go http.ListenAndServe(":8080", healthMux)
+
 	_, subnet, err := net.ParseCIDR(s.Subnet)
 	if err != nil {
 		return err
diff --git a/pkg/util/arm/deploy.go b/pkg/util/arm/deploy.go
index 3bad254fc..6a34d51a6 100644
--- a/pkg/util/arm/deploy.go
+++ b/pkg/util/arm/deploy.go
@@ -34,7 +34,8 @@ func DeployTemplate(ctx context.Context, log *logrus.Entry, deployments features
 	}
 
 	if azureerrors.HasAuthorizationFailedError(err) ||
-		azureerrors.HasLinkedAuthorizationFailedError(err) {
+		azureerrors.HasLinkedAuthorizationFailedError(err) ||
+		azureerrors.IsDeploymentMissingPermissionsError(err) {
 		return err
 	}
 
diff --git a/pkg/util/azureerrors/error.go b/pkg/util/azureerrors/error.go
index 9211ef695..1262558c0 100644
--- a/pkg/util/azureerrors/error.go
+++ b/pkg/util/azureerrors/error.go
@@ -14,6 +14,10 @@ import (
 	"github.com/Azure/ARO-RP/pkg/api"
 )
 
+const (
+	CodeInvalidTemplateDeployment = "InvalidTemplateDeployment"
+)
+
 // HasAuthorizationFailedError returns true it the error is, or contains, an
 // AuthorizationFailed error
 func HasAuthorizationFailedError(err error) bool {
@@ -60,6 +64,22 @@ func deploymentFailedDueToAuthError(err error, authCode string) bool {
 	return false
 }
 
+// IsDeploymentMissingPermissionsError returns true if the error indicates that
+// ARM rejected a template deployment pre-flight due to missing role
+// assignments.
+// This can be an indicator of role assignment propagation delay.
+func IsDeploymentMissingPermissionsError(err error) bool {
+	if detailedErr, ok := err.(autorest.DetailedError); ok {
+		if serviceErr, ok := detailedErr.Original.(*azure.ServiceError); ok {
+			if serviceErr.Code == CodeInvalidTemplateDeployment && strings.Contains(serviceErr.Message, "Authorization failed for template resource") {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
 // IsDeploymentActiveError returns true it the error is a DeploymentActive error
 func IsDeploymentActiveError(err error) bool {
 	if detailedErr, ok := err.(autorest.DetailedError); ok {
diff --git a/pkg/util/cluster/cluster.go b/pkg/util/cluster/cluster.go
index c7028507c..b71b54640 100644
--- a/pkg/util/cluster/cluster.go
+++ b/pkg/util/cluster/cluster.go
@@ -41,9 +41,6 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/authorization"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/features"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network"
-	redhatopenshift20200430 "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/redhatopenshift/2020-04-30/redhatopenshift"
-	redhatopenshift20210901preview "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/redhatopenshift/2021-09-01-preview/redhatopenshift"
-	redhatopenshift20220401 "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/redhatopenshift/2022-04-01/redhatopenshift"
 	redhatopenshift20230904 "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/redhatopenshift/2023-09-04/redhatopenshift"
 	utilgraph "github.com/Azure/ARO-RP/pkg/util/graph"
 	"github.com/Azure/ARO-RP/pkg/util/rbac"
@@ -57,20 +54,17 @@ type Cluster struct {
 	ci           bool
 	ciParentVnet string
 
-	spGraphClient                     *utilgraph.GraphServiceClient
-	deployments                       features.DeploymentsClient
-	groups                            features.ResourceGroupsClient
-	openshiftclustersv20200430        redhatopenshift20200430.OpenShiftClustersClient
-	openshiftclustersv20210901preview redhatopenshift20210901preview.OpenShiftClustersClient
-	openshiftclustersv20220401        redhatopenshift20220401.OpenShiftClustersClient
-	openshiftclustersv20230904        redhatopenshift20230904.OpenShiftClustersClient
-	securitygroups                    network.SecurityGroupsClient
-	subnets                           network.SubnetsClient
-	routetables                       network.RouteTablesClient
-	roleassignments                   authorization.RoleAssignmentsClient
-	peerings                          network.VirtualNetworkPeeringsClient
-	ciParentVnetPeerings              network.VirtualNetworkPeeringsClient
-	vaultsClient                      armkeyvault.VaultsClient
+	spGraphClient        *utilgraph.GraphServiceClient
+	deployments          features.DeploymentsClient
+	groups               features.ResourceGroupsClient
+	openshiftclusters    redhatopenshift20230904.OpenShiftClustersClient
+	securitygroups       network.SecurityGroupsClient
+	subnets              network.SubnetsClient
+	routetables          network.RouteTablesClient
+	roleassignments      authorization.RoleAssignmentsClient
+	peerings             network.VirtualNetworkPeeringsClient
+	ciParentVnetPeerings network.VirtualNetworkPeeringsClient
+	vaultsClient         armkeyvault.VaultsClient
 }
 
 func New(log *logrus.Entry, environment env.Core, ci bool) (*Cluster, error) {
@@ -111,19 +105,16 @@ func New(log *logrus.Entry, environment env.Core, ci bool) (*Cluster, error) {
 		env: environment,
 		ci:  ci,
 
-		spGraphClient:                     spGraphClient,
-		deployments:                       features.NewDeploymentsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
-		groups:                            features.NewResourceGroupsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
-		openshiftclustersv20200430:        redhatopenshift20200430.NewOpenShiftClustersClient(environment.Environment(), environment.SubscriptionID(), authorizer),
-		openshiftclustersv20210901preview: redhatopenshift20210901preview.NewOpenShiftClustersClient(environment.Environment(), environment.SubscriptionID(), authorizer),
-		openshiftclustersv20220401:        redhatopenshift20220401.NewOpenShiftClustersClient(environment.Environment(), environment.SubscriptionID(), authorizer),
-		openshiftclustersv20230904:        redhatopenshift20230904.NewOpenShiftClustersClient(environment.Environment(), environment.SubscriptionID(), authorizer),
-		securitygroups:                    network.NewSecurityGroupsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
-		subnets:                           network.NewSubnetsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
-		routetables:                       network.NewRouteTablesClient(environment.Environment(), environment.SubscriptionID(), authorizer),
-		roleassignments:                   authorization.NewRoleAssignmentsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
-		peerings:                          network.NewVirtualNetworkPeeringsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
-		vaultsClient:                      vaultClient,
+		spGraphClient:     spGraphClient,
+		deployments:       features.NewDeploymentsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
+		groups:            features.NewResourceGroupsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
+		openshiftclusters: redhatopenshift20230904.NewOpenShiftClustersClient(environment.Environment(), environment.SubscriptionID(), authorizer),
+		securitygroups:    network.NewSecurityGroupsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
+		subnets:           network.NewSubnetsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
+		routetables:       network.NewRouteTablesClient(environment.Environment(), environment.SubscriptionID(), authorizer),
+		roleassignments:   authorization.NewRoleAssignmentsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
+		peerings:          network.NewVirtualNetworkPeeringsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
+		vaultsClient:      vaultClient,
 	}
 
 	if ci && env.IsLocalDevelopmentMode() {
@@ -169,7 +160,7 @@ func (c *Cluster) DeleteApp(ctx context.Context) error {
 }
 
 func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName string, osClusterVersion string) error {
-	clusterGet, err := c.openshiftclustersv20230904.Get(ctx, vnetResourceGroup, clusterName)
+	clusterGet, err := c.openshiftclusters.Get(ctx, vnetResourceGroup, clusterName)
 	if err == nil {
 		if clusterGet.ProvisioningState == mgmtredhatopenshift20230904.Failed {
 			return fmt.Errorf("cluster exists and is in failed provisioning state, please delete and retry")
@@ -490,7 +481,7 @@ func (c *Cluster) createCluster(ctx context.Context, vnetResourceGroup, clusterN
 		return err
 	}
 
-	return c.openshiftclustersv20230904.CreateOrUpdateAndWait(ctx, vnetResourceGroup, clusterName, ocExt)
+	return c.openshiftclusters.CreateOrUpdateAndWait(ctx, vnetResourceGroup, clusterName, ocExt)
 }
 
 func (c *Cluster) registerSubscription(ctx context.Context) error {
@@ -609,7 +600,7 @@ func (c *Cluster) fixupNSGs(ctx context.Context, vnetResourceGroup, clusterName
 
 func (c *Cluster) deleteRoleAssignments(ctx context.Context, vnetResourceGroup, clusterName string) error {
 	c.log.Print("deleting role assignments")
-	oc, err := c.openshiftclustersv20200430.Get(ctx, vnetResourceGroup, clusterName)
+	oc, err := c.openshiftclusters.Get(ctx, vnetResourceGroup, clusterName)
 	if err != nil {
 		return fmt.Errorf("error getting cluster document: %w", err)
 	}
@@ -645,7 +636,7 @@ func (c *Cluster) deleteRoleAssignments(ctx context.Context, vnetResourceGroup,
 
 func (c *Cluster) deleteCluster(ctx context.Context, resourceGroup, clusterName string) error {
 	c.log.Printf("deleting cluster %s", clusterName)
-	if err := c.openshiftclustersv20200430.DeleteAndWait(ctx, resourceGroup, clusterName); err != nil {
+	if err := c.openshiftclusters.DeleteAndWait(ctx, resourceGroup, clusterName); err != nil {
 		return fmt.Errorf("error deleting cluster %s: %w", clusterName, err)
 	}
 	return nil
diff --git a/pkg/util/mocks/operator/deploy/deploy.go b/pkg/util/mocks/operator/deploy/deploy.go
index 99ad4f224..30f4efc25 100644
--- a/pkg/util/mocks/operator/deploy/deploy.go
+++ b/pkg/util/mocks/operator/deploy/deploy.go
@@ -62,6 +62,20 @@ func (mr *MockOperatorMockRecorder) CreateOrUpdateCredentialsRequest(arg0 interf
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateCredentialsRequest", reflect.TypeOf((*MockOperator)(nil).CreateOrUpdateCredentialsRequest), arg0)
 }
 
+// EnsureUpgradeAnnotation mocks base method.
+func (m *MockOperator) EnsureUpgradeAnnotation(arg0 context.Context) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "EnsureUpgradeAnnotation", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// EnsureUpgradeAnnotation indicates an expected call of EnsureUpgradeAnnotation.
+func (mr *MockOperatorMockRecorder) EnsureUpgradeAnnotation(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureUpgradeAnnotation", reflect.TypeOf((*MockOperator)(nil).EnsureUpgradeAnnotation), arg0)
+}
+
 // IsReady mocks base method.
 func (m *MockOperator) IsReady(arg0 context.Context) (bool, error) {
 	m.ctrl.T.Helper()
diff --git a/pkg/util/steps/refreshing.go b/pkg/util/steps/refreshing.go
index 96fbdf71f..8138ecda6 100644
--- a/pkg/util/steps/refreshing.go
+++ b/pkg/util/steps/refreshing.go
@@ -77,6 +77,7 @@ func (s *authorizationRefreshingActionStep) run(ctx context.Context, log *logrus
 			(azureerrors.IsUnauthorizedClientError(err) ||
 				azureerrors.HasAuthorizationFailedError(err) ||
 				azureerrors.IsInvalidSecretError(err) ||
+				azureerrors.IsDeploymentMissingPermissionsError(err) ||
 				err == ErrWantRefresh) {
 			log.Printf("auth error, refreshing and retrying: %v", err)
 			// Try refreshing auth.
diff --git a/pkg/util/version/const.go b/pkg/util/version/const.go
index e1b96a1b3..01f05a4f1 100644
--- a/pkg/util/version/const.go
+++ b/pkg/util/version/const.go
@@ -34,8 +34,8 @@ type Stream struct {
 // This default is left here ONLY for use by local development mode,
 // until we can come up with a better solution.
 var DefaultInstallStream = Stream{
-	Version:  NewVersion(4, 13, 23),
-	PullSpec: "quay.io/openshift-release-dev/ocp-release@sha256:ca556d3494d08765c90481f15dd965995371168ea7ee7a551000bed4481931c8",
+	Version:  NewVersion(4, 13, 40),
+	PullSpec: "quay.io/openshift-release-dev/ocp-release@sha256:c1f69e6137bc9cda2c6da56bafbc7ea969900acb5e5c349b1ebb2103b10b424f",
 }
 
 // FluentbitImage contains the location of the Fluentbit container image
diff --git a/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models.py b/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models.py
index 3e77e2302..b7e1fab63 100644
--- a/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models.py
+++ b/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models.py
@@ -1303,12 +1303,16 @@ class PlatformWorkloadIdentity(msrest.serialization.Model):
 class PlatformWorkloadIdentityProfile(msrest.serialization.Model):
     """PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.
 
+    :ivar upgradeable_to: UpgradeableTo stores a single OpenShift version a workload identity
+     cluster can be upgraded to.
+    :vartype upgradeable_to: str
     :ivar platform_workload_identities:
     :vartype platform_workload_identities:
      list[~azure.mgmt.redhatopenshift.v2024_08_12_preview.models.PlatformWorkloadIdentity]
     """
 
     _attribute_map = {
+        'upgradeable_to': {'key': 'upgradeableTo', 'type': 'str'},
         'platform_workload_identities': {'key': 'platformWorkloadIdentities', 'type': '[PlatformWorkloadIdentity]'},
     }
 
@@ -1317,11 +1321,15 @@ class PlatformWorkloadIdentityProfile(msrest.serialization.Model):
         **kwargs
     ):
         """
+        :keyword upgradeable_to: UpgradeableTo stores a single OpenShift version a workload identity
+         cluster can be upgraded to.
+        :paramtype upgradeable_to: str
         :keyword platform_workload_identities:
         :paramtype platform_workload_identities:
          list[~azure.mgmt.redhatopenshift.v2024_08_12_preview.models.PlatformWorkloadIdentity]
         """
         super(PlatformWorkloadIdentityProfile, self).__init__(**kwargs)
+        self.upgradeable_to = kwargs.get('upgradeable_to', None)
         self.platform_workload_identities = kwargs.get('platform_workload_identities', None)
 
 
diff --git a/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models_py3.py b/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models_py3.py
index 9828713ff..b8c2d9a1d 100644
--- a/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models_py3.py
+++ b/python/client/azure/mgmt/redhatopenshift/v2024_08_12_preview/models/_models_py3.py
@@ -1412,27 +1412,36 @@ class PlatformWorkloadIdentity(msrest.serialization.Model):
 class PlatformWorkloadIdentityProfile(msrest.serialization.Model):
     """PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.
 
+    :ivar upgradeable_to: UpgradeableTo stores a single OpenShift version a workload identity
+     cluster can be upgraded to.
+    :vartype upgradeable_to: str
     :ivar platform_workload_identities:
     :vartype platform_workload_identities:
      list[~azure.mgmt.redhatopenshift.v2024_08_12_preview.models.PlatformWorkloadIdentity]
     """
 
     _attribute_map = {
+        'upgradeable_to': {'key': 'upgradeableTo', 'type': 'str'},
         'platform_workload_identities': {'key': 'platformWorkloadIdentities', 'type': '[PlatformWorkloadIdentity]'},
     }
 
     def __init__(
         self,
         *,
+        upgradeable_to: Optional[str] = None,
         platform_workload_identities: Optional[List["PlatformWorkloadIdentity"]] = None,
         **kwargs
     ):
         """
+        :keyword upgradeable_to: UpgradeableTo stores a single OpenShift version a workload identity
+         cluster can be upgraded to.
+        :paramtype upgradeable_to: str
         :keyword platform_workload_identities:
         :paramtype platform_workload_identities:
          list[~azure.mgmt.redhatopenshift.v2024_08_12_preview.models.PlatformWorkloadIdentity]
         """
         super(PlatformWorkloadIdentityProfile, self).__init__(**kwargs)
+        self.upgradeable_to = upgradeable_to
         self.platform_workload_identities = platform_workload_identities
 
 
diff --git a/swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json b/swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
index c65beac00..95e0131f5 100644
--- a/swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
+++ b/swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
@@ -2349,6 +2349,9 @@
       "description": "PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters.",
       "type": "object",
       "properties": {
+        "upgradeableTo": {
+          "$ref": "#/definitions/UpgradeableTo"
+        },
         "platformWorkloadIdentities": {
           "type": "array",
           "items": {
@@ -2603,6 +2606,10 @@
         "type": "string"
       }
     },
+    "UpgradeableTo": {
+      "description": "UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to",
+      "type": "string"
+    },
     "UserAssignedIdentities": {
       "description": "UserAssignedIdentities stores a mapping from resource IDs of managed identities to their client/principal IDs.",
       "type": "object",
diff --git a/vendor/github.com/go-logr/logr/README.md b/vendor/github.com/go-logr/logr/README.md
index 8969526a6..7c7f0c69c 100644
--- a/vendor/github.com/go-logr/logr/README.md
+++ b/vendor/github.com/go-logr/logr/README.md
@@ -1,6 +1,7 @@
 # A minimal logging API for Go
 
 [![Go Reference](https://pkg.go.dev/badge/github.com/go-logr/logr.svg)](https://pkg.go.dev/github.com/go-logr/logr)
+[![Go Report Card](https://goreportcard.com/badge/github.com/go-logr/logr)](https://goreportcard.com/report/github.com/go-logr/logr)
 [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/go-logr/logr/badge)](https://securityscorecards.dev/viewer/?platform=github.com&org=go-logr&repo=logr)
 
 logr offers an(other) opinion on how Go programs and libraries can do logging
diff --git a/vendor/github.com/go-logr/logr/funcr/funcr.go b/vendor/github.com/go-logr/logr/funcr/funcr.go
index fb2f866f4..30568e768 100644
--- a/vendor/github.com/go-logr/logr/funcr/funcr.go
+++ b/vendor/github.com/go-logr/logr/funcr/funcr.go
@@ -236,15 +236,14 @@ func newFormatter(opts Options, outfmt outputFormat) Formatter {
 // implementation. It should be constructed with NewFormatter. Some of
 // its methods directly implement logr.LogSink.
 type Formatter struct {
-	outputFormat    outputFormat
-	prefix          string
-	values          []any
-	valuesStr       string
-	parentValuesStr string
-	depth           int
-	opts            *Options
-	group           string // for slog groups
-	groupDepth      int
+	outputFormat outputFormat
+	prefix       string
+	values       []any
+	valuesStr    string
+	depth        int
+	opts         *Options
+	groupName    string // for slog groups
+	groups       []groupDef
 }
 
 // outputFormat indicates which outputFormat to use.
@@ -257,6 +256,13 @@ const (
 	outputJSON
 )
 
+// groupDef represents a saved group.  The values may be empty, but we don't
+// know if we need to render the group until the final record is rendered.
+type groupDef struct {
+	name   string
+	values string
+}
+
 // PseudoStruct is a list of key-value pairs that gets logged as a struct.
 type PseudoStruct []any
 
@@ -264,76 +270,102 @@ type PseudoStruct []any
 func (f Formatter) render(builtins, args []any) string {
 	// Empirically bytes.Buffer is faster than strings.Builder for this.
 	buf := bytes.NewBuffer(make([]byte, 0, 1024))
+
 	if f.outputFormat == outputJSON {
-		buf.WriteByte('{') // for the whole line
+		buf.WriteByte('{') // for the whole record
 	}
 
+	// Render builtins
 	vals := builtins
 	if hook := f.opts.RenderBuiltinsHook; hook != nil {
 		vals = hook(f.sanitize(vals))
 	}
-	f.flatten(buf, vals, false, false) // keys are ours, no need to escape
+	f.flatten(buf, vals, false) // keys are ours, no need to escape
 	continuing := len(builtins) > 0
 
-	if f.parentValuesStr != "" {
-		if continuing {
-			buf.WriteByte(f.comma())
+	// Turn the inner-most group into a string
+	argsStr := func() string {
+		buf := bytes.NewBuffer(make([]byte, 0, 1024))
+
+		vals = args
+		if hook := f.opts.RenderArgsHook; hook != nil {
+			vals = hook(f.sanitize(vals))
 		}
-		buf.WriteString(f.parentValuesStr)
-		continuing = true
-	}
+		f.flatten(buf, vals, true) // escape user-provided keys
 
-	groupDepth := f.groupDepth
-	if f.group != "" {
-		if f.valuesStr != "" || len(args) != 0 {
-			if continuing {
-				buf.WriteByte(f.comma())
-			}
-			buf.WriteString(f.quoted(f.group, true)) // escape user-provided keys
-			buf.WriteByte(f.colon())
-			buf.WriteByte('{') // for the group
-			continuing = false
-		} else {
-			// The group was empty
-			groupDepth--
+		return buf.String()
+	}()
+
+	// Render the stack of groups from the inside out.
+	bodyStr := f.renderGroup(f.groupName, f.valuesStr, argsStr)
+	for i := len(f.groups) - 1; i >= 0; i-- {
+		grp := &f.groups[i]
+		if grp.values == "" && bodyStr == "" {
+			// no contents, so we must elide the whole group
+			continue
 		}
+		bodyStr = f.renderGroup(grp.name, grp.values, bodyStr)
 	}
 
-	if f.valuesStr != "" {
+	if bodyStr != "" {
 		if continuing {
 			buf.WriteByte(f.comma())
 		}
-		buf.WriteString(f.valuesStr)
-		continuing = true
+		buf.WriteString(bodyStr)
 	}
 
-	vals = args
-	if hook := f.opts.RenderArgsHook; hook != nil {
-		vals = hook(f.sanitize(vals))
+	if f.outputFormat == outputJSON {
+		buf.WriteByte('}') // for the whole record
 	}
-	f.flatten(buf, vals, continuing, true) // escape user-provided keys
 
-	for i := 0; i < groupDepth; i++ {
-		buf.WriteByte('}') // for the groups
+	return buf.String()
+}
+
+// renderGroup returns a string representation of the named group with rendered
+// values and args.  If the name is empty, this will return the values and args,
+// joined.  If the name is not empty, this will return a single key-value pair,
+// where the value is a grouping of the values and args.  If the values and
+// args are both empty, this will return an empty string, even if the name was
+// specified.
+func (f Formatter) renderGroup(name string, values string, args string) string {
+	buf := bytes.NewBuffer(make([]byte, 0, 1024))
+
+	needClosingBrace := false
+	if name != "" && (values != "" || args != "") {
+		buf.WriteString(f.quoted(name, true)) // escape user-provided keys
+		buf.WriteByte(f.colon())
+		buf.WriteByte('{')
+		needClosingBrace = true
 	}
 
-	if f.outputFormat == outputJSON {
-		buf.WriteByte('}') // for the whole line
+	continuing := false
+	if values != "" {
+		buf.WriteString(values)
+		continuing = true
+	}
+
+	if args != "" {
+		if continuing {
+			buf.WriteByte(f.comma())
+		}
+		buf.WriteString(args)
+	}
+
+	if needClosingBrace {
+		buf.WriteByte('}')
 	}
 
 	return buf.String()
 }
 
-// flatten renders a list of key-value pairs into a buffer.  If continuing is
-// true, it assumes that the buffer has previous values and will emit a
-// separator (which depends on the output format) before the first pair it
-// writes.  If escapeKeys is true, the keys are assumed to have
-// non-JSON-compatible characters in them and must be evaluated for escapes.
+// flatten renders a list of key-value pairs into a buffer.  If escapeKeys is
+// true, the keys are assumed to have non-JSON-compatible characters in them
+// and must be evaluated for escapes.
 //
 // This function returns a potentially modified version of kvList, which
 // ensures that there is a value for every key (adding a value if needed) and
 // that each key is a string (substituting a key if needed).
-func (f Formatter) flatten(buf *bytes.Buffer, kvList []any, continuing bool, escapeKeys bool) []any {
+func (f Formatter) flatten(buf *bytes.Buffer, kvList []any, escapeKeys bool) []any {
 	// This logic overlaps with sanitize() but saves one type-cast per key,
 	// which can be measurable.
 	if len(kvList)%2 != 0 {
@@ -354,7 +386,7 @@ func (f Formatter) flatten(buf *bytes.Buffer, kvList []any, continuing bool, esc
 		}
 		v := kvList[i+1]
 
-		if i > 0 || continuing {
+		if i > 0 {
 			if f.outputFormat == outputJSON {
 				buf.WriteByte(f.comma())
 			} else {
@@ -766,46 +798,17 @@ func (f Formatter) sanitize(kvList []any) []any {
 // startGroup opens a new group scope (basically a sub-struct), which locks all
 // the current saved values and starts them anew.  This is needed to satisfy
 // slog.
-func (f *Formatter) startGroup(group string) {
+func (f *Formatter) startGroup(name string) {
 	// Unnamed groups are just inlined.
-	if group == "" {
+	if name == "" {
 		return
 	}
 
-	// Any saved values can no longer be changed.
-	buf := bytes.NewBuffer(make([]byte, 0, 1024))
-	continuing := false
-
-	if f.parentValuesStr != "" {
-		buf.WriteString(f.parentValuesStr)
-		continuing = true
-	}
-
-	if f.group != "" && f.valuesStr != "" {
-		if continuing {
-			buf.WriteByte(f.comma())
-		}
-		buf.WriteString(f.quoted(f.group, true)) // escape user-provided keys
-		buf.WriteByte(f.colon())
-		buf.WriteByte('{') // for the group
-		continuing = false
-	}
-
-	if f.valuesStr != "" {
-		if continuing {
-			buf.WriteByte(f.comma())
-		}
-		buf.WriteString(f.valuesStr)
-	}
-
-	// NOTE: We don't close the scope here - that's done later, when a log line
-	// is actually rendered (because we have N scopes to close).
-
-	f.parentValuesStr = buf.String()
+	n := len(f.groups)
+	f.groups = append(f.groups[:n:n], groupDef{f.groupName, f.valuesStr})
 
 	// Start collecting new values.
-	f.group = group
-	f.groupDepth++
+	f.groupName = name
 	f.valuesStr = ""
 	f.values = nil
 }
@@ -900,7 +903,7 @@ func (f *Formatter) AddValues(kvList []any) {
 
 	// Pre-render values, so we don't have to do it on each Info/Error call.
 	buf := bytes.NewBuffer(make([]byte, 0, 1024))
-	f.flatten(buf, vals, false, true) // escape user-provided keys
+	f.flatten(buf, vals, true) // escape user-provided keys
 	f.valuesStr = buf.String()
 }
 
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 94eb75e3f..3bc9ff1a2 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -550,7 +550,7 @@ github.com/go-errors/errors
 github.com/go-jose/go-jose/v3
 github.com/go-jose/go-jose/v3/cipher
 github.com/go-jose/go-jose/v3/json
-# github.com/go-logr/logr v1.4.1
+# github.com/go-logr/logr v1.4.2
 ## explicit; go 1.18
 github.com/go-logr/logr
 github.com/go-logr/logr/funcr
@@ -1076,7 +1076,7 @@ github.com/openshift/cloud-credential-operator/pkg/apis/cloudcredential/v1
 # github.com/openshift/custom-resource-status v1.1.3-0.20220503160415-f2fdb4999d87
 ## explicit; go 1.12
 github.com/openshift/custom-resource-status/conditions/v1
-# github.com/openshift/hive/apis v0.0.0-20240510150258-83aedb9f6e73 => github.com/openshift/hive/apis v0.0.0-20231116161336-9dd47f8bfa1f
+# github.com/openshift/hive/apis v0.0.0-20240529172037-d7ead609f495 => github.com/openshift/hive/apis v0.0.0-20231116161336-9dd47f8bfa1f
 ## explicit; go 1.20
 github.com/openshift/hive/apis/hive/v1
 github.com/openshift/hive/apis/hive/v1/agent
@@ -1577,7 +1577,7 @@ gopkg.in/yaml.v2
 # gopkg.in/yaml.v3 v3.0.1
 ## explicit
 gopkg.in/yaml.v3
-# k8s.io/api v0.30.0 => k8s.io/api v0.25.16
+# k8s.io/api v0.30.1 => k8s.io/api v0.25.16
 ## explicit; go 1.19
 k8s.io/api/admission/v1
 k8s.io/api/admission/v1beta1
@@ -1646,7 +1646,7 @@ k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme
 k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1
 k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1
 k8s.io/apiextensions-apiserver/third_party/forked/celopenapi/model
-# k8s.io/apimachinery v0.30.0 => k8s.io/apimachinery v0.25.16
+# k8s.io/apimachinery v0.30.1 => k8s.io/apimachinery v0.25.16
 ## explicit; go 1.19
 k8s.io/apimachinery/pkg/api/equality
 k8s.io/apimachinery/pkg/api/errors


Test patch:
diff --git a/pkg/api/v20240812preview/openshiftcluster_validatestatic.go b/pkg/api/v20240812preview/openshiftcluster_validatestatic.go
index c027b1cd7..b5011fdd2 100644
--- a/pkg/api/v20240812preview/openshiftcluster_validatestatic.go
+++ b/pkg/api/v20240812preview/openshiftcluster_validatestatic.go
@@ -12,6 +12,7 @@ import (
 
 	azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
 	"github.com/Azure/go-autorest/autorest/azure"
+	"github.com/coreos/go-semver/semver"
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/api/util/immutable"
@@ -442,6 +443,14 @@ func (sv openShiftClusterStaticValidator) validatePlatformWorkloadIdentityProfil
 			return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, fmt.Sprintf("%s.PlatformWorkloadIdentities[%d].resourceID", path, n), "Resource must be a user assigned identity.")
 		}
 	}
+
+	if pwip.UpgradeableTo != nil {
+		_, err := semver.NewVersion(string(*pwip.UpgradeableTo))
+		if err != nil {
+			return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, fmt.Sprintf("%s.UpgradeableTo[%v]", path, *pwip.UpgradeableTo), "UpgradeableTo must be a valid OpenShift version in the format 'x.y.z'.")
+		}
+	}
+
 	return nil
 }
 
diff --git a/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go b/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go
index 97e0b802f..884df39ea 100644
--- a/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go
+++ b/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go
@@ -1163,6 +1163,9 @@ func TestOpenShiftClusterStaticValidateDelta(t *testing.T) {
 }
 
 func TestOpenShiftClusterStaticValidatePlatformWorkloadIdentityProfile(t *testing.T) {
+	validUpgradeableToValue := UpgradeableTo("4.14.29")
+	invalidUpgradeableToValue := UpgradeableTo("16.107.invalid")
+
 	createTests := []*validateTest{
 		{
 			name: "valid empty workloadIdentityProfile",
@@ -1323,6 +1326,27 @@ func TestOpenShiftClusterStaticValidatePlatformWorkloadIdentityProfile(t *testin
 			},
 			wantErr: "400: InvalidParameter: properties.servicePrincipalProfile: Must provide either an identity or service principal credentials.",
 		},
+		{
+			name: "valid UpgradeableTo value",
+			modify: func(oc *OpenShiftCluster) {
+				oc.Identity = &Identity{}
+				oc.Properties.ServicePrincipalProfile = nil
+				oc.Properties.PlatformWorkloadIdentityProfile = &PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &validUpgradeableToValue,
+				}
+			},
+		},
+		{
+			name: "invalid UpgradeableTo value",
+			modify: func(oc *OpenShiftCluster) {
+				oc.Identity = &Identity{}
+				oc.Properties.ServicePrincipalProfile = nil
+				oc.Properties.PlatformWorkloadIdentityProfile = &PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &invalidUpgradeableToValue,
+				}
+			},
+			wantErr: `400: InvalidParameter: properties.platformWorkloadIdentityProfile.UpgradeableTo[16.107.invalid]: UpgradeableTo must be a valid OpenShift version in the format 'x.y.z'.`,
+		},
 	}
 
 	runTests(t, testModeCreate, createTests)
diff --git a/pkg/frontend/openshiftcluster_putorpatch_test.go b/pkg/frontend/openshiftcluster_putorpatch_test.go
index f7c10ade9..540616ccf 100644
--- a/pkg/frontend/openshiftcluster_putorpatch_test.go
+++ b/pkg/frontend/openshiftcluster_putorpatch_test.go
@@ -3305,3 +3305,84 @@ func TestEnrichClusterSystemData(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateIdentityUrl(t *testing.T) {
+	for _, tt := range []struct {
+		name        string
+		identityURL string
+		cluster     *api.OpenShiftCluster
+		expected    *api.OpenShiftCluster
+		isCreate    bool
+		wantError   error
+	}{
+		{
+			name:        "identity URL is empty, is not wi/mi cluster create",
+			identityURL: "",
+			cluster:     &api.OpenShiftCluster{},
+			expected:    &api.OpenShiftCluster{},
+			isCreate:    false,
+		},
+		{
+			name:        "identity URL is empty, is wi/mi cluster create",
+			identityURL: "",
+			cluster:     &api.OpenShiftCluster{},
+			expected:    &api.OpenShiftCluster{},
+			isCreate:    true,
+			wantError:   errMissingIdentityURL,
+		},
+		{
+			name:        "cluster is not wi/mi, identityURL passed",
+			identityURL: "http://foo.bar",
+			cluster: &api.OpenShiftCluster{
+				Properties: api.OpenShiftClusterProperties{
+					ServicePrincipalProfile: &api.ServicePrincipalProfile{},
+				},
+			},
+			expected: &api.OpenShiftCluster{
+				Properties: api.OpenShiftClusterProperties{
+					ServicePrincipalProfile: &api.ServicePrincipalProfile{},
+				},
+			},
+			isCreate: true,
+		},
+		{
+			name:        "cluster is not wi/mi, identityURL not passed",
+			identityURL: "",
+			cluster: &api.OpenShiftCluster{
+				Properties: api.OpenShiftClusterProperties{
+					ServicePrincipalProfile: &api.ServicePrincipalProfile{},
+				},
+			},
+			expected: &api.OpenShiftCluster{
+				Properties: api.OpenShiftClusterProperties{
+					ServicePrincipalProfile: &api.ServicePrincipalProfile{},
+				},
+			},
+			isCreate: true,
+		},
+		{
+			name: "pass - identity URL passed on wi/mi cluster",
+			cluster: &api.OpenShiftCluster{
+				Identity: &api.Identity{},
+			},
+			identityURL: "http://foo.bar",
+			expected: &api.OpenShiftCluster{
+				Identity: &api.Identity{
+					IdentityURL: "http://foo.bar",
+				},
+			},
+			isCreate: true,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateIdentityUrl(tt.cluster, tt.identityURL, tt.isCreate)
+			if err != nil && err != tt.wantError {
+				t.Error(cmp.Diff(err, tt.wantError))
+			}
+
+			if !reflect.DeepEqual(tt.cluster, tt.expected) {
+				t.Error(cmp.Diff(tt.cluster, tt.expected))
+			}
+		})
+	}
+}
diff --git a/pkg/operator/deploy/deploy_test.go b/pkg/operator/deploy/deploy_test.go
index 6002e65dd..44f5e257a 100644
--- a/pkg/operator/deploy/deploy_test.go
+++ b/pkg/operator/deploy/deploy_test.go
@@ -10,6 +10,8 @@ import (
 
 	"github.com/golang/mock/gomock"
 	configv1 "github.com/openshift/api/config/v1"
+	operatorv1 "github.com/openshift/api/operator/v1"
+	operatorfake "github.com/openshift/client-go/operator/clientset/versioned/fake"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -461,3 +463,122 @@ func TestCheckPodImageVersion(t *testing.T) {
 		})
 	}
 }
+
+func TestTestEnsureUpgradeAnnotation(t *testing.T) {
+	UpgradeableTo1 := api.UpgradeableTo("4.14.59")
+
+	for _, tt := range []struct {
+		name                 string
+		cluster              api.OpenShiftClusterProperties
+		annotation           map[string]string
+		wantAnnotation       map[string]string
+		wantErr              string
+		cloudCredentialsName string
+	}{
+		{
+			name: "nil PlatformWorkloadIdentityProfile, no version persisted in cluster document",
+		},
+		{
+			name: "non-nil ServicePrincipalProfile, no version persisted in cluster document",
+			cluster: api.OpenShiftClusterProperties{
+				ServicePrincipalProfile: &api.ServicePrincipalProfile{
+					ClientID:     "",
+					ClientSecret: "",
+				},
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{},
+			},
+		},
+		{
+			name: "no version persisted in cluster document, persist it",
+			cluster: api.OpenShiftClusterProperties{
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &UpgradeableTo1,
+				},
+			},
+			annotation: nil,
+			wantAnnotation: map[string]string{
+				"cloudcredential.openshift.io/upgradeable-to": "4.14.59",
+			},
+		},
+		{
+			name: "cloud credential 'cluster' doesn't exist",
+			cluster: api.OpenShiftClusterProperties{
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &UpgradeableTo1,
+				},
+			},
+			cloudCredentialsName: "oh_no",
+			annotation:           nil,
+			wantAnnotation:       nil,
+			wantErr:              `cloudcredentials.operator.openshift.io "cluster" not found`,
+		},
+		{
+			name: "version persisted in cluster document, replace it",
+			cluster: api.OpenShiftClusterProperties{
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &UpgradeableTo1,
+				},
+			},
+			annotation: map[string]string{
+				"cloudcredential.openshift.io/upgradeable-to": "4.14.02",
+			},
+			wantAnnotation: map[string]string{
+				"cloudcredential.openshift.io/upgradeable-to": "4.14.59",
+			},
+		},
+		{
+			name: "annotations exist, append the upgradeable annotation",
+			cluster: api.OpenShiftClusterProperties{
+				PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{
+					UpgradeableTo: &UpgradeableTo1,
+				},
+			},
+			annotation: map[string]string{
+				"foo": "bar",
+			},
+			wantAnnotation: map[string]string{
+				"foo": "bar",
+				"cloudcredential.openshift.io/upgradeable-to": "4.14.59",
+			},
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := context.Background()
+			controller := gomock.NewController(t)
+			defer controller.Finish()
+
+			env := mock_env.NewMockInterface(controller)
+
+			oc := &api.OpenShiftCluster{
+				Properties: tt.cluster,
+			}
+
+			if tt.cloudCredentialsName == "" {
+				tt.cloudCredentialsName = "cluster"
+			}
+
+			cloudcredentialobject := &operatorv1.CloudCredential{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        tt.cloudCredentialsName,
+					Annotations: tt.annotation,
+				},
+			}
+
+			o := operator{
+				oc:          oc,
+				env:         env,
+				operatorcli: operatorfake.NewSimpleClientset(cloudcredentialobject),
+			}
+
+			err := o.EnsureUpgradeAnnotation(ctx)
+			utilerror.AssertErrorMessage(t, err, tt.wantErr)
+			result, _ := o.operatorcli.OperatorV1().CloudCredentials().List(ctx, metav1.ListOptions{})
+			for _, v := range result.Items {
+				actualAnnotations := v.ObjectMeta.Annotations
+				if !reflect.DeepEqual(actualAnnotations, tt.wantAnnotation) {
+					t.Errorf("actual annotation: %v, wanted %v", tt.annotation, tt.wantAnnotation)
+				}
+			}
+		})
+	}
+}
diff --git a/pkg/util/azureerrors/error_test.go b/pkg/util/azureerrors/error_test.go
index 77c9c333e..ee925060d 100644
--- a/pkg/util/azureerrors/error_test.go
+++ b/pkg/util/azureerrors/error_test.go
@@ -120,3 +120,36 @@ func TestIsDeploymentActiveError(t *testing.T) {
 		})
 	}
 }
+
+func TestIsDeploymentMissingPermissionsError(t *testing.T) {
+	for _, tt := range []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{
+			name: "Another error",
+			err:  errors.New("something happened"),
+		},
+		{
+			name: "Missing RoleAssignment",
+			err: autorest.DetailedError{
+				PackageType: "features.DeploymentsClient",
+				Method:      "CreateOrUpdate",
+				Message:     "Failure sending request",
+				Original: &azure.ServiceError{
+					Code:    CodeInvalidTemplateDeployment,
+					Message: "The template deployment failed with error: 'Authorization failed for template resource '$RESOURCE' of type 'Microsoft.Authorization/roleAssignments'. The client '$CLIENT' with object id '$CLIENT' does not have permission to perform action '$ACTION' at scope '$SCOPE'.'.",
+				},
+			},
+			want: true,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			got := IsDeploymentMissingPermissionsError(autorest.NewErrorWithError(tt.err, "", "", nil, ""))
+			if got != tt.want {
+				t.Errorf("got %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
diff --git a/pkg/util/pki/pki_test.go b/pkg/util/pki/pki_test.go
index 25676c561..c7a9d30bb 100644
--- a/pkg/util/pki/pki_test.go
+++ b/pkg/util/pki/pki_test.go
@@ -13,7 +13,7 @@ import (
 )
 
 func TestGetTlsConfig(t *testing.T) {
-	if os.Getenv("ARO_RUN_PKI_TESTS") != "" {
+	if os.Getenv("ARO_SKIP_PKI_TESTS") != "" {
 		t.Skip("")
 	}
 	kpiUrl := "https://issuer.pki.azure.com/dsms/issuercertificates?getissuersv3&caName=%s"