Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1894 (2023-11-24 14:49:35)

Change base image to rocky9
# Description

Change base image to Rocky9 Linux.

## Type of change

*Please select one of the options below.*

- Other

## Discussion

-

## Testing

Manual test runs and CI run.

## Documentation

-

## Follow-up

-


foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: a4b65c52bb95a70bd614ddee4934488d240dac1c
* Duration 2:51:48
* Result: :x: **FAILED**
* Error: `Error while executing command: if $fail_test; then exit 1; fi. Reason: exit status 1`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/20de941f-9975-40e8-9d5a-b12f8e452f6e.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/20de941f-9975-40e8-9d5a-b12f8e452f6e/workspace.zip) zip file of the working directory (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: 1aaceed4343fe6fa42c7c78e0d7ec2dc6ab86a3c
* Duration 2:48:15
* Result: :white_check_mark: **SUCCEEDED**
* Error: `N/A`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/7d4d4bd9-768d-4710-9831-760257a332d4.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/7d4d4bd9-768d-4710-9831-760257a332d4/workspace.zip) zip file of the working directory (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: b3961d7b09a9338353a1095c3d6cd9c55749ff64
* Duration 2:35:08
* Result: :x: **FAILED**
* Error: `Error while executing command: if $fail_test; then exit 1; fi. Reason: exit status 1`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/93538c8a-a0ec-4af0-93dd-209f3bfc72a4.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/93538c8a-a0ec-4af0-93dd-209f3bfc72a4/workspace.zip) zip file of the working directory (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: eac7f3e871924f3a63ea4e743d8406fdc1d2f21a
* Duration 2:48:48
* Result: :x: **FAILED**
* Error: `Error while executing command: if $fail_test; then exit 1; fi. Reason: exit status 1`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/88f115fe-4071-41f8-b5cf-2a1891795baa.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/88f115fe-4071-41f8-b5cf-2a1891795baa/workspace.zip) zip file of the working directory (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: c30b2277c32b09184aec5d1a3120ff26c2b40024
* Duration 2:31:32
* Result: :x: **FAILED**
* Error: `Error while executing command: if $fail_test; then exit 1; fi. Reason: exit status 1`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/c4698866-80eb-4ccd-aef8-a2c50007c478.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/c4698866-80eb-4ccd-aef8-a2c50007c478/workspace.zip) zip file of the working directory (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: edfbe23db69ed515a468a39d4722273410de05b0
* Duration 2:41:40
* Result: :x: **FAILED**
* Error: `Error while executing command: if $fail_test; then exit 1; fi. Reason: exit status 1`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/815b1c52-66ce-4e27-82e6-f0cd995f4df0.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/815b1c52-66ce-4e27-82e6-f0cd995f4df0/workspace.zip) zip file of the working directory (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: bca13cb3abd6cbd1cae014baa9de9747ec4e73c7
* Duration 2:37:58
* Result: :x: **FAILED**
* Error: `Error while executing command: if $fail_test; then exit 1; fi. Reason: exit status 1`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/45fd33e6-7e61-4639-a7a7-423c8cafc688.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/45fd33e6-7e61-4639-a7a7-423c8cafc688/workspace.zip) zip file of the working directory (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: a638f4841c50ecd52dcc0593172f75ce2eab73e6
* Duration 1:54:39
* Result: :white_check_mark: **SUCCEEDED**
* Error: `N/A`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/1ca3701b-25d6-451b-b3b5-7676516fa998.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/1ca3701b-25d6-451b-b3b5-7676516fa998/workspace.zip) zip file of the working directory (available for 30 days)


Patch:
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 9182a17c..5aedc0fc 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -14,17 +14,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Set up Go
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.20.6
-      - uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go
+          go-version: 1.20.11
       - name: Get dependencies
         run: curl -L --fail "https://github.com/apple/foundationdb/releases/download/${FDB_VER}/foundationdb-clients_${FDB_VER}-1_amd64.deb" -o fdb.deb
       - name: Install dependencies
@@ -41,22 +35,16 @@ jobs:
         # See https://kubernetes.io/releases for the current releases
         kubever: [ "v1.21.1", "v1.22.0", "v1.23.0" ]
     steps:
-    - name: Set up Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.20.6
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: 1.20.11
     - name: Fetch all tags
       run: git fetch --force --tags
-    - uses: actions/cache@v2
-      with:
-        path: ~/go/pkg/mod
-        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-go
     - name: Get dependencies
       env:
         KIND_VER: "v0.11.1"
@@ -108,22 +96,16 @@ jobs:
     name: Testing
     runs-on: ubuntu-latest
     steps:
-    - name: Set up Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.20.6
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: 1.20.11
     - name: Fetch all tags
       run: git fetch --force --tags
-    - uses: actions/cache@v2
-      with:
-        path: ~/go/pkg/mod
-        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-go
     - name: Get dependencies
       run: |
         # Only download all dependencies
@@ -145,47 +127,43 @@ jobs:
         image:
           - fdb-kubernetes-operator
           - fdb-data-loader
-          - fdb-kubernetes-operator-distroless
         include:
           - image: fdb-kubernetes-operator
             context: ./
             name: foundationdb/fdb-kubernetes-operator
-            tagSuffix: ""
-            file: ./Dockerfile
-            baseImage: "docker.io/debian:bullseye"
-          - image: fdb-kubernetes-operator-distroless
-            context: ./
-            name: foundationdb/fdb-kubernetes-operator
-            tagSuffix: -distrolesss
             file: ./Dockerfile
-            baseImage: "gcr.io/distroless/base"
           - image: fdb-data-loader
             context: ./sample-apps/data-loader
             name: foundationdb/fdb-data-loader
-            tagSuffix: ""
             file: ./sample-apps/data-loader/Dockerfile
-            baseImage: ""
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
         if: github.ref == 'refs/heads/main'
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Get the sha
-        id: get_sha
-        run: echo ::set-output name=TAG::${GITHUB_SHA}
       - name: Build image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           build-args: |
-            TAG=${{ steps.get_sha.outputs.TAG }}
-            BASE_IMAGE=${{ matrix.baseImage }}
+            TAG=${{ github.sha }}
           push: ${{ github.ref == 'refs/heads/main' }}
           context: ${{ matrix.context }}
-          tags: ${{ matrix.name }}:latest${{ matrix.tagSuffix }}
+          tags: ${{ matrix.name }}:latest
           file: ${{ matrix.file }}
+      - name: Run Trivy vulnerability scanner
+        if: ${{ matrix.image }} == 'fdb-kubernetes-operator'
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: 'docker.io/foundationdb/fdb-kubernetes-operator:latest'
+          format: 'table'
+          exit-code: '1'
+          ignore-unfixed: true
+          vuln-type: 'os,library'
+          severity: 'CRITICAL,HIGH'
+
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index e3c78f66..8791352f 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -11,7 +11,7 @@ jobs:
       upload_url: ${{ steps.create_release.outputs.upload_url }}
       tag: ${{ steps.get_tag.outputs.TAG  }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Get the tag without ref
         id: get_tag
         run: echo ::set-output name=TAG::${GITHUB_REF/refs\/tags\//}
@@ -35,15 +35,15 @@ jobs:
     needs: create-release
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Fetch all tags
         run: git fetch --force --tags
       - name: Set up Go
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.20.6
+          go-version: 1.20.11
       #  https://github.com/goreleaser/goreleaser/issues/1311
       - name: Get current semver tag
         run: echo "::set-output name=CURRENT_TAG::$(git describe --tags --match "v*" --abbrev=0)"
@@ -62,46 +62,34 @@ jobs:
         image:
           - fdb-kubernetes-operator
           - fdb-data-loader
-          - fdb-kubernetes-operator-distroless
         include:
           - image: fdb-kubernetes-operator
             context: ./
             name: foundationdb/fdb-kubernetes-operator
-            tagSuffix: ""
             file: ./Dockerfile
-            baseImage: "docker.io/debian:bookworm"
-          - image: fdb-kubernetes-operator-distroless
-            context: ./
-            name: foundationdb/fdb-kubernetes-operator
-            tagSuffix: -distrolesss
-            file: ./Dockerfile
-            baseImage: "gcr.io/distroless/base"
           - image: fdb-data-loader
             context: ./sample-apps/data-loader
             name: foundationdb/fdb-data-loader
-            tagSuffix: ""
             file: ./sample-apps/data-loader/Dockerfile
-            baseImage: ""
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Get the version
         id: get_tag
         run: echo ::set-output name=TAG::${GITHUB_REF/refs\/tags\//}
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Build and push to registry
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           build-args: |
             TAG=${{ steps.get_sha.outputs.TAG }}
-            BASE_IMAGE=${{ matrix.baseImage }}
           push: true
           context: ${{ matrix.context }}
-          tags: ${{ matrix.name }}:${{ steps.get_tag.outputs.TAG }}${{ matrix.tagSuffix }}
+          tags: ${{ matrix.name }}:${{ steps.get_tag.outputs.TAG }}
           file: ${{ matrix.file }}
diff --git a/Dockerfile b/Dockerfile
index e5f12fbc..1fb76070 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,17 +1,20 @@
-ARG BASE_IMAGE=docker.io/debian:bookworm
+ARG FDB_VERSION=6.2.29
+ARG FDB_WEBSITE=https://github.com/apple/foundationdb/releases/download
 
 # Build the manager binary
-FROM docker.io/library/golang:1.20.6 as builder
+FROM docker.io/library/golang:1.20.11 as builder
 
-# Install FDB this version is only required to compile the fdb operator
-ARG FDB_VERSION=6.2.29
-ARG FDB_WEBSITE=https://github.com/apple/foundationdb/releases/download
+ARG FDB_VERSION
+ARG FDB_WEBSITE
 ARG TAG="latest"
 
 RUN set -eux && \
-	curl --fail -L ${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb -o fdb.deb && \
-	dpkg -i fdb.deb && \
-    rm fdb.deb
+    curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb" -o foundationdb-clients_${FDB_VERSION}-1_amd64.deb && \
+    curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb" -o foundationdb-clients_${FDB_VERSION}-1_amd64.deb.sha256 && \
+    # TODO(johscheuer): The 6.2.29 sha256 file is not well formatted, enable this check again once 7.1 is used as base. \
+    # sha256sum -c foundationdb-clients_${FDB_VERSION}-1_amd64.deb.sha256 && \
+	dpkg -i foundationdb-clients_${FDB_VERSION}-1_amd64.deb && \
+    rm foundationdb-clients_${FDB_VERSION}-1_amd64.deb foundationdb-clients_${FDB_VERSION}-1_amd64.deb.sha256
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -42,28 +45,30 @@ RUN groupadd --gid 4059 fdb && \
 	mkdir -p /var/log/fdb && \
 	touch /var/log/fdb/.keep
 
-FROM $BASE_IMAGE
+FROM docker.io/rockylinux:9.2-minimal
+
+ARG FDB_VERSION
+ARG FDB_WEBSITE
 
 VOLUME /usr/lib/fdb
 
 WORKDIR /
 
+RUN set -eux && \
+    curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm" -o foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm && \
+    curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm.sha256" -o foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm.sha256 && \
+    microdnf install -y glibc && \
+    microdnf clean all && \
+    # TODO(johscheuer): The 6.2.29 sha256 file is not well formatted, enable this check again once 7.1 is used as base. \
+    # sha256sum -c foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm.sha256 && \
+	rpm -i foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm --excludepath=/usr/bin --excludepath=/usr/lib/foundationdb/backup_agent && \
+    rm foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm.sha256
+
 COPY --from=builder /etc/passwd /etc/passwd
 COPY --from=builder /etc/group /etc/group
 COPY --chown=fdb:fdb --from=builder /workspace/bin/manager .
-COPY --from=builder /usr/lib/libfdb_c.so /usr/lib/
 COPY --chown=fdb:fdb --from=builder /var/log/fdb/.keep /var/log/fdb/.keep
 
-# FoundationDB versions newer than 7.1.33 are complied with the AWS SDK per default and therefore require
-# some additional libraries that are not present in the default bookwork image. We copy those
-# libraries to make the distroless version of the operator work too, otherwise we could just install
-# the required packages.
-
-# This won't work with the current distroless image, as the image depends on debian11, but bookworm is debian12
-# see: https://github.com/GoogleContainerTools/distroless/issues/1342
-# For now we ignore this and once a debian12 based distroless image is available we can make use of that.
-COPY --from=builder /usr/lib/x86_64-linux-gnu/lib* /usr/lib/x86_64-linux-gnu/
-
 # Set to the numeric UID of fdb user to satisfy PodSecurityPolices which enforce runAsNonRoot
 USER 4059
 
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index b7dcd533..01619b83 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -584,7 +584,10 @@ func (processGroupStatus *ProcessGroupStatus) AllAddressesExcluded(logger logr.L
 	}
 
 	localityExclusionString := processGroupStatus.GetExclusionString()
-	if isRemaining, isPresent := remainingMap[localityExclusionString]; isPresent && !isRemaining {
+	if isRemaining, isPresent := remainingMap[localityExclusionString]; isPresent {
+		if isRemaining {
+			return false, fmt.Errorf("process has missing exclusion string in exclusion results: %s", localityExclusionString)
+		}
 		logger.V(1).Info("process group is fully excluded based on locality based exclusions", "processGroupID", processGroupStatus.ProcessGroupID, "exclusionString", localityExclusionString)
 		return true, nil
 	}
diff --git a/charts/fdb-operator/Chart.yaml b/charts/fdb-operator/Chart.yaml
index 6a085857..00e8c3e8 100644
--- a/charts/fdb-operator/Chart.yaml
+++ b/charts/fdb-operator/Chart.yaml
@@ -3,8 +3,7 @@ name: fdb-operator
 description: A Helm chart for foundationDB operator
 home: https://www.foundationdb.org/
 sources:
-- https://github.com/FoundationDB/fdb-kubernetes-operator/tree/master/helm/fdb-operator
-
+  - https://github.com/FoundationDB/fdb-kubernetes-operator/tree/master/helm/fdb-operator
 # A chart can be either an 'application' or a 'library' chart.
 #
 # Application charts are a collection of templates that can be packaged into versioned archives
@@ -14,14 +13,11 @@ sources:
 # a dependency of application charts to inject those utilities and functions into the rendering
 # pipeline. Library charts do not define any templates and therefore cannot be deployed.
 type: application
-
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 version: 0.2.0
-
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: v1.28.0
-
+appVersion: v1.30.0
 maintainers:
   - name: "foundationdb-ci"
diff --git a/charts/fdb-operator/values.yaml b/charts/fdb-operator/values.yaml
index f404a9d5..12188f21 100644
--- a/charts/fdb-operator/values.yaml
+++ b/charts/fdb-operator/values.yaml
@@ -1,9 +1,8 @@
 ---
 image:
   repository: foundationdb/fdb-kubernetes-operator
-  tag: v1.28.0
+  tag: v1.30.0
   pullPolicy: IfNotPresent
-
 initContainers:
   6.2:
     image:
@@ -20,45 +19,33 @@ initContainers:
       repository: foundationdb/foundationdb-kubernetes-sidecar
       tag: 7.1.15-1
       pullPolicy: IfNotPresent
-
 globalMode:
   enabled: false
-
 replicas: null
-
 imagePullSecrets: []
-
 annotations: {}
 podAnnotations: {}
 podLabels: {}
-
 serviceAccount:
   create: true
   name: null
   imagePullSecrets: []
   annotations: {}
-
 priorityClassName: null
-
 securityContext:
   runAsUser: 4059
   runAsGroup: 4059
   fsGroup: 4059
-
 containerSecurityContext:
   allowPrivilegeEscalation: false
   privileged: false
   capabilities:
     drop:
-    - all
+      - all
   readOnlyRootFilesystem: true
-
 nodeSelector: {}
-
 affinity: {}
-
 tolerations: {}
-
 resources:
   limits:
     cpu: 500m
@@ -66,7 +53,6 @@ resources:
   requests:
     cpu: 500m
     memory: 256Mi
-
 initContainersResources:
   limits:
     cpu: 10m
@@ -74,11 +60,10 @@ initContainersResources:
   requests:
     cpu: 10m
     memory: 50Mi
-
 initContainerSecurityContext:
   allowPrivilegeEscalation: false
   privileged: false
   capabilities:
     drop:
-    - all
+      - all
   readOnlyRootFilesystem: true
diff --git a/config/deployment/manager.yaml b/config/deployment/manager.yaml
index ed883c57..dafa2faa 100644
--- a/config/deployment/manager.yaml
+++ b/config/deployment/manager.yaml
@@ -21,12 +21,12 @@ spec:
         runAsGroup: 4059
         fsGroup: 4059
       volumes:
-      - name: tmp
-        emptyDir: {}
-      - name: logs
-        emptyDir: {}
-      - name: fdb-binaries
-        emptyDir: {}
+        - name: tmp
+          emptyDir: {}
+        - name: logs
+          emptyDir: {}
+        - name: fdb-binaries
+          emptyDir: {}
       serviceAccountName: fdb-kubernetes-operator-controller-manager
       initContainers:
         - name: foundationdb-kubernetes-init-6-2
@@ -81,36 +81,36 @@ spec:
             - name: fdb-binaries
               mountPath: /var/output-files
       containers:
-      - command:
-        - /manager
-        image: foundationdb/fdb-kubernetes-operator:v1.28.0
-        name: manager
-        env:
-          - name: WATCH_NAMESPACE
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.namespace
-        ports:
-          - name: metrics
-            containerPort: 8080
-        resources:
-          limits:
-            cpu: 500m
-            memory: 256Mi
-          requests:
-            cpu: 500m
-            memory: 256Mi
-        securityContext:
-          readOnlyRootFilesystem: true
-          allowPrivilegeEscalation: false
-          privileged: false
-        volumeMounts:
-        - name: tmp
-          mountPath: /tmp
-        - name: logs
-          mountPath: /var/log/fdb
-        - name: fdb-binaries
-          mountPath: /usr/bin/fdb
+        - command:
+            - /manager
+          image: foundationdb/fdb-kubernetes-operator:v1.30.0
+          name: manager
+          env:
+            - name: WATCH_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          ports:
+            - name: metrics
+              containerPort: 8080
+          resources:
+            limits:
+              cpu: 500m
+              memory: 256Mi
+            requests:
+              cpu: 500m
+              memory: 256Mi
+          securityContext:
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
+            privileged: false
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+            - name: logs
+              mountPath: /var/log/fdb
+            - name: fdb-binaries
+              mountPath: /usr/bin/fdb
       terminationGracePeriodSeconds: 10
 ---
 apiVersion: v1
diff --git a/config/samples/deployment.yaml b/config/samples/deployment.yaml
index 0b6e3ffd..6feeffe4 100644
--- a/config/samples/deployment.yaml
+++ b/config/samples/deployment.yaml
@@ -178,7 +178,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: foundationdb/fdb-kubernetes-operator:v1.28.0
+        image: foundationdb/fdb-kubernetes-operator:v1.30.0
         name: manager
         ports:
         - containerPort: 8080
diff --git a/controllers/add_pods.go b/controllers/add_pods.go
index 2267db9a..6840ed05 100644
--- a/controllers/add_pods.go
+++ b/controllers/add_pods.go
@@ -61,10 +61,8 @@ func (a addPods) reconcile(ctx context.Context, r *FoundationDBClusterReconciler
 		}
 
 		// Ignore the is not found error, as we are checking here if we should create Pods.
-		if err != nil {
-			if !k8serrors.IsNotFound(err) {
-				return &requeue{curError: err}
-			}
+		if err != nil && !k8serrors.IsNotFound(err) {
+			return &requeue{curError: err}
 		}
 
 		// If this process group is marked for removal, we normally don't want to spin it back up
diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go
index 58d83098..09769424 100644
--- a/controllers/cluster_controller.go
+++ b/controllers/cluster_controller.go
@@ -185,7 +185,7 @@ func (r *FoundationDBClusterReconciler) Reconcile(ctx context.Context, request c
 
 		if requeue.delayedRequeue {
 			clusterLog.Info("Delaying requeue for sub-reconciler",
-				"subReconciler", fmt.Sprintf("%T", subReconciler),
+				"reconciler", fmt.Sprintf("%T", subReconciler),
 				"message", requeue.message,
 				"error", requeue.curError)
 			delayedRequeue = true
@@ -252,6 +252,7 @@ func (r *FoundationDBClusterReconciler) SetupWithManager(mgr ctrl.Manager, maxCo
 	if err != nil {
 		return err
 	}
+
 	labelSelectorPredicate, err := predicate.LabelSelectorPredicate(selector)
 	if err != nil {
 		return err
diff --git a/controllers/controllers.go b/controllers/controllers.go
index e4d4eb10..e0eadd63 100644
--- a/controllers/controllers.go
+++ b/controllers/controllers.go
@@ -96,7 +96,7 @@ type requeue struct {
 
 // processRequeue interprets a requeue result from a subreconciler.
 func processRequeue(requeue *requeue, subReconciler interface{}, object runtime.Object, recorder record.EventRecorder, logger logr.Logger) (ctrl.Result, error) {
-	curLog := logger.WithValues("subReconciler", fmt.Sprintf("%T", subReconciler), "requeueAfter", requeue.delay)
+	curLog := logger.WithValues("reconciler", fmt.Sprintf("%T", subReconciler), "requeueAfter", requeue.delay)
 	if requeue.message == "" && requeue.curError != nil {
 		requeue.message = requeue.curError.Error()
 	}
diff --git a/controllers/update_status.go b/controllers/update_status.go
index 9b0d2ed4..412d6bfd 100644
--- a/controllers/update_status.go
+++ b/controllers/update_status.go
@@ -351,18 +351,21 @@ func checkAndSetProcessStatus(logger logr.Logger, r *FoundationDBClusterReconcil
 	}
 
 	var excluded, hasIncorrectCommandLine, hasMissingProcesses, sidecarUnreachable bool
+	var substitutions map[string]string
+	var err error
 
 	// Fetch the pod client and variables once per Pod.
 	podClient, message := r.getPodClient(cluster, pod)
 	if podClient == nil {
 		logger.Info("Unable to build pod client", "processGroupID", processGroupStatus.ProcessGroupID, "message", message)
-		return nil
-	}
-
-	substitutions, err := podClient.GetVariableSubstitutions()
-	if err != nil {
-		if internal.IsNetworkError(err) {
-			sidecarUnreachable = true
+		// As the Pod is not ready we can assume that the sidecar is not reachable.
+		sidecarUnreachable = true
+	} else {
+		substitutions, err = podClient.GetVariableSubstitutions()
+		if err != nil {
+			if internal.IsNetworkError(err) {
+				sidecarUnreachable = true
+			}
 		}
 	}
 
@@ -429,8 +432,8 @@ func checkAndSetProcessStatus(logger logr.Logger, r *FoundationDBClusterReconcil
 	if hasMissingProcesses {
 		return nil
 	}
-
 	processGroupStatus.UpdateCondition(fdbv1beta2.ProcessIsMarkedAsExcluded, excluded)
+	// If the sidecar is unreachable we are not able to compute the desired commandline.
 	if sidecarUnreachable {
 		return nil
 	}
diff --git a/docs/changelog/v1.29.0.md b/docs/changelog/v1.29.0.md
new file mode 100644
index 00000000..5c67c5fe
--- /dev/null
+++ b/docs/changelog/v1.29.0.md
@@ -0,0 +1,21 @@
+# v1.29.0
+
+## Changes
+
+### Operator
+
+* Fix logic when sidecar is unreachable for missing process detection [#1896](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1896)
+* Update the way the operator deployment is created and verified [#1895](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1895)
+* Add more logs when the fetched status has issues [#1893](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1893)
+* Allow to set the namespace in the three data hall example [#1892](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1892)
+* fixing error in AllAddressesExcluded [#1891](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1891)
+* Add cache to plugin [#1888](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1888)
+* Improve exclusion check to make sure that enough processes are up and running [#1887](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1887)
+* Ignore HA locality-based exclusion test if version doesn't support locality based exclusions [#1886](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1886)
+* Three data hall enable locking by default [#1885](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1885)
+* Allow to define creationTimestamp mock [#1882](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1882)
+* Add check for plugin version upgrades [#1881](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1881)
+* Make sure the exclusion is tried again [#1880](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1880)
+* Make sure the e2e test suite only runs locality based e2e tests if the provided versions support it [#1879](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1879)
+* Refactor the check process method and remove some duplicate work [#1870](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1870)
+* including processes using excluded server list [#1857](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1857)
diff --git a/docs/changelog/v1.30.0.md b/docs/changelog/v1.30.0.md
new file mode 100644
index 00000000..dd2afa25
--- /dev/null
+++ b/docs/changelog/v1.30.0.md
@@ -0,0 +1,14 @@
+# v1.30.0
+
+## Changes
+
+### Operator
+
+* Add indexer to fix exec command [#1917](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1917)
+* Add e2e test case for maintenance mode interaction [#1915](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1915)
+* Update dependencies [#1914](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1914)
+* Minor documentation improvements [#1913](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1913)
+* Add support for using the unified image as init container for the operator [#1911](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1911)
+* Add the missing list verb to Pods for testing the unified image [#1910](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1910)
+* Allow to modify the leader election timeouts [#1909](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1909)
+* format files changed during release with yq [#1907](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1907)
diff --git a/docs/design/imgs/fault_domain_multi_host.svg b/docs/design/imgs/fault_domain_multi_host.svg
index 36bedccf..0fdbd8df 100644
--- a/docs/design/imgs/fault_domain_multi_host.svg
+++ b/docs/design/imgs/fault_domain_multi_host.svg
@@ -63,7 +63,7 @@
         <rect x="385.75" y="96" width="204.5" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
         <text transform="translate(390.75 116.302)" fill="black">
           <tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="76.058" y="15">Pod 2</tspan>
-          <tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="72.354" y="33.448">Zone 3</tspan>
+          <tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="72.354" y="33.448">Zone 2</tspan>
         </text>
       </g>
       <g id="Graphic_8">
diff --git a/docs/design/imgs/increase_fault_domain_end.svg b/docs/design/imgs/increase_fault_domain_end.svg
index 9ef4077e..f02fc611 100644
--- a/docs/design/imgs/increase_fault_domain_end.svg
+++ b/docs/design/imgs/increase_fault_domain_end.svg
@@ -63,7 +63,7 @@
         <rect x="385.75" y="96" width="204.5" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
         <text transform="translate(390.75 116.302)" fill="black">
           <tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="76.058" y="15">Pod 2</tspan>
-          <tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="72.354" y="33.448">Zone 3</tspan>
+          <tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="72.354" y="33.448">Zone 2</tspan>
         </text>
       </g>
       <g id="Graphic_8">
diff --git a/docs/design/imgs/increase_fault_domain_initial.svg b/docs/design/imgs/increase_fault_domain_initial.svg
index b5053f7f..7ed368dd 100644
--- a/docs/design/imgs/increase_fault_domain_initial.svg
+++ b/docs/design/imgs/increase_fault_domain_initial.svg
@@ -63,7 +63,7 @@
         <rect x="385.75" y="96" width="204.5" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
         <text transform="translate(390.75 116.302)" fill="black">
           <tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="76.058" y="15">Pod 2</tspan>
-          <tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="72.354" y="33.448">Zone 3</tspan>
+          <tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="72.354" y="33.448">Zone 2</tspan>
         </text>
       </g>
       <g id="Graphic_8">
diff --git a/docs/design/imgs/reduce_fault_domains.svg b/docs/design/imgs/reduce_fault_domains.svg
index 18b1b4db..9ad204e5 100644
--- a/docs/design/imgs/reduce_fault_domains.svg
+++ b/docs/design/imgs/reduce_fault_domains.svg
@@ -63,7 +63,7 @@
         <rect x="385.75" y="96" width="204.5" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
         <text transform="translate(390.75 116.302)" fill="black">
           <tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="76.058" y="15">Pod 2</tspan>
-          <tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="72.354" y="33.448">Zone 3</tspan>
+          <tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="72.354" y="33.448">Zone 2</tspan>
         </text>
       </g>
       <g id="Graphic_8">
diff --git a/docs/manual/customization.md b/docs/manual/customization.md
index a8750d21..6ddc42d6 100644
--- a/docs/manual/customization.md
+++ b/docs/manual/customization.md
@@ -2,7 +2,7 @@
 
 This document covers some of the options the operator provides for customizing your FoundationDB deployment.
 
-Many of these customizations involve the `processes` field in the cluster spec, which we will refer to as the "process settings". This field is a dictionary, mapping a process class to a process settings object. This also supports a special key called `general` which is applied to all process classes. If a value is specified for a specific process class, the `general` value will be ignored. These values are merged at the top level of the process settings object. If you specify a `volumeClaimTemplate` object in the `storage` settings and a `podTemplate` object in the `general` settings, the storage processes will use both the custom `volumeClaimTemplate` and the general `podTemplate`. If you specify a `podTemplate` object in the `storage` settings and `podTemplate` object in the `general` settings, the storage processes will only use the values given in the `storage` settings, and will the pod template from the `general` settings completely.
+Many of these customizations involve the `processes` field in the cluster spec, which we will refer to as the "process settings". This field is a dictionary, mapping a process class to a process settings object. This also supports a special key called `general` which is applied to all process classes. If a value is specified for a specific process class, the `general` value will be ignored. These values are merged at the top level of the process settings object. If you specify a `volumeClaimTemplate` object in the `storage` settings and a `podTemplate` object in the `general` settings, the storage processes will use both the custom `volumeClaimTemplate` and the general `podTemplate`. If you specify a `podTemplate` object in the `storage` settings and `podTemplate` object in the `general` settings, the storage processes will only use the values given in the `storage` settings, and will ignore the pod template from the `general` settings completely.
 
 ## Running Multiple Storage Servers per Pod
 
diff --git a/docs/manual/debugging.md b/docs/manual/debugging.md
index bbc36765..306c30b7 100644
--- a/docs/manual/debugging.md
+++ b/docs/manual/debugging.md
@@ -1,5 +1,17 @@
 # Debugging
 
+## Logging
+
+The operator supports json-structured logging and will emit those logs to stdout and if configured to a file.
+The operator sets different log keys for easier log filtering in different contexts.
+All cluster related logs will contain the following fields:
+
+- `namespace`: Namespace of the `FoundationDBCluster` resource that is currently processed.
+- `cluster`: The name of the `FoundationDBCluster` resource that is currently processed.
+
+In addition the operator will set the `reconciler` field to the current sub-reconciler.
+If the operator iterates over all or a subset of `ProcessGroups`, the operator will set `processGroupID` to the current process group ID.
+
 ## Kubectl FDB Plugin
 
 You can use the [kubectl-fdb](/kubectl-fdb) plugin when investigating issues and running imperative commands.
diff --git a/docs/manual/replacements_and_deletions.md b/docs/manual/replacements_and_deletions.md
index 4253972a..e11691d3 100644
--- a/docs/manual/replacements_and_deletions.md
+++ b/docs/manual/replacements_and_deletions.md
@@ -1,9 +1,9 @@
 # Replacements and Deletions
 
-The operator has two different strategies it can take on process groups that are in an undesired state: replacement and deletion.
-In the case of replacement, we will create a brand new process group, move data off the old process group, and delete the resources for the old process group as well as the records of the process group itself.
+The operator has two different strategies it can use on process groups that are in an undesired state: replacement and deletion.
+In the case of replacement, we will create a brand new process group, move data off of the old process group, and delete the resources for the old process group as well as the records of the process group itself.
 In the case of deletion, we will delete some or all of the resources for the process group and then create new objects with the same names.
-We will cover details of when these different strategies are used in later sections.
+We will cover their details when these different strategies are used in later sections.
 
 A process group is marked for replacement by setting the `remove` flag on the process group.
 This flag is used during both replacements and shrinks, and a replacement is modeled as a grow followed by a shrink.
@@ -61,7 +61,7 @@ The following conditions are currently eligible for replacement:
 * `MissingService`: This indicates that a process group that doesn't have a Service assigned.
 * `PodPending`: This indicates that a process group where the Pod is in a pending state.
 * `NodeTaintReplacing`: This indicates a process group where the Pod has been running on a tainted Node for at least the configured duration. If a ProcessGroup has the `NodeTaintReplacing` condition, the replacement cannot be stopped, even after the Node taint was removed.
-* `ProcessIsMarkedAsExcluded`: This indicates a process group where at least on process is excluded. If the process group is not marked as removal, the operator will replace this process group to make sure the cluster runs at the right capacity.
+* `ProcessIsMarkedAsExcluded`: This indicates a process group where at least one process is excluded. If the process group is not marked for removal, the operator will replace this process group to make sure the cluster runs at the right capacity.
 
 Process groups that are set into the crash loop state with the `Buggify` setting won't be replaced by the operator.
 If the `cluster.Spec.Buggify.EmptyMonitorConf` setting is active the operator won't replace any process groups.
diff --git a/docs/manual/scaling.md b/docs/manual/scaling.md
index cda48412..f19c599a 100644
--- a/docs/manual/scaling.md
+++ b/docs/manual/scaling.md
@@ -19,13 +19,13 @@ spec:
     stateless: 4
 ```
 
-This will configure 6 storage processes, 5 log processes, and 4 stateless processes. This is fewer stateless processes that we had by default, which means that some processes will be running multiple roles. This is generally something you want to avoid in a production configuration, as it can lead to high activity on one role starving another role of resources.
+This will configure 6 storage processes, 5 log processes, and 4 stateless processes. This is fewer stateless processes than we had by default, which means that some processes will be running multiple roles. This is generally something you want to avoid in a production configuration, as it can lead to high activity on one role starving another role of resources.
 
 By default, the operator will provision processes with the following process types and counts:
 
 1. `storage`. Equal to the storage count in the database configuration. If no storage count is provided, this will be `2*F+1`, where `F` is the desired fault tolerance. For a double replicated cluster, the desired fault tolerance is 1.
 2. `log`. Equal to the `F+max(logs, remote_logs)`. The `logs` and `remote_logs` here are the counts specified in the database configuration. By default, `logs` is set to 3 and `remote_logs` is set to either `-1` or `logs`.
-3. `stateless`. Equal to the sum of all other roles in the database configuration + `F`. Currently, this is `max(proxies+resolvers+4, log_routers)`. The `4` is for the master, cluster controller, data distributor, and ratekeeper processes. This may change in the future as we add more roles to the database. By default, `proxies` is set to 3, `grv_proxies` is set 1, `commit_proxies` is set to 2, `resolvers` is set to 1, and `log_routers` is set to -1. If the version of FDB in the cluster is less than 7.0.0, the process counts will use the value of proxies set (or the default) and FDB will apply a ratio to how the individual proxy roles are distributed. If version of FDB is greater than 7.0.0 and the separated proxy counts are set while the old `proxies` are not, we will use `proxies=sum(grv_proxies + commit_proxies)`.
+3. `stateless`. Equal to the sum of all other roles in the database configuration + `F`. Currently, this is `max(proxies+resolvers+4, log_routers)`. The `4` is for the master, cluster controller, data distributor, and ratekeeper processes. This may change in the future as we add more roles to the database. By default, `proxies` is set to 3, `grv_proxies` is set to 1, `commit_proxies` is set to 2, `resolvers` is set to 1, and `log_routers` is set to -1. If the version of FDB in the cluster is less than 7.0.0, the process counts will use the value of proxies set (or the default) and FDB will apply a ratio to how the individual proxy roles are distributed. If version of FDB is greater than 7.0.0 and the separated proxy counts are set while the old `proxies` are not, we will use `proxies=sum(grv_proxies + commit_proxies)`.
 
 You can also set a process count to -1 to tell the operator not to provision any processes of that type.
 
diff --git a/docs/manual/upgrades.md b/docs/manual/upgrades.md
index 42cf9fdd..73b91d1e 100644
--- a/docs/manual/upgrades.md
+++ b/docs/manual/upgrades.md
@@ -20,7 +20,7 @@ The graphic below represents the different stages in a Pod during the upgrade:
 
 The first state on the left is the initial state before the operator is doing any work.
 The second state in the middle is the state of the Pod during the [Staging Phase](#staging-phase) and the [Restart Phase](#restart-phase).
-The third and last state is the sta te after the [Recreation of Pods Phase](#recreation-of-pods-phase) where the according Pod was recreated with the new desired image version.
+The third and last state is the state after the [Recreation of Pods Phase](#recreation-of-pods-phase) where the according Pod was recreated with the new desired image version.
 
 
 #### Pre Upgrade Check Phase
@@ -57,20 +57,21 @@ In order to replace the Pod the operator will wait for the Process Group to have
 #### Restart Phase
 
 The `BounceProcesses` subreconciler will handle the restart of all `fdbserver` processes.
-For FoundationDB clusters that are spanned across multiple Kubernetes clusters the operator will follow a special process which is document in the [technical design](technical_design.md#bounceprocesses).
+For FoundationDB clusters that are spanned across multiple Kubernetes clusters the operator will follow a special process which is documented in the [technical design](technical_design.md#bounceprocesses).
 The subreconciler will also ensure to wait until all `fdbserver` processes are ready to be restarted to prevent cases where only a subset of processes are restarted.
 In this case ready means that all Pods have the new `fdbmonitor` configuration present and that the new binary is present in the shared volume at `/var/dynamic-conf/bin/$fdb_version`.
-After the `kill` command the operator will initiate a new reconciliation loop to detect the new running version, this is handled in the `UpdateState` subreconciler and the version is detected based on the output of the [cluster status json](https://apple.github.io/foundationdb/mr-status.html).
+After the `kill` command the operator will initiate a new reconciliation loop to detect the new, upgraded running version, this is handled in the `UpdateState` subreconciler and the version is detected based on the output of the [cluster status json](https://apple.github.io/foundationdb/mr-status.html).
 
 #### Recreation of Pods Phase
 
-Technically the upgrade is now done and the cluster is running on the new version but the operator has to do some cleanup.
-The cleanup includes the recreation of all Pods to ensure they use the new FoundationDB image for the desired version and the configuration will be updated to point again to the binary present in the container.
+Technically the upgrade is now done and all the pods in the cluster are running on the new version but the operator has to do some cleanup.
+The cleanup includes the recreation of all Pods to ensure they use the new FoundationDB image for the desired version and the `fdbmonitor` configuration will be updated to point again to 
+the binary included in the image at `/bin/$fdb_version/fdb-server`, instead of the one at `/var/dynamic-conf/bin/$fdb_version` in the original, technically-upgraded pod.
 Depending on the defined [Pod update strategy](../cluster_spec.md#podupdatestrategy) the operator will handle the recreation of the Pods differently.
 In the default setting `ReplaceTransactionSystem` the operator will recreate the storage Pods zone by zone and replaces Pods of the transaction system (log and stateless processes).
 The replacement of the transaction system has the benefit that the recoveries of the transaction system is reduced to potentially one recovery.
 During this phase the operator will also run the `RemoveIncompatibleProcesses` subreconciler, this reconciler will recreate all Pods that host a `fdbserver` process that is in the list of the `incompatible_connections` and is missing the the process list of the [cluster status json](https://apple.github.io/foundationdb/mr-status.html).
-This reconciler should ensure that the cluster is moving faster towards the desired configuration if some processes where not restarted properly.
+This reconciler should ensure that the cluster is moving faster towards the desired configuration if some processes were not restarted properly.
 
 Once all Pods are updated to the new image the upgrade is done and the cluster status of the FoundationDB cluster resource in Kubernetes should show that the reconciliation is done.
 
@@ -89,7 +90,7 @@ If the process has this condition longer than the [IgnoreMissingProcessesSeconds
 This also means that for a short time the cluster might be running with less resources.
 
 Processes that are restarted too early can be identified with the `MissingProcesses` condition.
-Using `kubectl` to fetch the cluster status and searching for all Process Groups with this condition gives an overview of those processes restarted to early.
+Using `kubectl` to fetch the cluster status and searching for all Process Groups with this condition gives an overview of those processes restarted too early.
 
 The operator will handle those cases automatically and there is no need for manual intervention as long as not too many processes are restarted from the outside.
 In that case the best way forward is to restart all processes still running on the old version, this can either be done manually or by using the [kubectl fdb plugin](../../kubectl-fdb/Readme.md) with the following command:
diff --git a/fdbclient/command_runner.go b/fdbclient/command_runner.go
index 018b8b74..18366b0f 100644
--- a/fdbclient/command_runner.go
+++ b/fdbclient/command_runner.go
@@ -48,6 +48,7 @@ func getEnvironmentVariablesWithoutExcludedFdbEnv() []string {
 	excludedEnvironmentVariables := map[string]fdbv1beta2.None{
 		"FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY":       {},
 		"FDB_NETWORK_OPTION_IGNORE_EXTERNAL_CLIENT_FAILURES": {},
+		"FDB_NETWORK_OPTION_CLIENT_THREADS_PER_VERSION":      {},
 	}
 
 	osVariables := os.Environ()
diff --git a/fdbclient/common.go b/fdbclient/common.go
index ddd1a906..32ea2c56 100644
--- a/fdbclient/common.go
+++ b/fdbclient/common.go
@@ -59,12 +59,13 @@ func parseMachineReadableStatus(logger logr.Logger, contents []byte) (*fdbv1beta
 	}
 
 	if !status.Client.DatabaseStatus.Available {
-		logger.Info("database is unavailable")
+		logger.Info("database is unavailable", "status", status)
 		return nil, fdbv1beta2.TimeoutError{Err: fmt.Errorf("database is unavailable")}
 	}
 
 	if len(status.Cluster.Messages) > 0 {
 		logger.Info("found cluster message(s) in the machine-readable status", "messages", status.Cluster.Messages)
+		logger.V(1).Info("current status with cluster messages", "status", status)
 
 		// If the status is incomplete because of a timeout, return an error. This will force a new reconciliation.
 		for _, message := range status.Cluster.Messages {
diff --git a/go.mod b/go.mod
index 2ea2521c..3a3aac3f 100644
--- a/go.mod
+++ b/go.mod
@@ -19,14 +19,14 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.15.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
-	k8s.io/api v0.26.1
-	k8s.io/apimachinery v0.26.1
-	k8s.io/cli-runtime v0.26.1
-	k8s.io/client-go v0.26.1
+	k8s.io/api v0.26.10
+	k8s.io/apimachinery v0.26.10
+	k8s.io/cli-runtime v0.26.10
+	k8s.io/client-go v0.26.10
 	k8s.io/klog/v2 v2.90.0
-	k8s.io/kubectl v0.26.1
+	k8s.io/kubectl v0.26.10
 	k8s.io/utils v0.0.0-20230115233650-391b47cb4029
-	sigs.k8s.io/controller-runtime v0.14.6
+	sigs.k8s.io/controller-runtime v0.14.7
 	sigs.k8s.io/yaml v1.3.0
 )
 
@@ -105,8 +105,8 @@ require (
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.26.1 // indirect
-	k8s.io/component-base v0.26.1 // indirect
+	k8s.io/apiextensions-apiserver v0.26.10 // indirect
+	k8s.io/component-base v0.26.10 // indirect
 	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
 	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 	sigs.k8s.io/kustomize/api v0.12.1 // indirect
diff --git a/go.sum b/go.sum
index 610598aa..b822606a 100644
--- a/go.sum
+++ b/go.sum
@@ -743,20 +743,20 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8=
-k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ=
-k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg=
-k8s.io/apiextensions-apiserver v0.26.1 h1:cB8h1SRk6e/+i3NOrQgSFij1B2S0Y0wDoNl66bn8RMI=
-k8s.io/apiextensions-apiserver v0.26.1/go.mod h1:AptjOSXDGuE0JICx/Em15PaoO7buLwTs0dGleIHixSM=
+k8s.io/api v0.26.10 h1:skTnrDR0r8dg4MMLf6YZIzugxNM0BjFsWKPkNc5kOvk=
+k8s.io/api v0.26.10/go.mod h1:ou/H3yviqrHtP/DSPVTfsc7qNfmU06OhajytJfYXkXw=
+k8s.io/apiextensions-apiserver v0.26.10 h1:wAriTUc6l7gUqJKOxhmXnYo/VNJzk4oh4QLCUR4Uq+k=
+k8s.io/apiextensions-apiserver v0.26.10/go.mod h1:N2qhlxkhJLSoC4f0M1/1lNG627b45SYqnOPEVFoQXw4=
 k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
-k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ=
-k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74=
-k8s.io/cli-runtime v0.26.1 h1:f9+bRQ1V3elQsx37KmZy5fRAh56mVLbE9A7EMdlqVdI=
-k8s.io/cli-runtime v0.26.1/go.mod h1:+e5Ym/ARySKscUhZ8K3hZ+ZBo/wYPIcg+7b5sFYi6Gg=
+k8s.io/apimachinery v0.26.10 h1:aE+J2KIbjctFqPp3Y0q4Wh2PD+l1p2g3Zp4UYjSvtGU=
+k8s.io/apimachinery v0.26.10/go.mod h1:iT1ZP4JBP34wwM+ZQ8ByPEQ81u043iqAcsJYftX9amM=
+k8s.io/cli-runtime v0.26.10 h1:a5t8ejLCCjWBEny70uMDyPfOyOJH1qAxrrEo2a9fopU=
+k8s.io/cli-runtime v0.26.10/go.mod h1:i1UCYrl+n32ej4N2n2eacOMv4T94vRL0/ooOLopN23Q=
 k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE=
-k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU=
-k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE=
-k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4=
-k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU=
+k8s.io/client-go v0.26.10 h1:4mDzl+1IrfRxh4Ro0s65JRGJp14w77gSMUTjACYWVRo=
+k8s.io/client-go v0.26.10/go.mod h1:sh74ig838gCckU4ElYclWb24lTesPdEDPnlyg5vcbkA=
+k8s.io/component-base v0.26.10 h1:vl3Gfe5aC09mNxfnQtTng7u3rnBVrShOK3MAkqEleb0=
+k8s.io/component-base v0.26.10/go.mod h1:/IDdENUHG5uGxqcofZajovYXE9KSPzJ4yQbkYQt7oN0=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
@@ -765,16 +765,16 @@ k8s.io/klog/v2 v2.90.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
 k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E=
 k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4=
-k8s.io/kubectl v0.26.1 h1:K8A0Jjlwg8GqrxOXxAbjY5xtmXYeYjLU96cHp2WMQ7s=
-k8s.io/kubectl v0.26.1/go.mod h1:miYFVzldVbdIiXMrHZYmL/EDWwJKM+F0sSsdxsATFPo=
+k8s.io/kubectl v0.26.10 h1:UoHA2Apb/Ack+B3evJjokbQ1shq6WdAmVi9AtWiY1B8=
+k8s.io/kubectl v0.26.10/go.mod h1:U8Zb+jkWVI3H/LSbCDHQ0d70uYmOJtNQk9V2fmg7tGw=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20230115233650-391b47cb4029 h1:L8zDtT4jrxj+TaQYD0k8KNlr556WaVQylDXswKmX+dE=
 k8s.io/utils v0.0.0-20230115233650-391b47cb4029/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA=
-sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0=
+sigs.k8s.io/controller-runtime v0.14.7 h1:Vrnm2vk9ZFlRkXATHz0W0wXcqNl7kPat8q2JyxVy0Q8=
+sigs.k8s.io/controller-runtime v0.14.7/go.mod h1:ErTs3SJCOujNUnTz4AS+uh8hp6DHMo1gj6fFndJT1X8=
 sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k=
 sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM=
diff --git a/kubectl-fdb/cmd/analyze.go b/kubectl-fdb/cmd/analyze.go
index ff15ae35..c36b6cff 100644
--- a/kubectl-fdb/cmd/analyze.go
+++ b/kubectl-fdb/cmd/analyze.go
@@ -72,6 +72,11 @@ func newAnalyzeCmd(streams genericclioptions.IOStreams) *cobra.Command {
 				return err
 			}
 
+			shouldAnalyzeStatus, err := cmd.Flags().GetBool("analyze-status")
+			if err != nil {
+				return err
+			}
+
 			ignoreConditions, err := cmd.Flags().GetStringArray("ignore-condition")
 			if err != nil {
 				return err
@@ -90,7 +95,7 @@ func newAnalyzeCmd(streams genericclioptions.IOStreams) *cobra.Command {
 				return cmd.Help()
 			}
 
-			kubeClient, err := getKubeClient(o)
+			kubeClient, err := getKubeClient(cmd.Context(), o)
 			if err != nil {
 				return err
 			}
@@ -139,6 +144,10 @@ func newAnalyzeCmd(streams genericclioptions.IOStreams) *cobra.Command {
 					errs = append(errs, err)
 				}
 
+				if !shouldAnalyzeStatus {
+					continue
+				}
+
 				err = analyzeStatus(cmd, config, clientSet, kubeClient, cluster, autoFix)
 				if err != nil {
 					errs = append(errs, err)
@@ -182,6 +191,7 @@ kubectl fdb analyze --ignore-removals=false sample-cluster-1
 
 	cmd.Flags().Bool("auto-fix", false, "defines if the analyze tasks should try to fix the found issues (e.g. replace Pods).")
 	cmd.Flags().Bool("all-clusters", false, "defines all clusters in the given namespace should be analyzed.")
+	cmd.Flags().Bool("analyze-status", false, "defines if the command should also analyze the machine-readable status of the provided cluster(s).")
 	// We might want to move this into the root cmd if we need this in multiple places
 	cmd.Flags().Bool("no-color", false, "Disable color output.")
 	cmd.Flags().StringArray("ignore-condition", nil, "specify which process group conditions should be ignored and not be printed to stdout.")
diff --git a/kubectl-fdb/cmd/buggify_crash_loop.go b/kubectl-fdb/cmd/buggify_crash_loop.go
index 943cf7b4..23a351ad 100644
--- a/kubectl-fdb/cmd/buggify_crash_loop.go
+++ b/kubectl-fdb/cmd/buggify_crash_loop.go
@@ -61,7 +61,7 @@ func newBuggifyCrashLoop(streams genericclioptions.IOStreams) *cobra.Command {
 				return err
 			}
 
-			kubeClient, err := getKubeClient(o)
+			kubeClient, err := getKubeClient(cmd.Context(), o)
 			if err != nil {
 				return err
 			}
diff --git a/kubectl-fdb/cmd/buggify_empty_monitor_conf.go b/kubectl-fdb/cmd/buggify_empty_monitor_conf.go
index 9ae6baf8..5ca001f1 100644
--- a/kubectl-fdb/cmd/buggify_empty_monitor_conf.go
+++ b/kubectl-fdb/cmd/buggify_empty_monitor_conf.go
@@ -52,7 +52,7 @@ func newBuggifyEmptyMonitorConf(streams genericclioptions.IOStreams) *cobra.Comm
 				return err
 			}
 
-			kubeClient, err := getKubeClient(o)
+			kubeClient, err := getKubeClient(cmd.Context(), o)
 			if err != nil {
 				return err
 			}
diff --git a/kubectl-fdb/cmd/buggify_no_schedule.go b/kubectl-fdb/cmd/buggify_no_schedule.go
index 48656521..ca178040 100644
--- a/kubectl-fdb/cmd/buggify_no_schedule.go
+++ b/kubectl-fdb/cmd/buggify_no_schedule.go
@@ -56,7 +56,7 @@ func newBuggifyNoSchedule(streams genericclioptions.IOStreams) *cobra.Command {
 				return err
 			}
 
-			kubeClient, err := getKubeClient(o)
+			kubeClient, err := getKubeClient(cmd.Context(), o)
 			if err != nil {
 				return err
 			}
diff --git a/kubectl-fdb/cmd/configuration.go b/kubectl-fdb/cmd/configuration.go
index e3cda9b8..7bf6cc79 100644
--- a/kubectl-fdb/cmd/configuration.go
+++ b/kubectl-fdb/cmd/configuration.go
@@ -52,7 +52,7 @@ func newConfigurationCmd(streams genericclioptions.IOStreams) *cobra.Command {
 				return err
 			}
 
-			kubeClient, err := getKubeClient(o)
+			kubeClient, err := getKubeClient(cmd.Context(), o)
 			if err != nil {
 				return err
 			}
diff --git a/kubectl-fdb/cmd/cordon.go b/kubectl-fdb/cmd/cordon.go
index db063c8e..fd4c357b 100644
--- a/kubectl-fdb/cmd/cordon.go
+++ b/kubectl-fdb/cmd/cordon.go
@@ -64,7 +64,7 @@ func newCordonCmd(streams genericclioptions.IOStreams) *cobra.Command {
 				return err
 			}
 
-			kubeClient, err := getKubeClient(o)
+			kubeClient, err := getKubeClient(cmd.Context(), o)
 			if err != nil {
 				return err
 			}
@@ -182,6 +182,7 @@ func cordonNode(cmd *cobra.Command, kubeClient client.Client, inputClusterName s
 	if len(errors) > 0 {
 		return fmt.Errorf("following operation failed, please check and retry: \n, %s", errors)
 	}
+
 	return nil
 }
 
@@ -211,6 +212,7 @@ func fetchPods(kubeClient client.Client, clusterName string, namespace string, n
 	if err != nil {
 		return pods, fmt.Errorf("unable to fetch pods. Error: %w", err)
 	}
+
 	return pods, nil
 }
 
diff --git a/kubectl-fdb/cmd/deprecation.go b/kubectl-fdb/cmd/deprecation.go
index fefe1920..5f45d4a3 100644
--- a/kubectl-fdb/cmd/deprecation.go
+++ b/kubectl-fdb/cmd/deprecation.go
@@ -57,7 +57,7 @@ Deprecated settings that should be replace by a newer setting (or removed) are p
 				return err
 			}
 
-			kubeClient, err := getKubeClient(o)
+			kubeClient, err := getKubeClient(cmd.Context(), o)
 			if err != nil {
 				return err
 			}
diff --git a/kubectl-fdb/cmd/exclusion_status.go b/kubectl-fdb/cmd/exclusion_status.go
index 54fc5813..39c60de6 100644
--- a/kubectl-fdb/cmd/exclusion_status.go
+++ b/kubectl-fdb/cmd/exclusion_status.go
@@ -41,7 +41,7 @@ func newExclusionStatusCmd(streams genericclioptions.IOStreams) *cobra.Command {
 		Use:   "exclusion-status",
 		Short: "Get the exclusion status for all excluded processes.",
 		Long:  "Get the exclusion status for all excluded processes.",
-		Args:  cobra.ExactValidArgs(1),
+		Args:  cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ignoreFullyExcluded, err := cmd.Flags().GetBool("ignore-fully-excluded")
 			if err != nil {
@@ -63,7 +63,7 @@ func newExclusionStatusCmd(streams genericclioptions.IOStreams) *cobra.Command {
 				return err
 			}
 
-			kubeClient, err := getKubeClient(o)
+			kubeClient, err := getKubeClient(cmd.Context(), o)
 			if err != nil {
 				return err
 			}
diff --git a/kubectl-fdb/cmd/exec.go b/kubectl-fdb/cmd/exec.go
index dfde0c40..cba42df5 100644
--- a/kubectl-fdb/cmd/exec.go
+++ b/kubectl-fdb/cmd/exec.go
@@ -53,7 +53,7 @@ func newExecCmd(streams genericclioptions.IOStreams) *cobra.Command {
 				return err
 			}
 
-			kubeClient, err := getKubeClient(o)
+			kubeClient, err := getKubeClient(cmd.Context(), o)
 			if err != nil {
 				return err
 			}
diff --git a/kubectl-fdb/cmd/fix_coordinator_ips.go b/kubectl-fdb/cmd/fix_coordinator_ips.go
index 07952c31..1991ceac 100644
--- a/kubectl-fdb/cmd/fix_coordinator_ips.go
+++ b/kubectl-fdb/cmd/fix_coordinator_ips.go
@@ -59,7 +59,7 @@ func newFixCoordinatorIPsCmd(streams genericclioptions.IOStreams) *cobra.Command
 				return err
 			}
 
-			kubeClient, err := getKubeClient(o)
+			kubeClient, err := getKubeClient(cmd.Context(), o)
 			if err != nil {
 				return err
 			}
diff --git a/kubectl-fdb/cmd/k8s_client.go b/kubectl-fdb/cmd/k8s_client.go
index a4ba9716..aeded6a7 100644
--- a/kubectl-fdb/cmd/k8s_client.go
+++ b/kubectl-fdb/cmd/k8s_client.go
@@ -22,10 +22,13 @@ package cmd
 
 import (
 	"bytes"
-	ctx "context"
+	"context"
 	"fmt"
+	"github.com/go-logr/logr"
 	"io"
+	"k8s.io/klog/v2"
 	"math/rand"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"strings"
 
 	fdbv1beta1 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta1"
@@ -45,7 +48,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-func getKubeClient(o *fdbBOptions) (client.Client, error) {
+func getKubeClient(ctx context.Context, o *fdbBOptions) (client.Client, error) {
 	config, err := o.configFlags.ToRESTConfig()
 	if err != nil {
 		return nil, err
@@ -56,7 +59,66 @@ func getKubeClient(o *fdbBOptions) (client.Client, error) {
 	utilruntime.Must(fdbv1beta1.AddToScheme(scheme))
 	utilruntime.Must(fdbv1beta2.AddToScheme(scheme))
 
-	return client.New(config, client.Options{Scheme: scheme})
+	// Don't printout any log messages from client-go.
+	klog.SetLogger(logr.Discard())
+
+	internalClient, err := client.NewWithWatch(config, client.Options{
+		Scheme: scheme,
+		Opts: client.WarningHandlerOptions{
+			SuppressWarnings: true,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	namespace, err := getNamespace(*o.configFlags.Namespace)
+	if err != nil {
+		return nil, err
+	}
+
+	cacheBuilder := cache.MultiNamespacedCacheBuilder([]string{namespace})
+	internalCache, err := cacheBuilder(config, cache.Options{
+		Scheme: scheme,
+		Mapper: internalClient.RESTMapper(),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// Setup index field to allow access to the node name more efficiently. The indexer must be created before the
+	// informer is started.
+	err = internalCache.IndexField(ctx, &corev1.Pod{}, "spec.nodeName", func(object client.Object) []string {
+		return []string{object.(*corev1.Pod).Spec.NodeName}
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Setup index field to allow access to the status phase of a Pod more efficiently. The indexer must be created before the
+	// informer is started.
+	err = internalCache.IndexField(ctx, &corev1.Pod{}, "status.phase", func(object client.Object) []string {
+		return []string{string(object.(*corev1.Pod).Status.Phase)}
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Make sure the internal cache is started.
+	go func() {
+		_ = internalCache.Start(ctx)
+	}()
+
+	internalCache.WaitForCacheSync(ctx)
+
+	return client.NewDelegatingClient(client.NewDelegatingClientInput{
+		CacheReader:       internalCache,
+		Client:            internalClient,
+		UncachedObjects:   nil,
+		CacheUnstructured: false,
+	})
 }
 
 func getNamespace(namespace string) (string, error) {
@@ -69,9 +131,9 @@ func getNamespace(namespace string) (string, error) {
 		return "", err
 	}
 
-	if context, ok := clientCfg.Contexts[clientCfg.CurrentContext]; ok {
-		if context.Namespace != "" {
-			return context.Namespace, nil
+	if ctx, ok := clientCfg.Contexts[clientCfg.CurrentContext]; ok {
+		if ctx.Namespace != "" {
+			return ctx.Namespace, nil
 		}
 	}
 
@@ -80,13 +142,13 @@ func getNamespace(namespace string) (string, error) {
 
 func getOperator(kubeClient client.Client, operatorName string, namespace string) (*appsv1.Deployment, error) {
 	operator := &appsv1.Deployment{}
-	err := kubeClient.Get(ctx.Background(), client.ObjectKey{Namespace: namespace, Name: operatorName}, operator)
+	err := kubeClient.Get(context.Background(), client.ObjectKey{Namespace: namespace, Name: operatorName}, operator)
 	return operator, err
 }
 
 func loadCluster(kubeClient client.Client, namespace string, clusterName string) (*fdbv1beta2.FoundationDBCluster, error) {
 	cluster := &fdbv1beta2.FoundationDBCluster{}
-	err := kubeClient.Get(ctx.Background(), types.NamespacedName{Namespace: namespace, Name: clusterName}, cluster)
+	err := kubeClient.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: clusterName}, cluster)
 	if err != nil {
 		return nil, err
 	}
@@ -94,12 +156,13 @@ func loadCluster(kubeClient client.Client, namespace string, clusterName string)
 	if err != nil {
 		return nil, err
 	}
+
 	return cluster, err
 }
 
 func getNodes(kubeClient client.Client, nodeSelector map[string]string) ([]string, error) {
 	var nodesList corev1.NodeList
-	err := kubeClient.List(ctx.Background(), &nodesList, client.MatchingLabels(nodeSelector))
+	err := kubeClient.List(context.Background(), &nodesList, client.MatchingLabels(nodeSelector))
 	if err != nil {
 		return []string{}, err
 	}
@@ -116,7 +179,7 @@ func getNodes(kubeClient client.Client, nodeSelector map[string]string) ([]strin
 func getPodsForCluster(kubeClient client.Client, cluster *fdbv1beta2.FoundationDBCluster) (*corev1.PodList, error) {
 	var podList corev1.PodList
 	err := kubeClient.List(
-		ctx.Background(),
+		context.Background(),
 		&podList,
 		client.MatchingLabels(cluster.GetMatchLabels()),
 		client.InNamespace(cluster.GetNamespace()))
diff --git a/kubectl-fdb/cmd/remove_process_group.go b/kubectl-fdb/cmd/remove_process_group.go
index fcade941..2e7919cc 100644
--- a/kubectl-fdb/cmd/remove_process_group.go
+++ b/kubectl-fdb/cmd/remove_process_group.go
@@ -60,7 +60,7 @@ func newRemoveProcessGroupCmd(streams genericclioptions.IOStreams) *cobra.Comman
 				return err
 			}
 
-			kubeClient, err := getKubeClient(o)
+			kubeClient, err := getKubeClient(cmd.Context(), o)
 			if err != nil {
 				return err
 			}
diff --git a/kubectl-fdb/cmd/restart.go b/kubectl-fdb/cmd/restart.go
index 82e0fa8a..457fd50c 100644
--- a/kubectl-fdb/cmd/restart.go
+++ b/kubectl-fdb/cmd/restart.go
@@ -74,7 +74,7 @@ func newRestartCmd(streams genericclioptions.IOStreams) *cobra.Command {
 				return err
 			}
 
-			kubeClient, err := getKubeClient(o)
+			kubeClient, err := getKubeClient(cmd.Context(), o)
 			if err != nil {
 				return err
 			}
diff --git a/kubectl-fdb/cmd/version.go b/kubectl-fdb/cmd/version.go
index 2f83c916..fd6df61f 100644
--- a/kubectl-fdb/cmd/version.go
+++ b/kubectl-fdb/cmd/version.go
@@ -55,7 +55,7 @@ func newVersionCmd(streams genericclioptions.IOStreams) *cobra.Command {
 			}
 
 			if !clientOnly {
-				kubeClient, err := getKubeClient(o)
+				kubeClient, err := getKubeClient(cmd.Context(), o)
 				if err != nil {
 					return err
 				}
diff --git a/setup/setup.go b/setup/setup.go
index 4963dc1d..be704f2b 100644
--- a/setup/setup.go
+++ b/setup/setup.go
@@ -76,7 +76,17 @@ type Options struct {
 	LogFileMinAge                      time.Duration
 	GetTimeout                         time.Duration
 	PostTimeout                        time.Duration
-	DeprecationOptions                 internal.DeprecationOptions
+	// LeaseDuration is the duration that non-leader candidates will
+	// wait to force acquire leadership. This is measured against time of
+	// last observed ack. Default is 15 seconds.
+	LeaseDuration time.Duration
+	// RenewDeadline is the duration that the acting controlplane will retry
+	// refreshing leadership before giving up. Default is 10 seconds.
+	RenewDeadline time.Duration
+	// RetryPeriod is the duration the LeaderElector clients should wait
+	// between tries of actions. Default is 2 seconds.
+	RetryPeriod        time.Duration
+	DeprecationOptions internal.DeprecationOptions
 }
 
 // BindFlags will parse the given flagset for the operator option flags
@@ -106,6 +116,9 @@ func (o *Options) BindFlags(fs *flag.FlagSet) {
 	fs.StringVar(&o.WatchNamespace, "watch-namespace", os.Getenv("WATCH_NAMESPACE"), "Defines which namespace the operator should watch.")
 	fs.DurationVar(&o.GetTimeout, "get-timeout", 5*time.Second, "http timeout for get requests to the FDB sidecar.")
 	fs.DurationVar(&o.PostTimeout, "post-timeout", 10*time.Second, "http timeout for post requests to the FDB sidecar.")
+	fs.DurationVar(&o.LeaseDuration, "leader-election-lease-duration", 15*time.Second, "the duration that non-leader candidates will wait to force acquire leadership.")
+	fs.DurationVar(&o.RenewDeadline, "leader-election-renew-deadline", 10*time.Second, "the duration that the acting controlplane will retry refreshing leadership before giving up.")
+	fs.DurationVar(&o.RetryPeriod, "leader-election-retry-period", 2*time.Second, "the duration the LeaderElector clients should wait between tries of action.")
 	fs.BoolVar(&o.EnableRestartIncompatibleProcesses, "enable-restart-incompatible-processes", true, "This flag enables/disables in the operator to restart incompatible fdbserver processes.")
 	fs.BoolVar(&o.ServerSideApply, "server-side-apply", false, "This flag enables server side apply.")
 	fs.BoolVar(&o.EnableRecoveryState, "enable-recovery-state", true, "This flag enables the use of the recovery state for the minimum uptime between bounced if the FDB version supports it.")
@@ -152,6 +165,9 @@ func StartManager(
 		MetricsBindAddress: operatorOpts.MetricsAddr,
 		LeaderElection:     operatorOpts.EnableLeaderElection,
 		LeaderElectionID:   operatorOpts.LeaderElectionID,
+		LeaseDuration:      &operatorOpts.LeaseDuration,
+		RenewDeadline:      &operatorOpts.RenewDeadline,
+		RetryPeriod:        &operatorOpts.RetryPeriod,
 		Port:               9443,
 	}
 


Test patch:
diff --git a/api/v1beta2/foundationdbcluster_types_test.go b/api/v1beta2/foundationdbcluster_types_test.go
index aa0cac2f..9f3820ba 100644
--- a/api/v1beta2/foundationdbcluster_types_test.go
+++ b/api/v1beta2/foundationdbcluster_types_test.go
@@ -5552,6 +5552,26 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 			false,
 			fmt.Errorf("process has missing address in exclusion results: 192.168.0.2"),
 		),
+		Entry("process group is excluded using locality and it is present in the remaining map as excluded",
+			&ProcessGroupStatus{
+				ProcessGroupID: "storage-1",
+			},
+			map[string]bool{
+				fmt.Sprintf("%s:storage-1", FDBLocalityExclusionPrefix): false,
+			},
+			true,
+			nil,
+		),
+		Entry("process group is excluded using locality and it is present in the remaining map as not excluded",
+			&ProcessGroupStatus{
+				ProcessGroupID: "storage-1",
+			},
+			map[string]bool{
+				fmt.Sprintf("%s:storage-1", FDBLocalityExclusionPrefix): true,
+			},
+			false,
+			fmt.Errorf("process has missing exclusion string in exclusion results: %s", fmt.Sprintf("%s:storage-1", FDBLocalityExclusionPrefix)),
+		),
 	)
 
 	DescribeTable("when getting the removal mode", func(cluster *FoundationDBCluster, expected PodUpdateMode) {
diff --git a/config/tests/three_data_hall/Readme.md b/config/tests/three_data_hall/Readme.md
index 8f3a14a3..86ea2fda 100644
--- a/config/tests/three_data_hall/Readme.md
+++ b/config/tests/three_data_hall/Readme.md
@@ -3,6 +3,7 @@
 This example requires that your Kubernetes cluster has nodes which are labeled with `topology.kubernetes.io/zone`.
 The example requires at least 3 unique zones, those can be faked for testing, by adding the labels to a node.
 If you want to use cloud provider specific zone label values you can set the `AZ1`, `AZ2` and `AZ3` environment variables.
+You can also set the namespace with the `NAMESPACE` environment variable to deploy this setup in a specific namespace.
 
 ## Create the Three-Data-Hall cluster
 
diff --git a/config/tests/three_data_hall/create.bash b/config/tests/three_data_hall/create.bash
index e35f54b5..0ab2597e 100755
--- a/config/tests/three_data_hall/create.bash
+++ b/config/tests/three_data_hall/create.bash
@@ -20,6 +20,7 @@ DIR="${BASH_SOURCE%/*}"
 AZ1=${AZ1:-"az1"}
 AZ2=${AZ2:-"az2"}
 AZ3=${AZ3:-"az3"}
+NAMESPACE=${NAMESPACE=-"default"}
 
 applyFile "${DIR}/stage_1.yaml" "${AZ1}" '""'
 checkReconciliationLoop test-cluster-${AZ1}
diff --git a/config/tests/three_data_hall/delete.bash b/config/tests/three_data_hall/delete.bash
index 387266c1..84697380 100755
--- a/config/tests/three_data_hall/delete.bash
+++ b/config/tests/three_data_hall/delete.bash
@@ -2,4 +2,4 @@
 
 set -eu
 
-kubectl delete fdb -l cluster-group=test-cluster
+kubectl -n "${NAMESPACE}" delete fdb -l cluster-group=test-cluster
diff --git a/config/tests/three_data_hall/functions.bash b/config/tests/three_data_hall/functions.bash
index a2fc3626..0bdd2fa1 100644
--- a/config/tests/three_data_hall/functions.bash
+++ b/config/tests/three_data_hall/functions.bash
@@ -1,13 +1,13 @@
 function applyFile() {
 	az=${2}
 
-	az="${az}" connectionString="${3}" envsubst < "${1}"| kubectl apply -f -
+	az="${az}" connectionString="${3}" envsubst < "${1}"| kubectl -n "${NAMESPACE}" apply -f -
 }
 
 function checkReconciliation() {
 	clusterName=$1
 
-	generationsOutput=$(kubectl get fdb "${clusterName}" -o jsonpath='{.metadata.generation} {.status.generations.reconciled}')
+	generationsOutput=$(kubectl -n "${NAMESPACE}" get fdb "${clusterName}" -o jsonpath='{.metadata.generation} {.status.generations.reconciled}')
 	read -ra generations <<< "${generationsOutput}"
 	if [[ ("${#generations[@]}" -ge 2) && ("${generations[0]}" == "${generations[1]}") ]]; then
 		return 1
@@ -18,7 +18,7 @@ function checkReconciliation() {
 }
 
 function getConnectionString() {
-	kubectl get fdb "${1}" -o jsonpath='{.status.connectionString}'
+	kubectl -n "${NAMESPACE}" get fdb "${1}" -o jsonpath='{.status.connectionString}'
 }
 
 function checkReconciliationLoop() {
diff --git a/config/tests/unified_image/unified_image_role.yaml b/config/tests/unified_image/unified_image_role.yaml
index d66f3309..78694a45 100644
--- a/config/tests/unified_image/unified_image_role.yaml
+++ b/config/tests/unified_image/unified_image_role.yaml
@@ -17,6 +17,7 @@ rules:
       - "watch"
       - "update"
       - "patch"
+      - "list"
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
diff --git a/e2e/fixtures/cluster_config.go b/e2e/fixtures/cluster_config.go
index 66d2603d..8b374ffe 100644
--- a/e2e/fixtures/cluster_config.go
+++ b/e2e/fixtures/cluster_config.go
@@ -138,7 +138,7 @@ func (config *ClusterConfig) SetDefaults(factory *Factory) {
 	}
 
 	if config.StorageServerPerPod == 0 {
-		config.StorageServerPerPod = 1
+		config.StorageServerPerPod = 2
 	}
 
 	if config.CreationCallback == nil {
diff --git a/e2e/fixtures/fdb_cluster.go b/e2e/fixtures/fdb_cluster.go
index a14c06bb..e9d927d0 100644
--- a/e2e/fixtures/fdb_cluster.go
+++ b/e2e/fixtures/fdb_cluster.go
@@ -748,8 +748,8 @@ func (fdbCluster *FdbCluster) SetPodAsUnschedulable(pod corev1.Pod) error {
 
 // SetProcessGroupsAsUnschedulable sets the provided process groups on the NoSchedule list of the current FoundationDBCluster. This will make
 // sure that the Pod is stuck in Pending.
-func (fdbCluster *FdbCluster) SetProcessGroupsAsUnschedulable(procesGroups []fdbv1beta2.ProcessGroupID) {
-	fdbCluster.cluster.Spec.Buggify.NoSchedule = procesGroups
+func (fdbCluster *FdbCluster) SetProcessGroupsAsUnschedulable(processGroups []fdbv1beta2.ProcessGroupID) {
+	fdbCluster.cluster.Spec.Buggify.NoSchedule = processGroups
 	fdbCluster.UpdateClusterSpec()
 }
 
diff --git a/e2e/fixtures/fdb_operator_client.go b/e2e/fixtures/fdb_operator_client.go
index 0d5512ea..76a1199d 100644
--- a/e2e/fixtures/fdb_operator_client.go
+++ b/e2e/fixtures/fdb_operator_client.go
@@ -40,8 +40,6 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer/yaml"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/wait"
 	yamlutil "k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -50,9 +48,8 @@ import (
 const (
 	operatorDeploymentName     = "fdb-kubernetes-operator-controller-manager"
 	foundationdbServiceAccount = "fdb-kubernetes"
-	// operatorDeployment is a string that contains all the deployment settings used to deploy the operator.
-	// Embedding it as a string make the test suite portable.
-	operatorDeployment = `apiVersion: v1
+	// The configuration for the RBAC setup for the operator deployment
+	operatorRBAC = `apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: fdb-kubernetes-operator-controller-manager
@@ -193,8 +190,10 @@ rules:
   subjects:
   - kind: ServiceAccount
     name: fdb-kubernetes-operator-controller-manager
-    namespace: {{ .Namespace }}
----
+    namespace: {{ .Namespace }}`
+	// operatorDeployment is a string that contains all the deployment settings used to deploy the operator.
+	// Embedding it as a string make the test suite portable.
+	operatorDeployment = `
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -334,6 +333,141 @@ spec:
           secretName: {{ .SecretName }}
       - name: fdb-binaries
         emptyDir: {}`
+	// operatorDeploymentUnifiedImage ...
+	operatorDeploymentUnifiedImage = `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: fdb-kubernetes-operator-controller-manager
+    control-plane: controller-manager
+  name: fdb-kubernetes-operator-controller-manager
+  namespace: {{ .Namespace }}
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: fdb-kubernetes-operator-controller-manager
+  template:
+    metadata:
+      labels:
+        app: fdb-kubernetes-operator-controller-manager
+        control-plane: controller-manager
+      annotations:
+        prometheus.io/scrape: 'true'
+        prometheus.io/port: '8080'
+    spec:
+      initContainers:
+        {{ range $index, $version := .SidecarVersions }}
+        - name: foundationdb-kubernetes-init-{{ $index }}
+          image: {{ .BaseImage }}:{{ .SidecarTag}}
+          imagePullPolicy: {{ .ImagePullPolicy }}
+          args:
+            - --mode
+            - init
+            - --output-dir
+            - /var/output-files
+            - --copy-binary
+            - fdbcli
+            - --copy-binary
+            - fdbbackup
+            - --copy-binary
+            - fdbrestore
+            - --copy-library
+            - "{{ .FDBVersion.Compact }}"
+{{ if eq .FDBVersion.Compact "7.1" }}
+            - --copy-primary-library
+            - "{{ .FDBVersion.Compact }}"
+{{ end }}
+          volumeMounts:
+            - name: fdb-binaries
+              mountPath: /var/output-files
+          securityContext:
+            runAsUser: 0
+            runAsGroup: 0
+        {{ end }}
+      containers:
+      - command:
+        - /manager
+        args:
+        - --max-concurrent-reconciles=5
+        - --zap-log-level=debug
+        #- --server-side-apply
+        image: {{ .OperatorImage }}
+        name: manager
+        imagePullPolicy: Always
+        env:
+          - name: LD_LIBRARY_PATH
+            value: /usr/bin/fdb
+          - name: WATCH_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: FDB_TLS_CERTIFICATE_FILE
+            value: /tmp/fdb-certs/tls.crt
+          - name: FDB_TLS_CA_FILE
+            value: /tmp/fdb-certs/ca.pem
+          - name: FDB_TLS_KEY_FILE
+            value: /tmp/fdb-certs/tls.key
+          - name: FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY
+            value: /usr/bin/fdb
+          # FDB 7.3 adds a check for loading external client library, which doesn't work with 6.3.
+          # Consider remove this option once 6.3 is no longer being used.
+          - name: FDB_NETWORK_OPTION_IGNORE_EXTERNAL_CLIENT_FAILURES
+            value: ""
+          - name: FDB_BLOB_CREDENTIALS
+            value: /tmp/backup-credentials/credentials
+          # TODO (johscheuer): once we can generate certificates per Pod remove this!
+          - name: DISABLE_SIDECAR_TLS_CHECK
+            value: "1"
+          - name: FDB_NETWORK_OPTION_TRACE_ENABLE
+            value: "/var/log/fdb"
+          - name: FDB_NETWORK_OPTION_TRACE_FORMAT
+            value: json
+        ports:
+          - name: metrics
+            containerPort: 8080
+        resources:
+         requests:
+           cpu: {{ .CPURequests }}
+           memory: {{ .MemoryRequests }}
+        securityContext:
+          allowPrivilegeEscalation: false
+          privileged: false
+          readOnlyRootFilesystem: false
+        volumeMounts:
+        - mountPath: /tmp
+          name: tmp
+        - mountPath: /var/log/fdb
+          name: fdb-trace-logs
+        - name: fdb-certs
+          mountPath: /tmp/fdb-certs
+          readOnly: true
+        - name: fdb-binaries
+          mountPath: /usr/bin/fdb
+        - name: backup-credentials
+          mountPath: /tmp/backup-credentials
+          readOnly: true
+      securityContext:
+        fsGroup: 4059
+        runAsGroup: 4059
+        runAsUser: 4059
+      serviceAccountName: fdb-kubernetes-operator-controller-manager
+      terminationGracePeriodSeconds: 10
+      volumes:
+      - emptyDir: {}
+        name: tmp
+      - emptyDir: {}
+        name: fdb-trace-logs
+      - name: backup-credentials
+        secret:
+          secretName: {{ .BackupSecretName }}
+          optional: true
+      - name: fdb-certs
+        secret:
+          secretName: {{ .SecretName }}
+      - name: fdb-binaries
+        emptyDir: {}`
 )
 
 // operatorConfig represents the configuration of the operator Deployment.
@@ -375,12 +509,18 @@ func (factory *Factory) GetSidecarConfigs() []SidecarConfig {
 	additionalSidecarVersions := factory.GetAdditionalSidecarVersions()
 	sidecarConfigs := make([]SidecarConfig, 0, len(additionalSidecarVersions)+1)
 
+	image := factory.GetSidecarImage()
+	if factory.options.featureOperatorUnifiedImage {
+		image = factory.GetFoundationDBImage()
+	}
+
 	sidecarConfigs = append(
 		sidecarConfigs,
 		getDefaultSidecarConfig(
-			factory.GetSidecarImage(),
+			image,
 			factory.GetFDBVersion(),
 			factory.getImagePullPolicy(),
+			factory.options.featureOperatorUnifiedImage,
 		),
 	)
 	baseImage := sidecarConfigs[0].BaseImage
@@ -394,14 +534,14 @@ func (factory *Factory) GetSidecarConfigs() []SidecarConfig {
 
 		sidecarConfigs = append(
 			sidecarConfigs,
-			getSidecarConfig(baseImage, "", version, factory.getImagePullPolicy()),
+			getSidecarConfig(baseImage, "", version, factory.getImagePullPolicy(), factory.options.featureOperatorUnifiedImage),
 		)
 	}
 
 	return sidecarConfigs
 }
 
-func getDefaultSidecarConfig(sidecarImage string, version fdbv1beta2.Version, imagePullPolicy corev1.PullPolicy) SidecarConfig {
+func getDefaultSidecarConfig(sidecarImage string, version fdbv1beta2.Version, imagePullPolicy corev1.PullPolicy, useUnifiedImage bool) SidecarConfig {
 	defaultSidecarImage := strings.SplitN(sidecarImage, ":", 2)
 
 	var tag string
@@ -409,11 +549,11 @@ func getDefaultSidecarConfig(sidecarImage string, version fdbv1beta2.Version, im
 		tag = defaultSidecarImage[1]
 	}
 
-	return getSidecarConfig(defaultSidecarImage[0], tag, version, imagePullPolicy)
+	return getSidecarConfig(defaultSidecarImage[0], tag, version, imagePullPolicy, useUnifiedImage)
 }
 
-func getSidecarConfig(baseImage string, tag string, version fdbv1beta2.Version, imagePullPolicy corev1.PullPolicy) SidecarConfig {
-	if tag == "" {
+func getSidecarConfig(baseImage string, tag string, version fdbv1beta2.Version, imagePullPolicy corev1.PullPolicy, useUnifiedImage bool) SidecarConfig {
+	if tag == "" && !useUnifiedImage {
 		tag = fmt.Sprintf("%s-1", version)
 	}
 
@@ -454,10 +594,10 @@ func (factory *Factory) ensureFDBOperatorExists(namespace string) error {
 
 // CreateFDBOperatorIfAbsent creates the operator Deployment based on the template.
 func (factory *Factory) CreateFDBOperatorIfAbsent(namespace string) error {
-	t, err := template.New("operatorDeployment").Parse(operatorDeployment)
+	operatorRBACTemplate, err := template.New("operatorRBAC").Parse(operatorRBAC)
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 	buf := bytes.Buffer{}
-	gomega.Expect(t.Execute(&buf, factory.getOperatorConfig(namespace))).NotTo(gomega.HaveOccurred())
+	gomega.Expect(operatorRBACTemplate.Execute(&buf, factory.getOperatorConfig(namespace))).NotTo(gomega.HaveOccurred())
 	decoder := yamlutil.NewYAMLOrJSONDecoder(&buf, 100000)
 
 	for {
@@ -482,6 +622,39 @@ func (factory *Factory) CreateFDBOperatorIfAbsent(namespace string) error {
 		).NotTo(gomega.HaveOccurred())
 	}
 
+	deploymentTemplate := operatorDeployment
+	if factory.options.featureOperatorUnifiedImage {
+		deploymentTemplate = operatorDeploymentUnifiedImage
+	}
+
+	operatorDeploymentTemplate, err := template.New("operatorDeployment").Parse(deploymentTemplate)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	buf = bytes.Buffer{}
+	gomega.Expect(operatorDeploymentTemplate.Execute(&buf, factory.getOperatorConfig(namespace))).NotTo(gomega.HaveOccurred())
+	decoder = yamlutil.NewYAMLOrJSONDecoder(&buf, 100000)
+
+	for {
+		var rawObj runtime.RawExtension
+		err := decoder.Decode(&rawObj)
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return err
+		}
+
+		obj, _, err := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme).
+			Decode(rawObj.Raw, nil, nil)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		unstructuredObj := &unstructured.Unstructured{Object: unstructuredMap}
+
+		gomega.Expect(
+			factory.CreateIfAbsent(unstructuredObj),
+		).NotTo(gomega.HaveOccurred())
+	}
+
 	// Make sure the Operator Pods are running before moving forward.
 	factory.WaitUntilOperatorPodsRunning(namespace)
 	return nil
@@ -507,80 +680,38 @@ func (factory *Factory) WaitUntilOperatorPodsRunning(namespace string) {
 	).NotTo(gomega.HaveOccurred())
 
 	expectedReplicas := int(pointer.Int32Deref(deployment.Spec.Replicas, 1))
-
-	gomega.Eventually(func() (done bool) {
+	gomega.Eventually(func(g gomega.Gomega) int {
 		pods := factory.GetOperatorPods(namespace)
-		if len(pods.Items) != expectedReplicas {
-			return false
-		}
-
-		allRunning := true
+		var runningReplicas int
 		for _, pod := range pods.Items {
-			if pod.Status.Phase == corev1.PodRunning {
+			if pod.Status.Phase == corev1.PodRunning && pod.DeletionTimestamp.IsZero() {
+				runningReplicas++
 				continue
 			}
 
-			allRunning = false
-			// If the Pod is not running after 60 seconds we delete it and let the Deployment controller create a new Pod.
-			if time.Since(pod.CreationTimestamp.Time).Seconds() > 60.0 {
-				log.Println("operator Pod", pod.Name, "not running after 60 seconds, going to delete this Pod, status:", pod.Status)
+			// If the Pod is not running after 120 seconds we delete it and let the Deployment controller create a new Pod.
+			if time.Since(pod.CreationTimestamp.Time).Seconds() > 120.0 {
+				log.Println("operator Pod", pod.Name, "not running after 120 seconds, going to delete this Pod, status:", pod.Status)
 				err := factory.GetControllerRuntimeClient().Delete(ctx.TODO(), &pod)
 				if k8serrors.IsNotFound(err) {
 					continue
 				}
 
-				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+				g.Expect(err).NotTo(gomega.HaveOccurred())
 			}
 		}
 
-		return allRunning
-	}).WithTimeout(5 * time.Minute).WithPolling(2 * time.Second).Should(gomega.BeTrue())
+		return runningReplicas
+	}).WithTimeout(10 * time.Minute).WithPolling(2 * time.Second).Should(gomega.BeNumerically(">=", expectedReplicas))
 }
 
 // RecreateOperatorPods will recreate all operator Pods in the specified namespace and wait until the new Pods are
 // up and running.
 func (factory *Factory) RecreateOperatorPods(namespace string) {
-	initialList := factory.GetOperatorPods(namespace)
-
-	oldOperatorIds := map[types.UID]fdbv1beta2.None{}
-	for _, pod := range initialList.Items {
-		oldOperatorIds[pod.UID] = fdbv1beta2.None{}
-	}
-
 	gomega.Expect(
 		factory.GetControllerRuntimeClient().
 			DeleteAllOf(ctx.TODO(), &corev1.Pod{}, client.InNamespace(namespace), client.MatchingLabels(map[string]string{"app": operatorDeploymentName})),
 	).NotTo(gomega.HaveOccurred())
 
-	deployment := &appsv1.Deployment{}
-	gomega.Expect(
-		factory.GetControllerRuntimeClient().
-			Get(ctx.TODO(), client.ObjectKey{Name: operatorDeploymentName, Namespace: namespace}, deployment),
-	).NotTo(gomega.HaveOccurred())
-
-	expectedReplicas := int(pointer.Int32Deref(deployment.Spec.Replicas, 1))
-
-	gomega.Expect(wait.PollImmediate(1*time.Second, 10*time.Minute, func() (done bool, err error) {
-		podList := factory.GetOperatorPods(namespace)
-
-		if len(podList.Items) == 0 {
-			return false, nil
-		}
-
-		// Wait until the Pods are recreated
-		var runningPods int
-		for _, pod := range podList.Items {
-			if _, ok := oldOperatorIds[pod.UID]; ok {
-				continue
-			}
-
-			if pod.Status.Phase != corev1.PodRunning {
-				continue
-			}
-
-			runningPods++
-		}
-
-		return runningPods == expectedReplicas, nil
-	})).NotTo(gomega.HaveOccurred())
+	factory.WaitUntilOperatorPodsRunning(namespace)
 }
diff --git a/e2e/fixtures/kubernetes_fixtures.go b/e2e/fixtures/kubernetes_fixtures.go
index 12c9bd73..79878595 100644
--- a/e2e/fixtures/kubernetes_fixtures.go
+++ b/e2e/fixtures/kubernetes_fixtures.go
@@ -152,6 +152,7 @@ func (factory *Factory) ensureRBACSetupExists(namespace string) {
 					"watch",
 					"update",
 					"patch",
+					"list",
 				},
 			},
 		},
diff --git a/e2e/test_operator_maintenance_mode/operator_maintenance_mode_test.go b/e2e/test_operator_maintenance_mode/operator_maintenance_mode_test.go
new file mode 100644
index 00000000..7aa9268c
--- /dev/null
+++ b/e2e/test_operator_maintenance_mode/operator_maintenance_mode_test.go
@@ -0,0 +1,317 @@
+/*
+ * operator_maintenance_mode_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2024 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package operatorha
+
+/*
+This test suite includes tests around the interaction of the maintenance mode and the operator.
+*/
+
+import (
+	"fmt"
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	corev1 "k8s.io/api/core/v1"
+	"log"
+	"time"
+
+	"github.com/FoundationDB/fdb-kubernetes-operator/e2e/fixtures"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	factory     *fixtures.Factory
+	fdbCluster  *fixtures.FdbCluster
+	testOptions *fixtures.FactoryOptions
+)
+
+func init() {
+	testOptions = fixtures.InitFlags()
+}
+
+var _ = BeforeSuite(func() {
+	factory = fixtures.CreateFactory(testOptions)
+	fdbCluster = factory.CreateFdbCluster(
+		fixtures.DefaultClusterConfig(false),
+		factory.GetClusterOptions()...,
+	)
+
+	// Make sure the unschedulable Pod is not removed
+	Expect(fdbCluster.SetAutoReplacements(false, 12*time.Hour)).NotTo(HaveOccurred())
+
+	// Load some data into the cluster.
+	factory.CreateDataLoaderIfAbsent(fdbCluster)
+})
+
+var _ = AfterSuite(func() {
+	if CurrentSpecReport().Failed() {
+		log.Printf("failed due to %s", CurrentSpecReport().FailureMessage())
+	}
+	factory.Shutdown()
+})
+
+var _ = Describe("Operator maintenance mode tests", Label("e2e"), func() {
+	AfterEach(func() {
+		Expect(fdbCluster.ClearBuggifyNoSchedule(false)).NotTo(HaveOccurred())
+		if CurrentSpecReport().Failed() {
+			factory.DumpState(fdbCluster)
+		}
+		Expect(fdbCluster.WaitForReconciliation()).ToNot(HaveOccurred())
+		factory.StopInvariantCheck()
+		// Make sure all data is present in the cluster
+		fdbCluster.EnsureTeamTrackersAreHealthy()
+		fdbCluster.EnsureTeamTrackersHaveMinReplicas()
+	})
+
+	When("the maintenance mode is set", func() {
+		var failingStoragePod corev1.Pod
+		var faultDomain fdbv1beta2.FaultDomain
+
+		BeforeEach(func() {
+			failingStoragePod = fixtures.RandomPickOnePod(fdbCluster.GetStoragePods().Items)
+
+			// Set maintenance mode for this Pod
+			for _, processGroup := range fdbCluster.GetCluster().Status.ProcessGroups {
+				if processGroup.ProcessClass != fdbv1beta2.ProcessClassStorage {
+					continue
+				}
+
+				if processGroup.ProcessGroupID == fixtures.GetProcessGroupID(failingStoragePod) {
+					faultDomain = processGroup.FaultDomain
+				}
+			}
+
+			// Set the maintenance mode for 4 minutes.
+			fdbCluster.RunFdbCliCommandInOperator(fmt.Sprintf("maintenance on %s 240", faultDomain), false, 60)
+
+			// Set this Pod as unschedulable to keep it pending.
+			Expect(fdbCluster.SetPodAsUnschedulable(failingStoragePod)).NotTo(HaveOccurred())
+		})
+
+		AfterEach(func() {
+			// Make sure that the quota is deleted and new PVCs can be created.
+			Expect(fdbCluster.ClearBuggifyNoSchedule(true)).NotTo(HaveOccurred())
+			// Reset the maintenance mode
+			fdbCluster.RunFdbCliCommandInOperator("maintenance off", false, 60)
+		})
+
+		When("the Pod comes back before the maintenance mode times out", func() {
+			It("should not set the team tracker status to unhealthy", func() {
+				// Make sure the team tracker status shows healthy for the failed Pod and the maintenance zone is set.
+				Consistently(func(g Gomega) fdbv1beta2.FaultDomain {
+					status := fdbCluster.GetStatus()
+
+					for _, tracker := range status.Cluster.Data.TeamTrackers {
+						log.Println(tracker.State.Name, ":", tracker.State.Healthy)
+						g.Expect(tracker.State.Healthy).To(BeTrue())
+					}
+
+					log.Println("Maintenance Zone:", status.Cluster.MaintenanceZone)
+					return status.Cluster.MaintenanceZone
+				}).WithTimeout(2 * time.Minute).WithPolling(2 * time.Second).Should(Equal(faultDomain))
+			})
+		})
+
+		When("the maintenance mode times out", func() {
+			It("should update the team tracker status to unhealthy", func() {
+				// Make sure the team tracker status shows healthy for the failed Pod and the maintenance zone is set.
+				Consistently(func(g Gomega) fdbv1beta2.FaultDomain {
+					status := fdbCluster.GetStatus()
+
+					for _, tracker := range status.Cluster.Data.TeamTrackers {
+						g.Expect(tracker.State.Healthy).To(BeTrue())
+					}
+
+					return status.Cluster.MaintenanceZone
+				}).WithTimeout(2 * time.Minute).WithPolling(2 * time.Second).Should(Equal(faultDomain))
+
+				log.Println("Wait until maintenance mode times out")
+				// Wait until the maintenance zone is reset
+				Eventually(func(g Gomega) fdbv1beta2.FaultDomain {
+					return fdbCluster.GetStatus().Cluster.MaintenanceZone
+				}).WithTimeout(10 * time.Minute).WithPolling(2 * time.Second).Should(Equal(fdbv1beta2.FaultDomain("")))
+
+				startTime := time.Now()
+				log.Println("Wait until failure is detected")
+				// We would expect that the team tracker gets unhealthy once the maintenance mode is timed out.
+				Eventually(func(g Gomega) fdbv1beta2.FaultDomain {
+					status := fdbCluster.GetStatus()
+					for _, tracker := range status.Cluster.Data.TeamTrackers {
+						g.Expect(tracker.State.Healthy).To(BeFalse())
+					}
+
+					return status.Cluster.MaintenanceZone
+				}).WithTimeout(10 * time.Minute).WithPolling(2 * time.Second).Should(Equal(fdbv1beta2.FaultDomain("")))
+
+				log.Println("It took:", time.Since(startTime).String(), "to detected the failure")
+			})
+		})
+
+		When("there is a failure during maintenance mode", func() {
+			BeforeEach(func() {
+				// Set the maintenance mode for a long duration, e.g. 2h hours to make sure the mode is not timing out
+				// but actually is being reset.
+				fdbCluster.RunFdbCliCommandInOperator(fmt.Sprintf("maintenance on %s 7200", faultDomain), false, 60)
+			})
+
+			AfterEach(func() {
+				fdbCluster.SetCrashLoopContainers(nil, false)
+			})
+
+			It("should remove the maintenance mode", func() {
+				// Make sure the team tracker status shows healthy for the failed Pod and the maintenance zone is set.
+				Consistently(func(g Gomega) fdbv1beta2.FaultDomain {
+					status := fdbCluster.GetStatus()
+
+					for _, tracker := range status.Cluster.Data.TeamTrackers {
+						g.Expect(tracker.State.Healthy).To(BeTrue())
+					}
+
+					return status.Cluster.MaintenanceZone
+				}).WithTimeout(2 * time.Minute).WithPolling(2 * time.Second).Should(Equal(faultDomain))
+
+				log.Println("When another storage Pod is failing")
+				var podToRecreate corev1.Pod
+
+				for _, pod := range fdbCluster.GetStoragePods().Items {
+					if pod.Name == failingStoragePod.Name {
+						continue
+					}
+
+					podToRecreate = pod
+					break
+				}
+
+				// We delete a Pod and make it crash-looping, to make sure the process is failed for more than 60 seconds.
+				log.Println("Delete Pod", podToRecreate.Name)
+				fdbCluster.SetCrashLoopContainers([]fdbv1beta2.CrashLoopContainerObject{
+					{
+						ContainerName: fdbv1beta2.MainContainerName,
+						Targets:       []fdbv1beta2.ProcessGroupID{fixtures.GetProcessGroupID(podToRecreate)},
+					},
+				}, false)
+				factory.DeletePod(&podToRecreate)
+
+				log.Println("Wait until maintenance mode is reset")
+				startTime := time.Now()
+				// We would expect that the team tracker gets unhealthy once the maintenance mode is timed out.
+				Eventually(func(g Gomega) fdbv1beta2.FaultDomain {
+					status := fdbCluster.GetStatus()
+					for _, tracker := range status.Cluster.Data.TeamTrackers {
+						g.Expect(tracker.State.Healthy).To(BeFalse())
+					}
+
+					return status.Cluster.MaintenanceZone
+				}).WithTimeout(10 * time.Minute).WithPolling(2 * time.Second).Should(Equal(fdbv1beta2.FaultDomain("")))
+
+				log.Println("It took:", time.Since(startTime).String(), "to detected the failure")
+			})
+		})
+
+		When("there is additional load on the cluster", func() {
+			BeforeEach(func() {
+				// Set the maintenance mode for a long duration, e.g. 2h hours to make sure the mode is not timing out
+				// but actually is being reset.
+				fdbCluster.RunFdbCliCommandInOperator(fmt.Sprintf("maintenance on %s 7200", faultDomain), false, 60)
+			})
+
+			AfterEach(func() {
+				Consistently(func(g Gomega) fdbv1beta2.FaultDomain {
+					status := fdbCluster.GetStatus()
+
+					for _, tracker := range status.Cluster.Data.TeamTrackers {
+						g.Expect(tracker.State.Healthy).To(BeTrue())
+					}
+
+					return status.Cluster.MaintenanceZone
+				}).WithTimeout(2 * time.Minute).WithPolling(2 * time.Second).Should(Equal(faultDomain))
+			})
+
+			It("should not remove the maintenance mode", func() {
+				// Make sure the team tracker status shows healthy for the failed Pod and the maintenance zone is set.
+				Consistently(func(g Gomega) fdbv1beta2.FaultDomain {
+					status := fdbCluster.GetStatus()
+
+					for _, tracker := range status.Cluster.Data.TeamTrackers {
+						g.Expect(tracker.State.Healthy).To(BeTrue())
+					}
+
+					return status.Cluster.MaintenanceZone
+				}).WithTimeout(2 * time.Minute).WithPolling(2 * time.Second).Should(Equal(faultDomain))
+
+				log.Println("When loading additional data into the cluster")
+				factory.CreateDataLoaderIfAbsent(fdbCluster)
+				factory.CreateDataLoaderIfAbsent(fdbCluster)
+			})
+		})
+
+		When("the number of storage Pods is changed", func() {
+			var initialStoragePods int
+
+			BeforeEach(func() {
+				counts, err := fdbCluster.GetCluster().GetProcessCountsWithDefaults()
+				Expect(err).NotTo(HaveOccurred())
+
+				initialStoragePods = counts.Storage
+				// Set the maintenance mode for a long duration, e.g. 2h hours to make sure the mode is not timing out
+				// but actually is being reset.
+				fdbCluster.RunFdbCliCommandInOperator(fmt.Sprintf("maintenance on %s 7200", faultDomain), false, 60)
+			})
+
+			AfterEach(func() {
+				Expect(fdbCluster.ClearBuggifyNoSchedule(false)).NotTo(HaveOccurred())
+				spec := fdbCluster.GetCluster().Spec.DeepCopy()
+				// Add 3 additional storage Pods.
+				spec.ProcessCounts.Storage = initialStoragePods
+				fdbCluster.UpdateClusterSpecWithSpec(spec)
+			})
+
+			It("should not remove the maintenance mode", func() {
+				// Make sure the team tracker status shows healthy for the failed Pod and the maintenance zone is set.
+				Consistently(func(g Gomega) fdbv1beta2.FaultDomain {
+					status := fdbCluster.GetStatus()
+
+					for _, tracker := range status.Cluster.Data.TeamTrackers {
+						g.Expect(tracker.State.Healthy).To(BeTrue())
+					}
+
+					return status.Cluster.MaintenanceZone
+				}).WithTimeout(2 * time.Minute).WithPolling(2 * time.Second).Should(Equal(faultDomain))
+
+				log.Println("When adding additional storage processes to the cluster")
+				spec := fdbCluster.GetCluster().Spec.DeepCopy()
+				// Add 3 additional storage Pods.
+				spec.ProcessCounts.Storage = initialStoragePods + 3
+				fdbCluster.UpdateClusterSpecWithSpec(spec)
+
+				// Make sure the maintenance mode is kept and the team tracker shows healthy.
+				Consistently(func(g Gomega) fdbv1beta2.FaultDomain {
+					status := fdbCluster.GetStatus()
+
+					for _, tracker := range status.Cluster.Data.TeamTrackers {
+						g.Expect(tracker.State.Healthy).To(BeTrue())
+					}
+
+					return status.Cluster.MaintenanceZone
+				}).WithTimeout(5 * time.Minute).WithPolling(2 * time.Second).Should(Equal(faultDomain))
+			})
+		})
+	})
+})
diff --git a/e2e/test_operator_maintenance_mode/suite_test.go b/e2e/test_operator_maintenance_mode/suite_test.go
new file mode 100644
index 00000000..0298f6f6
--- /dev/null
+++ b/e2e/test_operator_maintenance_mode/suite_test.go
@@ -0,0 +1,34 @@
+/*
+ * suite_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2024 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package operatorha
+
+import (
+	"testing"
+	"time"
+
+	"github.com/FoundationDB/fdb-kubernetes-operator/e2e/fixtures"
+	"github.com/onsi/gomega"
+)
+
+func TestOperatorHA(t *testing.T) {
+	gomega.SetDefaultEventuallyTimeout(10 * time.Second)
+	fixtures.RunGinkgoTests(t, "Operator maintenance mode test suite")
+}
diff --git a/e2e/test_operator_upgrades/operator_upgrades_test.go b/e2e/test_operator_upgrades/operator_upgrades_test.go
index 437260a3..ffbe9014 100644
--- a/e2e/test_operator_upgrades/operator_upgrades_test.go
+++ b/e2e/test_operator_upgrades/operator_upgrades_test.go
@@ -244,9 +244,9 @@ var _ = Describe("Operator Upgrades", Label("e2e", "pr"), func() {
 
 			Expect(fdbCluster.SetAutoReplacements(false, 20*time.Minute)).ToNot(HaveOccurred())
 
-			// 1. Introduce crashloop into sidecar container to artificially create a partition.
+			// 1. Introduce crash-loop into sidecar container to artificially create a partition.
 			pickedPod := fixtures.ChooseRandomPod(fdbCluster.GetStoragePods())
-			log.Println("Injecting container fault to crash-loop sidecar process: ", pickedPod.Name)
+			log.Println("Injecting container fault to crash-loop sidecar process:", pickedPod.Name)
 
 			fdbCluster.SetCrashLoopContainers([]fdbv1beta2.CrashLoopContainerObject{
 				{
@@ -254,20 +254,32 @@ var _ = Describe("Operator Upgrades", Label("e2e", "pr"), func() {
 					Targets:       []fdbv1beta2.ProcessGroupID{fdbv1beta2.ProcessGroupID(pickedPod.Labels[fdbv1beta2.FDBProcessGroupIDLabel])},
 				},
 			}, false)
-			log.Println("Crash injected in pod: ", pickedPod.Name)
+			log.Println("Crash injected in pod:", pickedPod.Name)
 
-			// Wait until sidecar is crash-looping.
-			Eventually(func() bool {
+			// Wait until the Pod is running again and the sidecar is crash-looping.
+			Eventually(func(g Gomega) corev1.PodPhase {
 				pod := fdbCluster.GetPod(pickedPod.Name)
 				for _, container := range pod.Spec.Containers {
 					if container.Name == fdbv1beta2.SidecarContainerName {
-						log.Printf("Container: %s, Args: %s", container.Name, container.Args)
-						return container.Args[0] == "crash-loop"
+						log.Println("Container:", container.Name, "Args:", container.Args, "Phase:", pod.Status.Phase)
+						g.Expect(container.Args[0]).To(Equal("crash-loop"))
 					}
 				}
 
-				return false
-			}).WithPolling(2 * time.Second).WithTimeout(2 * time.Minute).MustPassRepeatedly(10).Should(BeTrue())
+				return pod.Status.Phase
+			}).WithPolling(2 * time.Second).WithTimeout(2 * time.Minute).MustPassRepeatedly(10).Should(Equal(corev1.PodRunning))
+
+			// Make sure we trigger a new reconciliation to make sure the process is up and running and only the sidecar
+			// is crash looping.
+			fdbCluster.ForceReconcile()
+
+			Eventually(func(g Gomega) bool {
+				for _, processGroup := range fdbCluster.GetCluster().Status.ProcessGroups {
+					g.Expect(processGroup.GetConditionTime(fdbv1beta2.MissingProcesses)).To(BeNil())
+				}
+
+				return true
+			}).WithPolling(2 * time.Second).WithTimeout(4 * time.Minute).MustPassRepeatedly(10).Should(BeTrue())
 
 			// 2. Start cluster upgrade.
 			log.Printf("Crash injected in sidecar container %s. Starting upgrade.", pickedPod.Name)
diff --git a/fdbclient/command_runner_test.go b/fdbclient/command_runner_test.go
index 662418b4..dd6c9f86 100644
--- a/fdbclient/command_runner_test.go
+++ b/fdbclient/command_runner_test.go
@@ -54,6 +54,8 @@ var _ = Describe("command_runner", func() {
 		BeforeEach(func() {
 			GinkgoT().Setenv("FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY", "")
 			GinkgoT().Setenv("FDB_NETWORK_OPTION_IGNORE_EXTERNAL_CLIENT_FAILURES", "")
+			GinkgoT().Setenv("FDB_NETWORK_OPTION_CLIENT_THREADS_PER_VERSION", "")
+
 			GinkgoT().Setenv("FDB_TLS_CERTIFICATE_FILE", "")
 
 			for _, env := range getEnvironmentVariablesWithoutExcludedFdbEnv() {
@@ -64,6 +66,7 @@ var _ = Describe("command_runner", func() {
 		It("should exclude the listed FDB variables but include all others", func() {
 			Expect(envVariablesKeys).NotTo(ContainElement("FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY"))
 			Expect(envVariablesKeys).NotTo(ContainElement("FDB_NETWORK_OPTION_IGNORE_EXTERNAL_CLIENT_FAILURES"))
+			Expect(envVariablesKeys).NotTo(ContainElement("FDB_NETWORK_OPTION_CLIENT_THREADS_PER_VERSION"))
 			Expect(envVariablesKeys).To(ContainElement("FDB_TLS_CERTIFICATE_FILE"))
 		})
 	})