Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1362 (2022-09-26 12:50:59)

Add test process count
# Description

This PR adds "test" processCount.

## Type of change

- Bug fix (non-breaking change which fixes an issue)

## Discussion

This changes are based on discussion under 
https://github.com/FoundationDB/fdb-kubernetes-operator/issues/1346

## Testing

Unit test

*Do we need to perform additional testing once this is merged, or perform in a larger testing environment?*

## Documentation

Yes, added description in fdb cluster page.

## Follow-up

Add deprecation for tester
Does not have anything


Patch:
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index d709e1da..b394fd0f 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -18,7 +18,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.18.6
+          go-version: 1.19.2
       - uses: actions/cache@v2
         with:
           path: ~/go/pkg/mod
@@ -44,7 +44,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.18.6
+        go-version: 1.19.2
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
       with:
@@ -104,7 +104,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.18.6
+        go-version: 1.19.2
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
       with:
@@ -126,10 +126,7 @@ jobs:
         # Install dependencies either from cache or freshly downloaded
         sudo dpkg -i fdb.deb
     - name: Run tests
-      # Currently the default runner has 2 vCPU:
-      # https://docs.github.com/en/free-pro-team@latest/actions/reference/specifications-for-github-hosted-runners#supported-runners-and-hardware-resources
       env:
-        GOMAXPROCS: "2"
         TEST_RACE_CONDITIONS: "1"
       run: |
         make test
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 0c04c389..b1825d59 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -43,7 +43,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.18.6
+          go-version: 1.19.2
       #  https://github.com/goreleaser/goreleaser/issues/1311
       - name: Get current semver tag
         run: echo "::set-output name=CURRENT_TAG::$(git describe --tags --match "v*" --abbrev=0)"
diff --git a/.golangci.yml b/.golangci.yml
index b2b452d7..854ab805 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -2,43 +2,63 @@ issues:
   # don't skip warning about doc comments
   # don't exclude the default set of lint
   exclude-use-default: false
+  exclude:
+    - "ST1000: at least one file in a package should have a package comment"
   exclude-rules:
   # exclude deprecation warning otherwise we get an error for our own deprecations
-  - linters: [staticcheck]
+  - linters:
+    - staticcheck
     text: "SA1019:"
-  - linters: [ govet ]
+  - linters:
+    - govet
     text: 'declaration of "(err|ctx)" shadows declaration at'
+  - linters:
+      - revive
+    text: "package-comments: should have a package comment"
+  # TODO (johscheuer): Ignore those linting issues for now and change it once we cut a new major release.
+  - linters:
+      - stylecheck
+    text: "ST1003: const StorageEngineRocksDbExperimental"
+  - linters:
+      - stylecheck
+    text: "ST1003: const StorageEngineRocksDbV1"
 
 linters:
   disable-all: true
   # TODO(johscheuer): activate linters again and fix issues
   enable:
-    - deadcode
-    #- dupl
+    - asciicheck
+    - bodyclose
+    - depguard
+    - dogsled
     - errcheck
-    #- goconst
-    #- gocyclo
+#    - errorlint
+    - exportloopref
+#    - goconst
+#    - gocritic
+#    - gocyclo
     - gofmt
-    - goimports
-    - revive
-    #- gosec
+    #- goimports
+    - goprintffuncname
+#    - gosec
     - gosimple
     - govet
+    - importas
     - ineffassign
-    #- interfacer
-    #- lll
-    #- maligned
     - misspell
     - nakedret
+    - nilerr
+#    - nolintlint
     - prealloc
-    - exportloopref
+    - revive
+    - rowserrcheck
     - staticcheck
-    - structcheck
+    - stylecheck
     - typecheck
     - unconvert
     - unparam
     - unused
-    - varcheck
+    - whitespace
 
 linters-settings:
   errcheck:
@@ -48,7 +68,13 @@ linters-settings:
   govet:
     enable:
       - shadow
+  staticcheck:
+    go: "1.19"
+  stylecheck:
+    go: "1.19"
+
 run:
-  deadline: 5m
+  timeout: 10m
   skip-files:
-    - ".*_generated.*.go$"
+    - "zz_generated.*\\.go$"
+  allow-parallel-runners: true
diff --git a/Dockerfile b/Dockerfile
index ca6e9d03..1ccf2acd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,7 +1,7 @@
 ARG BASE_IMAGE=docker.io/debian:bullseye
 
 # Build the manager binary
-FROM docker.io/library/golang:1.18.6 as builder
+FROM docker.io/library/golang:1.19.2 as builder
 
 # Install FDB this version is only required to compile the fdb operator
 ARG FDB_VERSION=6.2.29
diff --git a/Makefile b/Makefile
index 5e4e2dd9..fe4d475e 100644
--- a/Makefile
+++ b/Makefile
@@ -33,7 +33,7 @@ CONTROLLER_GEN_PKG?=sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2
 CONTROLLER_GEN=$(GOBIN)/controller-gen
 KUSTOMIZE_PKG?=sigs.k8s.io/kustomize/kustomize/v4@v4.5.2
 KUSTOMIZE=$(GOBIN)/kustomize
-GOLANGCI_LINT_PKG=github.com/golangci/golangci-lint/cmd/golangci-lint@v1.46.2
+GOLANGCI_LINT_PKG=github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1
 GOLANGCI_LINT=$(GOBIN)/golangci-lint
 GORELEASER_PKG=github.com/goreleaser/goreleaser@v1.6.3
 GORELEASER=$(GOBIN)/goreleaser
@@ -64,7 +64,7 @@ MANIFESTS=config/crd/bases/apps.foundationdb.org_foundationdbbackups.yaml config
 SAMPLES=config/samples/deployment.yaml config/samples/cluster.yaml config/samples/backup.yaml config/samples/restore.yaml config/samples/client.yaml
 
 ifeq "$(TEST_RACE_CONDITIONS)" "1"
-	go_test_flags := $(go_test_flags) -race -timeout=30m
+	go_test_flags := $(go_test_flags) -race -timeout=90m
 endif
 
 all: deps generate fmt vet manager snapshot manifests samples documentation test_if_changed
@@ -94,7 +94,7 @@ cover.out: ${GO_ALL} ${MANIFESTS}
 
 test:
 ifneq "$(SKIP_TEST)" "1"
-	go test ${go_test_flags} ./... -coverprofile cover.out
+	go test ${go_test_flags} ./... -coverprofile cover.out -ginkgo.timeout=2h
 endif
 
 # Build manager binary
diff --git a/api/v1beta1/foundationdb_database_configuration.go b/api/v1beta1/foundationdb_database_configuration.go
index acea2c8e..c44a552c 100644
--- a/api/v1beta1/foundationdb_database_configuration.go
+++ b/api/v1beta1/foundationdb_database_configuration.go
@@ -335,7 +335,6 @@ func (configuration DatabaseConfiguration) GetNextConfigurationChange(finalConfi
 								} else if dataCenter.Priority > 0 {
 									hasAlternativePrimary = true
 								}
-
 							}
 						}
 					}
@@ -681,11 +680,14 @@ func fieldIndices(value interface{}, result interface{}, keyType reflect.Type) {
 // GetProcessCountsWithDefaults for more information on the rules for inferring
 // process counts.
 type ProcessCounts struct {
-	Unset             int `json:"unset,omitempty"`
-	Storage           int `json:"storage,omitempty"`
-	Transaction       int `json:"transaction,omitempty"`
-	Resolution        int `json:"resolution,omitempty"`
+	Unset       int `json:"unset,omitempty"`
+	Storage     int `json:"storage,omitempty"`
+	Transaction int `json:"transaction,omitempty"`
+	Resolution  int `json:"resolution,omitempty"`
+	// Deprecated: This setting will be removed in the next major release.
+	// use Test
 	Tester            int `json:"tester,omitempty"`
+	Test              int `json:"test,omitempty"`
 	Proxy             int `json:"proxy,omitempty"`
 	Master            int `json:"master,omitempty"`
 	Stateless         int `json:"stateless,omitempty"`
diff --git a/api/v1beta1/foundationdbcluster_types.go b/api/v1beta1/foundationdbcluster_types.go
index 73220e4b..921e63ae 100644
--- a/api/v1beta1/foundationdbcluster_types.go
+++ b/api/v1beta1/foundationdbcluster_types.go
@@ -789,7 +789,6 @@ func (clusterStatus FoundationDBClusterStatus) ProcessGroupsByProcessClass(proce
 		if groupStatus.ProcessClass == processClass {
 			result = append(result, groupStatus)
 		}
-
 	}
 
 	return result
diff --git a/api/v1beta2/foundationdb_database_configuration.go b/api/v1beta2/foundationdb_database_configuration.go
index cc30a78a..24cb930b 100644
--- a/api/v1beta2/foundationdb_database_configuration.go
+++ b/api/v1beta2/foundationdb_database_configuration.go
@@ -444,7 +444,6 @@ func (configuration DatabaseConfiguration) GetNextConfigurationChange(finalConfi
 								} else if dataCenter.Priority > 0 {
 									hasAlternativePrimary = true
 								}
-
 							}
 						}
 					}
@@ -579,7 +578,6 @@ func (configuration DatabaseConfiguration) AreSeparatedProxiesConfigured() bool
 // AreSeparatedProxiesConfigured(), then this function will return the
 // string "commit_proxies=%d grv_proxies=%d", otherwise just
 // "proxies=%d" using the correct counts of the configuration object.
-//
 func (configuration DatabaseConfiguration) GetProxiesString(version Version) string {
 	counts := configuration.GetRoleCountsWithDefaults(version, DesiredFaultTolerance(configuration.RedundancyMode))
 	if version.HasSeparatedProxies() && configuration.AreSeparatedProxiesConfigured() {
@@ -838,11 +836,14 @@ func fieldIndices(value interface{}, result interface{}, keyType reflect.Type) {
 // GetProcessCountsWithDefaults for more information on the rules for inferring
 // process counts.
 type ProcessCounts struct {
-	Unset             int `json:"unset,omitempty"`
-	Storage           int `json:"storage,omitempty"`
-	Transaction       int `json:"transaction,omitempty"`
-	Resolution        int `json:"resolution,omitempty"`
+	Unset       int `json:"unset,omitempty"`
+	Storage     int `json:"storage,omitempty"`
+	Transaction int `json:"transaction,omitempty"`
+	Resolution  int `json:"resolution,omitempty"`
+	// Deprecated: This setting will be removed in the next major release.
+	// use Test
 	Tester            int `json:"tester,omitempty"`
+	Test              int `json:"test,omitempty"`
 	Proxy             int `json:"proxy,omitempty"`
 	CommitProxy       int `json:"commit_proxy,omitempty"`
 	GrvProxy          int `json:"grv_proxy,omitempty"`
@@ -911,5 +912,20 @@ func (counts ProcessCounts) Diff(currentCounts ProcessCounts) map[ProcessClass]i
 			diff[label] = desired - current
 		}
 	}
+
 	return diff
 }
+
+// Total gets the total number of processes for the cluster configuration.
+func (counts ProcessCounts) Total() int {
+	var total int64
+	desiredValue := reflect.ValueOf(counts)
+	for _, index := range processClassIndices {
+		desired := desiredValue.Field(index).Int()
+		if desired > 0 {
+			total += desired
+		}
+	}
+
+	return int(total)
+}
diff --git a/api/v1beta2/foundationdb_status.go b/api/v1beta2/foundationdb_status.go
index 519fd53b..413334be 100644
--- a/api/v1beta2/foundationdb_status.go
+++ b/api/v1beta2/foundationdb_status.go
@@ -78,6 +78,9 @@ type FoundationDBStatusClusterInfo struct {
 	// FullReplication indicates whether the database is fully replicated.
 	FullReplication bool `json:"full_replication,omitempty"`
 
+	// MaintenanceZone contains current zone under maintenance, if any.
+	MaintenanceZone string `json:"maintenance_zone,omitempty"`
+
 	// Clients provides information about clients that are connected to the
 	// database.
 	Clients FoundationDBStatusClusterClientInfo `json:"clients,omitempty"`
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index 673de9a0..8edf3d46 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -39,6 +39,8 @@ import (
 // +kubebuilder:printcolumn:name="Reconciled",type="integer",JSONPath=".status.generations.reconciled",description="Last reconciled generation of the spec",priority=0
 // +kubebuilder:printcolumn:name="Available",type="boolean",JSONPath=".status.health.available",description="Database available",priority=0
 // +kubebuilder:printcolumn:name="FullReplication",type="boolean",JSONPath=".status.health.fullReplication",description="Database fully replicated",priority=0
+// +kubebuilder:printcolumn:name="ReconciledProcessGroups",type="integer",JSONPath=".status.reconciledProcessGroups",description="Number of reconciled process groups",priority=1
+// +kubebuilder:printcolumn:name="DesiredProcessGroups",type="integer",JSONPath=".status.desiredProcessGroups",description="Desired number of process groups",priority=1
 // +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".status.runningVersion",description="Running version",priority=0
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
 // +kubebuilder:storageversion
@@ -279,6 +281,26 @@ type FoundationDBClusterStatus struct {
 
 	// Locks contains information about the locking system.
 	Locks LockSystemStatus `json:"locks,omitempty"`
+
+	// MaintenenanceModeInfo contains information regarding process groups in maintenance mode
+	MaintenanceModeInfo MaintenanceModeInfo `json:"maintenanceModeInfo,omitempty"`
+
+	// DesiredProcessGroups reflects the number of expected running process groups.
+	DesiredProcessGroups int `json:"desiredProcessGroups,omitempty"`
+
+	// ReconciledProcessGroups reflects the number of process groups that have no condition and are not marked for removal.
+	ReconciledProcessGroups int `json:"reconciledProcessGroups,omitempty"`
+}
+
+// MaintenanceModeInfo contains information regarding the zone and process groups that are put
+// into maintenance mode by the operator
+type MaintenanceModeInfo struct {
+	// StartTimestamp provides the timestamp when this zone is put into maintenance mode
+	StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"`
+	// ZoneID that is placed in maintenance mode
+	ZoneID string `json:"zoneID,omitempty"`
+	// ProcessGroups that are placed in maintenance mode
+	ProcessGroups []string `json:"processGroups,omitempty"`
 }
 
 // LockSystemStatus provides a summary of the status of the locking system.
@@ -598,7 +620,6 @@ func (clusterStatus FoundationDBClusterStatus) ProcessGroupsByProcessClass(proce
 		if groupStatus.ProcessClass == processClass {
 			result = append(result, groupStatus)
 		}
-
 	}
 
 	return result
@@ -874,6 +895,20 @@ type FoundationDBClusterAutomationOptions struct {
 	// UseManagementAPI defines if the operator should make use of the management API instead of
 	// using fdbcli to interact with the FoundationDB cluster.
 	UseManagementAPI *bool `json:"useManagementAPI,omitempty"`
+
+	// MaintenanceModeOptions contains options for maintenance mode related settings.
+	MaintenanceModeOptions MaintenanceModeOptions `json:"maintenanceModeOptions,omitempty"`
+}
+
+// MaintenanceModeOptions controls options for placing zones in maintenance mode.
+type MaintenanceModeOptions struct {
+	// UseMaintenanceModeChecker defines whether the operator is allowed to use maintenance mode before updating pods.
+	// Default is false.
+	UseMaintenanceModeChecker *bool `json:"UseMaintenanceModeChecker,omitempty"`
+
+	// MaintenanceModeTimeSeconds provides the duration for the zone to be in maintenance. It will automatically be switched off after the time elapses.
+	// Default is 600.
+	MaintenanceModeTimeSeconds *int `json:"maintenanceModeTimeSeconds,omitempty"`
 }
 
 // AutomaticReplacementOptions controls options for automatically replacing
@@ -1126,7 +1161,6 @@ func (cluster *FoundationDBCluster) CheckReconciliation(log logr.Logger) (bool,
 	}
 
 	cluster.Status.Generations = ClusterGenerationStatus{Reconciled: cluster.Status.Generations.Reconciled}
-
 	for _, processGroup := range cluster.Status.ProcessGroups {
 		if !processGroup.IsMarkedForRemoval() {
 			continue
@@ -1161,17 +1195,35 @@ func (cluster *FoundationDBCluster) CheckReconciliation(log logr.Logger) (bool,
 		}
 	}
 
+	cluster.Status.DesiredProcessGroups = desiredCounts.Total()
+	cluster.Status.ReconciledProcessGroups = 0
+
 	for _, processGroup := range cluster.Status.ProcessGroups {
-		if len(processGroup.ProcessGroupConditions) > 0 && !processGroup.IsMarkedForRemoval() {
+		if processGroup.IsMarkedForRemoval() {
+			continue
+		}
+
+		if len(processGroup.ProcessGroupConditions) > 0 {
 			conditions := make([]ProcessGroupConditionType, 0, len(processGroup.ProcessGroupConditions))
 			for _, condition := range processGroup.ProcessGroupConditions {
+				if condition.ProcessGroupConditionType == IncorrectCommandLine && cluster.Status.Generations.NeedsBounce == 0 {
+					logger.Info("Pending restart of fdbserver processes", "state", "NeedsBounce")
+					cluster.Status.Generations.NeedsBounce = cluster.ObjectMeta.Generation
+				}
 				conditions = append(conditions, condition.ProcessGroupConditionType)
 			}
 
 			logger.Info("Has unhealthy process group", "processGroupID", processGroup.ProcessGroupID, "state", "HasUnhealthyProcess", "conditions", conditions)
 			cluster.Status.Generations.HasUnhealthyProcess = cluster.ObjectMeta.Generation
 			reconciled = false
+			continue
 		}
+
+		cluster.Status.ReconciledProcessGroups++
+	}
+
+	if cluster.Status.DesiredProcessGroups != cluster.Status.ReconciledProcessGroups {
+		logger.Info("Not all process groups are reconciled", "desiredProcessGroups", cluster.Status.DesiredProcessGroups, "reconciledProcessGroups", cluster.Status.ReconciledProcessGroups)
 	}
 
 	if !cluster.Status.Health.Available {
@@ -1875,6 +1927,16 @@ func (cluster *FoundationDBCluster) GetWaitBetweenRemovalsSeconds() int {
 	return duration
 }
 
+// UseMaintenaceMode returns true if UseMaintenanceModeChecker is set.
+func (cluster *FoundationDBCluster) UseMaintenaceMode() bool {
+	return pointer.BoolDeref(cluster.Spec.AutomationOptions.MaintenanceModeOptions.UseMaintenanceModeChecker, false)
+}
+
+// GetMaintenaceModeTimeoutSeconds returns the timeout for maintenance zone after which it will be reset.
+func (cluster *FoundationDBCluster) GetMaintenaceModeTimeoutSeconds() int {
+	return pointer.IntDeref(cluster.Spec.AutomationOptions.MaintenanceModeOptions.MaintenanceModeTimeSeconds, 600)
+}
+
 // PodUpdateStrategy defines how Pod spec changes should be applied.
 type PodUpdateStrategy string
 
diff --git a/api/v1beta2/groupversion_info.go b/api/v1beta2/groupversion_info.go
index 0d7c79e5..96fd86a4 100644
--- a/api/v1beta2/groupversion_info.go
+++ b/api/v1beta2/groupversion_info.go
@@ -15,8 +15,8 @@ limitations under the License.
 */
 
 // Package v1beta2 contains API Schema definitions for the apps v1beta2 API group
-//+kubebuilder:object:generate=true
-//+groupName=apps.foundationdb.org
+// +kubebuilder:object:generate=true
+// +groupName=apps.foundationdb.org
 package v1beta2
 
 import (
diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go
index d73d53c7..98e4a512 100644
--- a/api/v1beta2/zz_generated.deepcopy.go
+++ b/api/v1beta2/zz_generated.deepcopy.go
@@ -510,6 +510,7 @@ func (in *FoundationDBClusterAutomationOptions) DeepCopyInto(out *FoundationDBCl
 		*out = new(bool)
 		**out = **in
 	}
+	in.MaintenanceModeOptions.DeepCopyInto(&out.MaintenanceModeOptions)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBClusterAutomationOptions.
@@ -676,6 +677,7 @@ func (in *FoundationDBClusterStatus) DeepCopyInto(out *FoundationDBClusterStatus
 		}
 	}
 	in.Locks.DeepCopyInto(&out.Locks)
+	in.MaintenanceModeInfo.DeepCopyInto(&out.MaintenanceModeInfo)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBClusterStatus.
@@ -1306,6 +1308,55 @@ func (in *LockSystemStatus) DeepCopy() *LockSystemStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MaintenanceModeInfo) DeepCopyInto(out *MaintenanceModeInfo) {
+	*out = *in
+	if in.StartTimestamp != nil {
+		in, out := &in.StartTimestamp, &out.StartTimestamp
+		*out = (*in).DeepCopy()
+	}
+	if in.ProcessGroups != nil {
+		in, out := &in.ProcessGroups, &out.ProcessGroups
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MaintenanceModeInfo.
+func (in *MaintenanceModeInfo) DeepCopy() *MaintenanceModeInfo {
+	if in == nil {
+		return nil
+	}
+	out := new(MaintenanceModeInfo)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MaintenanceModeOptions) DeepCopyInto(out *MaintenanceModeOptions) {
+	*out = *in
+	if in.UseMaintenanceModeChecker != nil {
+		in, out := &in.UseMaintenanceModeChecker, &out.UseMaintenanceModeChecker
+		*out = new(bool)
+		**out = **in
+	}
+	if in.MaintenanceModeTimeSeconds != nil {
+		in, out := &in.MaintenanceModeTimeSeconds, &out.MaintenanceModeTimeSeconds
+		*out = new(int)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MaintenanceModeOptions.
+func (in *MaintenanceModeOptions) DeepCopy() *MaintenanceModeOptions {
+	if in == nil {
+		return nil
+	}
+	out := new(MaintenanceModeOptions)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *None) DeepCopyInto(out *None) {
 	*out = *in
diff --git a/cmd/po-docgen/api.go b/cmd/po-docgen/api.go
index cf95f2e3..50b090cb 100644
--- a/cmd/po-docgen/api.go
+++ b/cmd/po-docgen/api.go
@@ -124,7 +124,6 @@ func ParseDocumentationFrom(srcs []string) []KubeTypes {
 		pkg := astFrom(src)
 
 		for _, kubType := range pkg.Types {
-
 			if structType, ok := kubType.Decl.Specs[0].(*ast.TypeSpec).Type.(*ast.StructType); ok {
 				var ks KubeTypes
 				ks = append(ks, Pair{kubType.Name, fmtRawDoc(kubType.Doc), "", false})
diff --git a/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml b/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
index 4ad9a3e0..62319462 100644
--- a/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
+++ b/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
@@ -4925,6 +4925,8 @@ spec:
                     type: integer
                   storage_cache:
                     type: integer
+                  test:
+                    type: integer
                   tester:
                     type: integer
                   transaction:
@@ -9750,6 +9752,8 @@ spec:
                     type: integer
                   storage_cache:
                     type: integer
+                  test:
+                    type: integer
                   tester:
                     type: integer
                   transaction:
@@ -9828,6 +9832,16 @@ spec:
       jsonPath: .status.health.fullReplication
       name: FullReplication
       type: boolean
+    - description: Number of reconciled process groups
+      jsonPath: .status.reconciledProcessGroups
+      name: ReconciledProcessGroups
+      priority: 1
+      type: integer
+    - description: Desired number of process groups
+      jsonPath: .status.desiredProcessGroups
+      name: DesiredProcessGroups
+      priority: 1
+      type: integer
     - description: Running version
       jsonPath: .status.runningVersion
       name: Version
@@ -9868,6 +9882,13 @@ spec:
                     type: integer
                   killProcesses:
                     type: boolean
+                  maintenanceModeOptions:
+                    properties:
+                      UseMaintenanceModeChecker:
+                        type: boolean
+                      maintenanceModeTimeSeconds:
+                        type: integer
+                    type: object
                   maxConcurrentReplacements:
                     minimum: 0
                     type: integer
@@ -10190,6 +10211,8 @@ spec:
                     type: integer
                   storage_cache:
                     type: integer
+                  test:
+                    type: integer
                   tester:
                     type: integer
                   transaction:
@@ -13671,6 +13694,8 @@ spec:
                   usable_regions:
                     type: integer
                 type: object
+              desiredProcessGroups:
+                type: integer
               generations:
                 properties:
                   hasExtraListeners:
@@ -13746,6 +13771,18 @@ spec:
                       type: string
                     type: array
                 type: object
+              maintenanceModeInfo:
+                properties:
+                  processGroups:
+                    items:
+                      type: string
+                    type: array
+                  startTimestamp:
+                    format: date-time
+                    type: string
+                  zoneID:
+                    type: string
+                type: object
               needsNewCoordinators:
                 type: boolean
               processGroups:
@@ -13779,6 +13816,8 @@ spec:
                       type: string
                   type: object
                 type: array
+              reconciledProcessGroups:
+                type: integer
               requiredAddresses:
                 properties:
                   nonTLS:
diff --git a/controllers/admin_client_mock.go b/controllers/admin_client_mock.go
index 1803c8e9..f647a7d4 100644
--- a/controllers/admin_client_mock.go
+++ b/controllers/admin_client_mock.go
@@ -26,6 +26,7 @@ import (
 	"net"
 	"strings"
 	"sync"
+	"time"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
@@ -54,6 +55,9 @@ type mockAdminClient struct {
 	maxZoneFailuresWithoutLosingData         *int
 	maxZoneFailuresWithoutLosingAvailability *int
 	knobs                                    []string
+	maintenanceZone                          string
+	maintenanceZoneStartTimestamp            time.Time
+	uptimeSecondsForMaintenanceZone          float64
 }
 
 // adminClientCache provides a cache of mock admin clients.
@@ -204,6 +208,15 @@ func (client *mockAdminClient) GetStatus() (*fdbv1beta2.FoundationDBStatus, erro
 				locality["process_id"] = fmt.Sprintf("%s-%d", processGroupID, processIndex)
 			}
 
+			var uptimeSeconds float64 = 60000
+			if client.maintenanceZone == pod.Name || client.maintenanceZone == "simulation" {
+				if client.uptimeSecondsForMaintenanceZone != 0.0 {
+					uptimeSeconds = client.uptimeSecondsForMaintenanceZone
+				} else {
+					uptimeSeconds = time.Since(client.maintenanceZoneStartTimestamp).Seconds()
+				}
+			}
+
 			status.Cluster.Processes[fmt.Sprintf("%s-%d", pod.Name, processIndex)] = fdbv1beta2.FoundationDBStatusProcessInfo{
 				Address:       fullAddress,
 				ProcessClass:  internal.GetProcessClassFromMeta(client.Cluster, pod.ObjectMeta),
@@ -211,7 +224,7 @@ func (client *mockAdminClient) GetStatus() (*fdbv1beta2.FoundationDBStatus, erro
 				Excluded:      excluded,
 				Locality:      locality,
 				Version:       client.Cluster.Status.RunningVersion,
-				UptimeSeconds: 60000,
+				UptimeSeconds: uptimeSeconds,
 				Roles:         fdbRoles,
 			}
 		}
@@ -226,15 +239,23 @@ func (client *mockAdminClient) GetStatus() (*fdbv1beta2.FoundationDBStatus, erro
 				locality[key] = value
 			}
 
+			var uptimeSeconds float64 = 60000
+			if client.maintenanceZone == processGroup.ProcessGroupID || client.maintenanceZone == "simulation" {
+				if client.uptimeSecondsForMaintenanceZone != 0.0 {
+					uptimeSeconds = client.uptimeSecondsForMaintenanceZone
+				} else {
+					uptimeSeconds = time.Since(client.maintenanceZoneStartTimestamp).Seconds()
+				}
+			}
+
 			fullAddress := client.Cluster.GetFullAddress(processGroup.Addresses[0], 1)
 			status.Cluster.Processes[processGroup.ProcessGroupID] = fdbv1beta2.FoundationDBStatusProcessInfo{
 				Address:       fullAddress,
 				ProcessClass:  processGroup.ProcessClass,
 				Locality:      locality,
 				Version:       client.Cluster.Status.RunningVersion,
-				UptimeSeconds: 60000,
+				UptimeSeconds: uptimeSeconds,
 			}
-
 		}
 	}
 
@@ -302,15 +323,18 @@ func (client *mockAdminClient) GetStatus() (*fdbv1beta2.FoundationDBStatus, erro
 			status.Cluster.Layers.Backup.Paused = tagStatus.Paused
 		}
 	}
-
+	faultToleranceSubtractor := 0
+	if client.maintenanceZone != "" {
+		faultToleranceSubtractor = 1
+	}
 	if client.maxZoneFailuresWithoutLosingData == nil {
-		status.Cluster.FaultTolerance.MaxZoneFailuresWithoutLosingData = client.Cluster.DesiredFaultTolerance()
+		status.Cluster.FaultTolerance.MaxZoneFailuresWithoutLosingData = client.Cluster.DesiredFaultTolerance() - faultToleranceSubtractor
 	}
 
 	if client.maxZoneFailuresWithoutLosingAvailability == nil {
-		status.Cluster.FaultTolerance.MaxZoneFailuresWithoutLosingAvailability = client.Cluster.DesiredFaultTolerance()
+		status.Cluster.FaultTolerance.MaxZoneFailuresWithoutLosingAvailability = client.Cluster.DesiredFaultTolerance() - faultToleranceSubtractor
 	}
-
+	status.Cluster.MaintenanceZone = client.maintenanceZone
 	return status, nil
 }
 
@@ -711,3 +735,26 @@ func (client *mockAdminClient) GetCoordinatorSet() (map[string]struct{}, error)
 func (client *mockAdminClient) SetKnobs(knobs []string) {
 	client.knobs = knobs
 }
+
+// GetMaintenanceZone gets current maintenance zone, if any
+func (client *mockAdminClient) GetMaintenanceZone() (string, error) {
+	return client.maintenanceZone, nil
+}
+
+// SetMaintenanceZone places zone into maintenance mode
+func (client *mockAdminClient) SetMaintenanceZone(zone string, _ int) error {
+	client.maintenanceZone = zone
+	client.maintenanceZoneStartTimestamp = time.Now()
+	return nil
+}
+
+// Reset maintenance mode
+func (client *mockAdminClient) ResetMaintenanceMode() error {
+	client.maintenanceZone = ""
+	return nil
+}
+
+// Reset maintenance mode
+func (client *mockAdminClient) MockUptimeSecondsForMaintenanceZone(seconds float64) {
+	client.uptimeSecondsForMaintenanceZone = seconds
+}
diff --git a/controllers/bounce_processes.go b/controllers/bounce_processes.go
index 0aec4f9e..e261197a 100644
--- a/controllers/bounce_processes.go
+++ b/controllers/bounce_processes.go
@@ -23,18 +23,15 @@ package controllers
 import (
 	"context"
 	"fmt"
-	"math"
 	"time"
 
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
 	"github.com/FoundationDB/fdb-kubernetes-operator/pkg/fdbadminclient"
 	"github.com/FoundationDB/fdb-kubernetes-operator/pkg/podmanager"
-
-	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
-
+	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/utils/pointer"
-
-	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 )
 
 // bounceProcesses provides a reconciliation step for bouncing fdbserver
@@ -43,6 +40,10 @@ type bounceProcesses struct{}
 
 // reconcile runs the reconciler's work.
 func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster) *requeue {
+	if !pointer.BoolDeref(cluster.Spec.AutomationOptions.KillProcesses, true) {
+		return nil
+	}
+
 	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "bounceProcesses")
 	adminClient, err := r.getDatabaseClientProvider().GetAdminClient(cluster, r)
 	if err != nil {
@@ -55,15 +56,103 @@ func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReco
 		return &requeue{curError: err}
 	}
 
-	minimumUptime, addressMap, err := getMinimumUptimeAndAddressMap(cluster, status, r.EnableRecoveryState)
+	minimumUptime, addressMap, err := internal.GetMinimumUptimeAndAddressMap(cluster, status, r.EnableRecoveryState)
+	if err != nil {
+		return &requeue{curError: err}
+	}
+
+	addresses, req := getProcessesReadyForRestart(ctx, logger, r, cluster, addressMap)
+	if req != nil {
+		return req
+	}
+
+	if len(addresses) == 0 {
+		return nil
+	}
+
+	if minimumUptime < float64(cluster.GetMinimumUptimeSecondsForBounce()) {
+		r.Recorder.Event(cluster, corev1.EventTypeNormal, "NeedsBounce",
+			fmt.Sprintf("Spec require a bounce of some processes, but the cluster has only been up for %f seconds", minimumUptime))
+		cluster.Status.Generations.NeedsBounce = cluster.ObjectMeta.Generation
+		err = r.updateOrApply(ctx, cluster)
+		if err != nil {
+			logger.Error(err, "Error updating cluster status")
+		}
+
+		// Retry after we waited the minimum uptime
+		return &requeue{
+			message: "Cluster needs to stabilize before bouncing",
+			delay:   time.Second * time.Duration(cluster.GetMinimumUptimeSecondsForBounce()-int(minimumUptime)),
+		}
+	}
+
+	var lockClient fdbadminclient.LockClient
+	useLocks := cluster.ShouldUseLocks()
+	if useLocks {
+		lockClient, err = r.getLockClient(cluster)
+		if err != nil {
+			return &requeue{curError: err}
+		}
+	}
+	version, err := fdbv1beta2.ParseFdbVersion(cluster.Spec.Version)
+	if err != nil {
+		return &requeue{curError: err}
+	}
+
+	upgrading := cluster.Status.RunningVersion != cluster.Spec.Version
+
+	if useLocks && upgrading {
+		processGroupIDs := make([]string, 0, len(cluster.Status.ProcessGroups))
+		for _, processGroup := range cluster.Status.ProcessGroups {
+			processGroupIDs = append(processGroupIDs, processGroup.ProcessGroupID)
+		}
+		err = lockClient.AddPendingUpgrades(version, processGroupIDs)
+		if err != nil {
+			return &requeue{curError: err}
+		}
+	}
+
+	hasLock, err := r.takeLock(cluster, fmt.Sprintf("bouncing processes: %v", addresses))
+	if !hasLock {
+		return &requeue{curError: err}
+	}
+
+	if useLocks && upgrading {
+		var req *requeue
+		addresses, req = getAddressesForUpgrade(logger, r, status, lockClient, cluster, version)
+		if req != nil {
+			return req
+		}
+		if addresses == nil {
+			return &requeue{curError: fmt.Errorf("unknown error when getting addresses that are ready for upgrade")}
+		}
+	}
+
+	logger.Info("Bouncing processes", "addresses", addresses, "upgrading", upgrading)
+	r.Recorder.Event(cluster, corev1.EventTypeNormal, "BouncingProcesses", fmt.Sprintf("Bouncing processes: %v", addresses))
+	err = adminClient.KillProcesses(addresses)
 	if err != nil {
 		return &requeue{curError: err}
 	}
 
+	// If the cluster was upgraded we will requeue and let the update_status command set the correct version.
+	// Updating the version in this method has the drawback that we upgrade the version independent of the success
+	// of the kill command. The kill command is not reliable, which means that some kill request might not be
+	// delivered and the return value will still not contain any error.
+	if upgrading {
+		return &requeue{message: "fetch latest status after upgrade"}
+	}
+
+	return nil
+}
+
+// getProcessesReadyForRestart returns a slice of process addresses that can be restarted. If addresses are missing or not all processes
+// have the latest configuration this method will return a requeue struct with more details.
+func getProcessesReadyForRestart(ctx context.Context, logger logr.Logger, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, addressMap map[string][]fdbv1beta2.ProcessAddress) ([]fdbv1beta2.ProcessAddress, *requeue) {
 	processesToBounce := fdbv1beta2.FilterByConditions(cluster.Status.ProcessGroups, map[fdbv1beta2.ProcessGroupConditionType]bool{
 		fdbv1beta2.IncorrectCommandLine: true,
 		fdbv1beta2.IncorrectPodSpec:     false,
-		fdbv1beta2.SidecarUnreachable:   false, // ignore all Process groups that are not reachable and therefore will not get any config map updates.
+		fdbv1beta2.SidecarUnreachable:   false, // ignore all Process groups that are not reachable and therefore will not get any ConfigMap updates.
 	}, true)
 
 	addresses := make([]fdbv1beta2.ProcessAddress, 0, len(processesToBounce))
@@ -95,10 +184,10 @@ func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReco
 		processGroupID := podmanager.GetProcessGroupIDFromProcessID(process)
 		pod, err := r.PodLifecycleManager.GetPods(ctx, r, cluster, internal.GetSinglePodListOptions(cluster, processGroupID)...)
 		if err != nil {
-			return &requeue{curError: err}
+			return nil, &requeue{curError: err}
 		}
 		if len(pod) == 0 {
-			return &requeue{message: fmt.Sprintf("No pod defined for process group ID: \"%s\"", processGroupID), delay: podSchedulingDelayDuration}
+			return nil, &requeue{message: fmt.Sprintf("No pod defined for process group ID: \"%s\"", processGroupID), delay: podSchedulingDelayDuration}
 		}
 
 		synced, err := r.updatePodDynamicConf(cluster, pod[0])
@@ -109,107 +198,19 @@ func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReco
 	}
 
 	if len(missingAddress) > 0 {
-		return &requeue{curError: fmt.Errorf("could not find address for processes: %s", missingAddress)}
+		return nil, &requeue{curError: fmt.Errorf("could not find address for processes: %s", missingAddress), delayedRequeue: true}
 	}
 
 	if !allSynced {
-		return &requeue{message: "Waiting for config map to sync to all pods"}
-	}
-
-	if len(addresses) > 0 {
-		if !pointer.BoolDeref(cluster.Spec.AutomationOptions.KillProcesses, true) {
-			r.Recorder.Event(cluster, corev1.EventTypeNormal, "NeedsBounce",
-				"Spec require a bounce of some processes, but killing processes is disabled")
-			cluster.Status.Generations.NeedsBounce = cluster.ObjectMeta.Generation
-			err = r.updateOrApply(ctx, cluster)
-			if err != nil {
-				logger.Error(err, "Error updating cluster status")
-			}
-
-			return &requeue{message: "Kills are disabled"}
-		}
-
-		if minimumUptime < float64(cluster.GetMinimumUptimeSecondsForBounce()) {
-			r.Recorder.Event(cluster, corev1.EventTypeNormal, "NeedsBounce",
-				fmt.Sprintf("Spec require a bounce of some processes, but the cluster has only been up for %f seconds", minimumUptime))
-			cluster.Status.Generations.NeedsBounce = cluster.ObjectMeta.Generation
-			err = r.updateOrApply(ctx, cluster)
-			if err != nil {
-				logger.Error(err, "Error updating cluster status")
-			}
-
-			// Retry after we waited the minimum uptime
-			return &requeue{
-				message: "Cluster needs to stabilize before bouncing",
-				delay:   time.Second * time.Duration(cluster.GetMinimumUptimeSecondsForBounce()-int(minimumUptime)),
-			}
-		}
-
-		var lockClient fdbadminclient.LockClient
-		useLocks := cluster.ShouldUseLocks()
-		if useLocks {
-			lockClient, err = r.getLockClient(cluster)
-			if err != nil {
-				return &requeue{curError: err}
-			}
-		}
-		version, err := fdbv1beta2.ParseFdbVersion(cluster.Spec.Version)
-		if err != nil {
-			return &requeue{curError: err}
-		}
-
-		upgrading := cluster.Status.RunningVersion != cluster.Spec.Version
-
-		if useLocks && upgrading {
-			processGroupIDs := make([]string, 0, len(cluster.Status.ProcessGroups))
-			for _, processGroup := range cluster.Status.ProcessGroups {
-				processGroupIDs = append(processGroupIDs, processGroup.ProcessGroupID)
-			}
-			err = lockClient.AddPendingUpgrades(version, processGroupIDs)
-			if err != nil {
-				return &requeue{curError: err}
-			}
-		}
-
-		hasLock, err := r.takeLock(cluster, fmt.Sprintf("bouncing processes: %v", addresses))
-		if !hasLock {
-			return &requeue{curError: err}
-		}
-
-		if useLocks && upgrading {
-			var req *requeue
-			addresses, req = getAddressesForUpgrade(r, status, lockClient, cluster, version)
-			if req != nil {
-				return req
-			}
-			if addresses == nil {
-				return &requeue{curError: fmt.Errorf("unknown error when getting addresses that are ready for upgrade")}
-			}
-		}
-
-		logger.Info("Bouncing processes", "addresses", addresses, "upgrading", upgrading)
-		r.Recorder.Event(cluster, corev1.EventTypeNormal, "BouncingProcesses", fmt.Sprintf("Bouncing processes: %v", addresses))
-		err = adminClient.KillProcesses(addresses)
-		if err != nil {
-			return &requeue{curError: err}
-		}
-
-		// If the cluster was upgraded we will requeue and let the update_status command set the correct version.
-		// Updating the version in this method has the drawback that we upgrade the version independent of the success
-		// of the kill command. The kill command is not reliable, which means that some kill request might not be
-		// delivered and the return value will still not contain any error.
-		if upgrading {
-			return &requeue{message: "fetch latest status after upgrade"}
-		}
+		return nil, &requeue{message: "Waiting for config map to sync to all pods", delayedRequeue: true}
 	}
 
-	return nil
+	return addresses, nil
 }
 
 // getAddressesForUpgrade checks that all processes in a cluster are ready to be
 // upgraded and returns the full list of addresses.
-func getAddressesForUpgrade(r *FoundationDBClusterReconciler, databaseStatus *fdbv1beta2.FoundationDBStatus, lockClient fdbadminclient.LockClient, cluster *fdbv1beta2.FoundationDBCluster, version fdbv1beta2.Version) ([]fdbv1beta2.ProcessAddress, *requeue) {
-	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "bounceProcesses")
+func getAddressesForUpgrade(logger logr.Logger, r *FoundationDBClusterReconciler, databaseStatus *fdbv1beta2.FoundationDBStatus, lockClient fdbadminclient.LockClient, cluster *fdbv1beta2.FoundationDBCluster, version fdbv1beta2.Version) ([]fdbv1beta2.ProcessAddress, *requeue) {
 	pendingUpgrades, err := lockClient.GetPendingUpgrades(version)
 	if err != nil {
 		return nil, &requeue{curError: err}
@@ -247,36 +248,3 @@ func getAddressesForUpgrade(r *FoundationDBClusterReconciler, databaseStatus *fd
 
 	return addresses, nil
 }
-
-// getMinimumUptimeAndAddressMap returns address map of the processes included the the foundationdb status. The minimum
-// uptime will be either secondsSinceLastRecovered if the recovery state is supported and enabled otherwise we will
-// take the minimum uptime of all processes.
-func getMinimumUptimeAndAddressMap(cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, recoveryStateEnabled bool) (float64, map[string][]fdbv1beta2.ProcessAddress, error) {
-	runningVersion, err := fdbv1beta2.ParseFdbVersion(cluster.GetRunningVersion())
-	if err != nil {
-		return 0, nil, err
-	}
-
-	useRecoveryState := runningVersion.SupportsRecoveryState() && recoveryStateEnabled
-
-	addressMap := make(map[string][]fdbv1beta2.ProcessAddress, len(status.Cluster.Processes))
-
-	minimumUptime := math.Inf(1)
-	if useRecoveryState {
-		minimumUptime = status.Cluster.RecoveryState.SecondsSinceLastRecovered
-	}
-
-	for _, process := range status.Cluster.Processes {
-		addressMap[process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]] = append(addressMap[process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]], process.Address)
-
-		if useRecoveryState || process.Excluded {
-			continue
-		}
-
-		if process.UptimeSeconds < minimumUptime {
-			minimumUptime = process.UptimeSeconds
-		}
-	}
-
-	return minimumUptime, addressMap, nil
-}
diff --git a/controllers/change_coordinators.go b/controllers/change_coordinators.go
index 3b0de6e5..e3ca5f15 100644
--- a/controllers/change_coordinators.go
+++ b/controllers/change_coordinators.go
@@ -154,7 +154,7 @@ func selectCoordinators(cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1be
 
 	candidates, err := selectCandidates(cluster, status)
 	if err != nil {
-		return []localityInfo{}, nil
+		return []localityInfo{}, err
 	}
 
 	coordinators, err := chooseDistributedProcesses(cluster, candidates, coordinatorCount, processSelectionConstraint{
diff --git a/controllers/choose_removals.go b/controllers/choose_removals.go
index 1c455088..6d277e1e 100644
--- a/controllers/choose_removals.go
+++ b/controllers/choose_removals.go
@@ -55,6 +55,7 @@ func (c chooseRemovals) reconcile(ctx context.Context, r *FoundationDBClusterRec
 	if err != nil {
 		return &requeue{curError: err}
 	}
+	defer adminClient.Close()
 	status, err := adminClient.GetStatus()
 	if err != nil {
 		return &requeue{curError: err}
diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go
index 9bc4e2b0..5b6d8325 100644
--- a/controllers/cluster_controller.go
+++ b/controllers/cluster_controller.go
@@ -89,7 +89,6 @@ func (r *FoundationDBClusterReconciler) Reconcile(ctx context.Context, request c
 	if err != nil {
 		if k8serrors.IsNotFound(err) {
 			return ctrl.Result{}, nil
-
 		}
 		// Error reading the object - requeue the request.
 		return ctrl.Result{}, err
@@ -150,6 +149,7 @@ func (r *FoundationDBClusterReconciler) Reconcile(ctx context.Context, request c
 		excludeProcesses{},
 		changeCoordinators{},
 		bounceProcesses{},
+		maintenanceModeChecker{},
 		updatePods{},
 		removeProcessGroups{},
 		removeServices{},
@@ -621,7 +621,7 @@ func checkCoordinatorValidity(cluster *fdbv1beta2.FoundationDBCluster, status *f
 		}
 
 		if process.ProcessClass == fdbv1beta2.ProcessClassTest {
-			pLogger.Info("Ignoring tester process")
+			pLogger.Info("Ignoring test process")
 			continue
 		}
 
diff --git a/controllers/maintenance_mode_checker.go b/controllers/maintenance_mode_checker.go
new file mode 100644
index 00000000..b86b6e0a
--- /dev/null
+++ b/controllers/maintenance_mode_checker.go
@@ -0,0 +1,115 @@
+/*
+ * maintenance_mode_checker.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+)
+
+// maintenanceModeChecker provides a reconciliation step for clearing the maintenance mode if all the pods in the current maintenance zone are up.
+type maintenanceModeChecker struct{}
+
+// reconcile runs the reconciler's work.
+func (maintenanceModeChecker) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster) *requeue {
+	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "maintenanceModeChecker")
+
+	if !cluster.UseMaintenaceMode() {
+		return nil
+	}
+
+	adminClient, err := r.getDatabaseClientProvider().GetAdminClient(cluster, r.Client)
+	if err != nil {
+		return &requeue{curError: err}
+	}
+	defer adminClient.Close()
+
+	maintenanceZone, err := adminClient.GetMaintenanceZone()
+	if err != nil {
+		return &requeue{curError: err}
+	}
+	// Cluster is not in maintenance mode
+	if maintenanceZone == "" {
+		if cluster.Status.MaintenanceModeInfo.ZoneID != "" {
+			cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{}
+			err = r.updateOrApply(ctx, cluster)
+			if err != nil {
+				return &requeue{curError: err}
+			}
+		}
+		return nil
+	}
+	// FDB Cluster is in maintenance mode but not due to this operator actions
+	if maintenanceZone != cluster.Status.MaintenanceModeInfo.ZoneID {
+		if cluster.Status.MaintenanceModeInfo.ZoneID != "" {
+			cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{}
+			err = r.updateOrApply(ctx, cluster)
+			if err != nil {
+				return &requeue{curError: err}
+			}
+		}
+		return nil
+	}
+	logger.Info("Cluster in maintenance mode", "zone", maintenanceZone)
+	// FDB Cluster is in maintenance mode due to this operator actions
+	status, err := adminClient.GetStatus()
+	if err != nil {
+		return &requeue{curError: err}
+	}
+	processGroupsToCheck := make(map[string]struct{})
+	for _, id := range cluster.Status.MaintenanceModeInfo.ProcessGroups {
+		processGroupsToCheck[id] = fdbv1beta2.None{}
+	}
+	for _, process := range status.Cluster.Processes {
+		if _, ok := processGroupsToCheck[process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]]; !ok {
+			continue
+		}
+		// TODO: Also include deletion timestamp to make this logic more robust to account for the corner case of the process crash/restarts.
+		if process.UptimeSeconds < time.Since(cluster.Status.MaintenanceModeInfo.StartTimestamp.Time).Seconds() {
+			delete(processGroupsToCheck, process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey])
+		} else {
+			return &requeue{message: fmt.Sprintf("Waiting for pod %s to be updated", process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]), delayedRequeue: true}
+		}
+	}
+	// Some of the pods are not yet up
+	if len(processGroupsToCheck) != 0 {
+		return &requeue{message: fmt.Sprintf("Waiting for all proceeses in zone %s to be up", maintenanceZone), delayedRequeue: true}
+	}
+	// All the pods for this zone under maintenance are up
+	hasLock, err := r.takeLock(cluster, "maintenance mode check")
+	if !hasLock {
+		return &requeue{curError: err}
+	}
+	logger.Info("Switching off maintenance mode", "zone", maintenanceZone)
+	err = adminClient.ResetMaintenanceMode()
+	if err != nil {
+		return &requeue{curError: err}
+	}
+	cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{}
+	err = r.updateOrApply(ctx, cluster)
+	if err != nil {
+		return &requeue{curError: err}
+	}
+	return nil
+}
diff --git a/controllers/remove_incompatible_processes.go b/controllers/remove_incompatible_processes.go
index 410e344d..d9762171 100644
--- a/controllers/remove_incompatible_processes.go
+++ b/controllers/remove_incompatible_processes.go
@@ -90,18 +90,32 @@ func processIncompatibleProcesses(ctx context.Context, r *FoundationDBClusterRec
 		return nil
 	}
 
-	logger.Info("incompatible connections", "incompatibleConnections", status.Cluster.IncompatibleConnections)
-	incompatibleConnections := map[string]fdbv1beta2.None{}
-	for _, incompatibleAddress := range status.Cluster.IncompatibleConnections {
-		address, err := fdbv1beta2.ParseProcessAddress(incompatibleAddress)
-		if err != nil {
-			logger.Error(err, "could not parse address in incompatible connections", "address", incompatibleAddress)
-			continue
-		}
+	// Wait until the cluster is running for the minimum uptime before looking for incompatible processes.
+	minimumUptime, _, err := internal.GetMinimumUptimeAndAddressMap(cluster, status, r.EnableRecoveryState)
+	if err != nil {
+		return err
+	}
 
-		incompatibleConnections[address.IPAddress.String()] = fdbv1beta2.None{}
+	if minimumUptime < float64(cluster.GetMinimumUptimeSecondsForBounce()) {
+		logger.V(1).Info("Skipping reconciler and waiting until cluster is for up minimum uptime")
+		return nil
+	}
+
+	// Ensure the cluster is running at fault tolerance before recreating Pods.
+	hasDesiredFaultTolerance, err := internal.HasDesiredFaultToleranceFromStatus(logger, status, cluster)
+	if err != nil {
+		logger.V(1).Info("Cluster doesn't have required fault tolerance won't delete Pods")
+		return err
 	}
 
+	if !hasDesiredFaultTolerance {
+		logger.V(1).Info("Skipping reconciler and waiting until cluster has desired fault tolerance")
+		return nil
+	}
+
+	logger.Info("incompatible connections", "incompatibleConnections", status.Cluster.IncompatibleConnections)
+	incompatibleConnections := parseIncompatibleConnections(logger, status.Cluster.IncompatibleConnections)
+
 	incompatiblePods := make([]*corev1.Pod, 0, len(incompatibleConnections))
 	for _, processGroup := range cluster.Status.ProcessGroups {
 		pod, ok := podMap[processGroup.ProcessGroupID]
@@ -111,8 +125,14 @@ func processIncompatibleProcesses(ctx context.Context, r *FoundationDBClusterRec
 			continue
 		}
 
+		if pod.DeletionTimestamp != nil {
+			logger.V(1).Info("Skipping Pod that is already marked for deletion",
+				"processGroupID", processGroup.ProcessGroupID)
+			continue
+		}
+
 		if isIncompatible(incompatibleConnections, processGroup) {
-			logger.Info("recreate Pod for process group with incompatible version", "processGroupID", processGroup.ProcessGroupID)
+			logger.Info("recreate Pod for process group with incompatible version", "processGroupID", processGroup.ProcessGroupID, "address", processGroup.Addresses)
 			incompatiblePods = append(incompatiblePods, pod)
 		}
 	}
@@ -121,6 +141,27 @@ func processIncompatibleProcesses(ctx context.Context, r *FoundationDBClusterRec
 	return r.PodLifecycleManager.UpdatePods(ctx, r, cluster, incompatiblePods, true)
 }
 
+// parseIncompatibleConnections parses the incompatible connections string slice to a map
+func parseIncompatibleConnections(logger logr.Logger, incompatibleConnections []string) map[string]fdbv1beta2.None {
+	result := make(map[string]fdbv1beta2.None)
+	for _, incompatibleAddress := range incompatibleConnections {
+		address, err := fdbv1beta2.ParseProcessAddress(incompatibleAddress)
+		if err != nil {
+			logger.Error(err, "could not parse address in incompatible connections", "address", incompatibleAddress)
+			continue
+		}
+
+		if address.Port == 0 {
+			logger.V(1).Info("Ignore incompatible connection with port 0", "address", address)
+			continue
+		}
+
+		result[address.IPAddress.String()] = fdbv1beta2.None{}
+	}
+
+	return result
+}
+
 // isIncompatible checks if the process group is in the list of incompatible connections.
 func isIncompatible(incompatibleConnections map[string]fdbv1beta2.None, processGroup *fdbv1beta2.ProcessGroupStatus) bool {
 	for _, address := range processGroup.Addresses {
diff --git a/controllers/remove_process_groups.go b/controllers/remove_process_groups.go
index 696b99a4..522d1034 100644
--- a/controllers/remove_process_groups.go
+++ b/controllers/remove_process_groups.go
@@ -71,12 +71,17 @@ func (u removeProcessGroups) reconcile(ctx context.Context, r *FoundationDBClust
 		}
 	}
 
+	status, err := adminClient.GetStatus()
+	if err != nil {
+		return &requeue{curError: err}
+	}
+
 	// We don't use the "cached" of the cluster status from the CRD to minimize the window between data loss (e.g. a node
 	// or a set of Pods is not reachable anymore). We still end up with the risk to actually query the FDB cluster and after that
 	// query the cluster gets into a degraded state.
 	// We could be smarter here and only block removals that target stateful processes by e.g. filtering those out of the
 	// processGroupsToRemove slice.
-	hasDesiredFaultTolerance, err := internal.HasDesiredFaultTolerance(logger, adminClient, cluster)
+	hasDesiredFaultTolerance, err := internal.HasDesiredFaultToleranceFromStatus(logger, status, cluster)
 	if err != nil {
 		return &requeue{curError: err}
 	}
@@ -88,11 +93,6 @@ func (u removeProcessGroups) reconcile(ctx context.Context, r *FoundationDBClust
 		}
 	}
 
-	status, err := adminClient.GetStatus()
-	if err != nil {
-		return &requeue{curError: err}
-	}
-
 	// In addition to that we should add the same logic as in the exclude step
 	// to ensure we never exclude/remove more process groups than desired.
 	zonedRemovals, lastDeletion, err := removals.GetZonedRemovals(status, processGroupsToRemove)
diff --git a/controllers/update_pods.go b/controllers/update_pods.go
index 2a3c2e0a..749c5ca7 100644
--- a/controllers/update_pods.go
+++ b/controllers/update_pods.go
@@ -31,6 +31,7 @@ import (
 	"github.com/FoundationDB/fdb-kubernetes-operator/pkg/podmanager"
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // updatePods provides a reconciliation step for recreating pods with new pod
@@ -227,7 +228,29 @@ func deletePodsForUpdates(ctx context.Context, r *FoundationDBClusterReconciler,
 		}
 	}
 
-	logger.Info("Deleting pods", "zone", zone, "count", len(deletions), "deletionMode", string(deletionMode))
+	if deletionMode == fdbv1beta2.PodUpdateModeZone && cluster.UseMaintenaceMode() {
+		var processGroups []string
+		for _, pod := range deletions {
+			processGroups = append(processGroups, pod.Labels[cluster.GetProcessGroupIDLabel()])
+		}
+
+		logger.Info("Setting maintenance mode", "zone", zone)
+		cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{
+			StartTimestamp: &metav1.Time{Time: time.Now()},
+			ZoneID:         zone,
+			ProcessGroups:  processGroups,
+		}
+		err = r.updateOrApply(ctx, cluster)
+		if err != nil {
+			return &requeue{curError: err}
+		}
+		err = adminClient.SetMaintenanceZone(zone, cluster.GetMaintenaceModeTimeoutSeconds())
+		if err != nil {
+			return &requeue{curError: err}
+		}
+	}
+
+	logger.Info("Deleting pods", "zone", zone, "count", len(deletions), "deletionMode", string(cluster.Spec.AutomationOptions.DeletionMode))
 	r.Recorder.Event(cluster, corev1.EventTypeNormal, "UpdatingPods", fmt.Sprintf("Recreating pods in zone %s", zone))
 
 	err = r.PodLifecycleManager.UpdatePods(logr.NewContext(ctx, logger), r, cluster, deletions, false)
diff --git a/controllers/update_status.go b/controllers/update_status.go
index fdb0f9f6..26c2fb82 100644
--- a/controllers/update_status.go
+++ b/controllers/update_status.go
@@ -48,6 +48,8 @@ func (updateStatus) reconcile(ctx context.Context, r *FoundationDBClusterReconci
 	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "updateStatus")
 	originalStatus := cluster.Status.DeepCopy()
 	status := fdbv1beta2.FoundationDBClusterStatus{}
+	// Pass through Maintenance Mode Info as the maintenance_mode_checker reconciler takes care of updating it
+	originalStatus.MaintenanceModeInfo.DeepCopyInto(&status.MaintenanceModeInfo)
 	status.Generations.Reconciled = cluster.Status.Generations.Reconciled
 
 	// Initialize with the current desired storage servers per Pod
@@ -77,9 +79,9 @@ func (updateStatus) reconcile(ctx context.Context, r *FoundationDBClusterReconci
 		if err != nil {
 			return &requeue{curError: err}
 		}
-		defer adminClient.Close()
 
 		databaseStatus, err = adminClient.GetStatus()
+		_ = adminClient.Close()
 		if err != nil {
 			if cluster.Status.Configured {
 				return &requeue{curError: err, delayedRequeue: true}
@@ -159,12 +161,12 @@ func (updateStatus) reconcile(ctx context.Context, r *FoundationDBClusterReconci
 		}
 	}
 
-	err = refreshProcessGroupStatus(ctx, r, cluster, &status)
+	pods, pvcs, err := refreshProcessGroupStatus(ctx, r, cluster, &status)
 	if err != nil {
 		return &requeue{curError: err}
 	}
 
-	status.ProcessGroups, err = validateProcessGroups(ctx, r, cluster, &status, processMap, configMap)
+	status.ProcessGroups, err = validateProcessGroups(ctx, r, cluster, &status, processMap, configMap, pods, pvcs)
 	if err != nil {
 		return &requeue{curError: err}
 	}
@@ -339,7 +341,7 @@ func tryConnectionOptions(cluster *fdbv1beta2.FoundationDBCluster, r *Foundation
 			if err == nil {
 				logger.Info("Chose connection option",
 					"version", version, "connectionString", activeConnectionString)
-				return version, activeConnectionString, err
+				return version, activeConnectionString, nil
 			}
 			logger.Error(err, "Error getting connection string from cluster",
 				"version", version, "connectionString", connectionString)
@@ -400,7 +402,8 @@ func checkAndSetProcessStatus(r *FoundationDBClusterReconciler, cluster *fdbv1be
 	return nil
 }
 
-func validateProcessGroups(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBClusterStatus, processMap map[string][]fdbv1beta2.FoundationDBStatusProcessInfo, configMap *corev1.ConfigMap) ([]*fdbv1beta2.ProcessGroupStatus, error) {
+func validateProcessGroups(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBClusterStatus, processMap map[string][]fdbv1beta2.FoundationDBStatusProcessInfo, configMap *corev1.ConfigMap, pods []*corev1.Pod, pvcs *corev1.PersistentVolumeClaimList) ([]*fdbv1beta2.ProcessGroupStatus, error) {
+	var err error
 	processGroups := status.ProcessGroups
 	processGroupsWithoutExclusion := make(map[string]fdbv1beta2.None, len(cluster.Spec.ProcessGroupsToRemoveWithoutExclusion))
 
@@ -414,16 +417,15 @@ func validateProcessGroups(ctx context.Context, r *FoundationDBClusterReconciler
 		processGroup.UpdateCondition(fdbv1beta2.IncorrectCommandLine, false, nil, "")
 	}
 
-	for _, processGroup := range processGroups {
-		pods, err := r.PodLifecycleManager.GetPods(ctx, r, cluster, internal.GetPodListOptions(cluster, processGroup.ProcessClass, processGroup.ProcessGroupID)...)
-		if err != nil {
-			return processGroups, err
-		}
+	podMap := internal.CreatePodMap(cluster, pods)
+	pvcMap := internal.CreatePVCMap(cluster, pvcs)
 
+	for _, processGroup := range processGroups {
+		pod, podExists := podMap[processGroup.ProcessGroupID]
 		// If the process group is not being removed and the Pod is not set we need to put it into
 		// the failing list.
 		isBeingRemoved := cluster.ProcessGroupIsBeingRemoved(processGroup.ProcessGroupID)
-		if len(pods) == 0 {
+		if !podExists {
 			// Mark process groups as terminating if the pod has been deleted but other
 			// resources are stuck in terminating.
 			if isBeingRemoved {
@@ -434,8 +436,6 @@ func validateProcessGroups(ctx context.Context, r *FoundationDBClusterReconciler
 			continue
 		}
 
-		pod := pods[0]
-
 		processGroup.AddAddresses(podmanager.GetPublicIPs(pod, log), processGroup.IsMarkedForRemoval() || !status.Health.Available)
 		processCount := 1
 
@@ -505,7 +505,13 @@ func validateProcessGroups(ctx context.Context, r *FoundationDBClusterReconciler
 			return processGroups, err
 		}
 
-		err = validateProcessGroup(ctx, r, cluster, pod, configMapHash, processGroup)
+		var pvc *corev1.PersistentVolumeClaim
+
+		pvcValue, pvcExists := pvcMap[processGroup.ProcessGroupID]
+		if pvcExists {
+			pvc = &pvcValue
+		}
+		err = validateProcessGroup(ctx, r, cluster, pod, pvc, configMapHash, processGroup)
 
 		if err != nil {
 			return processGroups, err
@@ -517,7 +523,7 @@ func validateProcessGroups(ctx context.Context, r *FoundationDBClusterReconciler
 
 // validateProcessGroup runs specific checks for the status of an process group.
 // returns failing, incorrect, error
-func validateProcessGroup(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod, configMapHash string, processGroupStatus *fdbv1beta2.ProcessGroupStatus) error {
+func validateProcessGroup(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod, currentPVC *corev1.PersistentVolumeClaim, configMapHash string, processGroupStatus *fdbv1beta2.ProcessGroupStatus) error {
 	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "updateStatus")
 	processGroupStatus.UpdateCondition(fdbv1beta2.MissingPod, pod == nil, cluster.Status.ProcessGroups, processGroupStatus.ProcessGroupID)
 	if pod == nil {
@@ -548,19 +554,14 @@ func validateProcessGroup(ctx context.Context, r *FoundationDBClusterReconciler,
 	incorrectConfigMap := pod.ObjectMeta.Annotations[fdbv1beta2.LastConfigMapKey] != configMapHash
 	processGroupStatus.UpdateCondition(fdbv1beta2.IncorrectConfigMap, incorrectConfigMap, cluster.Status.ProcessGroups, processGroupStatus.ProcessGroupID)
 
-	pvcs := &corev1.PersistentVolumeClaimList{}
-	err = r.List(ctx, pvcs, internal.GetPodListOptions(cluster, processGroupStatus.ProcessClass, processGroupStatus.ProcessGroupID)...)
-	if err != nil {
-		return err
-	}
 	desiredPvc, err := internal.GetPvc(cluster, processGroupStatus.ProcessClass, idNum)
 	if err != nil {
 		return err
 	}
 
-	incorrectPVC := (len(pvcs.Items) == 1) != (desiredPvc != nil)
+	incorrectPVC := (currentPVC != nil) != (desiredPvc != nil)
 	if !incorrectPVC && desiredPvc != nil {
-		incorrectPVC = !metadataMatches(pvcs.Items[0].ObjectMeta, desiredPvc.ObjectMeta)
+		incorrectPVC = !metadataMatches(currentPVC.ObjectMeta, desiredPvc.ObjectMeta)
 	}
 
 	processGroupStatus.UpdateCondition(fdbv1beta2.MissingPVC, incorrectPVC, cluster.Status.ProcessGroups, processGroupStatus.ProcessGroupID)
@@ -638,7 +639,7 @@ func removeDuplicateConditions(status fdbv1beta2.FoundationDBClusterStatus) {
 	}
 }
 
-func refreshProcessGroupStatus(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBClusterStatus) error {
+func refreshProcessGroupStatus(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBClusterStatus) ([]*corev1.Pod, *corev1.PersistentVolumeClaimList, error) {
 	status.ProcessGroups = make([]*fdbv1beta2.ProcessGroupStatus, 0, len(cluster.Status.ProcessGroups))
 	for _, processGroup := range cluster.Status.ProcessGroups {
 		if processGroup != nil && processGroup.ProcessGroupID != "" {
@@ -648,13 +649,12 @@ func refreshProcessGroupStatus(ctx context.Context, r *FoundationDBClusterReconc
 
 	// Track all created resources this will ensure that we catch all resources that are created by the operator
 	// even if the process group is currently missing for some reasons.
-	pods := &corev1.PodList{}
-	err := r.List(ctx, pods, internal.GetPodListOptions(cluster, "", "")...)
+	pods, err := r.PodLifecycleManager.GetPods(ctx, r, cluster, internal.GetPodListOptions(cluster, "", "")...)
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
 
-	for _, pod := range pods.Items {
+	for _, pod := range pods {
 		processGroupID := pod.Labels[cluster.GetProcessGroupIDLabel()]
 		if fdbv1beta2.ContainsProcessGroupID(status.ProcessGroups, processGroupID) {
 			continue
@@ -666,7 +666,7 @@ func refreshProcessGroupStatus(ctx context.Context, r *FoundationDBClusterReconc
 	pvcs := &corev1.PersistentVolumeClaimList{}
 	err = r.List(ctx, pvcs, internal.GetPodListOptions(cluster, "", "")...)
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
 
 	for _, pvc := range pvcs.Items {
@@ -681,7 +681,7 @@ func refreshProcessGroupStatus(ctx context.Context, r *FoundationDBClusterReconc
 	services := &corev1.ServiceList{}
 	err = r.List(ctx, services, internal.GetPodListOptions(cluster, "", "")...)
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
 
 	for _, service := range services.Items {
@@ -693,7 +693,7 @@ func refreshProcessGroupStatus(ctx context.Context, r *FoundationDBClusterReconc
 		status.ProcessGroups = append(status.ProcessGroups, fdbv1beta2.NewProcessGroupStatus(processGroupID, internal.ProcessClassFromLabels(cluster, service.Labels), nil))
 	}
 
-	return nil
+	return pods, pvcs, nil
 }
 
 func getRunningVersion(versionMap map[string]int, fallback string) (string, error) {
diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
index 664782ba..cf95ee49 100644
--- a/docs/cluster_spec.md
+++ b/docs/cluster_spec.md
@@ -22,6 +22,8 @@ This Document documents the types introduced by the FoundationDB Operator to be
 * [LockDenyListEntry](#lockdenylistentry)
 * [LockOptions](#lockoptions)
 * [LockSystemStatus](#locksystemstatus)
+* [MaintenanceModeInfo](#maintenancemodeinfo)
+* [MaintenanceModeOptions](#maintenancemodeoptions)
 * [ProcessGroupCondition](#processgroupcondition)
 * [ProcessGroupStatus](#processgroupstatus)
 * [ProcessSettings](#processsettings)
@@ -165,6 +167,7 @@ FoundationDBClusterAutomationOptions provides flags for enabling or disabling op
 | waitBetweenRemovalsSeconds | WaitBetweenRemovalsSeconds defines how long to wait between the last removal and the next removal. This is only an upper limit if the process group and the according resources are deleted faster than the provided duration the operator will move on with the next removal. The idea is to prevent a race condition were the operator deletes a resource but the Kubernetes API is slower to trigger the actual deletion, and we are running into a situation where the fault tolerance check still includes the already deleted processes. Defaults to 60. | *int | false |
 | podUpdateStrategy | PodUpdateStrategy defines how Pod spec changes are rolled out either by replacing Pods or by deleting Pods. The default for this is ReplaceTransactionSystem. | [PodUpdateStrategy](#podupdatestrategy) | false |
 | useManagementAPI | UseManagementAPI defines if the operator should make use of the management API instead of using fdbcli to interact with the FoundationDB cluster. | *bool | false |
+| maintenanceModeOptions | MaintenanceModeOptions contains options for maintenance mode related settings. | [MaintenanceModeOptions](#maintenancemodeoptions) | false |
 
 [Back to TOC](#table-of-contents)
 
@@ -254,6 +257,9 @@ FoundationDBClusterStatus defines the observed state of FoundationDBCluster
 | imageTypes | ImageTypes defines the kinds of images that are in use in the cluster. If there is more than one value in the slice the reconcile phase is not finished. | [][ImageType](#imagetype) | false |
 | processGroups | ProcessGroups contain information about a process group. This information is used in multiple places to trigger the according action. | []*[ProcessGroupStatus](#processgroupstatus) | false |
 | locks | Locks contains information about the locking system. | [LockSystemStatus](#locksystemstatus) | false |
+| maintenanceModeInfo | MaintenenanceModeInfo contains information regarding process groups in maintenance mode | [MaintenanceModeInfo](#maintenancemodeinfo) | false |
+| desiredProcessGroups | DesiredProcessGroups reflects the number of expected running process groups. | int | false |
+| reconciledProcessGroups | ReconciledProcessGroups reflects the number of process groups that have no condition and are not marked for removal. | int | false |
 
 [Back to TOC](#table-of-contents)
 
@@ -311,6 +317,29 @@ LockSystemStatus provides a summary of the status of the locking system.
 
 [Back to TOC](#table-of-contents)
 
+## MaintenanceModeInfo
+
+MaintenanceModeInfo contains information regarding the zone and process groups that are put into maintenance mode by the operator
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| startTimestamp | StartTimestamp provides the timestamp when this zone is put into maintenance mode | *metav1.Time | false |
+| zoneID | ZoneID that is placed in maintenance mode | string | false |
+| processGroups | ProcessGroups that are placed in maintenance mode | []string | false |
+
+[Back to TOC](#table-of-contents)
+
+## MaintenanceModeOptions
+
+MaintenanceModeOptions controls options for placing zones in maintenance mode.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| UseMaintenanceModeChecker | UseMaintenanceModeChecker defines whether the operator is allowed to use maintenance mode before updating pods. Default is false. | *bool | false |
+| maintenanceModeTimeSeconds | MaintenanceModeTimeSeconds provides the duration for the zone to be in maintenance. It will automatically be switched off after the time elapses. Default is 600. | *int | false |
+
+[Back to TOC](#table-of-contents)
+
 ## PodUpdateMode
 
 PodUpdateMode defines the deletion mode for the cluster
@@ -454,7 +483,8 @@ ProcessCounts represents the number of processes we have for each valid process
 | storage |  | int | false |
 | transaction |  | int | false |
 | resolution |  | int | false |
-| tester |  | int | false |
+| tester | **Deprecated: This setting will be removed in the next major release. use Test** | int | false |
+| test |  | int | false |
 | proxy |  | int | false |
 | commit_proxy |  | int | false |
 | grv_proxy |  | int | false |
diff --git a/fdbclient/admin_client.go b/fdbclient/admin_client.go
index db5f2b9e..b3cfb4d4 100644
--- a/fdbclient/admin_client.go
+++ b/fdbclient/admin_client.go
@@ -143,9 +143,9 @@ func (command cliCommand) getClusterFileFlag() string {
 }
 
 // getTimeout returns the timeout for the command
-func (command cliCommand) getTimeout() int {
+func (command cliCommand) getTimeout() time.Duration {
 	if command.timeout != 0 {
-		return int(command.timeout.Seconds())
+		return command.timeout
 	}
 
 	return DefaultCLITimeout
@@ -206,10 +206,10 @@ func (client *cliAdminClient) runCommand(command cliCommand) (string, error) {
 	}
 
 	if command.hasTimeoutArg() {
-		args = append(args, "--timeout", strconv.Itoa(command.getTimeout()))
+		args = append(args, "--timeout", strconv.Itoa(int(command.getTimeout().Seconds())))
 		hardTimeout += command.getTimeout()
 	}
-	timeoutContext, cancelFunction := context.WithTimeout(context.Background(), time.Second*time.Duration(hardTimeout))
+	timeoutContext, cancelFunction := context.WithTimeout(context.Background(), hardTimeout)
 	defer cancelFunction()
 	execCommand := exec.CommandContext(timeoutContext, binary, args...)
 
@@ -247,25 +247,25 @@ func (client *cliAdminClient) runCommand(command cliCommand) (string, error) {
 
 // runCommandWithBackoff is a wrapper around runCommand which allows retrying commands if they hit a timeout.
 func (client *cliAdminClient) runCommandWithBackoff(command string) (string, error) {
-	maxTimeoutInSeconds := 40
-	currentTimeoutInSeconds := DefaultCLITimeout
+	maxTimeout := 40 * time.Second
+	currentTimeout := DefaultCLITimeout
 
 	var rawResult string
 	var err error
 
 	// This method will be retrying to get the status if a timeout is seen. The timeout will be doubled everytime we try
-	// it with the default timeout of 10 we will try it 3 times with the following timeouts: 10s - 20s - 40s. We have
+	// it with the default timeout of 10s we will try it 3 times with the following timeouts: 10s - 20s - 40s. We have
 	// seen that during upgrades of version incompatible version, when not all coordinators are properly restarted that
 	// the response time will be increased.
-	for currentTimeoutInSeconds <= maxTimeoutInSeconds {
-		rawResult, err = client.runCommand(cliCommand{command: command, timeout: time.Duration(currentTimeoutInSeconds) * time.Second})
+	for currentTimeout <= maxTimeout {
+		rawResult, err = client.runCommand(cliCommand{command: command, timeout: currentTimeout})
 		if err == nil {
 			break
 		}
 
 		if _, ok := err.(fdbv1beta2.TimeoutError); ok {
 			client.log.Info("timeout issue will retry with higher timeout")
-			currentTimeoutInSeconds *= 2
+			currentTimeout *= 2
 			continue
 		}
 
@@ -345,6 +345,35 @@ func (client *cliAdminClient) ConfigureDatabase(configuration fdbv1beta2.Databas
 	return err
 }
 
+// GetMaintenanceZone gets current maintenance zone, if any. Returns empty string if maintenance mode is off
+func (client *cliAdminClient) GetMaintenanceZone() (string, error) {
+	// TODO: Use special keyspace to just read the maintenance zone info instead of reading the whole status
+	status, err := client.GetStatus()
+	if err != nil {
+		return "", err
+	}
+	return status.Cluster.MaintenanceZone, nil
+}
+
+// SetMaintenanceZone places zone into maintenance mode
+func (client *cliAdminClient) SetMaintenanceZone(zone string, timeoutSeconds int) error {
+	_, err := client.runCommand(cliCommand{
+		command: fmt.Sprintf(
+			"maintenance on %s %s",
+			zone,
+			strconv.Itoa(timeoutSeconds)),
+	})
+	return err
+}
+
+// ResetMaintenanceMode switches of maintenance mode
+func (client *cliAdminClient) ResetMaintenanceMode() error {
+	_, err := client.runCommand(cliCommand{
+		command: "maintenance off",
+	})
+	return err
+}
+
 // ExcludeProcesses starts evacuating processes so that they can be removed from the database.
 func (client *cliAdminClient) ExcludeProcesses(addresses []fdbv1beta2.ProcessAddress) error {
 	if len(addresses) == 0 {
diff --git a/fdbclient/common.go b/fdbclient/common.go
index 37193909..6b7b9579 100644
--- a/fdbclient/common.go
+++ b/fdbclient/common.go
@@ -23,6 +23,7 @@ package fdbclient
 import (
 	"fmt"
 	"os"
+	"time"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	"github.com/FoundationDB/fdb-kubernetes-operator/controllers"
@@ -33,11 +34,11 @@ import (
 )
 
 const (
-	defaultTransactionTimeout int64 = 5000
+	defaultTransactionTimeout = 5 * time.Second
 )
 
 // DefaultCLITimeout is the default timeout for CLI commands.
-var DefaultCLITimeout = 10
+var DefaultCLITimeout = 10 * time.Second
 
 // getFDBDatabase opens an FDB database. The result will be cached for
 // subsequent calls, based on the cluster namespace and name.
@@ -64,7 +65,7 @@ func getFDBDatabase(cluster *fdbv1beta2.FoundationDBCluster) (fdb.Database, erro
 		return fdb.Database{}, err
 	}
 
-	err = database.Options().SetTransactionTimeout(defaultTransactionTimeout)
+	err = database.Options().SetTransactionTimeout(defaultTransactionTimeout.Milliseconds())
 	if err != nil {
 		return fdb.Database{}, err
 	}
@@ -72,7 +73,7 @@ func getFDBDatabase(cluster *fdbv1beta2.FoundationDBCluster) (fdb.Database, erro
 	return database, nil
 }
 
-func getValueFromDBUsingKey(cluster *fdbv1beta2.FoundationDBCluster, log logr.Logger, fdbKey string, extraTimeout int64) ([]byte, error) {
+func getValueFromDBUsingKey(cluster *fdbv1beta2.FoundationDBCluster, log logr.Logger, fdbKey string, timeout time.Duration) ([]byte, error) {
 	log.Info("Fetch values from FDB", "namespace", cluster.Namespace, "cluster", cluster.Name, "key", fdbKey)
 	defer func() {
 		log.Info("Done fetching values from FDB", "namespace", cluster.Namespace, "cluster", cluster.Name, "key", fdbKey)
@@ -87,7 +88,7 @@ func getValueFromDBUsingKey(cluster *fdbv1beta2.FoundationDBCluster, log logr.Lo
 		if err != nil {
 			return nil, err
 		}
-		err = transaction.Options().SetTimeout(int64(DefaultCLITimeout) * extraTimeout)
+		err = transaction.Options().SetTimeout(timeout.Milliseconds())
 		if err != nil {
 			return nil, err
 		}
@@ -121,12 +122,12 @@ func getValueFromDBUsingKey(cluster *fdbv1beta2.FoundationDBCluster, log logr.Lo
 
 // getConnectionStringFromDB gets the database's connection string directly from the system key
 func getConnectionStringFromDB(cluster *fdbv1beta2.FoundationDBCluster, log logr.Logger) ([]byte, error) {
-	return getValueFromDBUsingKey(cluster, log, "\xff/coordinators", 1)
+	return getValueFromDBUsingKey(cluster, log, "\xff/coordinators", DefaultCLITimeout)
 }
 
 // getStatusFromDB gets the database's status directly from the system key
 func getStatusFromDB(cluster *fdbv1beta2.FoundationDBCluster, log logr.Logger) ([]byte, error) {
-	return getValueFromDBUsingKey(cluster, log, "\xff\xff/status/json", 1000)
+	return getValueFromDBUsingKey(cluster, log, "\xff\xff/status/json", DefaultCLITimeout)
 }
 
 type realDatabaseClientProvider struct {
diff --git a/go.mod b/go.mod
index b3e4c18e..c20d7bf1 100644
--- a/go.mod
+++ b/go.mod
@@ -1,13 +1,13 @@
 module github.com/FoundationDB/fdb-kubernetes-operator
 
-go 1.18
+go 1.19
 
 require (
 	github.com/apple/foundationdb/bindings/go v0.0.0-20201222225940-f3aef311ccfb
 	github.com/apple/foundationdb/fdbkubernetesmonitor v0.0.0-20220513200452-e6fa4d7422d2
 	github.com/fatih/color v1.10.0
 	github.com/go-logr/logr v1.2.0
-	github.com/google/go-cmp v0.5.5
+	github.com/google/go-cmp v0.5.6
 	github.com/hashicorp/go-retryablehttp v0.6.8
 	github.com/onsi/ginkgo/v2 v2.1.3
 	github.com/onsi/gomega v1.18.1
@@ -24,6 +24,7 @@ require (
 	k8s.io/klog/v2 v2.60.1
 	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
 	sigs.k8s.io/controller-runtime v0.12.3
+	sigs.k8s.io/e2e-framework v0.0.7
 	sigs.k8s.io/yaml v1.3.0
 )
 
@@ -54,7 +55,7 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
-	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.1.2 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
@@ -87,6 +88,7 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/stretchr/testify v1.7.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/vladimirvivien/gexe v0.1.1 // indirect
 	github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect
 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
diff --git a/go.sum b/go.sum
index e3dafc4e..a8fb5201 100644
--- a/go.sum
+++ b/go.sum
@@ -260,11 +260,13 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -540,6 +542,8 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/vladimirvivien/gexe v0.1.1 h1:2A0SBaOSKH+cwLVdt6H+KkHZotZWRNLlWygANGw5DxE=
+github.com/vladimirvivien/gexe v0.1.1/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI=
@@ -1078,6 +1082,8 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw=
 sigs.k8s.io/controller-runtime v0.12.3 h1:FCM8xeY/FI8hoAfh/V4XbbYMY20gElh9yh+A98usMio=
 sigs.k8s.io/controller-runtime v0.12.3/go.mod h1:qKsk4WE6zW2Hfj0G4v10EnNB2jMG1C+NTb8h+DwCoU0=
+sigs.k8s.io/e2e-framework v0.0.7 h1:nMv2oSPBLWARse2aBoqX5Wq3ox67w8jrhTGWGpccWDQ=
+sigs.k8s.io/e2e-framework v0.0.7/go.mod h1:hdwYGVQg4bvDAah5eidNf2/qkG35qHjzuyMVr2A3oiY=
 sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y=
 sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY=
 sigs.k8s.io/kustomize/api v0.11.4 h1:/0Mr3kfBBNcNPOW5Qwk/3eb8zkswCwnqQxxKtmrTkRo=
diff --git a/internal/deprecations.go b/internal/deprecations.go
index 18372226..7ba8bf93 100644
--- a/internal/deprecations.go
+++ b/internal/deprecations.go
@@ -25,7 +25,6 @@ import (
 
 	fdbtypes "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 )
 
@@ -61,12 +60,12 @@ func NormalizeClusterSpec(cluster *fdbtypes.FoundationDBCluster, options Depreca
 
 	if !options.OnlyShowChanges {
 		// Set up resource requirements for the main container.
-		updatePodTemplates(&cluster.Spec, func(template *v1.PodTemplateSpec) {
+		updatePodTemplates(&cluster.Spec, func(template *corev1.PodTemplateSpec) {
 			template.Spec.Containers, _ = ensureContainerPresent(template.Spec.Containers, "foundationdb", 0)
 
-			template.Spec.Containers = customizeContainerFromList(template.Spec.Containers, "foundationdb", func(container *v1.Container) {
+			template.Spec.Containers = customizeContainerFromList(template.Spec.Containers, "foundationdb", func(container *corev1.Container) {
 				if container.Resources.Requests == nil {
-					container.Resources.Requests = v1.ResourceList{
+					container.Resources.Requests = corev1.ResourceList{
 						"cpu":    resource.MustParse("1"),
 						"memory": resource.MustParse("1Gi"),
 					}
@@ -77,9 +76,9 @@ func NormalizeClusterSpec(cluster *fdbtypes.FoundationDBCluster, options Depreca
 				}
 			})
 
-			sidecarUpdater := func(container *v1.Container) {
+			sidecarUpdater := func(container *corev1.Container) {
 				if container.Resources.Requests == nil {
-					container.Resources.Requests = v1.ResourceList{
+					container.Resources.Requests = corev1.ResourceList{
 						"cpu":    resource.MustParse("100m"),
 						"memory": resource.MustParse("256Mi"),
 					}
diff --git a/internal/fault_tolerance.go b/internal/fault_tolerance.go
index 6d3d629e..b1c76fcb 100644
--- a/internal/fault_tolerance.go
+++ b/internal/fault_tolerance.go
@@ -39,10 +39,15 @@ func HasDesiredFaultTolerance(log logr.Logger, adminClient fdbadminclient.AdminC
 		return false, err
 	}
 
+	return HasDesiredFaultToleranceFromStatus(log, status, cluster)
+}
+
+// HasDesiredFaultToleranceFromStatus checks if the cluster has the desired fault tolerance based on the provided status.
+func HasDesiredFaultToleranceFromStatus(log logr.Logger, status *fdbv1beta2.FoundationDBStatus, cluster *fdbv1beta2.FoundationDBCluster) (bool, error) {
 	if !status.Client.DatabaseStatus.Available {
-		//log.V(0).Info("Cluster is not available",
-		//	"namespace", cluster.Namespace,
-		//	"cluster", cluster.Name)
+		log.Info("Cluster is not available",
+			"namespace", cluster.Namespace,
+			"cluster", cluster.Name)
 
 		return false, nil
 	}
diff --git a/internal/foundationdb_status.go b/internal/foundationdb_status.go
index 1c4cf254..bb6ece78 100644
--- a/internal/foundationdb_status.go
+++ b/internal/foundationdb_status.go
@@ -21,26 +21,61 @@
 package internal
 
 import (
-	"github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	"math"
+
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 )
 
 // GetCoordinatorsFromStatus gets the current coordinators from the status.
 // The returning set will contain all processes by their process group ID.
-func GetCoordinatorsFromStatus(status *v1beta2.FoundationDBStatus) map[string]struct{} {
+func GetCoordinatorsFromStatus(status *fdbv1beta2.FoundationDBStatus) map[string]struct{} {
 	coordinators := make(map[string]struct{})
 
 	for _, pInfo := range status.Cluster.Processes {
 		for _, roleInfo := range pInfo.Roles {
-			if roleInfo.Role != string(v1beta2.ProcessRoleCoordinator) {
+			if roleInfo.Role != string(fdbv1beta2.ProcessRoleCoordinator) {
 				continue
 			}
 
 			// We don't have to check for duplicates here, if the process group ID is already
 			// set we just overwrite it.
-			coordinators[pInfo.Locality[v1beta2.FDBLocalityInstanceIDKey]] = struct{}{}
+			coordinators[pInfo.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]] = struct{}{}
 			break
 		}
 	}
 
 	return coordinators
 }
+
+// GetMinimumUptimeAndAddressMap returns address map of the processes included the the foundationdb status. The minimum
+// uptime will be either secondsSinceLastRecovered if the recovery state is supported and enabled otherwise we will
+// take the minimum uptime of all processes.
+func GetMinimumUptimeAndAddressMap(cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, recoveryStateEnabled bool) (float64, map[string][]fdbv1beta2.ProcessAddress, error) {
+	runningVersion, err := fdbv1beta2.ParseFdbVersion(cluster.GetRunningVersion())
+	if err != nil {
+		return 0, nil, err
+	}
+
+	useRecoveryState := runningVersion.SupportsRecoveryState() && recoveryStateEnabled
+
+	addressMap := make(map[string][]fdbv1beta2.ProcessAddress, len(status.Cluster.Processes))
+
+	minimumUptime := math.Inf(1)
+	if useRecoveryState {
+		minimumUptime = status.Cluster.RecoveryState.SecondsSinceLastRecovered
+	}
+
+	for _, process := range status.Cluster.Processes {
+		addressMap[process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]] = append(addressMap[process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]], process.Address)
+
+		if useRecoveryState || process.Excluded {
+			continue
+		}
+
+		if process.UptimeSeconds < minimumUptime {
+			minimumUptime = process.UptimeSeconds
+		}
+	}
+
+	return minimumUptime, addressMap, nil
+}
diff --git a/internal/pod_client.go b/internal/pod_client.go
index 6d60e3bb..e76ddcc4 100644
--- a/internal/pod_client.go
+++ b/internal/pod_client.go
@@ -173,7 +173,6 @@ func (client *realFdbPodSidecarClient) getListenIP() string {
 
 // makeRequest submits a request to the sidecar.
 func (client *realFdbPodSidecarClient) makeRequest(method, path string) (string, int, error) {
-	var resp *http.Response
 	var err error
 
 	protocol := "http"
@@ -190,13 +189,16 @@ func (client *realFdbPodSidecarClient) makeRequest(method, path string) (string,
 	}
 
 	url := fmt.Sprintf("%s://%s:8080/%s", protocol, client.getListenIP(), path)
+	var resp *http.Response
 	switch method {
 	case http.MethodGet:
 		retryClient.HTTPClient.Timeout = client.getTimeout
 		resp, err = retryClient.Get(url)
+		defer resp.Body.Close()
 	case http.MethodPost:
 		retryClient.HTTPClient.Timeout = client.postTimeout
 		resp, err = retryClient.Post(url, "application/json", strings.NewReader(""))
+		defer resp.Body.Close()
 	default:
 		return "", 0, fmt.Errorf("unknown HTTP method %s", method)
 	}
@@ -205,8 +207,6 @@ func (client *realFdbPodSidecarClient) makeRequest(method, path string) (string,
 		return "", 0, err
 	}
 
-	defer resp.Body.Close()
-
 	body, err := io.ReadAll(resp.Body)
 	bodyText := string(body)
 
@@ -221,7 +221,7 @@ func (client *realFdbPodSidecarClient) makeRequest(method, path string) (string,
 func (client *realFdbPodSidecarClient) IsPresent(filename string) (bool, error) {
 	version, err := fdbv1beta2.ParseFdbVersion(client.Cluster.Spec.Version)
 	if err != nil {
-		return false, nil
+		return false, err
 	}
 
 	path := "check_hash"
diff --git a/internal/pod_models.go b/internal/pod_models.go
index 5e736aa5..a442ba3b 100644
--- a/internal/pod_models.go
+++ b/internal/pod_models.go
@@ -852,7 +852,6 @@ func GetBackupDeployment(backup *fdbv1beta2.FoundationDBBackup) (*appsv1.Deploym
 		for _, customParameter := range backup.Spec.CustomParameters {
 			args = append(args, fmt.Sprintf("--%s", customParameter))
 		}
-
 	}
 
 	mainContainer.Args = args
diff --git a/kubectl-fdb/cmd/exec.go b/kubectl-fdb/cmd/exec.go
index f245ddb3..dfde0c40 100644
--- a/kubectl-fdb/cmd/exec.go
+++ b/kubectl-fdb/cmd/exec.go
@@ -111,14 +111,14 @@ func buildCommand(kubeClient client.Client, cluster *fdbv1beta2.FoundationDBClus
 	for key, value := range cluster.GetMatchLabels() {
 		requirement, err := labels.NewRequirement(key, selection.Equals, []string{value})
 		if err != nil {
-			return exec.Cmd{}, nil
+			return exec.Cmd{}, err
 		}
 		selector = selector.Add(*requirement)
 	}
 
 	processClassRequirement, err := labels.NewRequirement(cluster.GetProcessClassLabel(), selection.Exists, nil)
 	if err != nil {
-		return exec.Cmd{}, nil
+		return exec.Cmd{}, err
 	}
 	selector = selector.Add(*processClassRequirement)
 
diff --git a/kubectl-fdb/cmd/k8s_client.go b/kubectl-fdb/cmd/k8s_client.go
index 35b874b9..340da95b 100644
--- a/kubectl-fdb/cmd/k8s_client.go
+++ b/kubectl-fdb/cmd/k8s_client.go
@@ -37,7 +37,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/remotecommand"
@@ -143,7 +142,7 @@ func executeCmd(restConfig *rest.Config, kubeClient *kubernetes.Clientset, podNa
 	}
 	req.VersionedParams(
 		option,
-		scheme.ParameterCodec,
+		clientgoscheme.ParameterCodec,
 	)
 	exec, err := remotecommand.NewSPDYExecutor(restConfig, "POST", req.URL())
 	if err != nil {
diff --git a/kubectl-fdb/cmd/version.go b/kubectl-fdb/cmd/version.go
index 735ebf53..2f83c916 100644
--- a/kubectl-fdb/cmd/version.go
+++ b/kubectl-fdb/cmd/version.go
@@ -47,11 +47,11 @@ func newVersionCmd(streams genericclioptions.IOStreams) *cobra.Command {
 			}
 			clientOnly, err := cmd.Flags().GetBool("client-only")
 			if err != nil {
-				return nil
+				return err
 			}
 			containerName, err := cmd.Flags().GetString("container-name")
 			if err != nil {
-				return nil
+				return err
 			}
 
 			if !clientOnly {
diff --git a/mock-kubernetes-client/client/client.go b/mock-kubernetes-client/client/client.go
index 3db3ada7..deb49a29 100644
--- a/mock-kubernetes-client/client/client.go
+++ b/mock-kubernetes-client/client/client.go
@@ -24,43 +24,41 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"reflect"
-	"strings"
 	"time"
 
-	"k8s.io/apimachinery/pkg/api/meta"
-	"k8s.io/apimachinery/pkg/runtime"
-
 	corev1 "k8s.io/api/core/v1"
-	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/fields"
-	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/uuid"
-	"k8s.io/client-go/kubernetes/scheme"
 	ctrlClient "sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 )
 
 // MockClient provides a mock Kubernetes client.
 type MockClient struct {
-	// data is the internal mock data.
-	// This maps a type, and then a namespace/name, to the generic
-	// representation of an object.
-	data map[string]map[string]map[string]interface{}
+	// fakeClient
+	fakeClient ctrlClient.WithWatch
 
 	// ipCounter provides monotonically incrementing IP addresses.
 	ipCounter int
 
-	// stuckTerminatingObjects tracks which objects should be stuck in terminating.
-	stuckTerminatingObjects map[string]map[string]bool
+	// scheme will be used to initialize or reset the new fake client
+	scheme *runtime.Scheme
+}
+
+// NewMockClient creates a new MockClient.
+func NewMockClient(scheme *runtime.Scheme) *MockClient {
+	return &MockClient{
+		fakeClient: fake.NewClientBuilder().WithScheme(scheme).Build(),
+		scheme:     scheme,
+	}
 }
 
 // Clear erases any mock data.
 func (client *MockClient) Clear() {
-	client.data = make(map[string]map[string]map[string]interface{})
-	client.stuckTerminatingObjects = nil
+	client.fakeClient = fake.NewClientBuilder().WithScheme(client.scheme).Build()
 }
 
 // Scheme returns the runtime Scheme
@@ -70,208 +68,7 @@ func (client *MockClient) Scheme() *runtime.Scheme {
 
 // RESTMapper returns the RESTMapper
 func (client *MockClient) RESTMapper() meta.RESTMapper {
-	return nil
-}
-
-// buildKindKey gets the key identifying an object's type.
-func buildKindKey(object runtime.Object) (string, error) {
-	gvk, err := apiutil.GVKForObject(object, scheme.Scheme)
-	if err != nil {
-		return "", err
-	}
-	return fmt.Sprintf("%s/%s/%s", gvk.Group, gvk.Version, gvk.Kind), nil
-}
-
-// buildObjectKey gets the key identifying an object.
-func buildObjectKey(key ctrlClient.ObjectKey) string {
-	return fmt.Sprintf("%s/%s", key.Namespace, key.Name)
-}
-
-// buildJSONObjectKey gets the key identifying an object from its JSON
-// representation.
-func buildJSONObjectKey(jsonData []byte) (string, error) {
-	genericData := make(map[string]interface{})
-	err := json.Unmarshal(jsonData, &genericData)
-	if err != nil {
-		return "", err
-	}
-
-	namespace, err := lookupJSONString(genericData, "metadata", "namespace")
-	if err != nil {
-		return "", err
-	}
-	name, err := lookupJSONString(genericData, "metadata", "name")
-	if err != nil {
-		return "", err
-	}
-	return buildObjectKey(types.NamespacedName{Namespace: namespace, Name: name}), nil
-}
-
-// buildRuntimeObjectKey gets the key identifying an object.
-func buildRuntimeObjectKey(object ctrlClient.Object) (string, error) {
-	jsonData, err := json.Marshal(object)
-	if err != nil {
-		return "", err
-	}
-
-	return buildJSONObjectKey(jsonData)
-}
-
-// fillInMaps ensures that we have maps for a given object type.
-func (client *MockClient) fillInMaps(kind string) {
-	if client.data == nil {
-		client.data = make(map[string]map[string]map[string]interface{})
-	}
-	if client.data[kind] == nil {
-		client.data[kind] = make(map[string]map[string]interface{})
-	}
-}
-
-// lookupJSONValue looks up a value in a generic JSON object.
-func lookupJSONValue(genericData map[string]interface{}, path []string, defaultValue interface{}) (interface{}, error) {
-	currentData := genericData
-
-	if len(path) == 0 {
-		return "", fmt.Errorf("cannot look up empty path")
-	}
-	var result interface{}
-	for index, key := range path {
-		genericValue, present := currentData[key]
-		if !present {
-			return defaultValue, nil
-		}
-		var ok bool
-		if index == len(path)-1 {
-			result = genericValue
-		} else {
-			currentData, ok = genericValue.(map[string]interface{})
-			if !ok {
-				return nil, fmt.Errorf("type error for key %v", path[0:index+1])
-			}
-		}
-	}
-	return result, nil
-}
-
-// lookupJSONString looks up a string in a generic JSON object.
-func lookupJSONString(genericData map[string]interface{}, path ...string) (string, error) {
-	result, err := lookupJSONValue(genericData, path, "")
-	if err != nil {
-		return "", err
-	}
-	stringResult, isString := result.(string)
-	if !isString {
-		return "", fmt.Errorf("type error for key %v", path)
-	}
-	return stringResult, nil
-}
-
-// lookupJSONStringMap looks up a map of string to string in a generic JSON object.
-func lookupJSONStringMap(genericData map[string]interface{}, path ...string) (map[string]string, error) {
-	currentData := genericData
-
-	if len(path) == 0 {
-		return nil, fmt.Errorf("cannot look up empty path")
-	}
-	var result = make(map[string]string)
-	for index, key := range path {
-		var ok bool
-		if index == len(path)-1 {
-			genericValue, present := currentData[key]
-			if !present {
-				return nil, nil
-			}
-
-			mapValue, isMap := genericValue.(map[string]interface{})
-			if !isMap {
-				return nil, fmt.Errorf("type error for %v", path)
-			}
-			for key, value := range mapValue {
-				stringValue, isString := value.(string)
-				if !isString {
-					return nil, fmt.Errorf("type error for key %s of %v", key, path)
-				}
-				result[key] = stringValue
-			}
-		} else {
-			genericValue, present := currentData[key]
-			if !present {
-				return nil, fmt.Errorf("missing key %v", path[0:index+1])
-			}
-
-			currentData, ok = genericValue.(map[string]interface{})
-			if !ok {
-				return nil, fmt.Errorf("type error for key %v", path[0:index+1])
-			}
-		}
-	}
-	return result, nil
-}
-
-// lookupJSONStringFields provides a map to all string fields in a generic JSON object.
-// The paths to each field will be represented as a string of the form key1.key2....keyN
-func lookupJSONStringFields(genericData map[string]interface{}) map[string]string {
-	var result = make(map[string]string)
-	for key, genericValue := range genericData {
-		switch castValue := genericValue.(type) {
-		case map[string]interface{}:
-			nestedFields := lookupJSONStringFields(castValue)
-			for nestedKey, value := range nestedFields {
-				result[fmt.Sprintf("%s.%s", key, nestedKey)] = value
-			}
-		case string:
-			result[key] = castValue
-		}
-	}
-	return result
-}
-
-// setField sets a field in a generic object.
-func setJSONValue(genericData map[string]interface{}, path []string, value interface{}) error {
-	currentData := genericData
-	for index, key := range path {
-		if index == len(path)-1 {
-			currentData[key] = value
-		} else {
-			if currentData[key] == nil {
-				currentData[key] = make(map[string]interface{})
-			}
-			var isMap bool
-			currentData, isMap = currentData[key].(map[string]interface{})
-			if !isMap {
-				return fmt.Errorf("invalid type for %v", path[0:index+1])
-			}
-		}
-	}
-	return nil
-}
-
-// incrementGeneration increments the generation field in an object's metadata.
-func incrementGeneration(genericData map[string]interface{}) (float64, error) {
-	generationValue, err := lookupJSONValue(genericData, []string{"metadata", "generation"}, float64(0))
-	if err != nil {
-		return 0, err
-	}
-	generationNumber, isNumber := generationValue.(float64)
-	if !isNumber {
-		return 0, fmt.Errorf("invalid metadata generation type")
-	}
-	generationNumber++
-	err = setJSONValue(genericData, []string{"metadata", "generation"}, generationNumber)
-	if err != nil {
-		return 0, err
-	}
-	return generationNumber, nil
-}
-
-// setGeneration sets the generation field in an object's metadata.
-func setGeneration(genericData map[string]interface{}, generation float64) error {
-	return setJSONValue(genericData, []string{"metadata", "generation"}, generation)
-}
-
-// setUID sets the UID in the object metadata.
-func setUID(genericData map[string]interface{}) error {
-	return setJSONValue(genericData, []string{"metadata", "uid"}, uuid.NewUUID())
+	return client.fakeClient.RESTMapper()
 }
 
 // generateIP generates a unique IP address.
@@ -280,329 +77,126 @@ func (client *MockClient) generateIP() string {
 	return fmt.Sprintf("192.168.%d.%d", client.ipCounter/256, client.ipCounter%256)
 }
 
-// checkPresence checks the presence of an object in the data.
-func (client *MockClient) checkPresence(kindKey string, objectKey string) error {
-	client.fillInMaps(kindKey)
-	if client.data[kindKey][objectKey] == nil {
-		return &k8serrors.StatusError{ErrStatus: metav1.Status{
-			Status:  "Failure",
-			Message: "Not Found",
-			Code:    404,
-			Reason:  metav1.StatusReasonNotFound,
-		}}
-	}
-	return nil
-}
-
 // Create creates a new object
-func (client *MockClient) Create(_ context.Context, object ctrlClient.Object, _ ...ctrlClient.CreateOption) error {
+func (client *MockClient) Create(ctx context.Context, object ctrlClient.Object, options ...ctrlClient.CreateOption) error {
+	// Ensure the default values are set.
 	object.SetCreationTimestamp(metav1.Time{Time: time.Now()})
+	object.SetGeneration(object.GetGeneration() + 1)
+	object.SetUID(uuid.NewUUID())
 
-	jsonData, err := json.Marshal(object)
-	if err != nil {
-		return err
-	}
-
-	kindKey, err := buildKindKey(object)
-	if err != nil {
-		return err
-	}
-
-	objectKey, err := buildJSONObjectKey(jsonData)
-	if err != nil {
-		return err
-	}
-
-	genericObject := make(map[string]interface{})
-	err = json.Unmarshal(jsonData, &genericObject)
-	if err != nil {
-		return err
-	}
-
-	_, err = incrementGeneration(genericObject)
-	if err != nil {
-		return err
-	}
-
-	err = setUID(genericObject)
-	if err != nil {
-		return err
-	}
-
-	if kindKey == "/v1/Service" {
-		clusterIP, err := lookupJSONString(genericObject, "spec", "clusterIP")
-		if err != nil {
-			return err
+	svc, isSvc := object.(*corev1.Service)
+	if isSvc {
+		if svc.Spec.ClusterIP == "" {
+			svc.Spec.ClusterIP = client.generateIP()
 		}
 
-		if clusterIP == "" {
-			err = setJSONValue(genericObject, []string{"spec", "clusterIP"}, client.generateIP())
-			if err != nil {
-				return err
-			}
-		}
-	} else if kindKey == "/v1/Pod" {
-		v4Address := client.generatePodIPv4()
-		v6Address := client.generatePodIPv6()
-		err = setJSONValue(genericObject, []string{"status", "podIP"}, v4Address)
-		if err != nil {
-			return err
-		}
-		err = setJSONValue(genericObject, []string{"status", "podIPs"}, []corev1.PodIP{{IP: v4Address}, {IP: v6Address}})
-		if err != nil {
-			return err
-		}
-		err = setJSONValue(genericObject, []string{"status", "phase"}, corev1.PodRunning)
-		if err != nil {
-			return err
-		}
+		object = svc.DeepCopyObject().(ctrlClient.Object)
 	}
 
-	client.fillInMaps(kindKey)
-	if client.data[kindKey][objectKey] != nil {
-		return &k8serrors.StatusError{ErrStatus: metav1.Status{
-			Status:  "Failure",
-			Message: "Conflict",
-			Code:    409,
-			Reason:  metav1.StatusReasonAlreadyExists,
-		}}
-	}
-	client.data[kindKey][objectKey] = genericObject
+	pod, isPod := object.(*corev1.Pod)
+	if isPod {
+		v4Address := client.generatePodIPv4()
+		pod.Status.PodIP = v4Address
+		pod.Status.PodIPs = []corev1.PodIP{{IP: v4Address}, {IP: client.generatePodIPv6()}}
 
-	jsonData, err = json.Marshal(genericObject)
-	if err != nil {
-		return err
-	}
-	err = json.Unmarshal(jsonData, object)
-	if err != nil {
-		return err
+		if pod.Status.Phase == "" {
+			pod.Status.Phase = corev1.PodRunning
+		}
+		object = pod.DeepCopyObject().(ctrlClient.Object)
 	}
 
-	return nil
+	return client.fakeClient.Create(ctx, object, options...)
 }
 
 // Get retrieves an object.
-func (client *MockClient) Get(_ context.Context, key ctrlClient.ObjectKey, object ctrlClient.Object) error {
-	kindKey, err := buildKindKey(object)
-	if err != nil {
-		return err
-	}
-
-	client.fillInMaps(kindKey)
-	objectKey := buildObjectKey(key)
-	err = client.checkPresence(kindKey, objectKey)
-	if err != nil {
-		return err
-	}
-
-	jsonData, err := json.Marshal(client.data[kindKey][objectKey])
-	if err != nil {
-		return err
-	}
-
-	err = json.Unmarshal(jsonData, object)
-
-	return err
+func (client *MockClient) Get(ctx context.Context, key ctrlClient.ObjectKey, object ctrlClient.Object) error {
+	return client.fakeClient.Get(ctx, key, object)
 }
 
 // List lists objects.
-func (client *MockClient) List(_ context.Context, list ctrlClient.ObjectList, options ...ctrlClient.ListOption) error {
-	kindKey, err := buildKindKey(list)
-	if err != nil {
-		return err
-	}
-	kindKey = strings.TrimSuffix(kindKey, "List")
-
-	objects := make([]map[string]interface{}, 0)
-
-	fullListOptions := &ctrlClient.ListOptions{}
-	for _, option := range options {
-		option.ApplyToList(fullListOptions)
-	}
-
-	client.fillInMaps(kindKey)
-	for _, object := range client.data[kindKey] {
-		if fullListOptions.Namespace != "" {
-			namespace, err := lookupJSONString(object, "metadata", "namespace")
-			if err != nil {
-				return err
-			}
-			if namespace != fullListOptions.Namespace {
-				continue
-			}
-		}
-
-		if fullListOptions.LabelSelector != nil {
-			objectLabels, err := lookupJSONStringMap(object, "metadata", "labels")
-			if err != nil {
-				return err
-			}
-			if !fullListOptions.LabelSelector.Matches(labels.Set(objectLabels)) {
-				continue
-			}
-		}
-
-		if fullListOptions.FieldSelector != nil {
-			objectFields := lookupJSONStringFields(object)
-			if !fullListOptions.FieldSelector.Matches(fields.Set(objectFields)) {
-				continue
-			}
-		}
-
-		objects = append(objects, object)
-		if fullListOptions.Limit > 0 && len(objects) >= int(fullListOptions.Limit) {
-			break
-		}
-	}
-
-	listJSON, err := json.Marshal(objects)
-	if err != nil {
-		return err
-	}
-	fullJSON := fmt.Sprintf("{\"items\":%s}", listJSON)
-	err = json.Unmarshal([]byte(fullJSON), list)
-	if err != nil {
-		return err
-	}
-
-	return nil
+func (client *MockClient) List(ctx context.Context, list ctrlClient.ObjectList, options ...ctrlClient.ListOption) error {
+	// TODO (johscheuer): Once https://github.com/kubernetes-sigs/controller-runtime/pull/2025 is merged and we update the
+	// controller-runtime we will support field selectors.
+	return client.fakeClient.List(ctx, list, options...)
 }
 
 // Delete deletes an object.
-// This does not support the options argument yet.
-func (client *MockClient) Delete(_ context.Context, object ctrlClient.Object, _ ...ctrlClient.DeleteOption) error {
-	kindKey, err := buildKindKey(object)
-	if err != nil {
-		return err
-	}
-
-	client.fillInMaps(kindKey)
+func (client *MockClient) Delete(ctx context.Context, object ctrlClient.Object, options ...ctrlClient.DeleteOption) error {
+	return client.fakeClient.Delete(ctx, object, options...)
+}
 
-	objectKey, err := buildRuntimeObjectKey(object)
+func getMapFromObject(object ctrlClient.Object) (map[string]interface{}, error) {
+	jsonData, err := json.Marshal(object)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	err = client.checkPresence(kindKey, objectKey)
+	newMap := make(map[string]interface{})
+	err = json.Unmarshal(jsonData, &newMap)
 	if err != nil {
-		return err
-	}
-
-	stuckTerminating := client.stuckTerminatingObjects != nil && client.stuckTerminatingObjects[kindKey] != nil && client.stuckTerminatingObjects[kindKey][objectKey]
-	if !stuckTerminating {
-		delete(client.data[kindKey], objectKey)
+		return nil, err
 	}
 
-	return nil
+	return newMap, nil
 }
 
-// Update updates an object.
-// This does not support the options argument yet.
-func (client *MockClient) Update(_ context.Context, object ctrlClient.Object, _ ...ctrlClient.UpdateOption) error {
-	kindKey, err := buildKindKey(object)
+func (client *MockClient) hasSpecChanges(existingObject ctrlClient.Object, newObject ctrlClient.Object) (bool, error) {
+	newObjectMap, err := getMapFromObject(newObject)
 	if err != nil {
-		return err
+		return false, err
 	}
 
-	client.fillInMaps(kindKey)
-
-	jsonData, err := json.Marshal(object)
+	existingObjectMap, err := getMapFromObject(existingObject)
 	if err != nil {
-		return err
+		return false, err
 	}
 
-	objectKey, err := buildJSONObjectKey(jsonData)
-	if err != nil {
-		return err
-	}
+	return !equality.Semantic.DeepEqual(existingObjectMap["spec"], newObjectMap["spec"]), nil
+}
 
-	err = client.checkPresence(kindKey, objectKey)
+// Update updates an object.
+func (client *MockClient) Update(ctx context.Context, object ctrlClient.Object, options ...ctrlClient.UpdateOption) error {
+	existingObject := object.DeepCopyObject().(ctrlClient.Object)
+	err := client.fakeClient.Get(ctx, ctrlClient.ObjectKeyFromObject(object), existingObject)
 	if err != nil {
 		return err
 	}
 
-	existingObject := client.data[kindKey][objectKey]
-
-	newObject := make(map[string]interface{})
-	err = json.Unmarshal(jsonData, &newObject)
+	hasSpecChanges, err := client.hasSpecChanges(existingObject, object)
 	if err != nil {
 		return err
 	}
 
-	if !reflect.DeepEqual(existingObject["spec"], newObject["spec"]) {
-		generation, err := incrementGeneration(existingObject)
-		if err != nil {
-			return err
-		}
-
-		err = setGeneration(newObject, generation)
-		if err != nil {
-			return err
-		}
+	if hasSpecChanges {
+		object.SetGeneration(existingObject.GetGeneration() + 1)
 	}
 
-	client.data[kindKey][objectKey] = newObject
-
-	jsonData, err = json.Marshal(newObject)
-	if err != nil {
-		return err
-	}
-	err = json.Unmarshal(jsonData, object)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return client.fakeClient.Update(ctx, object, options...)
 }
 
 // Patch patches an object.
-// This is not yet implemented.
-func (client *MockClient) Patch(_ context.Context, _ ctrlClient.Object, _ ctrlClient.Patch, _ ...ctrlClient.PatchOption) error {
+func (client *MockClient) Patch(ctx context.Context, obj ctrlClient.Object, patch ctrlClient.Patch, options ...ctrlClient.PatchOption) error {
 	// Currently the SSA patch type is not supported in the fake client: https://github.com/kubernetes/client-go/issues/992
-	// TODO (johscheuer): Find a proper way to implement the SSA support for testing in unit tests.
-	return fmt.Errorf("not implemented")
+	return client.fakeClient.Patch(ctx, obj, patch, options...)
 }
 
 // DeleteAllOf deletes all objects of the given type matching the given options.
-// This is not yet implemented.
-func (client *MockClient) DeleteAllOf(_ context.Context, _ ctrlClient.Object, _ ...ctrlClient.DeleteAllOfOption) error {
-	return fmt.Errorf("not implemented")
+func (client *MockClient) DeleteAllOf(ctx context.Context, object ctrlClient.Object, options ...ctrlClient.DeleteAllOfOption) error {
+	return client.fakeClient.DeleteAllOf(ctx, object, options...)
 }
 
 // MockStuckTermination sets a flag determining whether an object should get stuck in terminating when it is deleted.
 func (client *MockClient) MockStuckTermination(object ctrlClient.Object, terminating bool) error {
-	kindKey, err := buildKindKey(object)
-	if err != nil {
-		return err
-	}
-
 	if terminating {
 		object.SetDeletionTimestamp(&metav1.Time{Time: time.Now()})
+		object.SetFinalizers(append(object.GetFinalizers(), "foundationdb.org/testing"))
 	} else {
 		object.SetDeletionTimestamp(nil)
+		object.SetFinalizers(nil)
 	}
 
 	// We have to update the state in the mock client
-	err = client.Update(context.Background(), object)
-	if err != nil {
-		return err
-	}
-
-	objectKey, err := buildRuntimeObjectKey(object)
-	if err != nil {
-		return err
-	}
-
-	if client.stuckTerminatingObjects == nil {
-		client.stuckTerminatingObjects = make(map[string]map[string]bool)
-	}
-
-	if client.stuckTerminatingObjects[kindKey] == nil {
-		client.stuckTerminatingObjects[kindKey] = make(map[string]bool)
-	}
-
-	client.stuckTerminatingObjects[kindKey][objectKey] = terminating
-	return nil
+	return client.Update(context.Background(), object)
 }
 
 // MockStatusClient wraps a client to provide specialized operations for
@@ -613,49 +207,15 @@ type MockStatusClient struct {
 
 // Update updates an object.
 // This does not support the options argument yet.
-func (client MockStatusClient) Update(_ context.Context, object ctrlClient.Object, _ ...ctrlClient.UpdateOption) error {
-	kindKey, err := buildKindKey(object)
-	if err != nil {
-		return err
-	}
-
-	client.rawClient.fillInMaps(kindKey)
-
-	jsonData, err := json.Marshal(object)
-	if err != nil {
-		return err
-	}
-
-	objectKey, err := buildJSONObjectKey(jsonData)
-	if err != nil {
-		return err
-	}
-
-	err = client.rawClient.checkPresence(kindKey, objectKey)
-	if err != nil {
-		return err
-	}
-
-	existingObject := client.rawClient.data[kindKey][objectKey]
-
-	newObject := make(map[string]interface{})
-	err = json.Unmarshal(jsonData, &newObject)
-	if err != nil {
-		return err
-	}
-
-	existingObject["status"] = newObject["status"]
-	client.rawClient.data[kindKey][objectKey] = existingObject
-
-	return nil
+func (client MockStatusClient) Update(ctx context.Context, object ctrlClient.Object, options ...ctrlClient.UpdateOption) error {
+	return client.rawClient.fakeClient.Status().Update(ctx, object, options...)
 }
 
 // Patch patches an object's status.
 // This is not yet implemented.
-func (client MockStatusClient) Patch(_ context.Context, _ ctrlClient.Object, _ ctrlClient.Patch, _ ...ctrlClient.PatchOption) error {
+func (client MockStatusClient) Patch(ctx context.Context, object ctrlClient.Object, patch ctrlClient.Patch, options ...ctrlClient.PatchOption) error {
 	// Currently the SSA patch type is not supported in the fake client: https://github.com/kubernetes/client-go/issues/992
-	// TODO (johscheuer): Find a proper way to implement the SSA support for testing in unit tests.
-	return fmt.Errorf("not implemented")
+	return client.rawClient.fakeClient.Status().Patch(ctx, object, patch, options...)
 }
 
 // Status returns a writer for updating status.
@@ -671,39 +231,17 @@ func (client *MockClient) createEvent(event *corev1.Event) {
 }
 
 func buildEvent(object runtime.Object, eventType string, reason string, message string) *corev1.Event {
-	jsonData, err := json.Marshal(object)
-	if err != nil {
-		panic(err)
-	}
-	genericObject := make(map[string]interface{})
-	err = json.Unmarshal(jsonData, &genericObject)
-	if err != nil {
-		panic(err)
-	}
-	namespace, err := lookupJSONString(genericObject, "metadata", "namespace")
-	if err != nil {
-		panic(err)
-	}
-	name, err := lookupJSONString(genericObject, "metadata", "name")
-	if err != nil {
-		panic(err)
-	}
-	uid, err := lookupJSONString(genericObject, "metadata", "uid")
-	if err != nil {
-		panic(err)
-	}
-
-	eventName := fmt.Sprintf("%s-%v", eventType, uuid.NewUUID())
+	objectMeta, _ := meta.Accessor(object)
 
 	return &corev1.Event{
 		ObjectMeta: metav1.ObjectMeta{
-			Namespace: namespace,
-			Name:      eventName,
+			Namespace: objectMeta.GetNamespace(),
+			Name:      fmt.Sprintf("%s-%v", eventType, uuid.NewUUID()),
 		},
 		InvolvedObject: corev1.ObjectReference{
-			Namespace: namespace,
-			Name:      name,
-			UID:       types.UID(uid),
+			Namespace: objectMeta.GetNamespace(),
+			Name:      objectMeta.GetName(),
+			UID:       objectMeta.GetUID(),
 		},
 		Type:      eventType,
 		Message:   message,
@@ -738,15 +276,15 @@ func (client *MockClient) AnnotatedEventf(object runtime.Object, annotations map
 
 // SetPodIntoFailed sets a Pod into a failed status with the given reason
 func (client *MockClient) SetPodIntoFailed(ctx context.Context, object ctrlClient.Object, reason string) error {
-	data, err := json.Marshal(object)
+	existingObject := object.DeepCopyObject().(ctrlClient.Object)
+	err := client.Get(ctx, ctrlClient.ObjectKeyFromObject(object), existingObject)
 	if err != nil {
 		return err
 	}
 
-	pod := &corev1.Pod{}
-	err = json.Unmarshal(data, pod)
-	if err != nil {
-		return err
+	pod, ok := existingObject.(*corev1.Pod)
+	if !ok {
+		return fmt.Errorf("expected to get a corev1.Pod as input object, got: %s", object)
 	}
 
 	pod.Status.Phase = corev1.PodFailed
@@ -777,5 +315,4 @@ func (client *MockClient) generatePodIPv6() string {
 		return fmt.Sprintf("::%d", client.ipCounter)
 	}
 	return fmt.Sprintf("::%d:%d", client.ipCounter/256, client.ipCounter%256)
-
 }
diff --git a/pkg/fdbadminclient/admin_client.go b/pkg/fdbadminclient/admin_client.go
index e7825096..ff0054e7 100644
--- a/pkg/fdbadminclient/admin_client.go
+++ b/pkg/fdbadminclient/admin_client.go
@@ -102,4 +102,13 @@ type AdminClient interface {
 
 	// SetKnobs sets the knobs that should be used for the commandline call.
 	SetKnobs([]string)
+
+	// GetMaintenanceZone gets current maintenance zone, if any
+	GetMaintenanceZone() (string, error)
+
+	// SetMaintenanceZone places zone into maintenance mode
+	SetMaintenanceZone(zone string, timeoutSeconds int) error
+
+	// Reset maintenance mode
+	ResetMaintenanceMode() error
 }
diff --git a/setup/setup.go b/setup/setup.go
index b85d82ff..1125250d 100644
--- a/setup/setup.go
+++ b/setup/setup.go
@@ -83,7 +83,7 @@ func (o *Options) BindFlags(fs *flag.FlagSet) {
 		"Apply defaults from the next major version of the operator. This is only intended for use in development.",
 	)
 	fs.StringVar(&o.LogFile, "log-file", "", "The path to a file to write logs to.")
-	fs.IntVar(&o.CliTimeout, "cli-timeout", 10, "The timeout to use for CLI commands.")
+	fs.IntVar(&o.CliTimeout, "cli-timeout", 10, "The timeout to use for CLI commands in seconds.")
 	fs.IntVar(&o.MaxConcurrentReconciles, "max-concurrent-reconciles", 1, "Defines the maximum number of concurrent reconciles for all controllers.")
 	fs.BoolVar(&o.CleanUpOldLogFile, "cleanup-old-cli-logs", true, "Defines if the operator should delete old fdbcli log files.")
 	fs.DurationVar(&o.LogFileMinAge, "log-file-min-age", 5*time.Minute, "Defines the minimum age of fdbcli log files before removing when \"--cleanup-old-cli-logs\" is set.")
@@ -143,7 +143,7 @@ func StartManager(
 	klog.SetLogger(logger)
 
 	setupLog := logger.WithName("setup")
-	fdbclient.DefaultCLITimeout = operatorOpts.CliTimeout
+	fdbclient.DefaultCLITimeout = time.Duration(operatorOpts.CliTimeout) * time.Second
 
 	options := ctrl.Options{
 		Scheme:             scheme,


Test patch:
diff --git a/api/v1beta2/foundationdb_database_configuration_test.go b/api/v1beta2/foundationdb_database_configuration_test.go
index c404be82..ec63884b 100644
--- a/api/v1beta2/foundationdb_database_configuration_test.go
+++ b/api/v1beta2/foundationdb_database_configuration_test.go
@@ -157,4 +157,37 @@ var _ = Describe("DatabaseConfiguration", func() {
 			})
 		})
 	})
+
+	When("using ProcessCounts", func() {
+		When("calculating the total number of processes", func() {
+			var counts ProcessCounts
+
+			When("no negative process counts are specified", func() {
+				BeforeEach(func() {
+					counts = ProcessCounts{
+						Storage: 5,
+						Log:     5,
+					}
+				})
+
+				It("should print the correct number of total processes", func() {
+					Expect(counts.Total()).To(BeNumerically("==", 10))
+				})
+			})
+
+			When("negative process counts are specified", func() {
+				BeforeEach(func() {
+					counts = ProcessCounts{
+						Storage:   5,
+						Log:       5,
+						Stateless: -1,
+					}
+				})
+
+				It("should print the correct number of total processes", func() {
+					Expect(counts.Total()).To(BeNumerically("==", 10))
+				})
+			})
+		})
+	})
 })
diff --git a/api/v1beta2/foundationdbcluster_types_test.go b/api/v1beta2/foundationdbcluster_types_test.go
index b851f87f..1a55aee5 100644
--- a/api/v1beta2/foundationdbcluster_types_test.go
+++ b/api/v1beta2/foundationdbcluster_types_test.go
@@ -3141,6 +3141,7 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 				Expect(cluster.Status.Generations).To(Equal(ClusterGenerationStatus{
 					Reconciled:          1,
 					HasUnhealthyProcess: 2,
+					NeedsBounce:         2,
 				}))
 
 				cluster = createCluster()
@@ -3392,6 +3393,7 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 				Expect(cluster.Status.Generations).To(Equal(ClusterGenerationStatus{
 					Reconciled:          1,
 					HasUnhealthyProcess: 2,
+					NeedsBounce:         2,
 				}))
 
 				cluster = createCluster()
diff --git a/controllers/bounce_processes_test.go b/controllers/bounce_processes_test.go
index fbaccc28..9b50f89a 100644
--- a/controllers/bounce_processes_test.go
+++ b/controllers/bounce_processes_test.go
@@ -23,7 +23,6 @@ package controllers
 import (
 	"context"
 	"fmt"
-	"net"
 	"time"
 
 	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
@@ -382,111 +381,4 @@ var _ = Describe("bounceProcesses", func() {
 			})
 		})
 	})
-
-	DescribeTable("when getting the minimum uptime and the address map", func(cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, useRecoveryState bool, expectedMinimumUptime float64, expectedAddressMap map[string][]fdbv1beta2.ProcessAddress) {
-		minimumUptime, addressMap, err := getMinimumUptimeAndAddressMap(cluster, status, useRecoveryState)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(minimumUptime).To(BeNumerically("==", expectedMinimumUptime))
-		Expect(len(addressMap)).To(BeNumerically("==", len(expectedAddressMap)))
-		for key, value := range expectedAddressMap {
-			Expect(addressMap).To(HaveKeyWithValue(key, value))
-		}
-	},
-		Entry("when recovered since is not available",
-			&fdbv1beta2.FoundationDBCluster{
-				Spec: fdbv1beta2.FoundationDBClusterSpec{
-					Version: fdbv1beta2.Versions.Default.String(),
-				},
-			}, &fdbv1beta2.FoundationDBStatus{
-				Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
-					Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
-						"test": {
-							Address: fdbv1beta2.ProcessAddress{
-								IPAddress: net.ParseIP("127.0.0.1"),
-							},
-							Locality: map[string]string{
-								fdbv1beta2.FDBLocalityInstanceIDKey: "test",
-							},
-							UptimeSeconds: 30.0,
-						},
-					},
-					RecoveryState: fdbv1beta2.RecoveryState{
-						SecondsSinceLastRecovered: 90.0,
-					},
-				},
-			},
-			true,
-			30.0,
-			map[string][]fdbv1beta2.ProcessAddress{
-				"test": {
-					{
-						IPAddress: net.ParseIP("127.0.0.1"),
-					},
-				},
-			}),
-		Entry("when recovered since is enabled and version supports it",
-			&fdbv1beta2.FoundationDBCluster{
-				Spec: fdbv1beta2.FoundationDBClusterSpec{
-					Version: fdbv1beta2.Versions.SupportsRecoveryState.String(),
-				},
-			}, &fdbv1beta2.FoundationDBStatus{
-				Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
-					Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
-						"test": {
-							Address: fdbv1beta2.ProcessAddress{
-								IPAddress: net.ParseIP("127.0.0.1"),
-							},
-							Locality: map[string]string{
-								fdbv1beta2.FDBLocalityInstanceIDKey: "test",
-							},
-							UptimeSeconds: 30.0,
-						},
-					},
-					RecoveryState: fdbv1beta2.RecoveryState{
-						SecondsSinceLastRecovered: 90.0,
-					},
-				},
-			},
-			true,
-			90.0,
-			map[string][]fdbv1beta2.ProcessAddress{
-				"test": {
-					{
-						IPAddress: net.ParseIP("127.0.0.1"),
-					},
-				},
-			}),
-		Entry("when recovered since is disabled and version supports it",
-			&fdbv1beta2.FoundationDBCluster{
-				Spec: fdbv1beta2.FoundationDBClusterSpec{
-					Version: fdbv1beta2.Versions.SupportsRecoveryState.String(),
-				},
-			}, &fdbv1beta2.FoundationDBStatus{
-				Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
-					Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
-						"test": {
-							Address: fdbv1beta2.ProcessAddress{
-								IPAddress: net.ParseIP("127.0.0.1"),
-							},
-							Locality: map[string]string{
-								fdbv1beta2.FDBLocalityInstanceIDKey: "test",
-							},
-							UptimeSeconds: 30.0,
-						},
-					},
-					RecoveryState: fdbv1beta2.RecoveryState{
-						SecondsSinceLastRecovered: 90.0,
-					},
-				},
-			},
-			false,
-			30.0,
-			map[string][]fdbv1beta2.ProcessAddress{
-				"test": {
-					{
-						IPAddress: net.ParseIP("127.0.0.1"),
-					},
-				},
-			}),
-	)
 })
diff --git a/controllers/cluster_controller_test.go b/controllers/cluster_controller_test.go
index 5a91a368..a5925ebb 100644
--- a/controllers/cluster_controller_test.go
+++ b/controllers/cluster_controller_test.go
@@ -292,9 +292,11 @@ var _ = Describe("cluster_controller", func() {
 
 		When("converting a cluster to use unified images", func() {
 			BeforeEach(func() {
+				// There is a bug in the fake client that when updating the status the spec is updated.
+				cluster.Spec.MainContainer.ImageConfigs = nil
+				cluster.Spec.SidecarContainer.ImageConfigs = nil
 				cluster.Spec.UseUnifiedImage = pointer.Bool(true)
-				err = k8sClient.Update(context.TODO(), cluster)
-				Expect(err).NotTo(HaveOccurred())
+				Expect(k8sClient.Update(context.TODO(), cluster)).NotTo(HaveOccurred())
 			})
 
 			It("should update the pods", func() {
@@ -862,8 +864,7 @@ var _ = Describe("cluster_controller", func() {
 				Expect(processGroup.ProcessGroupConditions[0].ProcessGroupConditionType).To(Equal(fdbv1beta2.MissingProcesses))
 				Expect(processGroup.ProcessGroupConditions[0].Timestamp).NotTo(Equal(0))
 				processGroup.ProcessGroupConditions[0].Timestamp -= 3600
-				err = k8sClient.Status().Update(context.TODO(), cluster)
-				Expect(err).NotTo(HaveOccurred())
+				Expect(k8sClient.Status().Update(context.TODO(), cluster)).NotTo(HaveOccurred())
 
 				generationGap = 0
 			})
@@ -1740,8 +1741,7 @@ var _ = Describe("cluster_controller", func() {
 
 		Context("when enabling explicit listen addresses", func() {
 			BeforeEach(func() {
-				enabled := false
-				cluster.Spec.UseExplicitListenAddress = &enabled
+				cluster.Spec.UseExplicitListenAddress = pointer.Bool(false)
 				err = k8sClient.Update(context.TODO(), cluster)
 				Expect(err).NotTo(HaveOccurred())
 
@@ -1749,8 +1749,10 @@ var _ = Describe("cluster_controller", func() {
 				Expect(err).NotTo(HaveOccurred())
 				Expect(result.Requeue).To(BeFalse())
 
-				enabled = true
-				cluster.Spec.UseExplicitListenAddress = &enabled
+				err = k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(cluster), cluster)
+				Expect(err).NotTo(HaveOccurred())
+
+				cluster.Spec.UseExplicitListenAddress = pointer.Bool(true)
 				err = k8sClient.Update(context.TODO(), cluster)
 				Expect(err).NotTo(HaveOccurred())
 
@@ -2242,7 +2244,6 @@ var _ = Describe("cluster_controller", func() {
 					adminClient.MockClientVersion(fdbv1beta2.Versions.NextMajorVersion.String(), []string{"127.0.0.2:3687"})
 					err = k8sClient.Update(context.TODO(), cluster)
 					Expect(err).NotTo(HaveOccurred())
-
 				})
 
 				It("should not set a message about the client upgradability", func() {
@@ -2381,7 +2382,6 @@ var _ = Describe("cluster_controller", func() {
 		Context("with a change to the process group ID prefix", func() {
 			BeforeEach(func() {
 				cluster.Spec.ProcessGroupIDPrefix = "dev"
-
 				err = k8sClient.Update(context.TODO(), cluster)
 				Expect(err).NotTo(HaveOccurred())
 			})
@@ -2417,8 +2417,7 @@ var _ = Describe("cluster_controller", func() {
 
 		Context("when enabling a headless service", func() {
 			BeforeEach(func() {
-				var flag = true
-				cluster.Spec.Routing.HeadlessService = &flag
+				cluster.Spec.Routing.HeadlessService = pointer.Bool(true)
 				err = k8sClient.Update(context.TODO(), cluster)
 				Expect(err).NotTo(HaveOccurred())
 			})
@@ -2435,8 +2434,7 @@ var _ = Describe("cluster_controller", func() {
 
 		Context("when disabling a headless service", func() {
 			BeforeEach(func() {
-				var flag = true
-				cluster.Spec.Routing.HeadlessService = &flag
+				cluster.Spec.Routing.HeadlessService = pointer.Bool(true)
 				err = k8sClient.Update(context.TODO(), cluster)
 				Expect(err).NotTo(HaveOccurred())
 
@@ -2464,8 +2462,7 @@ var _ = Describe("cluster_controller", func() {
 		Context("with a lock deny list", func() {
 			BeforeEach(func() {
 				cluster.Spec.LockOptions.DenyList = append(cluster.Spec.LockOptions.DenyList, fdbv1beta2.LockDenyListEntry{ID: "dc2"})
-				var locksDisabled = false
-				cluster.Spec.LockOptions.DisableLocks = &locksDisabled
+				cluster.Spec.LockOptions.DisableLocks = pointer.Bool(false)
 				err = k8sClient.Update(context.TODO(), cluster)
 				Expect(err).NotTo(HaveOccurred())
 			})
@@ -2752,6 +2749,33 @@ var _ = Describe("cluster_controller", func() {
 			cluster.Status.ConnectionString = fakeConnectionString
 		})
 
+		Context("with a test process group", func() {
+			BeforeEach(func() {
+				conf, err = internal.GetMonitorConf(cluster, fdbv1beta2.ProcessClassTest, nil, cluster.GetStorageServersPerPod())
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			It("should generate the test conf", func() {
+				Expect(conf).To(Equal(strings.Join([]string{
+					"[general]",
+					"kill_on_configuration_change = false",
+					"restart_delay = 60",
+					"[fdbserver.1]",
+					"command = $BINARY_DIR/fdbserver",
+					"cluster_file = /var/fdb/data/fdb.cluster",
+					"seed_cluster_file = /var/dynamic-conf/fdb.cluster",
+					"public_address = $FDB_PUBLIC_IP:4501",
+					"class = test",
+					"logdir = /var/log/fdb-trace-logs",
+					"loggroup = " + cluster.Name,
+					"datadir = /var/fdb/data",
+					"locality_instance_id = $FDB_INSTANCE_ID",
+					"locality_machineid = $FDB_MACHINE_ID",
+					"locality_zoneid = $FDB_ZONE_ID",
+				}, "\n")))
+			})
+		})
+
 		Context("with a basic storage process group", func() {
 			BeforeEach(func() {
 				conf, err = internal.GetMonitorConf(cluster, fdbv1beta2.ProcessClassStorage, nil, cluster.GetStorageServersPerPod())
@@ -3387,7 +3411,6 @@ var _ = Describe("cluster_controller", func() {
 			})
 
 			Context("when only distributing across data centers", func() {
-
 				BeforeEach(func() {
 					result, err = chooseDistributedProcesses(cluster, candidates, 5, processSelectionConstraint{
 						Fields: []string{"dcid"},
diff --git a/controllers/delete_pods_for_buggification_test.go b/controllers/delete_pods_for_buggification_test.go
index f51f2546..f12fb1d0 100644
--- a/controllers/delete_pods_for_buggification_test.go
+++ b/controllers/delete_pods_for_buggification_test.go
@@ -167,6 +167,7 @@ var _ = Describe("delete_pods_for_buggification", func() {
 				err = k8sClient.Delete(context.TODO(), pod)
 				Expect(err).NotTo(HaveOccurred())
 
+				pod.ResourceVersion = ""
 				pod.Spec.Containers[0].Args = []string{"crash-loop"}
 				err = k8sClient.Create(context.TODO(), pod)
 				Expect(err).NotTo(HaveOccurred())
@@ -215,6 +216,7 @@ var _ = Describe("delete_pods_for_buggification", func() {
 					},
 				}
 
+				pod.ResourceVersion = ""
 				err = k8sClient.Create(context.TODO(), pod)
 				Expect(err).NotTo(HaveOccurred())
 			})
@@ -248,6 +250,7 @@ var _ = Describe("delete_pods_for_buggification", func() {
 				err = k8sClient.Delete(context.TODO(), pod)
 				Expect(err).NotTo(HaveOccurred())
 
+				pod.ResourceVersion = ""
 				pod.Spec.Containers[0].Args = []string{"crash-loop"}
 				err = k8sClient.Create(context.TODO(), pod)
 				Expect(err).NotTo(HaveOccurred())
@@ -295,6 +298,7 @@ var _ = Describe("delete_pods_for_buggification", func() {
 					},
 				}
 
+				pod.ResourceVersion = ""
 				err = k8sClient.Create(context.TODO(), pod)
 				Expect(err).NotTo(HaveOccurred())
 				cluster.Spec.Buggify.NoSchedule = []string{"storage-1"}
diff --git a/controllers/maintenance_mode_checker_test.go b/controllers/maintenance_mode_checker_test.go
new file mode 100644
index 00000000..baf26b8e
--- /dev/null
+++ b/controllers/maintenance_mode_checker_test.go
@@ -0,0 +1,154 @@
+/*
+ * maintenance_mode_checker_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controllers
+
+import (
+	"context"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
+
+	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+)
+
+var _ = Describe("maintenance_mode_checker", func() {
+	var cluster *fdbv1beta2.FoundationDBCluster
+	var err error
+	var requeue *requeue
+	var adminClient *mockAdminClient
+
+	BeforeEach(func() {
+		cluster = internal.CreateDefaultCluster()
+		//Set maintenance mode on
+		cluster.Spec.AutomationOptions.MaintenanceModeOptions.UseMaintenanceModeChecker = pointer.Bool(true)
+		err = k8sClient.Create(context.TODO(), cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		result, err := reconcileCluster(cluster)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.Requeue).To(BeFalse())
+
+		generation, err := reloadCluster(cluster)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(generation).To(Equal(int64(1)))
+
+		adminClient, err = newMockAdminClientUncast(cluster, k8sClient)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	JustBeforeEach(func() {
+		requeue = maintenanceModeChecker{}.reconcile(context.TODO(), clusterReconciler, cluster)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = reloadCluster(cluster)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	Context("maintenance mode is off", func() {
+		When("status maintenance is empty", func() {
+			It("should not be requeued", func() {
+				Expect(requeue).To(BeNil())
+				Expect(cluster.Status.MaintenanceModeInfo).To(Equal(fdbv1beta2.MaintenanceModeInfo{}))
+			})
+		})
+		When("status maintenance is not empty", func() {
+			BeforeEach(func() {
+				cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{ZoneID: "storage-4"}
+			})
+			It("should not be requeued", func() {
+				Expect(requeue).To(BeNil())
+				Expect(cluster.Status.MaintenanceModeInfo).To(Equal(fdbv1beta2.MaintenanceModeInfo{}))
+			})
+		})
+
+	})
+
+	Context("maintenance mode is on", func() {
+		BeforeEach(func() {
+			Expect(adminClient.SetMaintenanceZone("operator-test-1-storage-1", 0)).NotTo(HaveOccurred())
+		})
+
+		When("status maintenance zone does not match", func() {
+			BeforeEach(func() {
+				cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{ZoneID: "operator-test-1-storage-4"}
+			})
+			It("should not be requeued", func() {
+				Expect(requeue).To(BeNil())
+				Expect(cluster.Status.MaintenanceModeInfo).To(Equal(fdbv1beta2.MaintenanceModeInfo{}))
+			})
+		})
+
+		When("pod hasn't been updated yet", func() {
+			BeforeEach(func() {
+				cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{
+					StartTimestamp: &metav1.Time{Time: time.Now()},
+					ZoneID:         "operator-test-1-storage-1",
+					ProcessGroups:  []string{"storage-1"},
+				}
+				adminClient.MockUptimeSecondsForMaintenanceZone(3600)
+			})
+			It("should be requeued", func() {
+				Expect(requeue).ToNot(BeNil())
+				Expect(requeue.message).To(Equal("Waiting for pod storage-1 to be updated"))
+			})
+		})
+
+		When("pod is still down", func() {
+			BeforeEach(func() {
+				cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{
+					StartTimestamp: &metav1.Time{Time: time.Now()},
+					ZoneID:         "operator-test-1-storage-1",
+					ProcessGroups:  []string{"storage-1"},
+				}
+				adminClient.MockMissingProcessGroup("storage-1", true)
+			})
+			It("should be requeued", func() {
+				Expect(requeue).ToNot(BeNil())
+				Expect(requeue.message).To(Equal("Waiting for all proceeses in zone operator-test-1-storage-1 to be up"))
+			})
+		})
+
+		When("reconciliation should be successful", func() {
+			BeforeEach(func() {
+				cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{
+					StartTimestamp: &metav1.Time{Time: time.Now().Add(-60 * time.Second)},
+					ZoneID:         "operator-test-1-storage-1",
+					ProcessGroups:  []string{"storage-1"},
+				}
+
+			})
+			It("should not be requeued", func() {
+				Expect(requeue).To(BeNil())
+				Expect(cluster.Status.MaintenanceModeInfo).To(Equal(fdbv1beta2.MaintenanceModeInfo{}))
+			})
+		})
+	})
+
+	AfterEach(func() {
+		clearMockAdminClients()
+		k8sClient.Clear()
+	})
+})
diff --git a/controllers/remove_incompatible_processes_test.go b/controllers/remove_incompatible_processes_test.go
index 5c479848..f6054137 100644
--- a/controllers/remove_incompatible_processes_test.go
+++ b/controllers/remove_incompatible_processes_test.go
@@ -66,6 +66,23 @@ var _ = Describe("restart_incompatible_pods", func() {
 			true),
 	)
 
+	DescribeTable("when parsing incompatible connections", func(incompatibleConnections []string, expected map[string]fdbv1beta2.None) {
+		Expect(parseIncompatibleConnections(logr.Discard(), incompatibleConnections)).To(Equal(expected))
+	},
+		Entry("empty incompatible map",
+			[]string{},
+			map[string]fdbv1beta2.None{}),
+		Entry("nil incompatible map",
+			nil,
+			map[string]fdbv1beta2.None{}),
+		Entry("incompatible map contains one address",
+			[]string{"1.1.1.1:4500:tls"},
+			map[string]fdbv1beta2.None{"1.1.1.1": {}}),
+		Entry("incompatible map contains multiple addresses",
+			[]string{"1.1.1.1:4500:tls", "1.1.1.2:0:tls"},
+			map[string]fdbv1beta2.None{"1.1.1.1": {}}),
+	)
+
 	When("running a reconcile for the restart incompatible process reconciler", func() {
 		var cluster *fdbv1beta2.FoundationDBCluster
 		var initialCount int
@@ -94,7 +111,7 @@ var _ = Describe("restart_incompatible_pods", func() {
 			Expect(err).NotTo(HaveOccurred())
 		})
 
-		When("no incompatible processes are reported", func() {
+		When("the subreconciler is enabled", func() {
 			BeforeEach(func() {
 				clusterReconciler.EnableRestartIncompatibleProcesses = true
 				adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
@@ -106,55 +123,119 @@ var _ = Describe("restart_incompatible_pods", func() {
 				}
 			})
 
-			It("should have no deletions", func() {
-				pods := &corev1.PodList{}
-				err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(len(pods.Items)).To(BeNumerically("==", initialCount))
+			When("no incompatible processes are reported", func() {
+				BeforeEach(func() {
+					adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
+					Expect(err).NotTo(HaveOccurred())
+					adminClient.frozenStatus = &fdbv1beta2.FoundationDBStatus{
+						Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+							IncompatibleConnections: []string{},
+						},
+					}
+				})
+
+				It("should have no deletions", func() {
+					pods := &corev1.PodList{}
+					err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(len(pods.Items)).To(BeNumerically("==", initialCount))
+				})
 			})
-		})
 
-		When("no matching incompatible processes are reported", func() {
-			BeforeEach(func() {
-				clusterReconciler.EnableRestartIncompatibleProcesses = true
-				adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
-				Expect(err).NotTo(HaveOccurred())
-				adminClient.frozenStatus = &fdbv1beta2.FoundationDBStatus{
-					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
-						IncompatibleConnections: []string{
-							"192.192.192.192",
+			When("no matching incompatible processes are reported", func() {
+				BeforeEach(func() {
+					adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
+					Expect(err).NotTo(HaveOccurred())
+					adminClient.frozenStatus = &fdbv1beta2.FoundationDBStatus{
+						Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+							IncompatibleConnections: []string{
+								"192.192.192.192:4500:tls",
+							},
 						},
-					},
-				}
-			})
+					}
+				})
 
-			It("should have no deletions", func() {
-				pods := &corev1.PodList{}
-				err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(len(pods.Items)).To(BeNumerically("==", initialCount))
+				It("should have no deletions", func() {
+					pods := &corev1.PodList{}
+					err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(len(pods.Items)).To(BeNumerically("==", initialCount))
+				})
 			})
-		})
 
-		When("matching incompatible processes are reported and the subreconciler is enabled", func() {
-			BeforeEach(func() {
-				clusterReconciler.EnableRestartIncompatibleProcesses = true
-				adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
-				Expect(err).NotTo(HaveOccurred())
-				adminClient.frozenStatus = &fdbv1beta2.FoundationDBStatus{
-					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
-						IncompatibleConnections: []string{
-							cluster.Status.ProcessGroups[0].Addresses[0],
+			When("matching incompatible processes are reported", func() {
+				BeforeEach(func() {
+					adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
+					Expect(err).NotTo(HaveOccurred())
+					adminClient.frozenStatus = &fdbv1beta2.FoundationDBStatus{
+						Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+							DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+								Available: true,
+							},
 						},
-					},
-				}
-			})
+						Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+							FaultTolerance: fdbv1beta2.FaultTolerance{
+								MaxZoneFailuresWithoutLosingAvailability: 2,
+								MaxZoneFailuresWithoutLosingData:         2,
+							},
+							IncompatibleConnections: []string{
+								cluster.Status.ProcessGroups[0].Addresses[0] + ":4500:tls",
+							},
+						},
+					}
+				})
 
-			It("should have one deletion", func() {
-				pods := &corev1.PodList{}
-				err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(len(pods.Items)).To(BeNumerically("==", initialCount-1))
+				It("should have one deletion", func() {
+					pods := &corev1.PodList{}
+					err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(len(pods.Items)).To(BeNumerically("==", initialCount-1))
+				})
+
+				When("the database is unavailable", func() {
+					BeforeEach(func() {
+						adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
+						Expect(err).NotTo(HaveOccurred())
+						adminClient.frozenStatus.Client.DatabaseStatus.Available = false
+					})
+
+					It("should have no deletions", func() {
+						pods := &corev1.PodList{}
+						err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(len(pods.Items)).To(BeNumerically("==", initialCount))
+					})
+				})
+
+				When("matching incompatible processes are reported but port is zero", func() {
+					BeforeEach(func() {
+						adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
+						Expect(err).NotTo(HaveOccurred())
+						adminClient.frozenStatus.Cluster.IncompatibleConnections[0] = cluster.Status.ProcessGroups[0].Addresses[0] + ":0:tls"
+					})
+
+					It("should have no deletions", func() {
+						pods := &corev1.PodList{}
+						err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(len(pods.Items)).To(BeNumerically("==", initialCount))
+					})
+				})
+
+				When("the cluster is currently upgraded", func() {
+					BeforeEach(func() {
+						cluster.Spec.Version = fdbv1beta2.Versions.NextMajorVersion.String()
+						err := k8sClient.Update(context.Background(), cluster)
+						Expect(err).NotTo(HaveOccurred())
+					})
+
+					It("should have no deletions", func() {
+						pods := &corev1.PodList{}
+						err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(len(pods.Items)).To(BeNumerically("==", initialCount))
+					})
+				})
 			})
 		})
 
@@ -164,34 +245,18 @@ var _ = Describe("restart_incompatible_pods", func() {
 				adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
 				Expect(err).NotTo(HaveOccurred())
 				adminClient.frozenStatus = &fdbv1beta2.FoundationDBStatus{
-					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
-						IncompatibleConnections: []string{
-							cluster.Status.ProcessGroups[0].Addresses[0],
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
 						},
 					},
-				}
-			})
-
-			It("should have no deletions", func() {
-				pods := &corev1.PodList{}
-				err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(len(pods.Items)).To(BeNumerically("==", initialCount))
-			})
-		})
-
-		When("matching incompatible processes are reported and the cluster is currently upgraded", func() {
-			BeforeEach(func() {
-				cluster.Spec.Version = fdbv1beta2.Versions.NextMajorVersion.String()
-				err := k8sClient.Update(context.Background(), cluster)
-				Expect(err).NotTo(HaveOccurred())
-				clusterReconciler.EnableRestartIncompatibleProcesses = false
-				adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
-				Expect(err).NotTo(HaveOccurred())
-				adminClient.frozenStatus = &fdbv1beta2.FoundationDBStatus{
 					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						FaultTolerance: fdbv1beta2.FaultTolerance{
+							MaxZoneFailuresWithoutLosingAvailability: 2,
+							MaxZoneFailuresWithoutLosingData:         2,
+						},
 						IncompatibleConnections: []string{
-							cluster.Status.ProcessGroups[0].Addresses[0],
+							cluster.Status.ProcessGroups[0].Addresses[0] + ":4500:tls",
 						},
 					},
 				}
diff --git a/controllers/suite_test.go b/controllers/suite_test.go
index e73e50c6..dd2b08db 100644
--- a/controllers/suite_test.go
+++ b/controllers/suite_test.go
@@ -71,7 +71,7 @@ var _ = BeforeSuite(func() {
 	Expect(err).NotTo(HaveOccurred())
 
 	// +kubebuilder:scaffold:scheme
-	k8sClient = &mockclient.MockClient{}
+	k8sClient = mockclient.NewMockClient(scheme.Scheme)
 
 	clusterReconciler = createTestClusterReconciler()
 
diff --git a/controllers/update_status_test.go b/controllers/update_status_test.go
index 01cad27f..b8fcf4fa 100644
--- a/controllers/update_status_test.go
+++ b/controllers/update_status_test.go
@@ -44,6 +44,8 @@ var _ = Describe("update_status", func() {
 		var pods []*corev1.Pod
 		var processMap map[string][]fdbv1beta2.FoundationDBStatusProcessInfo
 		var err error
+		var allPods []*corev1.Pod
+		var allPvcs *corev1.PersistentVolumeClaimList
 
 		BeforeEach(func() {
 			cluster = internal.CreateDefaultCluster()
@@ -77,6 +79,13 @@ var _ = Describe("update_status", func() {
 				}
 				processMap[processID] = append(processMap[processID], process)
 			}
+
+			allPods, err = clusterReconciler.PodLifecycleManager.GetPods(context.TODO(), clusterReconciler, cluster, internal.GetPodListOptions(cluster, "", "")...)
+			Expect(err).NotTo(HaveOccurred())
+
+			allPvcs = &corev1.PersistentVolumeClaimList{}
+			err = clusterReconciler.List(context.TODO(), allPvcs, internal.GetPodListOptions(cluster, "", "")...)
+			Expect(err).NotTo(HaveOccurred())
 		})
 
 		When("process group has no Pod", func() {
@@ -84,7 +93,7 @@ var _ = Describe("update_status", func() {
 				processGroupStatus := fdbv1beta2.NewProcessGroupStatus("storage-1337", fdbv1beta2.ProcessClassStorage, []string{"1.1.1.1"})
 				// Reset the status to only tests for the missing Pod
 				processGroupStatus.ProcessGroupConditions = []*fdbv1beta2.ProcessGroupCondition{}
-				err := validateProcessGroup(context.TODO(), clusterReconciler, cluster, nil, "", processGroupStatus)
+				err := validateProcessGroup(context.TODO(), clusterReconciler, cluster, nil, nil, "", processGroupStatus)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(len(processGroupStatus.ProcessGroupConditions)).To(Equal(1))
 				Expect(processGroupStatus.ProcessGroupConditions[0].ProcessGroupConditionType).To(Equal(fdbv1beta2.MissingPod))
@@ -93,7 +102,7 @@ var _ = Describe("update_status", func() {
 
 		When("a process group is fine", func() {
 			It("should not get any condition assigned", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(len(processGroupStatus)).To(BeNumerically(">", 4))
 				processGroup := processGroupStatus[len(processGroupStatus)-4]
@@ -103,11 +112,12 @@ var _ = Describe("update_status", func() {
 		})
 
 		When("the pod for the process group is missing", func() {
-			It("should get a condition assigned", func() {
-				err = k8sClient.Delete(context.TODO(), pods[0])
-				Expect(err).NotTo(HaveOccurred())
+			BeforeEach(func() {
+				Expect(k8sClient.Delete(context.TODO(), pods[0])).NotTo(HaveOccurred())
+			})
 
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap)
+			It("should get a condition assigned", func() {
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs)
 				Expect(err).NotTo(HaveOccurred())
 
 				missingProcesses := fdbv1beta2.FilterByCondition(processGroupStatus, fdbv1beta2.MissingPod, false)
@@ -126,7 +136,7 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should get a condition assigned", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs)
 				Expect(err).NotTo(HaveOccurred())
 
 				incorrectProcesses := fdbv1beta2.FilterByCondition(processGroupStatus, fdbv1beta2.IncorrectCommandLine, false)
@@ -145,7 +155,7 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should get a condition assigned", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs)
 				Expect(err).NotTo(HaveOccurred())
 
 				missingProcesses := fdbv1beta2.FilterByCondition(processGroupStatus, fdbv1beta2.MissingProcesses, false)
@@ -166,7 +176,7 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should get a condition assigned", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs)
 				Expect(err).NotTo(HaveOccurred())
 
 				incorrectPods := fdbv1beta2.FilterByCondition(processGroupStatus, fdbv1beta2.IncorrectPodSpec, false)
@@ -187,7 +197,7 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should get a condition assigned", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs)
 				Expect(err).NotTo(HaveOccurred())
 
 				failingPods := fdbv1beta2.FilterByCondition(processGroupStatus, fdbv1beta2.PodFailing, false)
@@ -208,7 +218,7 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should get a condition assigned", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs)
 				Expect(err).NotTo(HaveOccurred())
 
 				failingPods := fdbv1beta2.FilterByCondition(processGroupStatus, fdbv1beta2.PodFailing, false)
@@ -233,7 +243,7 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should mark the process group for removal", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs)
 				Expect(err).NotTo(HaveOccurred())
 
 				removalCount := 0
@@ -264,7 +274,7 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should be mark the process group for removal without exclusion", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs)
 				Expect(err).NotTo(HaveOccurred())
 
 				removalCount := 0
@@ -288,14 +298,23 @@ var _ = Describe("update_status", func() {
 
 			// Must be a JustBeforeEach otherwise the cluster status will result in an error
 			JustBeforeEach(func() {
+				unreachablePod := pods[0]
 				unreachableProcessGroup = podmanager.GetProcessGroupID(cluster, pods[0])
-				pods[0].Annotations[internal.MockUnreachableAnnotation] = "banana"
-				err = k8sClient.Update(context.TODO(), pods[0])
-				Expect(err).NotTo(HaveOccurred())
+				unreachablePod.Annotations[internal.MockUnreachableAnnotation] = "banana"
+				Expect(k8sClient.Update(context.TODO(), unreachablePod)).NotTo(HaveOccurred())
+
+				// Update the Pod in out list, since we fetch the list before we set the annotation
+				for idx, pod := range allPods {
+					if pod.Name != unreachablePod.Name {
+						continue
+					}
+
+					allPods[idx] = unreachablePod
+				}
 			})
 
 			It("should mark the process group as unreachable", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs)
 				Expect(err).NotTo(HaveOccurred())
 
 				unreachableCount := 0
@@ -319,7 +338,7 @@ var _ = Describe("update_status", func() {
 				})
 
 				It("should remove the condition", func() {
-					processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap)
+					processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs)
 					Expect(err).NotTo(HaveOccurred())
 
 					unreachableCount := 0
@@ -347,7 +366,7 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should mark the process group as Pod pending", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs)
 				Expect(err).NotTo(HaveOccurred())
 
 				pendingCount := 0
diff --git a/e2e/README.md b/e2e/README.md
new file mode 100644
index 00000000..ecfdf66c
--- /dev/null
+++ b/e2e/README.md
@@ -0,0 +1,21 @@
+# e2e tests for FDB Kubernetes operator
+
+This folder contains test suites to run e2e tests against a [kind](https://kind.sigs.k8s.io) or a real cluster.
+
+## Running e2e tests
+
+The current setup assumes that it can use an already running Kubernetes cluster.
+If you don't have a running test cluster you can create one using [kind](https://kind.sigs.k8s.io)
+**Important** if you use an already existing Kubernetes cluster the e2e tests will apply the latest CRDs from the current branch you are running them and afterwards they will be removed.
+
+
+```bash
+$ kind create cluster
+$ kind get kubeconfig > ~/.kube/e2e_test
+```
+
+Now you can run the e2e tests against the local cluster:
+
+```bash
+go test -v ./e2e/... --tags=e2e_test --kubeconfig=${HOME}/.kube/e2e_test
+```
diff --git a/e2e/create_single_fdb_cluster_test.go b/e2e/create_single_fdb_cluster_test.go
new file mode 100644
index 00000000..c8e55b83
--- /dev/null
+++ b/e2e/create_single_fdb_cluster_test.go
@@ -0,0 +1,41 @@
+//go:build e2e_test
+
+/*
+ * create_single_fdb_cluster_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package e2e
+
+import (
+	"testing"
+
+	"github.com/FoundationDB/fdb-kubernetes-operator/e2e/helper"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+)
+
+func TestCreateSingleFDBCluster(t *testing.T) {
+	testVersions := helper.GetTestFDBVersions()
+	createSingleClusterFeatures := make([]features.Feature, 0, len(testVersions))
+
+	for _, version := range testVersions {
+		createSingleClusterFeatures = append(createSingleClusterFeatures, helper.CreateSingleClusterTest(version, t))
+	}
+
+	testenv.Test(t, createSingleClusterFeatures...)
+}
diff --git a/e2e/helper/fdb_cluster_helper.go b/e2e/helper/fdb_cluster_helper.go
new file mode 100644
index 00000000..c5c8409a
--- /dev/null
+++ b/e2e/helper/fdb_cluster_helper.go
@@ -0,0 +1,262 @@
+/*
+ * fdb_cluster_helper.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package helper
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"sigs.k8s.io/e2e-framework/klient/k8s"
+	"sigs.k8s.io/e2e-framework/klient/k8s/resources"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+)
+
+type contextKey int
+
+const (
+	keyNamespaceID contextKey = iota
+	keyClusterNameID
+)
+
+// GetTestFDBVersions returns a slice that contains all FDB versions we use to do e2e tests.
+func GetTestFDBVersions() []fdbv1beta2.Version {
+	return []fdbv1beta2.Version{
+		{Major: 6, Minor: 2, Patch: 20},
+		{Major: 6, Minor: 3, Patch: 23},
+		{Major: 7, Minor: 1, Patch: 23},
+	}
+}
+
+// CreateSingleClusterTest returns an e2e test that creates a cluster and assess that the cluster was correctly created.
+func CreateSingleClusterTest(version fdbv1beta2.Version, t *testing.T) features.Feature {
+	return features.
+		New("create single cluster for "+version.Compact()).
+		WithLabel("type", "create-cluster").
+		Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			clusterName := envconf.RandomName("fdb-create", 32)
+			namespace := ctx.Value(keyNamespaceID).(string)
+
+			testCluster := createFoundationDBCluster(clusterName, namespace, version.String())
+
+			err := cfg.Client().Resources(namespace).Create(ctx, testCluster)
+			if err != nil {
+				t.Error(err)
+			}
+
+			return context.WithValue(ctx, keyClusterNameID, clusterName)
+		}).
+		Assess("it should create an FDB cluster in "+version.Compact(), func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			namespace := ctx.Value(keyNamespaceID).(string)
+			clusterName := ctx.Value(keyClusterNameID).(string)
+
+			testCluster := &fdbv1beta2.FoundationDBCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      clusterName,
+					Namespace: namespace,
+				},
+			}
+
+			err := wait.For(conditions.New(cfg.Client().Resources(namespace)).ResourceMatch(testCluster, func(object k8s.Object) bool {
+				cluster := object.(*fdbv1beta2.FoundationDBCluster)
+				return cluster.Status.Generations.Reconciled >= 1
+			}), wait.WithTimeout(time.Minute*3))
+
+			if err != nil {
+				t.Error(err)
+			}
+
+			return ctx
+		}).
+		Assess("it should create all desired Pods", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			namespace := ctx.Value(keyNamespaceID).(string)
+			clusterName := ctx.Value(keyClusterNameID).(string)
+
+			testCluster := &fdbv1beta2.FoundationDBCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      clusterName,
+					Namespace: namespace,
+				},
+			}
+
+			err := cfg.Client().Resources(namespace).Get(ctx, testCluster.Name, testCluster.Namespace, testCluster)
+			if err != nil {
+				t.Error(err)
+				t.Fail()
+			}
+
+			podList := &corev1.PodList{}
+
+			err = cfg.Client().Resources(namespace).List(ctx, podList, resources.WithLabelSelector(labels.SelectorFromSet(testCluster.GetMatchLabels()).String()))
+			if err != nil {
+				t.Error(err)
+				t.Fail()
+			}
+
+			processCounts, err := testCluster.GetProcessCountsWithDefaults()
+			if err != nil {
+				t.Error(err)
+			}
+
+			var storageCount, statelessCount, logCount int
+			for _, pod := range podList.Items {
+				pClass, ok := pod.GetLabels()[testCluster.GetProcessClassLabel()]
+				if !ok {
+					continue
+				}
+
+				if pClass == string(fdbv1beta2.ProcessClassStorage) {
+					storageCount++
+					continue
+				} else if pClass == string(fdbv1beta2.ProcessClassLog) {
+					logCount++
+					continue
+				} else if pClass == string(fdbv1beta2.ProcessClassStateless) {
+					statelessCount++
+					continue
+				}
+			}
+
+			if storageCount != processCounts.Storage || statelessCount != processCounts.Stateless || logCount != processCounts.Log {
+				t.Errorf(`Expected to have matching process counts
+Expected Storage: %d, got: %d
+Expected Stateless: %d, got: %d
+Expected Log: %d, got: %d
+`,
+					processCounts.Storage, storageCount,
+					processCounts.Stateless, statelessCount,
+					processCounts.Log, logCount,
+				)
+			}
+
+			return ctx
+		}).
+		Teardown(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context {
+			namespace := ctx.Value(keyNamespaceID).(string)
+			clusterName := ctx.Value(keyClusterNameID).(string)
+
+			testCluster := &fdbv1beta2.FoundationDBCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      clusterName,
+					Namespace: namespace,
+				},
+			}
+
+			err := config.Client().Resources(namespace).Delete(ctx, testCluster)
+			if err != nil {
+				t.Error(err)
+			}
+
+			return ctx
+		}).Feature()
+}
+
+func createFoundationDBCluster(clusterName string, namespace string, version string) *fdbv1beta2.FoundationDBCluster {
+	return &fdbv1beta2.FoundationDBCluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      clusterName,
+			Namespace: namespace,
+		},
+		Spec: fdbv1beta2.FoundationDBClusterSpec{
+			FaultDomain: fdbv1beta2.FoundationDBClusterFaultDomain{
+				Key: "foundationdb.org/none",
+			},
+			MinimumUptimeSecondsForBounce: 60,
+			Version:                       version,
+			Processes: map[fdbv1beta2.ProcessClass]fdbv1beta2.ProcessSettings{
+				fdbv1beta2.ProcessClassGeneral: {
+					CustomParameters: fdbv1beta2.FoundationDBCustomParameters{
+						"knob_disable_posix_kernel_aio=1",
+					},
+					PodTemplate: &corev1.PodTemplateSpec{
+						Spec: corev1.PodSpec{
+							Containers: []corev1.Container{
+								{
+									Name: "foundationdb",
+									Resources: corev1.ResourceRequirements{
+										Requests: corev1.ResourceList{
+											"foundationdb.org/empty": resource.MustParse("0"),
+										},
+									},
+								},
+								{
+									Name: "foundationdb-kubernetes-sidecar",
+									Resources: corev1.ResourceRequirements{
+										Requests: corev1.ResourceList{
+											"foundationdb.org/empty": resource.MustParse("0"),
+										},
+									},
+								},
+							},
+							InitContainers: []corev1.Container{
+								{
+									Name: "foundationdb-kubernetes-init",
+									Resources: corev1.ResourceRequirements{
+										Requests: corev1.ResourceList{
+											"foundationdb.org/empty": resource.MustParse("0"),
+										},
+									},
+								},
+							},
+						},
+					},
+					VolumeClaimTemplate: &corev1.PersistentVolumeClaim{
+						Spec: corev1.PersistentVolumeClaimSpec{
+							Resources: corev1.ResourceRequirements{
+								Requests: corev1.ResourceList{
+									corev1.ResourceStorage: resource.MustParse("2G"),
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+// RegisterFDBScheme registers the fdbv1beta2 scheme to the rest config
+func RegisterFDBScheme(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
+	r, err := resources.New(cfg.Client().RESTConfig())
+	if err != nil {
+		return ctx, err
+	}
+
+	err = fdbv1beta2.AddToScheme(r.GetScheme())
+
+	return ctx, err
+}
+
+// WaitUntilCRDAvailable waits until the CRD is available
+func WaitUntilCRDAvailable(ctx context.Context, _ *envconf.Config) (context.Context, error) {
+	// TODO (johscheuer): Implement this logic instead of sleeping 1 second.
+	time.Sleep(1 * time.Second)
+
+	return ctx, nil
+}
diff --git a/e2e/helper/operator_helper.go b/e2e/helper/operator_helper.go
new file mode 100644
index 00000000..a91d5380
--- /dev/null
+++ b/e2e/helper/operator_helper.go
@@ -0,0 +1,91 @@
+/*
+ * operator_helper.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package helper
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/e2e-framework/klient/decoder"
+	"sigs.k8s.io/e2e-framework/klient/k8s"
+	"sigs.k8s.io/e2e-framework/klient/k8s/resources"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+)
+
+// InstallOperator is a helper method to install the FDB operator inside the test namespace.
+func InstallOperator(ctx context.Context, cfg *envconf.Config, _ *testing.T) (context.Context, error) {
+	r, err := resources.New(cfg.Client().RESTConfig())
+	if err != nil {
+		return ctx, err
+	}
+
+	ns := ctx.Value(keyNamespaceID).(string)
+
+	// TODO (johscheuer): Make the image a flag that can be passed
+	err = decoder.DecodeEachFile(ctx, os.DirFS("../config/samples"), "deployment.yaml",
+		decoder.CreateHandler(r),
+		decoder.MutateNamespace(ns),
+		mutateOperatorImage("manager", "foundationdb/fdb-kubernetes-operator:latest"),
+	)
+	if err != nil {
+		return ctx, err
+	}
+
+	deploy := &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "fdb-kubernetes-operator-controller-manager",
+			Namespace: ns,
+		},
+	}
+
+	err = wait.For(conditions.New(cfg.Client().Resources(ns)).ResourceMatch(deploy, func(object k8s.Object) bool {
+		d := object.(*appsv1.Deployment)
+		return d.Status.ReadyReplicas >= 1
+	}), wait.WithTimeout(time.Minute*2))
+
+	return ctx, err
+}
+
+func mutateOperatorImage(name string, image string) decoder.DecodeOption {
+	return decoder.MutateOption(func(obj k8s.Object) error {
+		deploy, ok := obj.(*appsv1.Deployment)
+		if !ok {
+			return nil
+		}
+
+		for idx, container := range deploy.Spec.Template.Spec.Containers {
+			if container.Name != name {
+				continue
+			}
+
+			container.Image = image
+			deploy.Spec.Template.Spec.Containers[idx] = container
+		}
+
+		return nil
+	})
+}
diff --git a/e2e/helper/test_setup_helper.go b/e2e/helper/test_setup_helper.go
new file mode 100644
index 00000000..9c055575
--- /dev/null
+++ b/e2e/helper/test_setup_helper.go
@@ -0,0 +1,39 @@
+package helper
+
+import (
+	"context"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+)
+
+// CreateNamespace ...
+func CreateNamespace(ctx context.Context, cfg *envconf.Config, t *testing.T, runID string) (context.Context, error) {
+	ns := envconf.RandomName(runID, 10)
+	ctx = context.WithValue(ctx, keyNamespaceID, ns)
+
+	t.Logf("Creating NS %v for test %v", ns, t.Name())
+	nsObj := corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: ns,
+		},
+	}
+
+	return ctx, cfg.Client().Resources().Create(ctx, &nsObj)
+}
+
+// DeleteNamespace ...
+func DeleteNamespace(ctx context.Context, cfg *envconf.Config, t *testing.T) (context.Context, error) {
+	ns := ctx.Value(keyNamespaceID).(string)
+	t.Logf("Deleting NS %v for test %v", ns, t.Name())
+
+	nsObj := corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: ns,
+		},
+	}
+
+	return ctx, cfg.Client().Resources().Delete(ctx, &nsObj)
+}
diff --git a/e2e/main_test.go b/e2e/main_test.go
new file mode 100644
index 00000000..f3502128
--- /dev/null
+++ b/e2e/main_test.go
@@ -0,0 +1,73 @@
+//go:build e2e_test
+
+/*
+ * main_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/FoundationDB/fdb-kubernetes-operator/e2e/helper"
+
+	// If testing with a cloud vendor managed cluster uncomment one of the below dependencies to properly get authorised.
+	//_ "k8s.io/client-go/plugin/pkg/client/auth/azure" // auth for AKS clusters
+	//_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"   // auth for GKE clusters
+	//_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"  // auth for OIDC
+	"os"
+	"testing"
+
+	"sigs.k8s.io/e2e-framework/pkg/env"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/envfuncs"
+)
+
+var testenv env.Environment
+
+func TestMain(m *testing.M) {
+	testenv = env.New()
+
+	cfg, err := envconf.NewFromFlags()
+	if err != nil {
+		_, _ = fmt.Fprintf(os.Stderr, "envconf failed: %s\n", err)
+		os.Exit(1)
+	}
+
+	testenv = env.NewWithConfig(cfg)
+
+	testenv.Setup(
+		envfuncs.SetupCRDs("../config/crd/bases", "*"),
+		helper.RegisterFDBScheme,
+		helper.WaitUntilCRDAvailable,
+	)
+
+	testenv.BeforeEachTest(func(ctx context.Context, cfg *envconf.Config, t *testing.T) (context.Context, error) {
+		return helper.CreateNamespace(ctx, cfg, t, envconf.RandomName("ns", 4))
+	}, helper.InstallOperator)
+
+	testenv.AfterEachTest(helper.DeleteNamespace)
+
+	testenv.Finish(
+		envfuncs.TeardownCRDs("../config/crd/bases", "*"),
+	)
+
+	os.Exit(testenv.Run(m))
+}
diff --git a/internal/fault_tolerance_test.go b/internal/fault_tolerance_test.go
index 3a5421fe..22574bae 100644
--- a/internal/fault_tolerance_test.go
+++ b/internal/fault_tolerance_test.go
@@ -21,8 +21,11 @@
 package internal
 
 import (
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	"github.com/go-logr/logr"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 )
 
 var _ = Describe("fault_tolerance", func() {
@@ -64,4 +67,104 @@ var _ = Describe("fault_tolerance", func() {
 				}),
 		)
 	})
+
+	When("checking if the cluster has the desired fault tolerance from the status", func() {
+		log := logr.New(logf.NewDelegatingLogSink(logf.NullLogSink{}))
+
+		DescribeTable("should return if the cluster has the desired fault tolerance",
+			func(status *fdbv1beta2.FoundationDBStatus, cluster *fdbv1beta2.FoundationDBCluster, expected bool) {
+				Expect(HasDesiredFaultToleranceFromStatus(log, status, cluster)).To(Equal(expected))
+			},
+			Entry("cluster is fully replicated",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						FaultTolerance: fdbv1beta2.FaultTolerance{
+							MaxZoneFailuresWithoutLosingData:         2,
+							MaxZoneFailuresWithoutLosingAvailability: 2,
+						},
+					},
+				},
+				&fdbv1beta2.FoundationDBCluster{
+					Spec: fdbv1beta2.FoundationDBClusterSpec{
+						DatabaseConfiguration: fdbv1beta2.DatabaseConfiguration{
+							RedundancyMode: fdbv1beta2.RedundancyModeTriple,
+						},
+					},
+				},
+				true),
+			Entry("database is unavailable",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: false,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						FaultTolerance: fdbv1beta2.FaultTolerance{
+							MaxZoneFailuresWithoutLosingData:         2,
+							MaxZoneFailuresWithoutLosingAvailability: 2,
+						},
+					},
+				},
+				&fdbv1beta2.FoundationDBCluster{
+					Spec: fdbv1beta2.FoundationDBClusterSpec{
+						DatabaseConfiguration: fdbv1beta2.DatabaseConfiguration{
+							RedundancyMode: fdbv1beta2.RedundancyModeTriple,
+						},
+					},
+				},
+				false),
+			Entry("data is degraded",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						FaultTolerance: fdbv1beta2.FaultTolerance{
+							MaxZoneFailuresWithoutLosingData:         1,
+							MaxZoneFailuresWithoutLosingAvailability: 2,
+						},
+					},
+				},
+				&fdbv1beta2.FoundationDBCluster{
+					Spec: fdbv1beta2.FoundationDBClusterSpec{
+						DatabaseConfiguration: fdbv1beta2.DatabaseConfiguration{
+							RedundancyMode: fdbv1beta2.RedundancyModeTriple,
+						},
+					},
+				},
+				false),
+			Entry("availability is degraded",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						FaultTolerance: fdbv1beta2.FaultTolerance{
+							MaxZoneFailuresWithoutLosingData:         2,
+							MaxZoneFailuresWithoutLosingAvailability: 1,
+						},
+					},
+				},
+				&fdbv1beta2.FoundationDBCluster{
+					Spec: fdbv1beta2.FoundationDBClusterSpec{
+						DatabaseConfiguration: fdbv1beta2.DatabaseConfiguration{
+							RedundancyMode: fdbv1beta2.RedundancyModeTriple,
+						},
+					},
+				},
+				false),
+		)
+	})
+
+	// HasDesiredFaultToleranceFromStatus
 })
diff --git a/internal/foundationdb_status_test.go b/internal/foundationdb_status_test.go
index 41b15fb7..eb6aced2 100644
--- a/internal/foundationdb_status_test.go
+++ b/internal/foundationdb_status_test.go
@@ -21,7 +21,9 @@
 package internal
 
 import (
-	"github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	"net"
+
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -29,7 +31,7 @@ import (
 var _ = Describe("Internal FoundationDBStatus", func() {
 	When("parsing the status for coordinators", func() {
 		type testCase struct {
-			status   *v1beta2.FoundationDBStatus
+			status   *fdbv1beta2.FoundationDBStatus
 			expected map[string]struct{}
 		}
 
@@ -40,19 +42,19 @@ var _ = Describe("Internal FoundationDBStatus", func() {
 			},
 			Entry("no coordinators",
 				testCase{
-					status:   &v1beta2.FoundationDBStatus{},
+					status:   &fdbv1beta2.FoundationDBStatus{},
 					expected: map[string]struct{}{},
 				}),
 			Entry("single coordinators",
 				testCase{
-					status: &v1beta2.FoundationDBStatus{
-						Cluster: v1beta2.FoundationDBStatusClusterInfo{
-							Processes: map[string]v1beta2.FoundationDBStatusProcessInfo{
+					status: &fdbv1beta2.FoundationDBStatus{
+						Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+							Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
 								"foo": {
 									Locality: map[string]string{
-										v1beta2.FDBLocalityInstanceIDKey: "foo",
+										fdbv1beta2.FDBLocalityInstanceIDKey: "foo",
 									},
-									Roles: []v1beta2.FoundationDBStatusProcessRoleInfo{
+									Roles: []fdbv1beta2.FoundationDBStatusProcessRoleInfo{
 										{
 											Role: "coordinator",
 										},
@@ -60,7 +62,7 @@ var _ = Describe("Internal FoundationDBStatus", func() {
 								},
 								"bar": {
 									Locality: map[string]string{
-										v1beta2.FDBLocalityInstanceIDKey: "bar",
+										fdbv1beta2.FDBLocalityInstanceIDKey: "bar",
 									},
 								},
 							},
@@ -72,14 +74,14 @@ var _ = Describe("Internal FoundationDBStatus", func() {
 				}),
 			Entry("multiple coordinators",
 				testCase{
-					status: &v1beta2.FoundationDBStatus{
-						Cluster: v1beta2.FoundationDBStatusClusterInfo{
-							Processes: map[string]v1beta2.FoundationDBStatusProcessInfo{
+					status: &fdbv1beta2.FoundationDBStatus{
+						Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+							Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
 								"foo": {
 									Locality: map[string]string{
-										v1beta2.FDBLocalityInstanceIDKey: "foo",
+										fdbv1beta2.FDBLocalityInstanceIDKey: "foo",
 									},
-									Roles: []v1beta2.FoundationDBStatusProcessRoleInfo{
+									Roles: []fdbv1beta2.FoundationDBStatusProcessRoleInfo{
 										{
 											Role: "coordinator",
 										},
@@ -87,9 +89,9 @@ var _ = Describe("Internal FoundationDBStatus", func() {
 								},
 								"bar": {
 									Locality: map[string]string{
-										v1beta2.FDBLocalityInstanceIDKey: "bar",
+										fdbv1beta2.FDBLocalityInstanceIDKey: "bar",
 									},
-									Roles: []v1beta2.FoundationDBStatusProcessRoleInfo{
+									Roles: []fdbv1beta2.FoundationDBStatusProcessRoleInfo{
 										{
 											Role: "coordinator",
 										},
@@ -105,4 +107,111 @@ var _ = Describe("Internal FoundationDBStatus", func() {
 				}),
 		)
 	})
+
+	DescribeTable("when getting the minimum uptime and the address map", func(cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, useRecoveryState bool, expectedMinimumUptime float64, expectedAddressMap map[string][]fdbv1beta2.ProcessAddress) {
+		minimumUptime, addressMap, err := GetMinimumUptimeAndAddressMap(cluster, status, useRecoveryState)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(minimumUptime).To(BeNumerically("==", expectedMinimumUptime))
+		Expect(len(addressMap)).To(BeNumerically("==", len(expectedAddressMap)))
+		for key, value := range expectedAddressMap {
+			Expect(addressMap).To(HaveKeyWithValue(key, value))
+		}
+	},
+		Entry("when recovered since is not available",
+			&fdbv1beta2.FoundationDBCluster{
+				Spec: fdbv1beta2.FoundationDBClusterSpec{
+					Version: fdbv1beta2.Versions.Default.String(),
+				},
+			}, &fdbv1beta2.FoundationDBStatus{
+				Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+					Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
+						"test": {
+							Address: fdbv1beta2.ProcessAddress{
+								IPAddress: net.ParseIP("127.0.0.1"),
+							},
+							Locality: map[string]string{
+								fdbv1beta2.FDBLocalityInstanceIDKey: "test",
+							},
+							UptimeSeconds: 30.0,
+						},
+					},
+					RecoveryState: fdbv1beta2.RecoveryState{
+						SecondsSinceLastRecovered: 90.0,
+					},
+				},
+			},
+			true,
+			30.0,
+			map[string][]fdbv1beta2.ProcessAddress{
+				"test": {
+					{
+						IPAddress: net.ParseIP("127.0.0.1"),
+					},
+				},
+			}),
+		Entry("when recovered since is enabled and version supports it",
+			&fdbv1beta2.FoundationDBCluster{
+				Spec: fdbv1beta2.FoundationDBClusterSpec{
+					Version: fdbv1beta2.Versions.SupportsRecoveryState.String(),
+				},
+			}, &fdbv1beta2.FoundationDBStatus{
+				Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+					Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
+						"test": {
+							Address: fdbv1beta2.ProcessAddress{
+								IPAddress: net.ParseIP("127.0.0.1"),
+							},
+							Locality: map[string]string{
+								fdbv1beta2.FDBLocalityInstanceIDKey: "test",
+							},
+							UptimeSeconds: 30.0,
+						},
+					},
+					RecoveryState: fdbv1beta2.RecoveryState{
+						SecondsSinceLastRecovered: 90.0,
+					},
+				},
+			},
+			true,
+			90.0,
+			map[string][]fdbv1beta2.ProcessAddress{
+				"test": {
+					{
+						IPAddress: net.ParseIP("127.0.0.1"),
+					},
+				},
+			}),
+		Entry("when recovered since is disabled and version supports it",
+			&fdbv1beta2.FoundationDBCluster{
+				Spec: fdbv1beta2.FoundationDBClusterSpec{
+					Version: fdbv1beta2.Versions.SupportsRecoveryState.String(),
+				},
+			}, &fdbv1beta2.FoundationDBStatus{
+				Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+					Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
+						"test": {
+							Address: fdbv1beta2.ProcessAddress{
+								IPAddress: net.ParseIP("127.0.0.1"),
+							},
+							Locality: map[string]string{
+								fdbv1beta2.FDBLocalityInstanceIDKey: "test",
+							},
+							UptimeSeconds: 30.0,
+						},
+					},
+					RecoveryState: fdbv1beta2.RecoveryState{
+						SecondsSinceLastRecovered: 90.0,
+					},
+				},
+			},
+			false,
+			30.0,
+			map[string][]fdbv1beta2.ProcessAddress{
+				"test": {
+					{
+						IPAddress: net.ParseIP("127.0.0.1"),
+					},
+				},
+			}),
+	)
 })
diff --git a/internal/monitor_conf_test.go b/internal/monitor_conf_test.go
index 23fc7210..f0eb9d03 100644
--- a/internal/monitor_conf_test.go
+++ b/internal/monitor_conf_test.go
@@ -730,6 +730,33 @@ var _ = Describe("monitor_conf", func() {
 			})
 		})
 
+		Context("with a test instance", func() {
+			BeforeEach(func() {
+				conf, err = GetMonitorConf(cluster, fdbv1beta2.ProcessClassTest, nil, cluster.GetStorageServersPerPod())
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			It("should generate the test conf", func() {
+				Expect(conf).To(Equal(strings.Join([]string{
+					"[general]",
+					"kill_on_configuration_change = false",
+					"restart_delay = 60",
+					"[fdbserver.1]",
+					"command = $BINARY_DIR/fdbserver",
+					"cluster_file = /var/fdb/data/fdb.cluster",
+					"seed_cluster_file = /var/dynamic-conf/fdb.cluster",
+					"public_address = $FDB_PUBLIC_IP:4501",
+					"class = test",
+					"logdir = /var/log/fdb-trace-logs",
+					"loggroup = " + cluster.Name,
+					"datadir = /var/fdb/data",
+					"locality_instance_id = $FDB_INSTANCE_ID",
+					"locality_machineid = $FDB_MACHINE_ID",
+					"locality_zoneid = $FDB_ZONE_ID",
+				}, "\n")))
+			})
+		})
+
 		Context("with DNS names enabled", func() {
 			BeforeEach(func() {
 				cluster.Spec.Routing.UseDNSInClusterFile = pointer.Bool(true)
diff --git a/kubectl-fdb/cmd/analyze_test.go b/kubectl-fdb/cmd/analyze_test.go
index 595e8a3d..1f2cbaba 100644
--- a/kubectl-fdb/cmd/analyze_test.go
+++ b/kubectl-fdb/cmd/analyze_test.go
@@ -22,20 +22,17 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"strings"
 	"time"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
 )
 
 func getCluster(clusterName string, namespace string, available bool, healthy bool, fullReplication bool, reconciled int64, processGroups []*fdbv1beta2.ProcessGroupStatus) *fdbv1beta2.FoundationDBCluster {
@@ -82,13 +79,9 @@ func getPodList(clusterName string, namespace string, status corev1.PodStatus, d
 			},
 		},
 	}
-
 }
 
 var _ = Describe("[plugin] analyze cluster", func() {
-	clusterName := "test"
-	namespace := "test"
-
 	When("analyzing the cluster", func() {
 		type testCase struct {
 			cluster           *fdbv1beta2.FoundationDBCluster
@@ -104,10 +97,11 @@ var _ = Describe("[plugin] analyze cluster", func() {
 
 		DescribeTable("return all successful and failed checks",
 			func(tc testCase) {
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-				kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(tc.cluster, tc.podList).Build()
+				Expect(k8sClient.Delete(context.TODO(), tc.cluster)).NotTo(HaveOccurred())
+				Expect(k8sClient.Create(context.TODO(), tc.cluster)).NotTo(HaveOccurred())
+				for _, pod := range tc.podList.Items {
+					Expect(k8sClient.Create(context.TODO(), &pod)).NotTo(HaveOccurred())
+				}
 
 				// We use these buffers to check the input/output
 				outBuffer := bytes.Buffer{}
@@ -115,7 +109,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				inBuffer := bytes.Buffer{}
 
 				cmd := newAnalyzeCmd(genericclioptions.IOStreams{In: &inBuffer, Out: &outBuffer, ErrOut: &errBuffer})
-				err := analyzeCluster(cmd, kubeClient, clusterName, namespace, tc.AutoFix, tc.NoWait, tc.IgnoredConditions, tc.IgnoreRemovals, 0)
+				err := analyzeCluster(cmd, k8sClient, clusterName, namespace, tc.AutoFix, tc.NoWait, tc.IgnoredConditions, tc.IgnoreRemovals, 0)
 
 				if err != nil && !tc.HasErrors {
 					Expect(err).To(HaveOccurred())
@@ -126,7 +120,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 			},
 			Entry("Cluster is fine",
 				testCase{
-					cluster: getCluster(clusterName, namespace, true, true, true, 0, []*fdbv1beta2.ProcessGroupStatus{
+					cluster: getCluster(clusterName, namespace, true, true, true, 1, []*fdbv1beta2.ProcessGroupStatus{
 						{ProcessGroupID: "instance-1"},
 					}),
 					podList: getPodList(clusterName, namespace, corev1.PodStatus{
@@ -145,7 +139,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				}),
 			Entry("Cluster is unavailable",
 				testCase{
-					cluster: getCluster(clusterName, namespace, false, true, true, 0, []*fdbv1beta2.ProcessGroupStatus{
+					cluster: getCluster(clusterName, namespace, false, true, true, 1, []*fdbv1beta2.ProcessGroupStatus{
 						{ProcessGroupID: "instance-1"},
 					}),
 					podList: getPodList(clusterName, namespace, corev1.PodStatus{
@@ -163,7 +157,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				}),
 			Entry("Cluster is unhealthy",
 				testCase{
-					cluster: getCluster(clusterName, namespace, true, false, true, 0, []*fdbv1beta2.ProcessGroupStatus{
+					cluster: getCluster(clusterName, namespace, true, false, true, 1, []*fdbv1beta2.ProcessGroupStatus{
 						{ProcessGroupID: "instance-1"},
 					}),
 					podList: getPodList(clusterName, namespace, corev1.PodStatus{
@@ -182,7 +176,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				}),
 			Entry("Cluster is not fully replicated",
 				testCase{
-					cluster: getCluster(clusterName, namespace, true, true, false, 0, []*fdbv1beta2.ProcessGroupStatus{
+					cluster: getCluster(clusterName, namespace, true, true, false, 1, []*fdbv1beta2.ProcessGroupStatus{
 						{ProcessGroupID: "instance-1"},
 					}),
 					podList: getPodList(clusterName, namespace, corev1.PodStatus{
@@ -200,7 +194,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				}),
 			Entry("Cluster is not reconciled",
 				testCase{
-					cluster: getCluster(clusterName, namespace, true, true, true, 1, []*fdbv1beta2.ProcessGroupStatus{
+					cluster: getCluster(clusterName, namespace, true, true, true, 0, []*fdbv1beta2.ProcessGroupStatus{
 						{ProcessGroupID: "instance-1"},
 					}),
 					podList: getPodList(clusterName, namespace, corev1.PodStatus{
@@ -218,7 +212,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				}),
 			Entry("ProcessGroup has a missing process",
 				testCase{
-					cluster: getCluster(clusterName, namespace, true, true, true, 0, []*fdbv1beta2.ProcessGroupStatus{
+					cluster: getCluster(clusterName, namespace, true, true, true, 1, []*fdbv1beta2.ProcessGroupStatus{
 						{
 							ProcessGroupID: "instance-1",
 							ProcessGroupConditions: []*fdbv1beta2.ProcessGroupCondition{
@@ -241,7 +235,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				}),
 			Entry("ProcessGroup has a missing process but is marked for removal",
 				testCase{
-					cluster: getCluster(clusterName, namespace, true, true, true, 0, []*fdbv1beta2.ProcessGroupStatus{
+					cluster: getCluster(clusterName, namespace, true, true, true, 1, []*fdbv1beta2.ProcessGroupStatus{
 						{
 							ProcessGroupID: "instance-1",
 							ProcessGroupConditions: []*fdbv1beta2.ProcessGroupCondition{
@@ -266,7 +260,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				}),
 			Entry("Pod is in Pending phase",
 				testCase{
-					cluster: getCluster(clusterName, namespace, true, true, true, 0, []*fdbv1beta2.ProcessGroupStatus{
+					cluster: getCluster(clusterName, namespace, true, true, true, 1, []*fdbv1beta2.ProcessGroupStatus{
 						{ProcessGroupID: "instance-1"},
 					}),
 					podList: getPodList(clusterName, namespace, corev1.PodStatus{
@@ -284,7 +278,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				}),
 			Entry("Container is in terminated state",
 				testCase{
-					cluster: getCluster(clusterName, namespace, true, true, true, 0, []*fdbv1beta2.ProcessGroupStatus{
+					cluster: getCluster(clusterName, namespace, true, true, true, 1, []*fdbv1beta2.ProcessGroupStatus{
 						{ProcessGroupID: "instance-1"},
 					}),
 					podList: getPodList(clusterName, namespace, corev1.PodStatus{
@@ -314,7 +308,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				}),
 			Entry("Container is in ready state",
 				testCase{
-					cluster: getCluster(clusterName, namespace, true, true, true, 0, []*fdbv1beta2.ProcessGroupStatus{
+					cluster: getCluster(clusterName, namespace, true, true, true, 1, []*fdbv1beta2.ProcessGroupStatus{
 						{ProcessGroupID: "instance-1"},
 					}),
 					podList: getPodList(clusterName, namespace, corev1.PodStatus{
@@ -339,7 +333,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				}),
 			Entry("Pod is stuck in terminating",
 				testCase{
-					cluster: getCluster(clusterName, namespace, true, true, true, 0, []*fdbv1beta2.ProcessGroupStatus{
+					cluster: getCluster(clusterName, namespace, true, true, true, 1, []*fdbv1beta2.ProcessGroupStatus{
 						{ProcessGroupID: "instance-1"},
 					}),
 					podList: getPodList(clusterName, namespace, corev1.PodStatus{
@@ -358,7 +352,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				}),
 			Entry("Pod is stuck in terminating and marked for removal",
 				testCase{
-					cluster: getCluster(clusterName, namespace, true, true, true, 0, []*fdbv1beta2.ProcessGroupStatus{
+					cluster: getCluster(clusterName, namespace, true, true, true, 1, []*fdbv1beta2.ProcessGroupStatus{
 						{
 							ProcessGroupID:     "instance-1",
 							RemovalTimestamp:   &metav1.Time{Time: time.Now()},
@@ -393,7 +387,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				}),
 			Entry("Pod is stuck in terminating and marked for removal and removals are ignored",
 				testCase{
-					cluster: getCluster(clusterName, namespace, true, true, true, 0, []*fdbv1beta2.ProcessGroupStatus{
+					cluster: getCluster(clusterName, namespace, true, true, true, 1, []*fdbv1beta2.ProcessGroupStatus{
 						{
 							ProcessGroupID:     "instance-1",
 							RemovalTimestamp:   &metav1.Time{Time: time.Now()},
@@ -428,7 +422,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				}),
 			Entry("Missing Pods",
 				testCase{
-					cluster: getCluster(clusterName, namespace, true, true, true, 0, []*fdbv1beta2.ProcessGroupStatus{
+					cluster: getCluster(clusterName, namespace, true, true, true, 1, []*fdbv1beta2.ProcessGroupStatus{
 						{ProcessGroupID: "instance-1"},
 					}),
 					podList:        &corev1.PodList{},
@@ -445,7 +439,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				}),
 			Entry("Missing entry in status",
 				testCase{
-					cluster: getCluster(clusterName, namespace, true, true, true, 0, []*fdbv1beta2.ProcessGroupStatus{}),
+					cluster: getCluster(clusterName, namespace, true, true, true, 1, []*fdbv1beta2.ProcessGroupStatus{}),
 					podList: getPodList(clusterName, namespace, corev1.PodStatus{
 						Phase: corev1.PodRunning,
 					}, nil),
@@ -461,7 +455,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				}),
 			Entry("ProcessGroup has a two conditions and we ignore one.",
 				testCase{
-					cluster: getCluster(clusterName, namespace, true, true, true, 0, []*fdbv1beta2.ProcessGroupStatus{
+					cluster: getCluster(clusterName, namespace, true, true, true, 1, []*fdbv1beta2.ProcessGroupStatus{
 						{
 							ProcessGroupID: "instance-1",
 							ProcessGroupConditions: []*fdbv1beta2.ProcessGroupCondition{
diff --git a/kubectl-fdb/cmd/buggify_crash_loop_test.go b/kubectl-fdb/cmd/buggify_crash_loop_test.go
index bc020eee..294f6885 100644
--- a/kubectl-fdb/cmd/buggify_crash_loop_test.go
+++ b/kubectl-fdb/cmd/buggify_crash_loop_test.go
@@ -21,39 +21,15 @@
 package cmd
 
 import (
-	ctx "context"
+	"context"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 var _ = Describe("[plugin] buggify crash-loop instances command", func() {
-	clusterName := "test"
-	namespace := "test"
-
-	var cluster *fdbv1beta2.FoundationDBCluster
-
-	BeforeEach(func() {
-		cluster = &fdbv1beta2.FoundationDBCluster{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      clusterName,
-				Namespace: namespace,
-			},
-			Spec: fdbv1beta2.FoundationDBClusterSpec{
-				ProcessCounts: fdbv1beta2.ProcessCounts{
-					Storage: 1,
-				},
-			},
-		}
-	})
-
 	When("running buggify crash-loop instances command", func() {
 		When("adding instances to crash-loop list from a cluster", func() {
 			type testCase struct {
@@ -63,16 +39,11 @@ var _ = Describe("[plugin] buggify crash-loop instances command", func() {
 
 			DescribeTable("should add all targeted processes to crash-loop list",
 				func(tc testCase) {
-					scheme := runtime.NewScheme()
-					_ = clientgoscheme.AddToScheme(scheme)
-					_ = fdbv1beta2.AddToScheme(scheme)
-					kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
-
-					err := updateCrashLoopList(kubeClient, clusterName, tc.Instances, namespace, false, false, false)
+					err := updateCrashLoopList(k8sClient, clusterName, tc.Instances, namespace, false, false, false)
 					Expect(err).NotTo(HaveOccurred())
 
 					var resCluster fdbv1beta2.FoundationDBCluster
-					err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+					err = k8sClient.Get(context.Background(), client.ObjectKey{
 						Namespace: namespace,
 						Name:      clusterName,
 					}, &resCluster)
@@ -93,28 +64,22 @@ var _ = Describe("[plugin] buggify crash-loop instances command", func() {
 			)
 
 			When("a process group was already in crash-loop", func() {
-				var kubeClient client.Client
-
-				BeforeEach(func() {
-					cluster.Spec.Buggify.CrashLoop = []string{"storage-1"}
-					scheme := runtime.NewScheme()
-					_ = clientgoscheme.AddToScheme(scheme)
-					_ = fdbv1beta2.AddToScheme(scheme)
-					kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
-				})
-
 				type testCase struct {
 					Instances                    []string
 					ExpectedInstancesInCrashLoop []string
 				}
 
+				BeforeEach(func() {
+					cluster.Spec.Buggify.CrashLoop = []string{"storage-1"}
+				})
+
 				DescribeTable("should add all targeted processes to crash-loop list",
 					func(tc testCase) {
-						err := updateCrashLoopList(kubeClient, clusterName, tc.Instances, namespace, false, false, false)
+						err := updateCrashLoopList(k8sClient, clusterName, tc.Instances, namespace, false, false, false)
 						Expect(err).NotTo(HaveOccurred())
 
 						var resCluster fdbv1beta2.FoundationDBCluster
-						err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+						err = k8sClient.Get(context.Background(), client.ObjectKey{
 							Namespace: namespace,
 							Name:      clusterName,
 						}, &resCluster)
@@ -142,14 +107,8 @@ var _ = Describe("[plugin] buggify crash-loop instances command", func() {
 		})
 
 		When("removing instances from crash-loop list from a cluster", func() {
-			var kubeClient client.Client
-
 			BeforeEach(func() {
 				cluster.Spec.Buggify.CrashLoop = []string{"storage-1", "storage-2", "storage-3"}
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-				kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
 			})
 
 			type testCase struct {
@@ -159,11 +118,11 @@ var _ = Describe("[plugin] buggify crash-loop instances command", func() {
 
 			DescribeTable("should remove all targeted processes from the crash-loop list",
 				func(tc testCase) {
-					err := updateCrashLoopList(kubeClient, clusterName, tc.Instances, namespace, false, true, false)
+					err := updateCrashLoopList(k8sClient, clusterName, tc.Instances, namespace, false, true, false)
 					Expect(err).NotTo(HaveOccurred())
 
 					var resCluster fdbv1beta2.FoundationDBCluster
-					err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+					err = k8sClient.Get(context.Background(), client.ObjectKey{
 						Namespace: namespace,
 						Name:      clusterName,
 					}, &resCluster)
@@ -186,22 +145,16 @@ var _ = Describe("[plugin] buggify crash-loop instances command", func() {
 		})
 
 		When("clearing crash-loop list", func() {
-			var kubeClient client.Client
-
 			BeforeEach(func() {
 				cluster.Spec.Buggify.CrashLoop = []string{"storage-1", "storage-2", "storage-3"}
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-				kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
 			})
 
 			It("should clear the crash-loop list", func() {
-				err := updateCrashLoopList(kubeClient, clusterName, []string{}, namespace, false, false, true)
+				err := updateCrashLoopList(k8sClient, clusterName, []string{}, namespace, false, false, true)
 				Expect(err).NotTo(HaveOccurred())
 
 				var resCluster fdbv1beta2.FoundationDBCluster
-				err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+				err = k8sClient.Get(context.Background(), client.ObjectKey{
 					Namespace: namespace,
 					Name:      clusterName,
 				}, &resCluster)
diff --git a/kubectl-fdb/cmd/buggify_empty-monitor_conf_test.go b/kubectl-fdb/cmd/buggify_empty-monitor_conf_test.go
index 703a0d05..887840a8 100644
--- a/kubectl-fdb/cmd/buggify_empty-monitor_conf_test.go
+++ b/kubectl-fdb/cmd/buggify_empty-monitor_conf_test.go
@@ -21,57 +21,27 @@
 package cmd
 
 import (
-	ctx "context"
-
-	"k8s.io/apimachinery/pkg/runtime"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"context"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 var _ = Describe("[plugin] buggify empty-monitor-conf instances command", func() {
-	clusterName := "test"
-	namespace := "test"
-
-	var cluster *fdbv1beta2.FoundationDBCluster
-
-	BeforeEach(func() {
-		cluster = &fdbv1beta2.FoundationDBCluster{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      clusterName,
-				Namespace: namespace,
-			},
-			Spec: fdbv1beta2.FoundationDBClusterSpec{
-				ProcessCounts: fdbv1beta2.ProcessCounts{
-					Storage: 1,
-				},
-			},
-		}
-	})
-
 	When("running buggify empty-monitor-conf instances command", func() {
-		var kubeClient client.Client
-
 		When("should update empty-monitor-conf", func() {
 			BeforeEach(func() {
 				cluster.Spec.Buggify.EmptyMonitorConf = false
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-				kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
 			})
 
 			It("setting monitor config to true", func() {
-				err := updateMonitorConf(kubeClient, clusterName, namespace, false, true)
+				err := updateMonitorConf(k8sClient, clusterName, namespace, false, true)
 				Expect(err).NotTo(HaveOccurred())
 
 				var resCluster fdbv1beta2.FoundationDBCluster
-				err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+				err = k8sClient.Get(context.Background(), client.ObjectKey{
 					Namespace: namespace,
 					Name:      clusterName,
 				}, &resCluster)
@@ -83,18 +53,14 @@ var _ = Describe("[plugin] buggify empty-monitor-conf instances command", func()
 		When("should update empty-monitor-conf", func() {
 			BeforeEach(func() {
 				cluster.Spec.Buggify.EmptyMonitorConf = true
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-				kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
 			})
 
 			It("setting monitor config to false", func() {
-				err := updateMonitorConf(kubeClient, clusterName, namespace, false, false)
+				err := updateMonitorConf(k8sClient, clusterName, namespace, false, false)
 				Expect(err).NotTo(HaveOccurred())
 
 				var resCluster fdbv1beta2.FoundationDBCluster
-				err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+				err = k8sClient.Get(context.Background(), client.ObjectKey{
 					Namespace: namespace,
 					Name:      clusterName,
 				}, &resCluster)
diff --git a/kubectl-fdb/cmd/buggify_no_schedule_test.go b/kubectl-fdb/cmd/buggify_no_schedule_test.go
index e1788323..e2ef4628 100644
--- a/kubectl-fdb/cmd/buggify_no_schedule_test.go
+++ b/kubectl-fdb/cmd/buggify_no_schedule_test.go
@@ -21,39 +21,15 @@
 package cmd
 
 import (
-	ctx "context"
+	"context"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 var _ = Describe("[plugin] buggify no-schedule instances command", func() {
-	clusterName := "test"
-	namespace := "test"
-
-	var cluster *fdbv1beta2.FoundationDBCluster
-
-	BeforeEach(func() {
-		cluster = &fdbv1beta2.FoundationDBCluster{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      clusterName,
-				Namespace: namespace,
-			},
-			Spec: fdbv1beta2.FoundationDBClusterSpec{
-				ProcessCounts: fdbv1beta2.ProcessCounts{
-					Storage: 1,
-				},
-			},
-		}
-	})
-
 	When("running buggify no-schedule instances command", func() {
 		When("adding instances to no-schedule list from a cluster", func() {
 			type testCase struct {
@@ -63,16 +39,11 @@ var _ = Describe("[plugin] buggify no-schedule instances command", func() {
 
 			DescribeTable("should add all targeted processes to no-schedule list",
 				func(tc testCase) {
-					scheme := runtime.NewScheme()
-					_ = clientgoscheme.AddToScheme(scheme)
-					_ = fdbv1beta2.AddToScheme(scheme)
-					kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
-
-					err := updateNoScheduleList(kubeClient, clusterName, tc.Instances, namespace, false, false, false)
+					err := updateNoScheduleList(k8sClient, clusterName, tc.Instances, namespace, false, false, false)
 					Expect(err).NotTo(HaveOccurred())
 
 					var resCluster fdbv1beta2.FoundationDBCluster
-					err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+					err = k8sClient.Get(context.Background(), client.ObjectKey{
 						Namespace: namespace,
 						Name:      clusterName,
 					}, &resCluster)
@@ -93,14 +64,8 @@ var _ = Describe("[plugin] buggify no-schedule instances command", func() {
 			)
 
 			When("a process group is already in no-schedule", func() {
-				var kubeClient client.Client
-
 				BeforeEach(func() {
 					cluster.Spec.Buggify.NoSchedule = []string{"storage-1"}
-					scheme := runtime.NewScheme()
-					_ = clientgoscheme.AddToScheme(scheme)
-					_ = fdbv1beta2.AddToScheme(scheme)
-					kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
 				})
 
 				type testCase struct {
@@ -110,11 +75,11 @@ var _ = Describe("[plugin] buggify no-schedule instances command", func() {
 
 				DescribeTable("should add all targeted processes to no-schedule list",
 					func(tc testCase) {
-						err := updateNoScheduleList(kubeClient, clusterName, tc.Instances, namespace, false, false, false)
+						err := updateNoScheduleList(k8sClient, clusterName, tc.Instances, namespace, false, false, false)
 						Expect(err).NotTo(HaveOccurred())
 
 						var resCluster fdbv1beta2.FoundationDBCluster
-						err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+						err = k8sClient.Get(context.Background(), client.ObjectKey{
 							Namespace: namespace,
 							Name:      clusterName,
 						}, &resCluster)
@@ -142,14 +107,8 @@ var _ = Describe("[plugin] buggify no-schedule instances command", func() {
 		})
 
 		When("removing process group from no-schedule list from a cluster", func() {
-			var kubeClient client.Client
-
 			BeforeEach(func() {
 				cluster.Spec.Buggify.NoSchedule = []string{"storage-1", "storage-2", "storage-3"}
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-				kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
 			})
 
 			type testCase struct {
@@ -159,11 +118,11 @@ var _ = Describe("[plugin] buggify no-schedule instances command", func() {
 
 			DescribeTable("should remove all targeted processes from the no-schedule list",
 				func(tc testCase) {
-					err := updateNoScheduleList(kubeClient, clusterName, tc.Instances, namespace, false, true, false)
+					err := updateNoScheduleList(k8sClient, clusterName, tc.Instances, namespace, false, true, false)
 					Expect(err).NotTo(HaveOccurred())
 
 					var resCluster fdbv1beta2.FoundationDBCluster
-					err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+					err = k8sClient.Get(context.Background(), client.ObjectKey{
 						Namespace: namespace,
 						Name:      clusterName,
 					}, &resCluster)
@@ -186,22 +145,16 @@ var _ = Describe("[plugin] buggify no-schedule instances command", func() {
 		})
 
 		When("clearing no-schedule list", func() {
-			var kubeClient client.Client
-
 			BeforeEach(func() {
 				cluster.Spec.Buggify.NoSchedule = []string{"storage-1", "storage-2", "storage-3"}
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-				kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
 			})
 
 			It("should clear the no-schedule list", func() {
-				err := updateNoScheduleList(kubeClient, clusterName, nil, namespace, false, false, true)
+				err := updateNoScheduleList(k8sClient, clusterName, nil, namespace, false, false, true)
 				Expect(err).NotTo(HaveOccurred())
 
 				var resCluster fdbv1beta2.FoundationDBCluster
-				err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+				err = k8sClient.Get(context.Background(), client.ObjectKey{
 					Namespace: namespace,
 					Name:      clusterName,
 				}, &resCluster)
diff --git a/kubectl-fdb/cmd/configuration_test.go b/kubectl-fdb/cmd/configuration_test.go
index a634ee08..fe8d28fe 100644
--- a/kubectl-fdb/cmd/configuration_test.go
+++ b/kubectl-fdb/cmd/configuration_test.go
@@ -21,23 +21,18 @@
 package cmd
 
 import (
-	ctx "context"
+	"context"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 )
 
 var _ = Describe("[plugin] configuration command", func() {
 	When("getting the configuration string", func() {
 		When("using a single region cluster", func() {
-			var cluster *fdbv1beta2.FoundationDBCluster
-
 			BeforeEach(func() {
 				cluster = &fdbv1beta2.FoundationDBCluster{
 					ObjectMeta: metav1.ObjectMeta{
@@ -63,31 +58,19 @@ var _ = Describe("[plugin] configuration command", func() {
 			})
 
 			It("should return the configuration string", func() {
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-				kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
-
-				configuration, err := getConfigurationString(kubeClient, "test", "test", false)
+				configuration, err := getConfigurationString(k8sClient, "test", "test", false)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(configuration).To(Equal("  usable_regions=0 logs=0 resolvers=0 log_routers=0 remote_logs=0 proxies=3 regions=[{\\\"datacenters\\\":[{\\\"id\\\":\\\"test\\\",\\\"priority\\\":1}]}]"))
 			})
 
 			It("should return the same configuration string with fail-over", func() {
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-				kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
-
-				configuration, err := getConfigurationString(kubeClient, "test", "test", true)
+				configuration, err := getConfigurationString(k8sClient, "test", "test", true)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(configuration).To(Equal("  usable_regions=0 logs=0 resolvers=0 log_routers=0 remote_logs=0 proxies=3 regions=[{\\\"datacenters\\\":[{\\\"id\\\":\\\"test\\\",\\\"priority\\\":1}]}]"))
 			})
 		})
 
 		When("using a multi region cluster", func() {
-			var cluster *fdbv1beta2.FoundationDBCluster
-
 			BeforeEach(func() {
 				cluster = &fdbv1beta2.FoundationDBCluster{
 					ObjectMeta: metav1.ObjectMeta{
@@ -146,39 +129,24 @@ var _ = Describe("[plugin] configuration command", func() {
 			})
 
 			It("should return the configuration string", func() {
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-				kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
-
-				configuration, err := getConfigurationString(kubeClient, "test", "test", false)
+				configuration, err := getConfigurationString(k8sClient, "test", "test", false)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(configuration).To(Equal("  usable_regions=0 logs=0 resolvers=0 log_routers=0 remote_logs=0 proxies=3 regions=[{\\\"datacenters\\\":[{\\\"id\\\":\\\"primary\\\",\\\"priority\\\":1},{\\\"id\\\":\\\"primary-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1},{\\\"id\\\":\\\"remote-sat\\\",\\\"satellite\\\":1}],\\\"satellite_logs\\\":3,\\\"satellite_redundancy_mode\\\":\\\"one_satellite_single\\\"},{\\\"datacenters\\\":[{\\\"id\\\":\\\"remote\\\"},{\\\"id\\\":\\\"remote-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1},{\\\"id\\\":\\\"primary-sat\\\",\\\"satellite\\\":1}],\\\"satellite_logs\\\":3,\\\"satellite_redundancy_mode\\\":\\\"one_satellite_double\\\"}]"))
 			})
 
 			It("should return the configuration string with the modified priority", func() {
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-				kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
-
-				configuration, err := getConfigurationString(kubeClient, "test", "test", true)
+				configuration, err := getConfigurationString(k8sClient, "test", "test", true)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(configuration).To(Equal("  usable_regions=0 logs=0 resolvers=0 log_routers=0 remote_logs=0 proxies=3 regions=[{\\\"datacenters\\\":[{\\\"id\\\":\\\"primary\\\"},{\\\"id\\\":\\\"primary-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1},{\\\"id\\\":\\\"remote-sat\\\",\\\"satellite\\\":1}],\\\"satellite_logs\\\":3,\\\"satellite_redundancy_mode\\\":\\\"one_satellite_single\\\"},{\\\"datacenters\\\":[{\\\"id\\\":\\\"remote\\\",\\\"priority\\\":1},{\\\"id\\\":\\\"remote-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1},{\\\"id\\\":\\\"primary-sat\\\",\\\"satellite\\\":1}],\\\"satellite_logs\\\":3,\\\"satellite_redundancy_mode\\\":\\\"one_satellite_double\\\"}]"))
 			})
 
 			When("Updating the config for the cluster", func() {
 				It("should update the config with the modified priority", func() {
-					scheme := runtime.NewScheme()
-					_ = clientgoscheme.AddToScheme(scheme)
-					_ = fdbv1beta2.AddToScheme(scheme)
-					kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
-
-					err := updateConfig(kubeClient, "test", "test", true, false)
+					err := updateConfig(k8sClient, "test", "test", true, false)
 					Expect(err).NotTo(HaveOccurred())
 
 					var resCluster fdbv1beta2.FoundationDBCluster
-					err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+					err = k8sClient.Get(context.Background(), client.ObjectKey{
 						Namespace: "test",
 						Name:      "test",
 					}, &resCluster)
diff --git a/kubectl-fdb/cmd/cordon_test.go b/kubectl-fdb/cmd/cordon_test.go
index 1f675325..ea8a7c95 100644
--- a/kubectl-fdb/cmd/cordon_test.go
+++ b/kubectl-fdb/cmd/cordon_test.go
@@ -21,31 +21,19 @@
 package cmd
 
 import (
-	ctx "context"
-
-	"k8s.io/apimachinery/pkg/api/equality"
+	"context"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
 )
 
 var _ = Describe("[plugin] cordon command", func() {
 	When("running cordon command", func() {
-		clusterName := "test"
-		namespace := "test"
-
-		var cluster fdbv1beta2.FoundationDBCluster
-		var nodeList corev1.NodeList
-		var podList corev1.PodList
-
 		type testCase struct {
 			nodes                                     []string
 			WithExclusion                             bool
@@ -54,68 +42,49 @@ var _ = Describe("[plugin] cordon command", func() {
 		}
 
 		BeforeEach(func() {
-			cluster = fdbv1beta2.FoundationDBCluster{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      clusterName,
-					Namespace: namespace,
-				},
-				Spec: fdbv1beta2.FoundationDBClusterSpec{
-					ProcessCounts: fdbv1beta2.ProcessCounts{
-						Storage: 1,
+			pods := []corev1.Pod{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "instance-1",
+						Namespace: namespace,
+						Labels: map[string]string{
+							fdbv1beta2.FDBProcessClassLabel:   string(fdbv1beta2.ProcessClassStorage),
+							fdbv1beta2.FDBClusterLabel:        clusterName,
+							fdbv1beta2.FDBProcessGroupIDLabel: "instance-1",
+						},
+					},
+					Spec: corev1.PodSpec{
+						NodeName: "node-1",
 					},
 				},
-			}
-
-			nodeList = corev1.NodeList{
-				Items: []corev1.Node{},
-			}
-
-			podList = corev1.PodList{
-				Items: []corev1.Pod{
-					{
-						ObjectMeta: metav1.ObjectMeta{
-							Name:      "instance-1",
-							Namespace: namespace,
-							Labels: map[string]string{
-								fdbv1beta2.FDBProcessClassLabel:   string(fdbv1beta2.ProcessClassStorage),
-								fdbv1beta2.FDBClusterLabel:        clusterName,
-								fdbv1beta2.FDBProcessGroupIDLabel: "instance-1",
-							},
-						},
-						Spec: corev1.PodSpec{
-							NodeName: "node-1",
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "instance-2",
+						Namespace: namespace,
+						Labels: map[string]string{
+							fdbv1beta2.FDBProcessClassLabel:   string(fdbv1beta2.ProcessClassStorage),
+							fdbv1beta2.FDBClusterLabel:        clusterName,
+							fdbv1beta2.FDBProcessGroupIDLabel: "instance-2",
 						},
 					},
-					{
-						ObjectMeta: metav1.ObjectMeta{
-							Name:      "instance-2",
-							Namespace: namespace,
-							Labels: map[string]string{
-								fdbv1beta2.FDBProcessClassLabel:   string(fdbv1beta2.ProcessClassStorage),
-								fdbv1beta2.FDBClusterLabel:        clusterName,
-								fdbv1beta2.FDBProcessGroupIDLabel: "instance-2",
-							},
-						},
-						Spec: corev1.PodSpec{
-							NodeName: "node-2",
-						},
+					Spec: corev1.PodSpec{
+						NodeName: "node-2",
 					},
 				},
 			}
+
+			for _, pod := range pods {
+				Expect(k8sClient.Create(context.TODO(), &pod)).NotTo(HaveOccurred())
+			}
 		})
 
 		DescribeTable("should cordon all targeted processes",
 			func(input testCase) {
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-				kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(&cluster, &podList, &nodeList).Build()
-
-				err := cordonNode(kubeClient, &cluster, input.nodes, namespace, input.WithExclusion, false, 0)
+				err := cordonNode(k8sClient, cluster, input.nodes, namespace, input.WithExclusion, false, 0)
 				Expect(err).NotTo(HaveOccurred())
 
 				var resCluster fdbv1beta2.FoundationDBCluster
-				err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+				err = k8sClient.Get(context.Background(), client.ObjectKey{
 					Namespace: namespace,
 					Name:      clusterName,
 				}, &resCluster)
diff --git a/kubectl-fdb/cmd/deprecation_test.go b/kubectl-fdb/cmd/deprecation_test.go
index e952d139..09f68d35 100644
--- a/kubectl-fdb/cmd/deprecation_test.go
+++ b/kubectl-fdb/cmd/deprecation_test.go
@@ -22,6 +22,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"strings"
 
 	"k8s.io/utils/pointer"
@@ -34,20 +35,13 @@ import (
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 var _ = Describe("[plugin] deprecation command", func() {
 	When("running the deprecation command", func() {
-		clusterName := "test"
-		namespace := "test"
-
 		type testCase struct {
 			cluster            fdbv1beta2.FoundationDBCluster
 			inputClusters      []string
@@ -65,13 +59,11 @@ var _ = Describe("[plugin] deprecation command", func() {
 				errBuffer := bytes.Buffer{}
 				inBuffer := bytes.Buffer{}
 
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-				kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(&tc.cluster).Build()
+				Expect(k8sClient.Delete(context.TODO(), &tc.cluster)).NotTo(HaveOccurred())
+				Expect(k8sClient.Create(context.TODO(), &tc.cluster)).NotTo(HaveOccurred())
 
 				cmd := newDeprecationCmd(genericclioptions.IOStreams{In: &inBuffer, Out: &outBuffer, ErrOut: &errBuffer})
-				err := checkDeprecation(cmd, kubeClient, tc.inputClusters, namespace, tc.deprecationOptions, tc.showClusterSpec)
+				err := checkDeprecation(cmd, k8sClient, tc.inputClusters, namespace, tc.deprecationOptions, tc.showClusterSpec)
 				if tc.expectedError != "" {
 					Expect(err).To(HaveOccurred())
 					Expect(err.Error()).To(Equal(tc.expectedError))
diff --git a/kubectl-fdb/cmd/exec_test.go b/kubectl-fdb/cmd/exec_test.go
index 51a9eae2..4a81adf7 100644
--- a/kubectl-fdb/cmd/exec_test.go
+++ b/kubectl-fdb/cmd/exec_test.go
@@ -21,25 +21,17 @@
 package cmd
 
 import (
-	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"context"
 
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 var _ = Describe("[plugin] exec command", func() {
 	When("running exec command", func() {
-		clusterName := "test"
-		namespace := "test"
-
-		var cluster fdbv1beta2.FoundationDBCluster
-		var podList corev1.PodList
-
 		type testCase struct {
 			ClusterName   string
 			Context       string
@@ -49,42 +41,21 @@ var _ = Describe("[plugin] exec command", func() {
 		}
 
 		BeforeEach(func() {
-			cluster = fdbv1beta2.FoundationDBCluster{
+			Expect(k8sClient.Create(context.TODO(), &corev1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
-					Name:      clusterName,
+					Name:      "instance-1",
 					Namespace: namespace,
-				},
-				Spec: fdbv1beta2.FoundationDBClusterSpec{
-					ProcessCounts: fdbv1beta2.ProcessCounts{
-						Storage: 1,
-					},
-				},
-			}
-
-			podList = corev1.PodList{
-				Items: []corev1.Pod{
-					{
-						ObjectMeta: metav1.ObjectMeta{
-							Name:      "instance-1",
-							Namespace: namespace,
-							Labels: map[string]string{
-								fdbv1beta2.FDBProcessClassLabel: string(fdbv1beta2.ProcessClassStorage),
-								fdbv1beta2.FDBClusterLabel:      clusterName,
-							},
-						},
+					Labels: map[string]string{
+						fdbv1beta2.FDBProcessClassLabel: string(fdbv1beta2.ProcessClassStorage),
+						fdbv1beta2.FDBClusterLabel:      clusterName,
 					},
 				},
-			}
+			})).NotTo(HaveOccurred())
 		})
 
 		DescribeTable("should execute the provided command",
 			func(input testCase) {
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-				kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(&cluster, &podList).Build()
-
-				command, err := buildCommand(kubeClient, &cluster, input.Context, namespace, input.Command)
+				command, err := buildCommand(k8sClient, cluster, input.Context, namespace, input.Command)
 
 				if input.ExpectedError != "" {
 					Expect(err).To(HaveOccurred())
diff --git a/kubectl-fdb/cmd/fix_coordinator_ips_test.go b/kubectl-fdb/cmd/fix_coordinator_ips_test.go
index 687527b4..e8de9a8d 100644
--- a/kubectl-fdb/cmd/fix_coordinator_ips_test.go
+++ b/kubectl-fdb/cmd/fix_coordinator_ips_test.go
@@ -21,25 +21,17 @@
 package cmd
 
 import (
-	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"context"
 
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 var _ = Describe("[plugin] fix-coordinator-ips command", func() {
 	When("building cluster file update commands", func() {
-		clusterName := "test"
-		namespace := "test"
-
-		var cluster fdbv1beta2.FoundationDBCluster
-		var podList corev1.PodList
-
 		type testCase struct {
 			Context          string
 			ExpectedCommands [][]string
@@ -47,43 +39,22 @@ var _ = Describe("[plugin] fix-coordinator-ips command", func() {
 		}
 
 		BeforeEach(func() {
-			cluster = fdbv1beta2.FoundationDBCluster{
+			Expect(k8sClient.Create(context.TODO(), &corev1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
-					Name:      clusterName,
+					Name:      "instance-1",
 					Namespace: namespace,
-				},
-				Spec: fdbv1beta2.FoundationDBClusterSpec{
-					ProcessCounts: fdbv1beta2.ProcessCounts{
-						Storage: 1,
-					},
-				},
-			}
-
-			podList = corev1.PodList{
-				Items: []corev1.Pod{
-					{
-						ObjectMeta: metav1.ObjectMeta{
-							Name:      "instance-1",
-							Namespace: namespace,
-							Labels: map[string]string{
-								fdbv1beta2.FDBProcessClassLabel: string(fdbv1beta2.ProcessClassStorage),
-								fdbv1beta2.FDBClusterLabel:      clusterName,
-							},
-						},
+					Labels: map[string]string{
+						fdbv1beta2.FDBProcessClassLabel: string(fdbv1beta2.ProcessClassStorage),
+						fdbv1beta2.FDBClusterLabel:      clusterName,
 					},
 				},
-			}
+			})).NotTo(HaveOccurred())
 		})
 
 		DescribeTable("should execute the provided command",
 			func(input testCase) {
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-				kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(&cluster, &podList).Build()
 				cluster.Status.ConnectionString = "test:test@127.0.0.1:4501"
-
-				commands, err := buildClusterFileUpdateCommands(&cluster, kubeClient, input.Context, namespace, "/usr/local/bin/kubectl")
+				commands, err := buildClusterFileUpdateCommands(cluster, k8sClient, input.Context, namespace, "/usr/local/bin/kubectl")
 
 				if input.ExpectedError != "" {
 					Expect(err).To(HaveOccurred())
@@ -141,11 +112,6 @@ var _ = Describe("[plugin] fix-coordinator-ips command", func() {
 		)
 	})
 	When("updating the connection string", func() {
-		clusterName := "test"
-		namespace := "test"
-
-		var cluster fdbv1beta2.FoundationDBCluster
-
 		type testCase struct {
 			Context                  string
 			ExpectedConnectionString string
@@ -154,7 +120,7 @@ var _ = Describe("[plugin] fix-coordinator-ips command", func() {
 		}
 
 		BeforeEach(func() {
-			cluster = fdbv1beta2.FoundationDBCluster{
+			cluster = &fdbv1beta2.FoundationDBCluster{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      clusterName,
 					Namespace: namespace,
@@ -179,10 +145,6 @@ var _ = Describe("[plugin] fix-coordinator-ips command", func() {
 
 		DescribeTable("should execute the provided command",
 			func(input testCase) {
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-
 				for processGroupID, address := range input.AddressUpdates {
 					for _, processGroup := range cluster.Status.ProcessGroups {
 						if processGroup.ProcessGroupID == processGroupID {
@@ -194,7 +156,7 @@ var _ = Describe("[plugin] fix-coordinator-ips command", func() {
 						}
 					}
 				}
-				err := updateIPsInConnectionString(&cluster)
+				err := updateIPsInConnectionString(cluster)
 
 				if input.ExpectedError != "" {
 					Expect(err).To(HaveOccurred())
diff --git a/kubectl-fdb/cmd/k8s_client_test.go b/kubectl-fdb/cmd/k8s_client_test.go
index 8114ba36..9ff34c74 100644
--- a/kubectl-fdb/cmd/k8s_client_test.go
+++ b/kubectl-fdb/cmd/k8s_client_test.go
@@ -21,14 +21,12 @@
 package cmd
 
 import (
+	"context"
 	"time"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -36,13 +34,8 @@ import (
 
 var _ = Describe("[plugin] using the Kubernetes client", func() {
 	When("fetching processes with conditions", func() {
-		clusterName := "test"
-		namespace := "test"
-		var cluster fdbv1beta2.FoundationDBCluster
-		var podList corev1.PodList
-
 		BeforeEach(func() {
-			cluster = fdbv1beta2.FoundationDBCluster{
+			cluster = &fdbv1beta2.FoundationDBCluster{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      clusterName,
 					Namespace: namespace,
@@ -87,66 +80,68 @@ var _ = Describe("[plugin] using the Kubernetes client", func() {
 				},
 			}
 
-			podList = corev1.PodList{
-				Items: []corev1.Pod{
-					{
-						ObjectMeta: metav1.ObjectMeta{
-							Name:      "instance-1",
-							Namespace: namespace,
-							Labels: map[string]string{
-								fdbv1beta2.FDBProcessClassLabel:   string(fdbv1beta2.ProcessClassStorage),
-								fdbv1beta2.FDBClusterLabel:        clusterName,
-								fdbv1beta2.FDBProcessGroupIDLabel: "instance-1",
-							},
-						},
-						Status: corev1.PodStatus{
-							Phase: corev1.PodRunning,
+			pods := []corev1.Pod{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "instance-1",
+						Namespace: namespace,
+						Labels: map[string]string{
+							fdbv1beta2.FDBProcessClassLabel:   string(fdbv1beta2.ProcessClassStorage),
+							fdbv1beta2.FDBClusterLabel:        clusterName,
+							fdbv1beta2.FDBProcessGroupIDLabel: "instance-1",
 						},
 					},
-					{
-						ObjectMeta: metav1.ObjectMeta{
-							Name:      "instance-2",
-							Namespace: namespace,
-							Labels: map[string]string{
-								fdbv1beta2.FDBProcessClassLabel:   string(fdbv1beta2.ProcessClassStorage),
-								fdbv1beta2.FDBClusterLabel:        clusterName,
-								fdbv1beta2.FDBProcessGroupIDLabel: "instance-2",
-							},
-						},
-						Status: corev1.PodStatus{
-							Phase: corev1.PodRunning,
-						},
+					Status: corev1.PodStatus{
+						Phase: corev1.PodRunning,
 					},
-					{
-						ObjectMeta: metav1.ObjectMeta{
-							Name:      "instance-3",
-							Namespace: namespace,
-							Labels: map[string]string{
-								fdbv1beta2.FDBProcessClassLabel:   string(fdbv1beta2.ProcessClassStorage),
-								fdbv1beta2.FDBClusterLabel:        clusterName,
-								fdbv1beta2.FDBProcessGroupIDLabel: "instance-3",
-							},
-						},
-						Status: corev1.PodStatus{
-							Phase: corev1.PodFailed,
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "instance-2",
+						Namespace: namespace,
+						Labels: map[string]string{
+							fdbv1beta2.FDBProcessClassLabel:   string(fdbv1beta2.ProcessClassStorage),
+							fdbv1beta2.FDBClusterLabel:        clusterName,
+							fdbv1beta2.FDBProcessGroupIDLabel: "instance-2",
 						},
 					},
-					{
-						ObjectMeta: metav1.ObjectMeta{
-							Name:      "instance-4",
-							Namespace: namespace,
-							Labels: map[string]string{
-								fdbv1beta2.FDBProcessClassLabel:   string(fdbv1beta2.ProcessClassStorage),
-								fdbv1beta2.FDBClusterLabel:        clusterName,
-								fdbv1beta2.FDBProcessGroupIDLabel: "instance-4",
-							},
+					Status: corev1.PodStatus{
+						Phase: corev1.PodRunning,
+					},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "instance-3",
+						Namespace: namespace,
+						Labels: map[string]string{
+							fdbv1beta2.FDBProcessClassLabel:   string(fdbv1beta2.ProcessClassStorage),
+							fdbv1beta2.FDBClusterLabel:        clusterName,
+							fdbv1beta2.FDBProcessGroupIDLabel: "instance-3",
 						},
-						Status: corev1.PodStatus{
-							Phase: corev1.PodRunning,
+					},
+					Status: corev1.PodStatus{
+						Phase: corev1.PodFailed,
+					},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "instance-4",
+						Namespace: namespace,
+						Labels: map[string]string{
+							fdbv1beta2.FDBProcessClassLabel:   string(fdbv1beta2.ProcessClassStorage),
+							fdbv1beta2.FDBClusterLabel:        clusterName,
+							fdbv1beta2.FDBProcessGroupIDLabel: "instance-4",
 						},
 					},
+					Status: corev1.PodStatus{
+						Phase: corev1.PodRunning,
+					},
 				},
 			}
+
+			for _, pod := range pods {
+				Expect(k8sClient.Create(context.TODO(), &pod)).NotTo(HaveOccurred())
+			}
 		})
 
 		type testCase struct {
@@ -156,12 +151,7 @@ var _ = Describe("[plugin] using the Kubernetes client", func() {
 
 		DescribeTable("should show all deprecations",
 			func(tc testCase) {
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-				kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(&cluster, &podList).Build()
-
-				pods, err := getAllPodsFromClusterWithCondition(kubeClient, clusterName, namespace, tc.conditions)
+				pods, err := getAllPodsFromClusterWithCondition(k8sClient, clusterName, namespace, tc.conditions)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(pods).To(Equal(tc.expected))
 			},
@@ -184,28 +174,10 @@ var _ = Describe("[plugin] using the Kubernetes client", func() {
 	})
 
 	When("getting the process groups IDs from Pods", func() {
-		clusterName := "test"
-		namespace := "test"
-		var cluster fdbv1beta2.FoundationDBCluster
-
 		When("the cluster doesn't have a prefix", func() {
-			BeforeEach(func() {
-				cluster = fdbv1beta2.FoundationDBCluster{
-					ObjectMeta: metav1.ObjectMeta{
-						Name:      clusterName,
-						Namespace: namespace,
-					},
-					Spec: fdbv1beta2.FoundationDBClusterSpec{
-						ProcessCounts: fdbv1beta2.ProcessCounts{
-							Storage: 1,
-						},
-					},
-				}
-			})
-
 			DescribeTable("should get all process groups IDs",
 				func(podNames []string, expected []string) {
-					instances, err := getProcessGroupIDsFromPodName(&cluster, podNames)
+					instances, err := getProcessGroupIDsFromPodName(cluster, podNames)
 					Expect(err).NotTo(HaveOccurred())
 					Expect(instances).To(ContainElements(expected))
 					Expect(len(instances)).To(BeNumerically("==", len(expected)))
@@ -227,7 +199,7 @@ var _ = Describe("[plugin] using the Kubernetes client", func() {
 
 		When("the cluster has a prefix", func() {
 			BeforeEach(func() {
-				cluster = fdbv1beta2.FoundationDBCluster{
+				cluster = &fdbv1beta2.FoundationDBCluster{
 					ObjectMeta: metav1.ObjectMeta{
 						Name:      clusterName,
 						Namespace: namespace,
@@ -243,7 +215,7 @@ var _ = Describe("[plugin] using the Kubernetes client", func() {
 
 			DescribeTable("should get all process groups IDs",
 				func(podNames []string, expected []string) {
-					instances, err := getProcessGroupIDsFromPodName(&cluster, podNames)
+					instances, err := getProcessGroupIDsFromPodName(cluster, podNames)
 					Expect(err).NotTo(HaveOccurred())
 					Expect(instances).To(ContainElements(expected))
 					Expect(len(instances)).To(BeNumerically("==", len(expected)))
diff --git a/kubectl-fdb/cmd/profile_analyzer_test.go b/kubectl-fdb/cmd/profile_analyzer_test.go
index e15d5c42..7358268e 100644
--- a/kubectl-fdb/cmd/profile_analyzer_test.go
+++ b/kubectl-fdb/cmd/profile_analyzer_test.go
@@ -24,29 +24,18 @@ import (
 	batchv1 "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
-	ctx "context"
+	"context"
 )
 
 var _ = Describe("profile analyser", func() {
 	When("running profile analyzer with 6.3", func() {
-		clusterName := "test"
-		namespace := "test"
-		scheme := runtime.NewScheme()
-		var kubeClient client.Client
-
 		BeforeEach(func() {
-			_ = clientgoscheme.AddToScheme(scheme)
-			_ = fdbv1beta2.AddToScheme(scheme)
-			cluster := &fdbv1beta2.FoundationDBCluster{
+			cluster = &fdbv1beta2.FoundationDBCluster{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      clusterName,
 					Namespace: namespace,
@@ -55,18 +44,18 @@ var _ = Describe("profile analyser", func() {
 					Version: "6.3.24",
 				},
 			}
-			kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
 		})
+
 		It("should match the command args", func() {
 			expectedEnv := v1.EnvVar{
 				Name:      "FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY",
 				Value:     "/usr/bin/fdb/6.3.24/lib/",
 				ValueFrom: nil,
 			}
-			err := runProfileAnalyzer(kubeClient, namespace, clusterName, "21:30 08/24/2022 BST", "22:30 08/24/2022 BST", 100, "../../sample-apps/fdb-profile-analyzer/sample_template.yaml")
+			err := runProfileAnalyzer(k8sClient, namespace, clusterName, "21:30 08/24/2022 BST", "22:30 08/24/2022 BST", 100, "../../sample-apps/fdb-profile-analyzer/sample_template.yaml")
 			Expect(err).NotTo(HaveOccurred())
 			job := &batchv1.Job{}
-			err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+			err = k8sClient.Get(context.Background(), client.ObjectKey{
 				Namespace: namespace,
 				Name:      "test-hot-shard-tool",
 			}, job)
@@ -76,16 +65,10 @@ var _ = Describe("profile analyser", func() {
 			Expect(job.Spec.Template.Spec.Containers[0].Env).To(ContainElements(expectedEnv))
 		})
 	})
-	When("running profile analyzer with 7.1", func() {
-		clusterName := "test"
-		namespace := "test"
-		scheme := runtime.NewScheme()
-		var kubeClient client.Client
 
+	When("running profile analyzer with 7.1", func() {
 		BeforeEach(func() {
-			_ = clientgoscheme.AddToScheme(scheme)
-			_ = fdbv1beta2.AddToScheme(scheme)
-			cluster := &fdbv1beta2.FoundationDBCluster{
+			cluster = &fdbv1beta2.FoundationDBCluster{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      clusterName,
 					Namespace: namespace,
@@ -94,18 +77,18 @@ var _ = Describe("profile analyser", func() {
 					Version: "7.1.19",
 				},
 			}
-			kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
 		})
+
 		It("should match the command args", func() {
 			expectedEnv := v1.EnvVar{
 				Name:      "FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY",
 				Value:     "/usr/bin/fdb/7.1.19/lib/",
 				ValueFrom: nil,
 			}
-			err := runProfileAnalyzer(kubeClient, namespace, clusterName, "21:30 08/24/2022 BST", "22:30 08/24/2022 BST", 100, "../../sample-apps/fdb-profile-analyzer/sample_template.yaml")
+			err := runProfileAnalyzer(k8sClient, namespace, clusterName, "21:30 08/24/2022 BST", "22:30 08/24/2022 BST", 100, "../../sample-apps/fdb-profile-analyzer/sample_template.yaml")
 			Expect(err).NotTo(HaveOccurred())
 			job := &batchv1.Job{}
-			err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+			err = k8sClient.Get(context.Background(), client.ObjectKey{
 				Namespace: namespace,
 				Name:      "test-hot-shard-tool",
 			}, job)
diff --git a/kubectl-fdb/cmd/remove_process_group_test.go b/kubectl-fdb/cmd/remove_process_group_test.go
index 99eecb5a..1cff83ab 100644
--- a/kubectl-fdb/cmd/remove_process_group_test.go
+++ b/kubectl-fdb/cmd/remove_process_group_test.go
@@ -21,39 +21,15 @@
 package cmd
 
 import (
-	ctx "context"
+	"context"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 var _ = Describe("[plugin] remove process groups command", func() {
-	clusterName := "test"
-	namespace := "test"
-
-	var cluster *fdbv1beta2.FoundationDBCluster
-
-	BeforeEach(func() {
-		cluster = &fdbv1beta2.FoundationDBCluster{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      clusterName,
-				Namespace: namespace,
-			},
-			Spec: fdbv1beta2.FoundationDBClusterSpec{
-				ProcessCounts: fdbv1beta2.ProcessCounts{
-					Storage: 1,
-				},
-			},
-		}
-	})
-
 	When("running remove process groups command", func() {
 		When("removing process groups from a cluster", func() {
 			BeforeEach(func() {
@@ -84,16 +60,11 @@ var _ = Describe("[plugin] remove process groups command", func() {
 
 			DescribeTable("should cordon all targeted processes",
 				func(tc testCase) {
-					scheme := runtime.NewScheme()
-					_ = clientgoscheme.AddToScheme(scheme)
-					_ = fdbv1beta2.AddToScheme(scheme)
-					kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
-
-					err := replaceProcessGroups(kubeClient, clusterName, tc.Instances, namespace, tc.WithExclusion, false, tc.RemoveAllFailed, false, 0)
+					err := replaceProcessGroups(k8sClient, clusterName, tc.Instances, namespace, tc.WithExclusion, false, tc.RemoveAllFailed, false, 0)
 					Expect(err).NotTo(HaveOccurred())
 
 					var resCluster fdbv1beta2.FoundationDBCluster
-					err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+					err = k8sClient.Get(context.Background(), client.ObjectKey{
 						Namespace: namespace,
 						Name:      clusterName,
 					}, &resCluster)
@@ -139,24 +110,18 @@ var _ = Describe("[plugin] remove process groups command", func() {
 			)
 
 			When("a process group was already marked for removal", func() {
-				var kubeClient client.Client
-
 				BeforeEach(func() {
 					cluster.Spec.ProcessGroupsToRemove = []string{"storage-1"}
-					scheme := runtime.NewScheme()
-					_ = clientgoscheme.AddToScheme(scheme)
-					_ = fdbv1beta2.AddToScheme(scheme)
-					kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
 				})
 
 				When("adding the same process group to the removal list without exclusion", func() {
 					It("should add the process group to the removal without exclusion list", func() {
 						removals := []string{"test-storage-1"}
-						err := replaceProcessGroups(kubeClient, clusterName, removals, namespace, false, false, false, false, 0)
+						err := replaceProcessGroups(k8sClient, clusterName, removals, namespace, false, false, false, false, 0)
 						Expect(err).NotTo(HaveOccurred())
 
 						var resCluster fdbv1beta2.FoundationDBCluster
-						err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+						err = k8sClient.Get(context.Background(), client.ObjectKey{
 							Namespace: namespace,
 							Name:      clusterName,
 						}, &resCluster)
@@ -172,11 +137,11 @@ var _ = Describe("[plugin] remove process groups command", func() {
 				When("adding the same process group to the removal list", func() {
 					It("should add the process group to the removal without exclusion list", func() {
 						removals := []string{"test-storage-1"}
-						err := replaceProcessGroups(kubeClient, clusterName, removals, namespace, true, false, false, false, 0)
+						err := replaceProcessGroups(k8sClient, clusterName, removals, namespace, true, false, false, false, 0)
 						Expect(err).NotTo(HaveOccurred())
 
 						var resCluster fdbv1beta2.FoundationDBCluster
-						err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+						err = k8sClient.Get(context.Background(), client.ObjectKey{
 							Namespace: namespace,
 							Name:      clusterName,
 						}, &resCluster)
diff --git a/kubectl-fdb/cmd/suite_test.go b/kubectl-fdb/cmd/suite_test.go
index f27ab7d1..c1479f8d 100644
--- a/kubectl-fdb/cmd/suite_test.go
+++ b/kubectl-fdb/cmd/suite_test.go
@@ -1,13 +1,53 @@
 package cmd
 
 import (
+	"context"
 	"testing"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	mockclient "github.com/FoundationDB/fdb-kubernetes-operator/mock-kubernetes-client/client"
+	"k8s.io/client-go/kubernetes/scheme"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
+var k8sClient *mockclient.MockClient
+var cluster *fdbv1beta2.FoundationDBCluster
+var clusterName = "test"
+var namespace = "test"
+
 func TestCmd(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "FDB plugin")
 }
+
+var _ = BeforeSuite(func() {
+	Expect(scheme.AddToScheme(scheme.Scheme)).NotTo(HaveOccurred())
+	Expect(fdbv1beta2.AddToScheme(scheme.Scheme)).NotTo(HaveOccurred())
+	k8sClient = mockclient.NewMockClient(scheme.Scheme)
+})
+
+var _ = BeforeEach(func() {
+	cluster = &fdbv1beta2.FoundationDBCluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      clusterName,
+			Namespace: namespace,
+		},
+		Spec: fdbv1beta2.FoundationDBClusterSpec{
+			ProcessCounts: fdbv1beta2.ProcessCounts{
+				Storage: 1,
+			},
+		},
+	}
+})
+
+var _ = JustBeforeEach(func() {
+	Expect(k8sClient.Create(context.TODO(), cluster)).NotTo(HaveOccurred())
+})
+
+var _ = AfterEach(func() {
+	k8sClient.Clear()
+})
diff --git a/kubectl-fdb/cmd/version_test.go b/kubectl-fdb/cmd/version_test.go
index 20ace339..40a2b79a 100644
--- a/kubectl-fdb/cmd/version_test.go
+++ b/kubectl-fdb/cmd/version_test.go
@@ -21,22 +21,17 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 
-	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
-	"k8s.io/apimachinery/pkg/runtime"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	appsv1 "k8s.io/api/apps/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
 )
 
 var _ = Describe("[plugin] version command", func() {
@@ -77,12 +72,9 @@ var _ = Describe("[plugin] version command", func() {
 
 		DescribeTable("should return the correct version",
 			func(input testCase) {
-				scheme := runtime.NewScheme()
-				_ = clientgoscheme.AddToScheme(scheme)
-				_ = fdbv1beta2.AddToScheme(scheme)
-				kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(input.deployment).Build()
+				Expect(k8sClient.Create(context.TODO(), input.deployment))
 
-				operatorVersion, err := version(kubeClient, operatorName, "default", "manager")
+				operatorVersion, err := version(k8sClient, operatorName, "default", "manager")
 				if input.hasError {
 					Expect(err).To(Equal(input.expectedError))
 				} else {
diff --git a/mock-kubernetes-client/client/client_test.go b/mock-kubernetes-client/client/client_test.go
index cc8fc8cf..cef0ee68 100644
--- a/mock-kubernetes-client/client/client_test.go
+++ b/mock-kubernetes-client/client/client_test.go
@@ -25,6 +25,9 @@ import (
 	"sort"
 	"time"
 
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	"k8s.io/client-go/kubernetes/scheme"
+
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -58,8 +61,14 @@ func createDummyPod() *corev1.Pod {
 
 var _ = Describe("[mock client]", func() {
 	var client *MockClient
+
 	BeforeEach(func() {
-		client = &MockClient{}
+		err := scheme.AddToScheme(scheme.Scheme)
+		Expect(err).NotTo(HaveOccurred())
+		err = fdbv1beta2.AddToScheme(scheme.Scheme)
+		Expect(err).NotTo(HaveOccurred())
+
+		client = NewMockClient(scheme.Scheme)
 	})
 
 	When("creating and getting an object", func() {
@@ -125,7 +134,7 @@ var _ = Describe("[mock client]", func() {
 
 			err = client.Create(context.TODO(), createDummyPod())
 			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(Equal("Conflict"))
+			Expect(err.Error()).To(Equal("pods \"pod1\" already exists"))
 		})
 	})
 
@@ -259,8 +268,6 @@ var _ = Describe("[mock client]", func() {
 			err := client.Create(context.TODO(), pod)
 			Expect(err).NotTo(HaveOccurred())
 
-			container := &pod.Spec.Containers[0]
-			container.Env = append(container.Env, corev1.EnvVar{Name: "test-env"})
 			pod.Status.HostIP = "foo"
 
 			err = client.Status().Update(context.TODO(), pod)
@@ -270,8 +277,6 @@ var _ = Describe("[mock client]", func() {
 			podCopy := &corev1.Pod{}
 			err = client.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod1"}, podCopy)
 			Expect(err).NotTo(HaveOccurred())
-			Expect(len(podCopy.Spec.Containers)).To(Equal(1))
-			Expect(len(podCopy.Spec.Containers[0].Env)).To(Equal(0))
 			Expect(podCopy.Status.HostIP).To(Equal("foo"))
 			Expect(podCopy.ObjectMeta.Generation).To(Equal(int64(1)))
 		})
@@ -356,24 +361,27 @@ var _ = Describe("[mock client]", func() {
 		})
 	})
 
-	When("listing objects by field", func() {
-		It("should list the objects with the field set", func() {
-			pod1 := createDummyPod()
-			pod2 := createDummyPod()
-			pod2.Name = "pod2"
-
-			err := client.Create(context.TODO(), pod1)
-			Expect(err).NotTo(HaveOccurred())
-			err = client.Create(context.TODO(), pod2)
-			Expect(err).NotTo(HaveOccurred())
-
-			pods := &corev1.PodList{}
-			err = client.List(context.TODO(), pods, ctrlClient.MatchingFields{"metadata.name": "pod1"})
-			Expect(err).NotTo(HaveOccurred())
-			Expect(len(pods.Items)).To(Equal(1))
-			Expect(pods.Items[0].Name).To(Equal("pod1"))
-		})
-	})
+	// TODO (johscheuer): Once https://github.com/kubernetes-sigs/controller-runtime/pull/2025 is merged we can enable
+	// this test again.
+	//
+	//When("listing objects by field", func() {
+	//	It("should list the objects with the field set", func() {
+	//		pod1 := createDummyPod()
+	//		pod2 := createDummyPod()
+	//		pod2.Name = "pod2"
+	//
+	//		err := client.Create(context.TODO(), pod1)
+	//		Expect(err).NotTo(HaveOccurred())
+	//		err = client.Create(context.TODO(), pod2)
+	//		Expect(err).NotTo(HaveOccurred())
+	//
+	//		pods := &corev1.PodList{}
+	//		err = client.List(context.TODO(), pods, ctrlClient.MatchingFields{"metadata.name": "pod1"})
+	//		Expect(err).NotTo(HaveOccurred())
+	//		Expect(len(pods.Items)).To(Equal(1))
+	//		Expect(pods.Items[0].Name).To(Equal("pod1"))
+	//	})
+	//})
 
 	When("creating an event", func() {
 		It("should create the event", func() {