Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1164 (2022-04-06 12:06:57)

Fix wrong setting for test case
# Description

Fixes: https://github.com/FoundationDB/fdb-kubernetes-operator/issues/1163

## Type of change

*Please select one of the options below.*

- Bug fix (non-breaking change which fixes an issue)

## Discussion

The issue was a underlying race condition on how fast the process groups were removed.

## Testing

Unit tests.

## Documentation

-

## Follow-up

-


foundationdb-ci: 
### AWS CodeBuild CI Report for Linux CentOS 7

* CodeBuild project: fdb-kubernetes-operator-pr
* Commit ID: 1a205f62149e4da23c320c4db93d11721d616e9e
* Result: FAILED
* Error: `reference not found for primary source and source version 1a205f62149e4da23c320c4db93d11721d616e9e`
* [Build Logs](https://d1e0l78xbkh3xa.cloudfront.net/build-logs/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/7dd239ed-3b86-48ef-97f1-1ccda2300ec4.gz) (available for 30 days)


Patch:
diff --git a/Makefile b/Makefile
index 461d0d99..74b37f6d 100644
--- a/Makefile
+++ b/Makefile
@@ -189,13 +189,13 @@ config/samples/client.yaml: config/tests/client/*.yaml
 bin/po-docgen: cmd/po-docgen/*.go
 	go build -o bin/po-docgen cmd/po-docgen/main.go  cmd/po-docgen/api.go
 
-CLUSTER_DOCS_INPUT=api/v1beta2/foundationdbcluster_types.go api/v1beta2/foundationdb_custom_parameter.go api/v1beta2/foundationdb_database_configuration.go api/v1beta2/foundationdb_process_class.go
+CLUSTER_DOCS_INPUT=api/v1beta2/foundationdbcluster_types.go api/v1beta2/foundationdb_custom_parameter.go api/v1beta2/foundationdb_database_configuration.go api/v1beta2/foundationdb_process_class.go api/v1beta2/image_config.go
 
 docs/cluster_spec.md: bin/po-docgen $(CLUSTER_DOCS_INPUT)
 	bin/po-docgen api $(CLUSTER_DOCS_INPUT) > $@
 
 docs/backup_spec.md: bin/po-docgen api/v1beta2/foundationdbbackup_types.go
-	bin/po-docgen api api/v1beta2/foundationdbbackup_types.go api/v1beta2/foundationdb_custom_parameter.go > $@
+	bin/po-docgen api api/v1beta2/foundationdbbackup_types.go api/v1beta2/foundationdb_custom_parameter.go api/v1beta2/image_config.go > $@
 
 docs/restore_spec.md: bin/po-docgen api/v1beta2/foundationdbrestore_types.go
 	bin/po-docgen api api/v1beta2/foundationdbrestore_types.go api/v1beta2/foundationdb_custom_parameter.go > $@
diff --git a/api/v1beta2/foundationdb_database_configuration.go b/api/v1beta2/foundationdb_database_configuration.go
index 239b8b35..ed6fe9a1 100644
--- a/api/v1beta2/foundationdb_database_configuration.go
+++ b/api/v1beta2/foundationdb_database_configuration.go
@@ -196,6 +196,31 @@ func (configuration DatabaseConfiguration) NormalizeConfiguration() DatabaseConf
 	return *result
 }
 
+// NormalizeConfigurationWithSeparatedProxies ensures a standardized
+// format and defaults when comparing database configuration in the
+// cluster spec with database configuration in the cluster status,
+// taking into account if the current running version of FDB supports
+// them and if we need them configured.
+//
+// This will fill in defaults of -1 for some fields that have a default
+// of 0, and will ensure that the region configuration is ordered
+// consistently.
+func (configuration DatabaseConfiguration) NormalizeConfigurationWithSeparatedProxies(version string, areSeparatedProxiesConfigured bool) DatabaseConfiguration {
+	result := configuration.NormalizeConfiguration()
+
+	parsedVersion, _ := ParseFdbVersion(version)
+	if parsedVersion.HasSeparatedProxies() {
+		if !areSeparatedProxiesConfigured {
+			result.GrvProxies = 0
+			result.CommitProxies = 0
+		} else {
+			result.Proxies = 0
+		}
+	}
+
+	return result
+}
+
 func (configuration DatabaseConfiguration) getRegion(id string, priority int) Region {
 	var matchingRegion Region
 
@@ -246,16 +271,24 @@ func (configuration *DatabaseConfiguration) GetRoleCountsWithDefaults(version Ve
 	if counts.Logs == 0 {
 		counts.Logs = 3
 	}
-	if counts.Proxies == 0 {
-		counts.Proxies = 3
-	}
+
 	if version.HasSeparatedProxies() {
-		if counts.CommitProxies == 0 {
-			counts.CommitProxies = 2
+		if counts.CommitProxies == 0 && counts.GrvProxies == 0 && counts.Proxies == 0 {
+			counts.Proxies = 3
+		} else if counts.Proxies == 0 {
+			if counts.GrvProxies == 0 {
+				counts.GrvProxies = 1
+			}
+			if counts.CommitProxies == 0 {
+				counts.CommitProxies = 2
+			}
 		}
-		if counts.GrvProxies == 0 {
-			counts.GrvProxies = 1
+	} else {
+		if counts.Proxies == 0 {
+			counts.Proxies = 3
 		}
+		counts.CommitProxies = 0
+		counts.GrvProxies = 0
 	}
 	if counts.Resolvers == 0 {
 		counts.Resolvers = 1
@@ -519,7 +552,7 @@ func (configuration DatabaseConfiguration) getRegionPriorities() map[string]int
 // to 0
 func (configuration DatabaseConfiguration) AreSeparatedProxiesConfigured() bool {
 	counts := configuration.RoleCounts
-	return counts.Proxies == 0 && (counts.GrvProxies > 0 && counts.CommitProxies > 0)
+	return counts.Proxies == 0 && (counts.GrvProxies > 0 || counts.CommitProxies > 0)
 }
 
 // GetProxiesString returns a string that contains the correct fdbcli
diff --git a/api/v1beta2/foundationdbbackup_types.go b/api/v1beta2/foundationdbbackup_types.go
index 82b74ac2..e9bb212c 100644
--- a/api/v1beta2/foundationdbbackup_types.go
+++ b/api/v1beta2/foundationdbbackup_types.go
@@ -89,10 +89,15 @@ type FoundationDBBackupSpec struct {
 	// You have to ensure that the specified version in the Spec is compatible
 	// with the given version in your custom image.
 	// +kubebuilder:default:=false
+	// Deprecated: use ImageConfigs instead.
 	AllowTagOverride *bool `json:"allowTagOverride,omitempty"`
 
 	// This is the configuration of the target blobstore for this backup.
 	BlobStoreConfiguration *BlobStoreConfiguration `json:"blobStoreConfiguration,omitempty"`
+
+	// ImageConfigs allows customizing the image that we use for
+	// a container.
+	ImageConfigs []ImageConfig `json:"imageConfigs,omitempty"`
 }
 
 // FoundationDBBackupStatus describes the current status of the backup for a cluster.
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index 96b2340c..712a4594 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -1060,12 +1060,11 @@ func (cluster *FoundationDBCluster) GetProcessCountsWithDefaults() (ProcessCount
 			return *processCounts, err
 		}
 
-		if fdbVersion.HasSeparatedProxies() {
+		if fdbVersion.HasSeparatedProxies() && cluster.Spec.DatabaseConfiguration.AreSeparatedProxiesConfigured() {
 			primaryStatelessCount += cluster.calculateProcessCountFromRole(roleCounts.GrvProxies, processCounts.GrvProxy)
 			primaryStatelessCount += cluster.calculateProcessCountFromRole(roleCounts.CommitProxies, processCounts.CommitProxy)
 		} else {
 			primaryStatelessCount += cluster.calculateProcessCountFromRole(roleCounts.Proxies, processCounts.Proxy)
-
 		}
 
 		processCounts.Stateless = cluster.calculateProcessCount(true,
@@ -1412,66 +1411,20 @@ type ContainerOverrides struct {
 	ImageConfigs []ImageConfig `json:"imageConfigs,omitempty"`
 }
 
-// ImageConfig provides a policy for customizing an image.
-//
-// When multiple image configs are provided, they will be merged into a single
-// config that will be used to define the final image. For each field, we select
-// the value from the first entry in the config list that defines a value for
-// that field, and matches the version of FoundationDB the image is for. Any
-// config that specifies a different version than the one under consideration
-// will be ignored for the purposes of defining that image.
-type ImageConfig struct {
-	// Version is the version of FoundationDB this policy applies to. If this is
-	// blank, the policy applies to all FDB versions.
-	Version string `json:"version,omitempty"`
-
-	// BaseImage specifies the part of the image before the tag.
-	BaseImage string `json:"baseImage,omitempty"`
-
-	// Tag specifies a full image tag.
-	Tag string `json:"tag,omitempty"`
-
-	// TagSuffix specifies a suffix that will be added after the version to form
-	// the full tag.
-	TagSuffix string `json:"tagSuffix,omitempty"`
-}
-
-// SelectImageConfig selects image configs that apply to a version of FDB and
-// merges them into a single config.
-func SelectImageConfig(allConfigs []ImageConfig, versionString string) ImageConfig {
-	config := ImageConfig{Version: versionString}
-	for _, nextConfig := range allConfigs {
-		if nextConfig.Version != "" && nextConfig.Version != versionString {
-			continue
-		}
-		if config.BaseImage == "" {
-			config.BaseImage = nextConfig.BaseImage
-		}
-		if config.Tag == "" {
-			config.Tag = nextConfig.Tag
-		}
-		if config.TagSuffix == "" {
-			config.TagSuffix = nextConfig.TagSuffix
-		}
-	}
-	return config
-}
-
-// Image generates an image using a config.
-func (config ImageConfig) Image() string {
-	if config.Tag == "" {
-		return fmt.Sprintf("%s:%s%s", config.BaseImage, config.Version, config.TagSuffix)
-	}
-	return fmt.Sprintf("%s:%s", config.BaseImage, config.Tag)
-}
-
 // DesiredDatabaseConfiguration builds the database configuration for the
 // cluster based on its spec.
 func (cluster *FoundationDBCluster) DesiredDatabaseConfiguration() DatabaseConfiguration {
-	configuration := cluster.Spec.DatabaseConfiguration.NormalizeConfiguration()
-
+	configuration := cluster.Spec.DatabaseConfiguration.NormalizeConfigurationWithSeparatedProxies(cluster.Spec.Version, cluster.Spec.DatabaseConfiguration.AreSeparatedProxiesConfigured())
 	configuration.RoleCounts = cluster.GetRoleCountsWithDefaults()
 	configuration.RoleCounts.Storage = 0
+
+	if cluster.Spec.DatabaseConfiguration.AreSeparatedProxiesConfigured() {
+		configuration.RoleCounts.Proxies = 0
+	} else {
+		configuration.RoleCounts.GrvProxies = 0
+		configuration.RoleCounts.CommitProxies = 0
+	}
+
 	if configuration.StorageEngine == StorageEngineSSD {
 		configuration.StorageEngine = StorageEngineSSD2
 	}
diff --git a/api/v1beta2/image_config.go b/api/v1beta2/image_config.go
new file mode 100644
index 00000000..bf582b3e
--- /dev/null
+++ b/api/v1beta2/image_config.go
@@ -0,0 +1,72 @@
+/*
+Copyright 2020-2022 FoundationDB project authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta2
+
+import "fmt"
+
+// ImageConfig provides a policy for customizing an image.
+//
+// When multiple image configs are provided, they will be merged into a single
+// config that will be used to define the final image. For each field, we select
+// the value from the first entry in the config list that defines a value for
+// that field, and matches the version of FoundationDB the image is for. Any
+// config that specifies a different version than the one under consideration
+// will be ignored for the purposes of defining that image.
+type ImageConfig struct {
+	// Version is the version of FoundationDB this policy applies to. If this is
+	// blank, the policy applies to all FDB versions.
+	Version string `json:"version,omitempty"`
+
+	// BaseImage specifies the part of the image before the tag.
+	BaseImage string `json:"baseImage,omitempty"`
+
+	// Tag specifies a full image tag.
+	Tag string `json:"tag,omitempty"`
+
+	// TagSuffix specifies a suffix that will be added after the version to form
+	// the full tag.
+	TagSuffix string `json:"tagSuffix,omitempty"`
+}
+
+// SelectImageConfig selects image configs that apply to a version of FDB and
+// merges them into a single config.
+func SelectImageConfig(allConfigs []ImageConfig, versionString string) ImageConfig {
+	config := ImageConfig{Version: versionString}
+	for _, nextConfig := range allConfigs {
+		if nextConfig.Version != "" && nextConfig.Version != versionString {
+			continue
+		}
+		if config.BaseImage == "" {
+			config.BaseImage = nextConfig.BaseImage
+		}
+		if config.Tag == "" {
+			config.Tag = nextConfig.Tag
+		}
+		if config.TagSuffix == "" {
+			config.TagSuffix = nextConfig.TagSuffix
+		}
+	}
+	return config
+}
+
+// Image generates an image using a config.
+func (config ImageConfig) Image() string {
+	if config.Tag == "" {
+		return fmt.Sprintf("%s:%s%s", config.BaseImage, config.Version, config.TagSuffix)
+	}
+	return fmt.Sprintf("%s:%s", config.BaseImage, config.Tag)
+}
diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go
index 3e7a7e21..880fc6d6 100644
--- a/api/v1beta2/zz_generated.deepcopy.go
+++ b/api/v1beta2/zz_generated.deepcopy.go
@@ -364,6 +364,11 @@ func (in *FoundationDBBackupSpec) DeepCopyInto(out *FoundationDBBackupSpec) {
 		*out = new(BlobStoreConfiguration)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.ImageConfigs != nil {
+		in, out := &in.ImageConfigs, &out.ImageConfigs
+		*out = make([]ImageConfig, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBBackupSpec.
diff --git a/config/crd/bases/apps.foundationdb.org_foundationdbbackups.yaml b/config/crd/bases/apps.foundationdb.org_foundationdbbackups.yaml
index e20fcfdf..460e5ee7 100644
--- a/config/crd/bases/apps.foundationdb.org_foundationdbbackups.yaml
+++ b/config/crd/bases/apps.foundationdb.org_foundationdbbackups.yaml
@@ -3197,6 +3197,19 @@ spec:
                   type: string
                 maxItems: 100
                 type: array
+              imageConfigs:
+                items:
+                  properties:
+                    baseImage:
+                      type: string
+                    tag:
+                      type: string
+                    tagSuffix:
+                      type: string
+                    version:
+                      type: string
+                  type: object
+                type: array
               podTemplateSpec:
                 properties:
                   metadata:
diff --git a/config/deployment/manager.yaml b/config/deployment/manager.yaml
index e7d1569d..2e7b5752 100644
--- a/config/deployment/manager.yaml
+++ b/config/deployment/manager.yaml
@@ -66,7 +66,7 @@ spec:
       containers:
       - command:
         - /manager
-        image: foundationdb/fdb-kubernetes-operator:v1.2.0
+        image: foundationdb/fdb-kubernetes-operator:v1.3.1
         name: manager
         env:
           - name: WATCH_NAMESPACE
diff --git a/config/samples/deployment.yaml b/config/samples/deployment.yaml
index 4f454121..c25c4f6b 100644
--- a/config/samples/deployment.yaml
+++ b/config/samples/deployment.yaml
@@ -150,7 +150,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: foundationdb/fdb-kubernetes-operator:v1.2.0
+        image: foundationdb/fdb-kubernetes-operator:v1.3.1
         name: manager
         ports:
         - containerPort: 8080
diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go
index 7c5e5b65..44b39aef 100644
--- a/controllers/cluster_controller.go
+++ b/controllers/cluster_controller.go
@@ -28,6 +28,7 @@ import (
 	"math"
 	"regexp"
 	"sort"
+	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -59,14 +60,18 @@ type FoundationDBClusterReconciler struct {
 	PodClientProvider      func(*fdbv1beta2.FoundationDBCluster, *corev1.Pod) (podclient.FdbPodClient, error)
 	DatabaseClientProvider DatabaseClientProvider
 	DeprecationOptions     internal.DeprecationOptions
+	GetTimeout             time.Duration
+	PostTimeout            time.Duration
 }
 
 // NewFoundationDBClusterReconciler creates a new FoundationDBClusterReconciler with defaults.
 func NewFoundationDBClusterReconciler(podLifecycleManager podmanager.PodLifecycleManager) *FoundationDBClusterReconciler {
-	return &FoundationDBClusterReconciler{
+	r := &FoundationDBClusterReconciler{
 		PodLifecycleManager: podLifecycleManager,
-		PodClientProvider:   newFdbPodClient,
 	}
+	r.PodClientProvider = r.newFdbPodClient
+
+	return r
 }
 
 // +kubebuilder:rbac:groups=apps.foundationdb.org,resources=foundationdbclusters,verbs=get;list;watch;create;update;patch;delete
@@ -309,6 +314,7 @@ func (r *FoundationDBClusterReconciler) getPodClient(cluster *fdbv1beta2.Foundat
 		return nil, fmt.Sprintf("Process group in cluster %s/%s does not have pod defined", cluster.Namespace, cluster.Name)
 	}
 
+	// TODO how to pass this down?
 	podClient, err := r.PodClientProvider(cluster, pod)
 	if err != nil {
 		return nil, err.Error()
@@ -731,8 +737,8 @@ func checkCoordinatorValidity(cluster *fdbv1beta2.FoundationDBCluster, status *f
 }
 
 // newFdbPodClient builds a client for working with an FDB Pod
-func newFdbPodClient(cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod) (podclient.FdbPodClient, error) {
-	return internal.NewFdbPodClient(cluster, pod, log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "pod", pod.Name))
+func (r *FoundationDBClusterReconciler) newFdbPodClient(cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod) (podclient.FdbPodClient, error) {
+	return internal.NewFdbPodClient(cluster, pod, log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "pod", pod.Name), r.GetTimeout, r.PostTimeout)
 }
 
 func (r *FoundationDBClusterReconciler) getCoordinatorSet(cluster *fdbv1beta2.FoundationDBCluster) (map[string]struct{}, error) {
diff --git a/controllers/replace_failed_process_groups.go b/controllers/replace_failed_process_groups.go
index 5116e3e2..2aa8436a 100644
--- a/controllers/replace_failed_process_groups.go
+++ b/controllers/replace_failed_process_groups.go
@@ -33,13 +33,21 @@ type replaceFailedProcessGroups struct{}
 
 // reconcile runs the reconciler's work.
 func (c replaceFailedProcessGroups) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster) *requeue {
+	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "replaceFailedProcessGroups")
+	// If the EmptyMonitorConf setting is set we expect that all fdb processes of this cluster part are missing in order
+	// to prevent the operator from replacing any process groups we skip this reconciliation here.
+	if cluster.Spec.Buggify.EmptyMonitorConf {
+		logger.V(1).Info("Skipping because EmptyMonitorConf is set to true")
+		return nil
+	}
+
 	adminClient, err := r.DatabaseClientProvider.GetAdminClient(cluster, r)
 	if err != nil {
 		return &requeue{curError: err}
 	}
 	defer adminClient.Close()
 
-	if replacements.ReplaceFailedProcessGroups(log, cluster, adminClient) {
+	if replacements.ReplaceFailedProcessGroups(logger, cluster, adminClient) {
 		err := r.Status().Update(ctx, cluster)
 		if err != nil {
 			return &requeue{curError: err}
diff --git a/controllers/update_backup_status.go b/controllers/update_backup_status.go
index 07fdfed3..63794c64 100644
--- a/controllers/update_backup_status.go
+++ b/controllers/update_backup_status.go
@@ -22,9 +22,9 @@ package controllers
 
 import (
 	"context"
-	"reflect"
 
 	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
+	"k8s.io/apimachinery/pkg/api/equality"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	appsv1 "k8s.io/api/apps/v1"
@@ -62,7 +62,7 @@ func (s updateBackupStatus) reconcile(ctx context.Context, r *FoundationDBBackup
 		annotationChange := mergeAnnotations(&backupDeployment.ObjectMeta, desiredBackupDeployment.ObjectMeta)
 
 		metadataMatch := !annotationChange &&
-			reflect.DeepEqual(backupDeployment.ObjectMeta.Labels, desiredBackupDeployment.ObjectMeta.Labels)
+			equality.Semantic.DeepEqual(backupDeployment.ObjectMeta.Labels, desiredBackupDeployment.ObjectMeta.Labels)
 
 		status.DeploymentConfigured = generationsMatch && metadataMatch
 
@@ -103,7 +103,7 @@ func (s updateBackupStatus) reconcile(ctx context.Context, r *FoundationDBBackup
 		return &requeue{curError: err}
 	}
 
-	if !reflect.DeepEqual(backup.Status, *originalStatus) {
+	if !equality.Semantic.DeepEqual(backup.Status, *originalStatus) {
 		err = r.Status().Update(ctx, backup)
 		if err != nil {
 			log.Error(err, "Error updating backup status", "namespace", backup.Namespace, "backup", backup.Name)
diff --git a/controllers/update_database_configuration.go b/controllers/update_database_configuration.go
index 35896ab4..4368e636 100644
--- a/controllers/update_database_configuration.go
+++ b/controllers/update_database_configuration.go
@@ -65,7 +65,7 @@ func (u updateDatabaseConfiguration) reconcile(ctx context.Context, r *Foundatio
 		return nil
 	}
 
-	currentConfiguration = status.Cluster.DatabaseConfiguration.NormalizeConfiguration()
+	currentConfiguration = status.Cluster.DatabaseConfiguration.NormalizeConfigurationWithSeparatedProxies(cluster.Spec.Version, cluster.Spec.DatabaseConfiguration.AreSeparatedProxiesConfigured())
 	cluster.ClearMissingVersionFlags(&currentConfiguration)
 	needsChange = initialConfig || !reflect.DeepEqual(desiredConfiguration, currentConfiguration)
 
diff --git a/controllers/update_pods.go b/controllers/update_pods.go
index 51e5b980..df7f6cb8 100644
--- a/controllers/update_pods.go
+++ b/controllers/update_pods.go
@@ -76,7 +76,7 @@ func (updatePods) reconcile(ctx context.Context, r *FoundationDBClusterReconcile
 		}
 
 		if shouldRequeueDueToTerminatingPod(pod, cluster, processGroup.ProcessGroupID) {
-			return &requeue{message: "Cluster has pod that is pending deletion", delay: podSchedulingDelayDuration}
+			return &requeue{message: "Cluster has pod that is pending deletion", delay: podSchedulingDelayDuration, delayedRequeue: true}
 		}
 
 		_, idNum, err := podmanager.ParseProcessGroupID(processGroup.ProcessGroupID)
diff --git a/controllers/update_status.go b/controllers/update_status.go
index 10b5dae8..139bbdf9 100644
--- a/controllers/update_status.go
+++ b/controllers/update_status.go
@@ -104,7 +104,7 @@ func (updateStatus) reconcile(ctx context.Context, r *FoundationDBClusterReconci
 	}
 
 	status.HasListenIPsForAllPods = cluster.NeedsExplicitListenAddress()
-	status.DatabaseConfiguration = databaseStatus.Cluster.DatabaseConfiguration.NormalizeConfiguration()
+	status.DatabaseConfiguration = databaseStatus.Cluster.DatabaseConfiguration.NormalizeConfigurationWithSeparatedProxies(cluster.Spec.Version, cluster.Spec.DatabaseConfiguration.AreSeparatedProxiesConfigured())
 	cluster.ClearMissingVersionFlags(&status.DatabaseConfiguration)
 	status.Configured = cluster.Status.Configured || (databaseStatus.Client.DatabaseStatus.Available && databaseStatus.Cluster.Layers.Error != "configurationMissing")
 
diff --git a/docs/backup_spec.md b/docs/backup_spec.md
index 4a1b6ce2..7af087cc 100644
--- a/docs/backup_spec.md
+++ b/docs/backup_spec.md
@@ -14,6 +14,7 @@ This Document documents the types introduced by the FoundationDB Operator to be
 * [FoundationDBBackupStatusBackupDetails](#foundationdbbackupstatusbackupdetails)
 * [FoundationDBLiveBackupStatus](#foundationdblivebackupstatus)
 * [FoundationDBLiveBackupStatusState](#foundationdblivebackupstatusstate)
+* [ImageConfig](#imageconfig)
 
 ## BackupGenerationStatus
 
@@ -86,8 +87,9 @@ FoundationDBBackupSpec describes the desired state of the backup for a cluster.
 | backupDeploymentMetadata | BackupDeploymentMetadata allows customizing labels and annotations on the deployment for the backup agents. | *[metav1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#objectmeta-v1-meta) | false |
 | podTemplateSpec | PodTemplateSpec allows customizing the pod template for the backup agents. | *[corev1.PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#podtemplatespec-v1-core) | false |
 | customParameters | CustomParameters defines additional parameters to pass to the backup agents. | FoundationDBCustomParameters | false |
-| allowTagOverride | This setting defines if a user provided image can have it's own tag rather than getting the provided version appended. You have to ensure that the specified version in the Spec is compatible with the given version in your custom image. | *bool | false |
+| allowTagOverride | This setting defines if a user provided image can have it's own tag rather than getting the provided version appended. You have to ensure that the specified version in the Spec is compatible with the given version in your custom image. **Deprecated: use ImageConfigs instead.** | *bool | false |
 | blobStoreConfiguration | This is the configuration of the target blobstore for this backup. | *[BlobStoreConfiguration](#blobstoreconfiguration) | false |
+| imageConfigs | ImageConfigs allows customizing the image that we use for a container. | [][ImageConfig](#imageconfig) | false |
 
 [Back to TOC](#table-of-contents)
 
@@ -151,3 +153,16 @@ URLParameter defines a single URL parameter to pass to the blobstore.
 FoundationDBCustomParameter defines a single custom knob
 
 [Back to TOC](#table-of-contents)
+
+## ImageConfig
+
+ImageConfig provides a policy for customizing an image.  When multiple image configs are provided, they will be merged into a single config that will be used to define the final image. For each field, we select the value from the first entry in the config list that defines a value for that field, and matches the version of FoundationDB the image is for. Any config that specifies a different version than the one under consideration will be ignored for the purposes of defining that image.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| version | Version is the version of FoundationDB this policy applies to. If this is blank, the policy applies to all FDB versions. | string | false |
+| baseImage | BaseImage specifies the part of the image before the tag. | string | false |
+| tag | Tag specifies a full image tag. | string | false |
+| tagSuffix | TagSuffix specifies a suffix that will be added after the version to form the full tag. | string | false |
+
+[Back to TOC](#table-of-contents)
diff --git a/docs/changelog/v1.3.0.md b/docs/changelog/v1.3.0.md
new file mode 100644
index 00000000..aa28bec7
--- /dev/null
+++ b/docs/changelog/v1.3.0.md
@@ -0,0 +1,16 @@
+# v1.3.0
+
+## Changes
+
+### Operator
+
+* Ensure we delete all resources when a process group is in resource terminating state [#1145](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1145)
+* Update ginkgo version to v2 [#1156](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1156)
+* fix replacements with empty monitor conf setting [#1141](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1141)
+* make use of constants for storage engine references [#1165](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1165)
+
+### Plugin
+
+* add flag to ignore removals [#1166](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1166)
+* experimental command for the plugin to show the state of exclusions [#1133](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1133)
+* skip Pods that are part of a process group marked for removal [#1132](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1132)
diff --git a/docs/changelog/v1.3.1.md b/docs/changelog/v1.3.1.md
new file mode 100644
index 00000000..b328c415
--- /dev/null
+++ b/docs/changelog/v1.3.1.md
@@ -0,0 +1,7 @@
+# v1.3.1
+
+## Changes
+
+### Operator
+
+* Fix handling of role counts when running 7.1. [#1172](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1172)
diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
index 42d023e3..5a3d1e0d 100644
--- a/docs/cluster_spec.md
+++ b/docs/cluster_spec.md
@@ -18,7 +18,6 @@ This Document documents the types introduced by the FoundationDB Operator to be
 * [FoundationDBClusterList](#foundationdbclusterlist)
 * [FoundationDBClusterSpec](#foundationdbclusterspec)
 * [FoundationDBClusterStatus](#foundationdbclusterstatus)
-* [ImageConfig](#imageconfig)
 * [LabelConfig](#labelconfig)
 * [LockDenyListEntry](#lockdenylistentry)
 * [LockOptions](#lockoptions)
@@ -34,6 +33,7 @@ This Document documents the types introduced by the FoundationDB Operator to be
 * [Region](#region)
 * [RoleCounts](#rolecounts)
 * [VersionFlags](#versionflags)
+* [ImageConfig](#imageconfig)
 
 ## AutomaticReplacementOptions
 
@@ -253,19 +253,6 @@ FoundationDBClusterStatus defines the observed state of FoundationDBCluster
 
 [Back to TOC](#table-of-contents)
 
-## ImageConfig
-
-ImageConfig provides a policy for customizing an image.  When multiple image configs are provided, they will be merged into a single config that will be used to define the final image. For each field, we select the value from the first entry in the config list that defines a value for that field, and matches the version of FoundationDB the image is for. Any config that specifies a different version than the one under consideration will be ignored for the purposes of defining that image.
-
-| Field | Description | Scheme | Required |
-| ----- | ----------- | ------ | -------- |
-| version | Version is the version of FoundationDB this policy applies to. If this is blank, the policy applies to all FDB versions. | string | false |
-| baseImage | BaseImage specifies the part of the image before the tag. | string | false |
-| tag | Tag specifies a full image tag. | string | false |
-| tagSuffix | TagSuffix specifies a suffix that will be added after the version to form the full tag. | string | false |
-
-[Back to TOC](#table-of-contents)
-
 ## ImageType
 
 ImageType defines a single kind of images used in the cluster.
@@ -526,3 +513,16 @@ VersionFlags defines internal flags for new features in the database.
 ProcessClass models the class of a pod
 
 [Back to TOC](#table-of-contents)
+
+## ImageConfig
+
+ImageConfig provides a policy for customizing an image.  When multiple image configs are provided, they will be merged into a single config that will be used to define the final image. For each field, we select the value from the first entry in the config list that defines a value for that field, and matches the version of FoundationDB the image is for. Any config that specifies a different version than the one under consideration will be ignored for the purposes of defining that image.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| version | Version is the version of FoundationDB this policy applies to. If this is blank, the policy applies to all FDB versions. | string | false |
+| baseImage | BaseImage specifies the part of the image before the tag. | string | false |
+| tag | Tag specifies a full image tag. | string | false |
+| tagSuffix | TagSuffix specifies a suffix that will be added after the version to form the full tag. | string | false |
+
+[Back to TOC](#table-of-contents)
diff --git a/helm/fdb-operator/values.yaml b/helm/fdb-operator/values.yaml
index 9f6de5b1..32decc2a 100644
--- a/helm/fdb-operator/values.yaml
+++ b/helm/fdb-operator/values.yaml
@@ -1,7 +1,7 @@
 ---
 image:
   repository: foundationdb/fdb-kubernetes-operator
-  tag: v1.2.0
+  tag: v1.3.1
   pullPolicy: IfNotPresent
 
 initContainers:
diff --git a/internal/pod_client.go b/internal/pod_client.go
index 2fec4d7a..0ea4a9a5 100644
--- a/internal/pod_client.go
+++ b/internal/pod_client.go
@@ -90,6 +90,12 @@ type realFdbPodSidecarClient struct {
 
 	// logger is used to add common fields to log messages.
 	logger logr.Logger
+
+	// getTimeout defines the timeout for get requests
+	getTimeout time.Duration
+
+	// postTimeout defines the timeout for post requests
+	postTimeout time.Duration
 }
 
 // realPodSidecarClient provides a client for use in real environments, using
@@ -106,7 +112,7 @@ type realFdbPodAnnotationClient struct {
 }
 
 // NewFdbPodClient builds a client for working with an FDB Pod
-func NewFdbPodClient(cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod, log logr.Logger) (podclient.FdbPodClient, error) {
+func NewFdbPodClient(cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod, log logr.Logger, getTimeout time.Duration, postTimeout time.Duration) (podclient.FdbPodClient, error) {
 	if GetImageType(pod) == FDBImageTypeUnified {
 		return &realFdbPodAnnotationClient{Cluster: cluster, Pod: pod, logger: log}, nil
 	}
@@ -152,7 +158,7 @@ func NewFdbPodClient(cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod, l
 		tlsConfig.RootCAs = certPool
 	}
 
-	return &realFdbPodSidecarClient{Cluster: cluster, Pod: pod, useTLS: useTLS, tlsConfig: tlsConfig, logger: log}, nil
+	return &realFdbPodSidecarClient{Cluster: cluster, Pod: pod, useTLS: useTLS, tlsConfig: tlsConfig, logger: log, getTimeout: getTimeout, postTimeout: postTimeout}, nil
 }
 
 // getListenIP gets the IP address that a pod listens on.
@@ -186,12 +192,10 @@ func (client *realFdbPodSidecarClient) makeRequest(method string, path string) (
 	url := fmt.Sprintf("%s://%s:8080/%s", protocol, client.getListenIP(), path)
 	switch method {
 	case http.MethodGet:
-		// We assume that a get request should be relative fast.
-		retryClient.HTTPClient.Timeout = 5 * time.Second
+		retryClient.HTTPClient.Timeout = client.getTimeout
 		resp, err = retryClient.Get(url)
 	case http.MethodPost:
-		// A post request could take a little bit longer since we copy sometimes files.
-		retryClient.HTTPClient.Timeout = 10 * time.Second
+		retryClient.HTTPClient.Timeout = client.postTimeout
 		resp, err = retryClient.Post(url, "application/json", strings.NewReader(""))
 	default:
 		return "", fmt.Errorf("unknown HTTP method %s", method)
diff --git a/internal/pod_models.go b/internal/pod_models.go
index fbf999b9..3b8cd7ac 100644
--- a/internal/pod_models.go
+++ b/internal/pod_models.go
@@ -825,7 +825,11 @@ func GetBackupDeployment(backup *fdbv1beta2.FoundationDBBackup) (*appsv1.Deploym
 		podTemplate.Spec.Containers = containers
 	}
 
-	image, err := GetImage(mainContainer.Image, []fdbv1beta2.ImageConfig{{BaseImage: "foundationdb/foundationdb"}}, backup.Spec.Version)
+	if backup.Spec.ImageConfigs == nil {
+		backup.Spec.ImageConfigs = []fdbv1beta2.ImageConfig{{BaseImage: "foundationdb/foundationdb"}}
+	}
+
+	image, err := GetImage(mainContainer.Image, backup.Spec.ImageConfigs, backup.Spec.Version)
 	if err != nil {
 		return nil, err
 	}
diff --git a/internal/replacements/replace_failed_process_groups.go b/internal/replacements/replace_failed_process_groups.go
index 12ce6376..b6b2e086 100644
--- a/internal/replacements/replace_failed_process_groups.go
+++ b/internal/replacements/replace_failed_process_groups.go
@@ -49,7 +49,6 @@ func getMaxReplacements(cluster *fdbv1beta2.FoundationDBCluster, maxReplacements
 // ReplaceFailedProcessGroups flags failed processes groups for removal and returns an indicator
 // of whether any processes were thus flagged.
 func ReplaceFailedProcessGroups(log logr.Logger, cluster *fdbv1beta2.FoundationDBCluster, adminClient fdbadminclient.AdminClient) bool {
-	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "replaceFailedProcessGroups")
 	// Automatic replacements are disabled so we don't have to check anything further
 	if !cluster.GetEnableAutomaticReplacements() {
 		return false
@@ -94,7 +93,7 @@ func ReplaceFailedProcessGroups(log logr.Logger, cluster *fdbv1beta2.FoundationD
 				"failureTime", time.Unix(missingTime, 0).UTC().String())
 		}
 
-		logger.Info("Replace process group",
+		log.Info("Replace process group",
 			"processGroupID", processGroupStatus.ProcessGroupID,
 			"reason", fmt.Sprintf("automatic replacement detected failure time: %s", time.Unix(missingTime, 0).UTC().String()))
 
diff --git a/kubectl-fdb/cmd/analyze.go b/kubectl-fdb/cmd/analyze.go
index 9822ec4e..3848ec19 100644
--- a/kubectl-fdb/cmd/analyze.go
+++ b/kubectl-fdb/cmd/analyze.go
@@ -63,6 +63,11 @@ func newAnalyzeCmd(streams genericclioptions.IOStreams) *cobra.Command {
 				return err
 			}
 
+			ignoreRemovals, err := cmd.Flags().GetBool("ignore-removals")
+			if err != nil {
+				return err
+			}
+
 			ignoreConditions, err := cmd.Flags().GetStringArray("ignore-condition")
 			if err != nil {
 				return err
@@ -108,7 +113,7 @@ func newAnalyzeCmd(streams genericclioptions.IOStreams) *cobra.Command {
 
 			var errs []error
 			for _, cluster := range clusters {
-				err := analyzeCluster(cmd, kubeClient, cluster, namespace, autoFix, wait, ignoreConditions)
+				err := analyzeCluster(cmd, kubeClient, cluster, namespace, autoFix, wait, ignoreConditions, ignoreRemovals)
 				if err != nil {
 					errs = append(errs, err)
 				}
@@ -139,6 +144,10 @@ kubectl fdb analyze --auto-fix sample-cluster-1
 
 # Analyze the cluster "sample-cluster-1" in the current namespace and ignore the IncorrectCommandLine and IncorrectPodSpec condition
 kubectl fdb analyze cluster --ignore-condition=IncorrectCommandLine --ignore-condition=IncorrectPodSpec sample-cluster-1
+
+# Per default the plugin will print out how many process groups are marked for removal instead of printing out each process group.
+# This can be disabled by using the ignore-removals flag to print out the details about process groups that are marked for removal.
+kubectl fdb analyze cluster --ignore-removals=false sample-cluster-1
 `,
 	}
 	cmd.SetOut(o.Out)
@@ -149,7 +158,8 @@ kubectl fdb analyze cluster --ignore-condition=IncorrectCommandLine --ignore-con
 	cmd.Flags().Bool("all-clusters", false, "defines all clusters in the given namespace should be analyzed.")
 	// We might want to move this into the root cmd if we need this in multiple places
 	cmd.Flags().Bool("no-color", false, "Disable color output.")
-	cmd.Flags().StringArray("ignore-condition", nil, "specify which process group conditions should be ignored and not be printed to stdout")
+	cmd.Flags().StringArray("ignore-condition", nil, "specify which process group conditions should be ignored and not be printed to stdout.")
+	cmd.Flags().Bool("ignore-removals", true, "specify if process groups marked for removal should be ignored.")
 
 	o.configFlags.AddFlags(cmd.Flags())
 
@@ -208,7 +218,7 @@ func printStatement(cmd *cobra.Command, line string, mesType messageType) {
 	color.Unset()
 }
 
-func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName string, namespace string, autoFix bool, wait bool, ignoreConditions []string) error {
+func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName string, namespace string, autoFix bool, wait bool, ignoreConditions []string, ignoreRemovals bool) error {
 	foundIssues := false
 	cluster, err := loadCluster(kubeClient, namespace, clusterName)
 
@@ -258,9 +268,12 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 		// Skip if the processGroup should be removed
 		// or should we check for how long they are marked as removed e.g. stuck in removal?
 		if processGroup.IsMarkedForRemoval() {
-			statement := fmt.Sprintf("ProcessGroup: %s is marked for removal, excluded state: %t", processGroup.ProcessGroupID, processGroup.IsExcluded())
 			removedProcessGroups[processGroup.ProcessGroupID] = fdbv1beta2.None{}
-			printStatement(cmd, statement, warnMessage)
+			if !ignoreRemovals {
+				statement := fmt.Sprintf("ProcessGroup: %s is marked for removal, excluded state: %t", processGroup.ProcessGroupID, processGroup.IsExcluded())
+				printStatement(cmd, statement, warnMessage)
+			}
+
 			continue
 		}
 
@@ -293,8 +306,12 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 		}
 	}
 
+	if ignoreRemovals && len(removedProcessGroups) > 0 {
+		printStatement(cmd, fmt.Sprintf("Ignored %d process groups marked for removal", len(removedProcessGroups)), warnMessage)
+	}
+
 	if ignoredConditions > 0 {
-		printStatement(cmd, fmt.Sprintf("ignored %d conditions", ignoredConditions), warnMessage)
+		printStatement(cmd, fmt.Sprintf("Ignored %d conditions", ignoredConditions), warnMessage)
 	}
 
 	if !processGroupIssue {
diff --git a/setup/setup.go b/setup/setup.go
index 5d05671c..45d25559 100644
--- a/setup/setup.go
+++ b/setup/setup.go
@@ -66,6 +66,8 @@ type Options struct {
 	PrintVersion            bool
 	LabelSelector           string
 	WatchNamespace          string
+	GetTimeout              time.Duration
+	PostTimeout             time.Duration
 }
 
 // BindFlags will parse the given flagset for the operator option flags
@@ -90,6 +92,8 @@ func (o *Options) BindFlags(fs *flag.FlagSet) {
 	fs.BoolVar(&o.PrintVersion, "version", false, "Prints the version of the operator and exits.")
 	fs.StringVar(&o.LabelSelector, "label-selector", "", "Defines a label-selector that will be used to select resources.")
 	fs.StringVar(&o.WatchNamespace, "watch-namespace", os.Getenv("WATCH_NAMESPACE"), "Defines which namespace the operator should watch")
+	fs.DurationVar(&o.GetTimeout, "get-timeout", 5*time.Second, "http timeout for get requests to the FDB sidecar")
+	fs.DurationVar(&o.PostTimeout, "post-timeout", 10*time.Second, "http timeout for post requests to the FDB sidecar")
 }
 
 // StartManager will start the FoundationDB operator manager.
@@ -173,6 +177,8 @@ func StartManager(
 		clusterReconciler.Recorder = mgr.GetEventRecorderFor("foundationdbcluster-controller")
 		clusterReconciler.DeprecationOptions = operatorOpts.DeprecationOptions
 		clusterReconciler.DatabaseClientProvider = fdbclient.NewDatabaseClientProvider(logger)
+		clusterReconciler.GetTimeout = operatorOpts.GetTimeout
+		clusterReconciler.PostTimeout = operatorOpts.PostTimeout
 		clusterReconciler.Log = logr.WithName("controllers").WithName("FoundationDBCluster")
 
 		if err := clusterReconciler.SetupWithManager(mgr, operatorOpts.MaxConcurrentReconciles, *labelSelector, watchedObjects...); err != nil {


Test patch:
diff --git a/api/v1beta2/foundationdb_status_test.go b/api/v1beta2/foundationdb_status_test.go
index 84183642..0b570e95 100644
--- a/api/v1beta2/foundationdb_status_test.go
+++ b/api/v1beta2/foundationdb_status_test.go
@@ -76,7 +76,7 @@ var _ = Describe("FoundationDBStatus", func() {
 					},
 					DatabaseConfiguration: DatabaseConfiguration{
 						RedundancyMode: RedundancyModeDouble,
-						StorageEngine:  "ssd-2",
+						StorageEngine:  StorageEngineSSD2,
 						UsableRegions:  1,
 						Regions:        nil,
 						RoleCounts:     RoleCounts{Storage: 0, Logs: 3, Proxies: 3, Resolvers: 1, LogRouters: 0, RemoteLogs: 0},
@@ -460,7 +460,7 @@ var _ = Describe("FoundationDBStatus", func() {
 		status := FoundationDBStatusClusterInfo{
 			DatabaseConfiguration: DatabaseConfiguration{
 				RedundancyMode: "double",
-				StorageEngine:  "ssd-2",
+				StorageEngine:  StorageEngineSSD2,
 				UsableRegions:  1,
 				Regions:        nil,
 				RoleCounts:     RoleCounts{Storage: 0, Logs: 3, Proxies: 3, CommitProxies: 2, GrvProxies: 1, Resolvers: 1, LogRouters: -1, RemoteLogs: -1},
diff --git a/api/v1beta2/foundationdbcluster_types_test.go b/api/v1beta2/foundationdbcluster_types_test.go
index 55e792ed..7d61fa2e 100644
--- a/api/v1beta2/foundationdbcluster_types_test.go
+++ b/api/v1beta2/foundationdbcluster_types_test.go
@@ -140,8 +140,10 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 		})
 
 		When("the version supports grv and commit proxies", func() {
-			It("should return the default role counts", func() {
-				cluster := &FoundationDBCluster{
+			var cluster *FoundationDBCluster
+
+			BeforeEach(func() {
+				cluster = &FoundationDBCluster{
 					ObjectMeta: metav1.ObjectMeta{
 						Name:      "foo",
 						Namespace: "default",
@@ -153,14 +155,16 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 						Version: "7.0.0",
 					},
 				}
+			})
 
+			It("should return the default role counts", func() {
 				counts := cluster.GetRoleCountsWithDefaults()
 				Expect(counts).To(Equal(RoleCounts{
 					Storage:       3,
 					Logs:          3,
 					Proxies:       3,
-					CommitProxies: 2,
-					GrvProxies:    1,
+					CommitProxies: 0,
+					GrvProxies:    0,
 					Resolvers:     1,
 					RemoteLogs:    -1,
 					LogRouters:    -1,
@@ -171,8 +175,8 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 					"resolvers":      1,
 					"remote_logs":    -1,
 					"log_routers":    -1,
-					"commit_proxies": 2,
-					"grv_proxies":    1,
+					"commit_proxies": 0,
+					"grv_proxies":    0,
 				}))
 				Expect(cluster.Spec.DatabaseConfiguration.RoleCounts).To(Equal(RoleCounts{}))
 
@@ -181,8 +185,8 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 				Expect(counts).To(Equal(RoleCounts{
 					Storage:       3,
 					Logs:          3,
-					CommitProxies: 2,
-					GrvProxies:    1,
+					CommitProxies: 0,
+					GrvProxies:    0,
 					Proxies:       3,
 					Resolvers:     1,
 					RemoteLogs:    3,
@@ -194,8 +198,8 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 					"resolvers":      1,
 					"remote_logs":    3,
 					"log_routers":    3,
-					"commit_proxies": 2,
-					"grv_proxies":    1,
+					"commit_proxies": 0,
+					"grv_proxies":    0,
 				}))
 
 				cluster.Spec.DatabaseConfiguration.RoleCounts = RoleCounts{
@@ -207,8 +211,8 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 					Storage:       5,
 					Logs:          3,
 					Proxies:       3,
-					GrvProxies:    1,
-					CommitProxies: 2,
+					GrvProxies:    0,
+					CommitProxies: 0,
 					Resolvers:     1,
 					RemoteLogs:    3,
 					LogRouters:    3,
@@ -221,8 +225,8 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 				Expect(counts).To(Equal(RoleCounts{
 					Storage:       3,
 					Logs:          8,
-					CommitProxies: 2,
-					GrvProxies:    1,
+					CommitProxies: 0,
+					GrvProxies:    0,
 					Proxies:       3,
 					Resolvers:     1,
 					RemoteLogs:    8,
@@ -239,19 +243,112 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 					Storage:       3,
 					Logs:          4,
 					Proxies:       3,
-					GrvProxies:    1,
-					CommitProxies: 2,
+					GrvProxies:    0,
+					CommitProxies: 0,
 					Resolvers:     1,
 					RemoteLogs:    5,
 					LogRouters:    6,
 				}))
+
+			})
+
+			It("should return the correct counts when all proxies are unconfigured", func() {
+				cluster.Spec.DatabaseConfiguration.RoleCounts = RoleCounts{
+					Proxies:       0,
+					CommitProxies: 0,
+					GrvProxies:    0,
+				}
+				Expect(cluster.GetRoleCountsWithDefaults()).To(Equal(RoleCounts{
+					Storage:       3,
+					Logs:          3,
+					Proxies:       3,
+					GrvProxies:    0,
+					CommitProxies: 0,
+					Resolvers:     1,
+					RemoteLogs:    -1,
+					LogRouters:    -1,
+				}))
+			})
+
+			It("should return the correct counts when only proxies are configured", func() {
+				cluster.Spec.DatabaseConfiguration.RoleCounts = RoleCounts{
+					Proxies:       3,
+					CommitProxies: 0,
+					GrvProxies:    0,
+				}
+				Expect(cluster.GetRoleCountsWithDefaults()).To(Equal(RoleCounts{
+					Storage:       3,
+					Logs:          3,
+					Proxies:       3,
+					GrvProxies:    0,
+					CommitProxies: 0,
+					Resolvers:     1,
+					RemoteLogs:    -1,
+					LogRouters:    -1,
+				}))
 			})
+
+			It("should return the correct counts when only proxies are unconfigured and grv_proxies/commit_proxies are configured", func() {
+				cluster.Spec.DatabaseConfiguration.RoleCounts = RoleCounts{
+					Proxies:       0,
+					CommitProxies: 9,
+					GrvProxies:    4,
+				}
+				Expect(cluster.GetRoleCountsWithDefaults()).To(Equal(RoleCounts{
+					Storage:       3,
+					Logs:          3,
+					Proxies:       0,
+					GrvProxies:    4,
+					CommitProxies: 9,
+					Resolvers:     1,
+					RemoteLogs:    -1,
+					LogRouters:    -1,
+				}))
+			})
+
+			It("should return the correct counts when proxies, grv_proxies are unconfigured and commit_proxies configured", func() {
+				cluster.Spec.DatabaseConfiguration.RoleCounts = RoleCounts{
+					Proxies:       0,
+					CommitProxies: 3,
+					GrvProxies:    0,
+				}
+				Expect(cluster.GetRoleCountsWithDefaults()).To(Equal(RoleCounts{
+					Storage:       3,
+					Logs:          3,
+					Proxies:       0,
+					GrvProxies:    1,
+					CommitProxies: 3,
+					Resolvers:     1,
+					RemoteLogs:    -1,
+					LogRouters:    -1,
+				}))
+			})
+
+			It("should return the correct counts when proxies, commit_proxies are unconfigured and grv_proxies configured", func() {
+				cluster.Spec.DatabaseConfiguration.RoleCounts = RoleCounts{
+					Proxies:       0,
+					CommitProxies: 0,
+					GrvProxies:    5,
+				}
+				Expect(cluster.GetRoleCountsWithDefaults()).To(Equal(RoleCounts{
+					Storage:       3,
+					Logs:          3,
+					Proxies:       0,
+					GrvProxies:    5,
+					CommitProxies: 2,
+					Resolvers:     1,
+					RemoteLogs:    -1,
+					LogRouters:    -1,
+				}))
+			})
+
 		})
 	})
 
 	When("getting the default process counts", func() {
-		It("should return the default process counts", func() {
-			cluster := &FoundationDBCluster{
+		var cluster *FoundationDBCluster
+		BeforeEach(func() {
+			cluster = &FoundationDBCluster{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "foo",
 					Namespace: "default",
@@ -271,7 +368,9 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 					},
 				},
 			}
+		})
 
+		It("should return the default process counts", func() {
 			counts, err := cluster.GetProcessCountsWithDefaults()
 			Expect(err).NotTo(HaveOccurred())
 			Expect(counts).To(Equal(ProcessCounts{
@@ -332,6 +431,18 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 			Expect(counts).To(Equal(ProcessCounts{
 				Storage:   5,
 				Log:       5,
+				Stateless: 9,
+			}))
+		})
+
+		It("should return the default process counts when proxies are unset", func() {
+			cluster.Spec.Version = "7.1.0-rc2"
+			cluster.Spec.DatabaseConfiguration.Proxies = 0
+			counts, err := cluster.GetProcessCountsWithDefaults()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(counts).To(Equal(ProcessCounts{
+				Storage:   5,
+				Log:       4,
 				Stateless: 22,
 			}))
 		})
@@ -680,7 +791,7 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 					Spec: FoundationDBClusterSpec{
 						DatabaseConfiguration: DatabaseConfiguration{
 							RedundancyMode: RedundancyModeDouble,
-							StorageEngine:  "ssd",
+							StorageEngine:  StorageEngineSSD,
 							RoleCounts: RoleCounts{
 								Storage: 5,
 								Logs:    4,
@@ -692,16 +803,16 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 				}
 			})
 
-			It("should be parsed correctly", func() {
+			It("should be parsed correctly when proxies are set", func() {
 				Expect(cluster.DesiredDatabaseConfiguration()).To(Equal(DatabaseConfiguration{
 					RedundancyMode: RedundancyModeDouble,
-					StorageEngine:  "ssd-2",
+					StorageEngine:  StorageEngineSSD2,
 					UsableRegions:  1,
 					RoleCounts: RoleCounts{
 						Logs:          4,
 						Proxies:       5,
-						CommitProxies: 2,
-						GrvProxies:    1,
+						CommitProxies: 0,
+						GrvProxies:    0,
 						Resolvers:     1,
 						LogRouters:    -1,
 						RemoteLogs:    -1,
@@ -714,12 +825,68 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 
 				Expect(cluster.DesiredDatabaseConfiguration()).To(Equal(DatabaseConfiguration{
 					RedundancyMode: RedundancyModeDouble,
-					StorageEngine:  "ssd-2",
+					StorageEngine:  StorageEngineSSD2,
 					UsableRegions:  1,
 					RoleCounts: RoleCounts{
 						Logs:          3,
 						Proxies:       3,
+						CommitProxies: 0,
+						GrvProxies:    0,
+						Resolvers:     1,
+						LogRouters:    -1,
+						RemoteLogs:    -1,
+					},
+				}))
+			})
+
+			It("should be parsed correctly when no proxies are set", func() {
+				cluster.Spec.DatabaseConfiguration.RoleCounts.Proxies = 0
+				Expect(cluster.DesiredDatabaseConfiguration()).To(Equal(DatabaseConfiguration{
+					RedundancyMode: RedundancyModeDouble,
+					StorageEngine:  StorageEngineSSD2,
+					UsableRegions:  1,
+					RoleCounts: RoleCounts{
+						Logs:          4,
+						Proxies:       3,
+						CommitProxies: 0,
+						GrvProxies:    0,
+						Resolvers:     1,
+						LogRouters:    -1,
+						RemoteLogs:    -1,
+					},
+				}))
+			})
+
+			It("should be parsed correctly when only grv_proxies are set", func() {
+				cluster.Spec.DatabaseConfiguration.RoleCounts.Proxies = 0
+				cluster.Spec.DatabaseConfiguration.RoleCounts.GrvProxies = 4
+				Expect(cluster.DesiredDatabaseConfiguration()).To(Equal(DatabaseConfiguration{
+					RedundancyMode: RedundancyModeDouble,
+					StorageEngine:  StorageEngineSSD2,
+					UsableRegions:  1,
+					RoleCounts: RoleCounts{
+						Logs:          4,
+						Proxies:       0,
 						CommitProxies: 2,
+						GrvProxies:    4,
+						Resolvers:     1,
+						LogRouters:    -1,
+						RemoteLogs:    -1,
+					},
+				}))
+			})
+
+			It("should be parsed correctly when only grv_proxies are set", func() {
+				cluster.Spec.DatabaseConfiguration.RoleCounts.Proxies = 0
+				cluster.Spec.DatabaseConfiguration.RoleCounts.CommitProxies = 4
+				Expect(cluster.DesiredDatabaseConfiguration()).To(Equal(DatabaseConfiguration{
+					RedundancyMode: RedundancyModeDouble,
+					StorageEngine:  StorageEngineSSD2,
+					UsableRegions:  1,
+					RoleCounts: RoleCounts{
+						Logs:          4,
+						Proxies:       0,
+						CommitProxies: 4,
 						GrvProxies:    1,
 						Resolvers:     1,
 						LogRouters:    -1,
@@ -727,9 +894,29 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 					},
 				}))
 			})
+
+			It("should be parsed correctly when both grv_proxies/commit_proxies are set", func() {
+				cluster.Spec.DatabaseConfiguration.RoleCounts.Proxies = 0
+				cluster.Spec.DatabaseConfiguration.RoleCounts.CommitProxies = 4
+				cluster.Spec.DatabaseConfiguration.RoleCounts.GrvProxies = 4
+				Expect(cluster.DesiredDatabaseConfiguration()).To(Equal(DatabaseConfiguration{
+					RedundancyMode: RedundancyModeDouble,
+					StorageEngine:  StorageEngineSSD2,
+					UsableRegions:  1,
+					RoleCounts: RoleCounts{
+						Logs:          4,
+						Proxies:       0,
+						CommitProxies: 4,
+						GrvProxies:    4,
+						Resolvers:     1,
+						LogRouters:    -1,
+						RemoteLogs:    -1,
+					},
+				}))
+			})
 		})
 
-		When("the version supports grv and commit proxies", func() {
+		When("the version does not support grv and commit proxies", func() {
 			BeforeEach(func() {
 				cluster = &FoundationDBCluster{
 					ObjectMeta: metav1.ObjectMeta{
@@ -739,7 +926,7 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 					Spec: FoundationDBClusterSpec{
 						DatabaseConfiguration: DatabaseConfiguration{
 							RedundancyMode: RedundancyModeDouble,
-							StorageEngine:  "ssd",
+							StorageEngine:  StorageEngineSSD,
 							RoleCounts: RoleCounts{
 								Storage: 5,
 								Logs:    4,
@@ -754,7 +941,7 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 			It("should be parsed correctly", func() {
 				Expect(cluster.DesiredDatabaseConfiguration()).To(Equal(DatabaseConfiguration{
 					RedundancyMode: RedundancyModeDouble,
-					StorageEngine:  "ssd-2",
+					StorageEngine:  StorageEngineSSD2,
 					UsableRegions:  1,
 					RoleCounts: RoleCounts{
 						Logs:          4,
@@ -771,7 +958,7 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 
 				Expect(cluster.DesiredDatabaseConfiguration()).To(Equal(DatabaseConfiguration{
 					RedundancyMode: RedundancyModeDouble,
-					StorageEngine:  "ssd-2",
+					StorageEngine:  StorageEngineSSD2,
 					UsableRegions:  1,
 					RoleCounts: RoleCounts{
 						Logs:          3,
@@ -791,7 +978,7 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 		It("should be parsed correctly", func() {
 			configuration := DatabaseConfiguration{
 				RedundancyMode: RedundancyModeDouble,
-				StorageEngine:  "ssd",
+				StorageEngine:  StorageEngineSSD,
 				UsableRegions:  1,
 				RoleCounts: RoleCounts{
 					Logs:          5,
@@ -824,7 +1011,7 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 		When("CommitProxies and GrvProxies are not configured", func() {
 			configuration := DatabaseConfiguration{
 				RedundancyMode: RedundancyModeDouble,
-				StorageEngine:  "ssd",
+				StorageEngine:  StorageEngineSSD,
 				UsableRegions:  1,
 				RoleCounts: RoleCounts{
 					Logs:    5,
@@ -846,7 +1033,7 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 		When("CommitProxies and GrvProxies are configured but not proxies", func() {
 			configuration := DatabaseConfiguration{
 				RedundancyMode: RedundancyModeDouble,
-				StorageEngine:  "ssd",
+				StorageEngine:  StorageEngineSSD,
 				UsableRegions:  1,
 				RoleCounts: RoleCounts{
 					Logs:          5,
@@ -2275,11 +2462,12 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 	})
 
 	Context("Normalize cluster spec", func() {
+		version := "7.1.0"
 		When("log routers are missing", func() {
-			It("should set the correct value (-1) for og routers", func() {
+			It("should set the correct value (-1) for log routers", func() {
 				spec := DatabaseConfiguration{}
 				spec.RemoteLogs = 9
-				normalized := spec.NormalizeConfiguration()
+				normalized := spec.NormalizeConfigurationWithSeparatedProxies(version, false)
 				Expect(normalized.LogRouters).To(Equal(-1))
 				Expect(normalized.RemoteLogs).To(Equal(9))
 			})
@@ -2327,7 +2515,7 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 						},
 					},
 				}
-				normalized := spec.NormalizeConfiguration()
+				normalized := spec.NormalizeConfigurationWithSeparatedProxies(version, false)
 				Expect(normalized.Regions).To(Equal([]Region{
 					{
 						DataCenters: []DataCenter{
@@ -2411,7 +2599,7 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 						},
 					},
 				}
-				normalized := spec.NormalizeConfiguration()
+				normalized := spec.NormalizeConfigurationWithSeparatedProxies(version, false)
 				Expect(normalized.Regions).To(Equal([]Region{
 					{
 						DataCenters: []DataCenter{
@@ -2701,13 +2889,13 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 							},
 							DatabaseConfiguration: DatabaseConfiguration{
 								RedundancyMode: RedundancyModeDouble,
-								StorageEngine:  "ssd-2",
+								StorageEngine:  StorageEngineSSD2,
 								UsableRegions:  1,
 								RoleCounts: RoleCounts{
 									Logs:          3,
 									Proxies:       3,
-									GrvProxies:    1,
-									CommitProxies: 2,
+									GrvProxies:    0,
+									CommitProxies: 0,
 									Resolvers:     1,
 									LogRouters:    -1,
 									RemoteLogs:    -1,
@@ -2952,7 +3140,7 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 							},
 							DatabaseConfiguration: DatabaseConfiguration{
 								RedundancyMode: RedundancyModeDouble,
-								StorageEngine:  "ssd-2",
+								StorageEngine:  StorageEngineSSD2,
 								UsableRegions:  1,
 								RoleCounts: RoleCounts{
 									Logs:          3,
@@ -3765,77 +3953,6 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 		)
 	})
 
-	When("merging image configs", func() {
-		It("applies chooses the first value for each field", func() {
-			configs := []ImageConfig{
-				{
-					BaseImage: "foundationdb/foundationdb",
-					Version:   Versions.Default.String(),
-				},
-				{
-					BaseImage: "foundationdb/foundationdb-slim",
-					Version:   Versions.Default.String(),
-					Tag:       "abcdef",
-					TagSuffix: "-1",
-				},
-			}
-
-			finalConfig := SelectImageConfig(configs, Versions.Default.String())
-			Expect(finalConfig).To(Equal(ImageConfig{
-				BaseImage: "foundationdb/foundationdb",
-				Version:   Versions.Default.String(),
-				Tag:       "abcdef",
-				TagSuffix: "-1",
-			}))
-		})
-
-		It("ignores configs that are for different versions", func() {
-			configs := []ImageConfig{
-				{
-					BaseImage: "foundationdb/foundationdb",
-					Version:   Versions.Default.String(),
-				},
-				{
-					Version: Versions.NextMajorVersion.String(),
-					Tag:     "abcdef",
-				},
-				{
-					TagSuffix: "-1",
-				},
-			}
-
-			finalConfig := SelectImageConfig(configs, Versions.Default.String())
-			Expect(finalConfig).To(Equal(ImageConfig{
-				BaseImage: "foundationdb/foundationdb",
-				Version:   Versions.Default.String(),
-				TagSuffix: "-1",
-			}))
-		})
-	})
-
-	When("building image names", func() {
-		It("applies the fields", func() {
-			config := ImageConfig{
-				BaseImage: "foundationdb/foundationdb-kubernetes-sidecar",
-				Version:   Versions.Default.String(),
-				TagSuffix: "-2",
-			}
-			image := config.Image()
-			Expect(image).To(Equal(fmt.Sprintf("foundationdb/foundationdb-kubernetes-sidecar:%s-2", Versions.Default)))
-		})
-
-		It("uses the tag to override the version and tag suffix", func() {
-			config := ImageConfig{
-				BaseImage: "foundationdb/foundationdb-kubernetes-sidecar",
-				Version:   Versions.Default.String(),
-				Tag:       "abcdef",
-				TagSuffix: "-2",
-			}
-			image := config.Image()
-			Expect(image).To(Equal("foundationdb/foundationdb-kubernetes-sidecar:abcdef"))
-		})
-	})
-
 	When("checking if the process group needs a replacement", func() {
 		var processGroup *ProcessGroupStatus
 		var needsReplacement bool
diff --git a/api/v1beta2/image_config_test.go b/api/v1beta2/image_config_test.go
new file mode 100644
index 00000000..b6adf120
--- /dev/null
+++ b/api/v1beta2/image_config_test.go
@@ -0,0 +1,81 @@
+package v1beta2
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("[api] ImageConfig", func() {
+	When("merging image configs", func() {
+		It("applies chooses the first value for each field", func() {
+			configs := []ImageConfig{
+				{
+					BaseImage: "foundationdb/foundationdb",
+					Version:   Versions.Default.String(),
+				},
+				{
+					BaseImage: "foundationdb/foundationdb-slim",
+					Version:   Versions.Default.String(),
+					Tag:       "abcdef",
+					TagSuffix: "-1",
+				},
+			}
+
+			finalConfig := SelectImageConfig(configs, Versions.Default.String())
+			Expect(finalConfig).To(Equal(ImageConfig{
+				BaseImage: "foundationdb/foundationdb",
+				Version:   Versions.Default.String(),
+				Tag:       "abcdef",
+				TagSuffix: "-1",
+			}))
+		})
+
+		It("ignores configs that are for different versions", func() {
+			configs := []ImageConfig{
+				{
+					BaseImage: "foundationdb/foundationdb",
+					Version:   Versions.Default.String(),
+				},
+				{
+					Version: Versions.NextMajorVersion.String(),
+					Tag:     "abcdef",
+				},
+				{
+					TagSuffix: "-1",
+				},
+			}
+
+			finalConfig := SelectImageConfig(configs, Versions.Default.String())
+			Expect(finalConfig).To(Equal(ImageConfig{
+				BaseImage: "foundationdb/foundationdb",
+				Version:   Versions.Default.String(),
+				TagSuffix: "-1",
+			}))
+		})
+	})
+
+	When("building image names", func() {
+		It("applies the fields", func() {
+			config := ImageConfig{
+				BaseImage: "foundationdb/foundationdb-kubernetes-sidecar",
+				Version:   Versions.Default.String(),
+				TagSuffix: "-2",
+			}
+			image := config.Image()
+			Expect(image).To(Equal(fmt.Sprintf("foundationdb/foundationdb-kubernetes-sidecar:%s-2", Versions.Default)))
+		})
+
+		It("uses the tag to override the version and tag suffix", func() {
+			config := ImageConfig{
+				BaseImage: "foundationdb/foundationdb-kubernetes-sidecar",
+				Version:   Versions.Default.String(),
+				Tag:       "abcdef",
+				TagSuffix: "-2",
+			}
+			image := config.Image()
+			Expect(image).To(Equal("foundationdb/foundationdb-kubernetes-sidecar:abcdef"))
+		})
+	})
+})
diff --git a/controllers/admin_client_test.go b/controllers/admin_client_test.go
index d65397ee..6d995dde 100644
--- a/controllers/admin_client_test.go
+++ b/controllers/admin_client_test.go
@@ -86,13 +86,13 @@ var _ = Describe("admin_client_test", func() {
 				It("should generate the status", func() {
 					Expect(status.Cluster.DatabaseConfiguration).To(Equal(fdbv1beta2.DatabaseConfiguration{
 						RedundancyMode: fdbv1beta2.RedundancyModeDouble,
-						StorageEngine:  "ssd-2",
+						StorageEngine:  fdbv1beta2.StorageEngineSSD2,
 						UsableRegions:  1,
 						RoleCounts: fdbv1beta2.RoleCounts{
 							Logs:          3,
 							Proxies:       3,
-							CommitProxies: 2,
-							GrvProxies:    1,
+							CommitProxies: 0,
+							GrvProxies:    0,
 							Resolvers:     1,
 							LogRouters:    -1,
 							RemoteLogs:    -1,
@@ -128,7 +128,7 @@ var _ = Describe("admin_client_test", func() {
 				It("should generate the status", func() {
 					Expect(status.Cluster.DatabaseConfiguration).To(Equal(fdbv1beta2.DatabaseConfiguration{
 						RedundancyMode: fdbv1beta2.RedundancyModeDouble,
-						StorageEngine:  "ssd-2",
+						StorageEngine:  fdbv1beta2.StorageEngineSSD2,
 						UsableRegions:  1,
 						RoleCounts: fdbv1beta2.RoleCounts{
 							Logs:          3,
diff --git a/controllers/cluster_controller_test.go b/controllers/cluster_controller_test.go
index 0aebdf48..6ae56417 100644
--- a/controllers/cluster_controller_test.go
+++ b/controllers/cluster_controller_test.go
@@ -2539,7 +2539,7 @@ var _ = Describe("cluster_controller", func() {
 			When("using rocksdb-v1 engine", func() {
 				When("using 6.3.26", func() {
 					BeforeEach(func() {
-						cluster.Spec.DatabaseConfiguration.StorageEngine = "ssd-rocksdb-v1"
+						cluster.Spec.DatabaseConfiguration.StorageEngine = fdbv1beta2.StorageEngineRocksDbV1
 						cluster.Spec.Version = "6.3.26"
 						err := k8sClient.Update(context.TODO(), cluster)
 						Expect(err).NotTo(HaveOccurred())
@@ -2553,7 +2553,7 @@ var _ = Describe("cluster_controller", func() {
 				})
 				When("using 7.1.0-rc3", func() {
 					BeforeEach(func() {
-						cluster.Spec.DatabaseConfiguration.StorageEngine = "ssd-rocksdb-v1"
+						cluster.Spec.DatabaseConfiguration.StorageEngine = fdbv1beta2.StorageEngineRocksDbV1
 						cluster.Spec.Version = "7.1.0-rc3"
 						err := k8sClient.Update(context.TODO(), cluster)
 						Expect(err).NotTo(HaveOccurred())
@@ -2567,7 +2567,7 @@ var _ = Describe("cluster_controller", func() {
 				})
 				When("using 7.1.0", func() {
 					BeforeEach(func() {
-						cluster.Spec.DatabaseConfiguration.StorageEngine = "ssd-rocksdb-v1"
+						cluster.Spec.DatabaseConfiguration.StorageEngine = fdbv1beta2.StorageEngineRocksDbV1
 						cluster.Spec.Version = "7.1.0"
 						err := k8sClient.Update(context.TODO(), cluster)
 						Expect(err).NotTo(HaveOccurred())
@@ -2580,7 +2580,7 @@ var _ = Describe("cluster_controller", func() {
 				})
 				When("using 7.1.0-rc4", func() {
 					BeforeEach(func() {
-						cluster.Spec.DatabaseConfiguration.StorageEngine = "ssd-rocksdb-v1"
+						cluster.Spec.DatabaseConfiguration.StorageEngine = fdbv1beta2.StorageEngineRocksDbV1
 						cluster.Spec.Version = "7.1.0-rc4"
 						err := k8sClient.Update(context.TODO(), cluster)
 						Expect(err).NotTo(HaveOccurred())
@@ -2596,7 +2596,7 @@ var _ = Describe("cluster_controller", func() {
 			When("using ssd-rocksdb-experimental", func() {
 				When("using 7.1.0-rc4", func() {
 					BeforeEach(func() {
-						cluster.Spec.DatabaseConfiguration.StorageEngine = "ssd-rocksdb-experimental"
+						cluster.Spec.DatabaseConfiguration.StorageEngine = fdbv1beta2.StorageEngineRocksDbExperimental
 						cluster.Spec.Version = "7.1.0-rc4"
 						err := k8sClient.Update(context.TODO(), cluster)
 						Expect(err).NotTo(HaveOccurred())
@@ -2610,7 +2610,7 @@ var _ = Describe("cluster_controller", func() {
 				})
 				When("using 7.1.0", func() {
 					BeforeEach(func() {
-						cluster.Spec.DatabaseConfiguration.StorageEngine = "ssd-rocksdb-experimental"
+						cluster.Spec.DatabaseConfiguration.StorageEngine = fdbv1beta2.StorageEngineRocksDbExperimental
 						cluster.Spec.Version = "7.1.0"
 						err := k8sClient.Update(context.TODO(), cluster)
 						Expect(err).NotTo(HaveOccurred())
@@ -2624,7 +2624,7 @@ var _ = Describe("cluster_controller", func() {
 				})
 				When("using 7.1.0-rc3", func() {
 					BeforeEach(func() {
-						cluster.Spec.DatabaseConfiguration.StorageEngine = "ssd-rocksdb-experimental"
+						cluster.Spec.DatabaseConfiguration.StorageEngine = fdbv1beta2.StorageEngineRocksDbExperimental
 						cluster.Spec.Version = "7.1.0-rc3"
 						err := k8sClient.Update(context.TODO(), cluster)
 						Expect(err).NotTo(HaveOccurred())
@@ -2637,7 +2637,7 @@ var _ = Describe("cluster_controller", func() {
 				})
 				When("using 6.3.24", func() {
 					BeforeEach(func() {
-						cluster.Spec.DatabaseConfiguration.StorageEngine = "ssd-rocksdb-experimental"
+						cluster.Spec.DatabaseConfiguration.StorageEngine = fdbv1beta2.StorageEngineRocksDbExperimental
 						cluster.Spec.Version = "6.3.24"
 						err := k8sClient.Update(context.TODO(), cluster)
 						Expect(err).NotTo(HaveOccurred())
diff --git a/controllers/remove_process_groups_test.go b/controllers/remove_process_groups_test.go
index 4466b9e4..cc024259 100644
--- a/controllers/remove_process_groups_test.go
+++ b/controllers/remove_process_groups_test.go
@@ -175,6 +175,11 @@ var _ = Describe("remove_process_groups", func() {
 			var secondRemovedProcessGroup *fdbv1beta2.ProcessGroupStatus
 
 			BeforeEach(func() {
+				// To allow multiple process groups to be removed we have to use the update mode all
+				cluster.Spec.AutomationOptions.RemovalMode = fdbv1beta2.PodUpdateModeAll
+				err := k8sClient.Update(context.TODO(), cluster)
+				Expect(err).NotTo(HaveOccurred())
+
 				initialCnt = len(cluster.Status.ProcessGroups)
 				secondRemovedProcessGroup = cluster.Status.ProcessGroups[1]
 				marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, secondRemovedProcessGroup.ProcessGroupID, secondRemovedProcessGroup.ProcessClass, removedProcessGroup.Addresses[0])
diff --git a/controllers/replace_failed_process_groups_test.go b/controllers/replace_failed_process_groups_test.go
index 9be14b24..df83f559 100644
--- a/controllers/replace_failed_process_groups_test.go
+++ b/controllers/replace_failed_process_groups_test.go
@@ -21,11 +21,9 @@
 package controllers
 
 import (
-	"context"
+	ctx "context"
 	"time"
 
-	"github.com/FoundationDB/fdb-kubernetes-operator/internal/replacements"
-
 	"k8s.io/utils/pointer"
 
 	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
@@ -38,11 +36,11 @@ import (
 var _ = Describe("replace_failed_process_groups", func() {
 	var cluster *fdbv1beta2.FoundationDBCluster
 	var err error
-	var result bool
+	var result *requeue
 
 	BeforeEach(func() {
 		cluster = internal.CreateDefaultCluster()
-		err = k8sClient.Create(context.TODO(), cluster)
+		err = k8sClient.Create(ctx.TODO(), cluster)
 		Expect(err).NotTo(HaveOccurred())
 
 		result, err := reconcileCluster(cluster)
@@ -58,12 +56,12 @@ var _ = Describe("replace_failed_process_groups", func() {
 		adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(adminClient).NotTo(BeNil())
-		result = replacements.ReplaceFailedProcessGroups(log, cluster, adminClient)
+		result = replaceFailedProcessGroups{}.reconcile(ctx.Background(), clusterReconciler, cluster)
 	})
 
 	Context("with no missing processes", func() {
-		It("should return false", func() {
-			Expect(result).To(BeFalse())
+		It("should return nil", func() {
+			Expect(result).To(BeNil())
 		})
 
 		It("should not mark anything for removal", func() {
@@ -81,8 +79,9 @@ var _ = Describe("replace_failed_process_groups", func() {
 		})
 
 		Context("with no other removals", func() {
-			It("should return true", func() {
-				Expect(result).To(BeTrue())
+			It("should requeue", func() {
+				Expect(result).NotTo(BeNil())
+				Expect(result.message).To(Equal("Removals have been updated in the cluster status"))
 			})
 
 			It("should mark the process group for removal", func() {
@@ -98,6 +97,20 @@ var _ = Describe("replace_failed_process_groups", func() {
 					Expect(pg.ExclusionSkipped).To(BeFalse())
 				}
 			})
+
+			When("EmptyMonitorConf is set to true", func() {
+				BeforeEach(func() {
+					cluster.Spec.Buggify.EmptyMonitorConf = true
+				})
+
+				It("should return nil", func() {
+					Expect(result).To(BeNil())
+				})
+
+				It("should not mark the process group for removal", func() {
+					Expect(getRemovedProcessGroupIDs(cluster)).To(Equal([]string{}))
+				})
+			})
 		})
 
 		Context("with multiple failed processes", func() {
@@ -109,8 +122,9 @@ var _ = Describe("replace_failed_process_groups", func() {
 				})
 			})
 
-			It("should return true", func() {
-				Expect(result).To(BeTrue())
+			It("should requeue", func() {
+				Expect(result).NotTo(BeNil())
+				Expect(result.message).To(Equal("Removals have been updated in the cluster status"))
 			})
 
 			It("should mark the first process group for removal", func() {
@@ -134,8 +148,8 @@ var _ = Describe("replace_failed_process_groups", func() {
 				processGroup.MarkForRemoval()
 			})
 
-			It("should return false", func() {
-				Expect(result).To(BeFalse())
+			It("should return nil", func() {
+				Expect(result).To(BeNil())
 			})
 
 			It("should not mark the process group for removal", func() {
@@ -148,8 +162,9 @@ var _ = Describe("replace_failed_process_groups", func() {
 					cluster.Spec.AutomationOptions.Replacements.MaxConcurrentReplacements = &replacements
 				})
 
-				It("should return true", func() {
-					Expect(result).To(BeTrue())
+				It("should requeue", func() {
+					Expect(result).NotTo(BeNil())
+					Expect(result.message).To(Equal("Removals have been updated in the cluster status"))
 				})
 
 				It("should mark the process group for removal", func() {
@@ -163,8 +178,8 @@ var _ = Describe("replace_failed_process_groups", func() {
 					cluster.Spec.AutomationOptions.Replacements.MaxConcurrentReplacements = &replacements
 				})
 
-				It("should return false", func() {
-					Expect(result).To(BeFalse())
+				It("should return nil", func() {
+					Expect(result).To(BeNil())
 				})
 
 				It("should not mark the process group for removal", func() {
@@ -180,8 +195,9 @@ var _ = Describe("replace_failed_process_groups", func() {
 				processGroup.SetExclude()
 			})
 
-			It("should return true", func() {
-				Expect(result).To(BeTrue())
+			It("should requeue", func() {
+				Expect(result).NotTo(BeNil())
+				Expect(result.message).To(Equal("Removals have been updated in the cluster status"))
 			})
 
 			It("should mark the process group for removal", func() {
@@ -195,8 +211,9 @@ var _ = Describe("replace_failed_process_groups", func() {
 				processGroup.Addresses = nil
 			})
 
-			It("should return true", func() {
-				Expect(result).To(BeTrue())
+			It("should requeue", func() {
+				Expect(result).NotTo(BeNil())
+				Expect(result.message).To(Equal("Removals have been updated in the cluster status"))
 			})
 
 			It("should mark the process group for removal", func() {
@@ -229,8 +246,8 @@ var _ = Describe("replace_failed_process_groups", func() {
 					}
 				})
 
-				It("should return false", func() {
-					Expect(result).To(BeFalse())
+				It("should return nil", func() {
+					Expect(result).To(BeNil())
 				})
 
 				It("should not mark the process group for removal", func() {
@@ -248,8 +265,8 @@ var _ = Describe("replace_failed_process_groups", func() {
 					adminClient.maxZoneFailuresWithoutLosingData = pointer.Int(0)
 				})
 
-				It("should return false", func() {
-					Expect(result).To(BeFalse())
+				It("should return nil", func() {
+					Expect(result).To(BeNil())
 				})
 
 				It("should not mark the process group for removal", func() {
@@ -268,8 +285,8 @@ var _ = Describe("replace_failed_process_groups", func() {
 			})
 		})
 
-		It("should return false", func() {
-			Expect(result).To(BeFalse())
+		It("should return nil", func() {
+			Expect(result).To(BeNil())
 		})
 
 		It("should not mark the process group for removal", func() {
@@ -286,8 +303,8 @@ var _ = Describe("replace_failed_process_groups", func() {
 			})
 		})
 
-		It("should return false", func() {
-			Expect(result).To(BeFalse())
+		It("should return nil", func() {
+			Expect(result).To(BeNil())
 		})
 
 		It("should not mark the process group for removal", func() {
diff --git a/internal/pod_models_test.go b/internal/pod_models_test.go
index a3370bbe..d88d5bc5 100644
--- a/internal/pod_models_test.go
+++ b/internal/pod_models_test.go
@@ -2947,6 +2947,19 @@ var _ = Describe("pod_models", func() {
 				Expect(deployment.Spec.Template.Spec.Containers[0].Args).To(ContainElement("--customParameter=1337"))
 			})
 		})
+
+		When("defining an image config", func() {
+			BeforeEach(func() {
+				backup.Spec.ImageConfigs = []fdbv1beta2.ImageConfig{{BaseImage: "foundationdb/foundationdb", Tag: "dev"}}
+				deployment, err = GetBackupDeployment(backup)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(deployment).NotTo(BeNil())
+			})
+
+			It("should set the image based on the image configs", func() {
+				Expect(deployment.Spec.Template.Spec.Containers[0].Image).To(Equal("foundationdb/foundationdb:dev"))
+			})
+		})
 	})
 
 	Context("Get image for container", func() {
diff --git a/kubectl-fdb/cmd/analyze_test.go b/kubectl-fdb/cmd/analyze_test.go
index 358701f0..a8880ff5 100644
--- a/kubectl-fdb/cmd/analyze_test.go
+++ b/kubectl-fdb/cmd/analyze_test.go
@@ -94,11 +94,12 @@ var _ = Describe("[plugin] analyze cluster", func() {
 			cluster           *fdbv1beta2.FoundationDBCluster
 			podList           *corev1.PodList
 			ExpectedErrMsg    string
-			ExpectedStdouMsg  string
+			ExpectedStdoutMsg string
 			AutoFix           bool
 			NoWait            bool
 			HasErrors         bool
 			IgnoredConditions []string
+			IgnoreRemovals    bool
 		}
 
 		DescribeTable("return all successful and failed checks",
@@ -114,14 +115,14 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				inBuffer := bytes.Buffer{}
 
 				cmd := newAnalyzeCmd(genericclioptions.IOStreams{In: &inBuffer, Out: &outBuffer, ErrOut: &errBuffer})
-				err := analyzeCluster(cmd, kubeClient, clusterName, namespace, tc.AutoFix, tc.NoWait, tc.IgnoredConditions)
+				err := analyzeCluster(cmd, kubeClient, clusterName, namespace, tc.AutoFix, tc.NoWait, tc.IgnoredConditions, tc.IgnoreRemovals)
 
 				if err != nil && !tc.HasErrors {
 					Expect(err).To(HaveOccurred())
 				}
 
 				Expect(strings.TrimSpace(tc.ExpectedErrMsg)).To(Equal(strings.TrimSpace(errBuffer.String())))
-				Expect(strings.TrimSpace(tc.ExpectedStdouMsg)).To(Equal(strings.TrimSpace(outBuffer.String())))
+				Expect(strings.TrimSpace(tc.ExpectedStdoutMsg)).To(Equal(strings.TrimSpace(outBuffer.String())))
 			},
 			Entry("Cluster is fine",
 				testCase{
@@ -132,14 +133,15 @@ var _ = Describe("[plugin] analyze cluster", func() {
 						Phase: corev1.PodRunning,
 					}, nil),
 					ExpectedErrMsg: "",
-					ExpectedStdouMsg: `Checking cluster: test/test
+					ExpectedStdoutMsg: `Checking cluster: test/test
 ✔ Cluster is available
 ✔ Cluster is fully replicated
 ✔ Cluster is reconciled
 ✔ ProcessGroups are all in ready condition
 ✔ Pods are all running and available`,
-					AutoFix:   false,
-					HasErrors: false,
+					AutoFix:        false,
+					HasErrors:      false,
+					IgnoreRemovals: true,
 				}),
 			Entry("Cluster is unavailable",
 				testCase{
@@ -150,13 +152,14 @@ var _ = Describe("[plugin] analyze cluster", func() {
 						Phase: corev1.PodRunning,
 					}, nil),
 					ExpectedErrMsg: "✖ Cluster is not available",
-					ExpectedStdouMsg: `Checking cluster: test/test
+					ExpectedStdoutMsg: `Checking cluster: test/test
 ✔ Cluster is fully replicated
 ✔ Cluster is reconciled
 ✔ ProcessGroups are all in ready condition
 ✔ Pods are all running and available`,
-					AutoFix:   false,
-					HasErrors: true,
+					AutoFix:        false,
+					HasErrors:      true,
+					IgnoreRemovals: true,
 				}),
 			Entry("Cluster is unhealthy",
 				testCase{
@@ -167,14 +170,15 @@ var _ = Describe("[plugin] analyze cluster", func() {
 						Phase: corev1.PodRunning,
 					}, nil),
 					ExpectedErrMsg: "",
-					ExpectedStdouMsg: `Checking cluster: test/test
+					ExpectedStdoutMsg: `Checking cluster: test/test
 ✔ Cluster is available
 ✔ Cluster is fully replicated
 ✔ Cluster is reconciled
 ✔ ProcessGroups are all in ready condition
 ✔ Pods are all running and available`,
-					AutoFix:   false,
-					HasErrors: true,
+					AutoFix:        false,
+					HasErrors:      true,
+					IgnoreRemovals: true,
 				}),
 			Entry("Cluster is not fully replicated",
 				testCase{
@@ -185,13 +189,14 @@ var _ = Describe("[plugin] analyze cluster", func() {
 						Phase: corev1.PodRunning,
 					}, nil),
 					ExpectedErrMsg: "✖ Cluster is not fully replicated",
-					ExpectedStdouMsg: `Checking cluster: test/test
+					ExpectedStdoutMsg: `Checking cluster: test/test
 ✔ Cluster is available
 ✔ Cluster is reconciled
 ✔ ProcessGroups are all in ready condition
 ✔ Pods are all running and available`,
-					AutoFix:   false,
-					HasErrors: true,
+					AutoFix:        false,
+					HasErrors:      true,
+					IgnoreRemovals: true,
 				}),
 			Entry("Cluster is not reconciled",
 				testCase{
@@ -202,13 +207,14 @@ var _ = Describe("[plugin] analyze cluster", func() {
 						Phase: corev1.PodRunning,
 					}, nil),
 					ExpectedErrMsg: "✖ Cluster is not reconciled",
-					ExpectedStdouMsg: `Checking cluster: test/test
+					ExpectedStdoutMsg: `Checking cluster: test/test
 ✔ Cluster is available
 ✔ Cluster is fully replicated
 ✔ ProcessGroups are all in ready condition
 ✔ Pods are all running and available`,
-					AutoFix:   false,
-					HasErrors: true,
+					AutoFix:        false,
+					HasErrors:      true,
+					IgnoreRemovals: true,
 				}),
 			Entry("ProcessGroup has a missing process",
 				testCase{
@@ -224,13 +230,14 @@ var _ = Describe("[plugin] analyze cluster", func() {
 						Phase: corev1.PodRunning,
 					}, nil),
 					ExpectedErrMsg: fmt.Sprintf("✖ ProcessGroup: instance-1 has the following condition: MissingProcesses since %s", time.Unix(time.Now().Unix(), 0).String()),
-					ExpectedStdouMsg: `Checking cluster: test/test
+					ExpectedStdoutMsg: `Checking cluster: test/test
 ✔ Cluster is available
 ✔ Cluster is fully replicated
 ✔ Cluster is reconciled
 ✔ Pods are all running and available`,
-					AutoFix:   false,
-					HasErrors: true,
+					AutoFix:        false,
+					HasErrors:      true,
+					IgnoreRemovals: true,
 				}),
 			Entry("ProcessGroup has a missing process but is marked for removal",
 				testCase{
@@ -247,14 +254,15 @@ var _ = Describe("[plugin] analyze cluster", func() {
 						Phase: corev1.PodRunning,
 					}, nil),
 					ExpectedErrMsg: "⚠ ProcessGroup: instance-1 is marked for removal, excluded state: false",
-					ExpectedStdouMsg: `Checking cluster: test/test
+					ExpectedStdoutMsg: `Checking cluster: test/test
 ✔ Cluster is available
 ✔ Cluster is fully replicated
 ✔ Cluster is reconciled
 ✔ ProcessGroups are all in ready condition
 ✔ Pods are all running and available`,
-					AutoFix:   false,
-					HasErrors: false,
+					AutoFix:        false,
+					HasErrors:      false,
+					IgnoreRemovals: false,
 				}),
 			Entry("Pod is in Pending phase",
 				testCase{
@@ -265,13 +273,14 @@ var _ = Describe("[plugin] analyze cluster", func() {
 						Phase: corev1.PodPending,
 					}, nil),
 					ExpectedErrMsg: "✖ Pod test/instance-1 has unexpected Phase Pending with Reason:",
-					ExpectedStdouMsg: `Checking cluster: test/test
+					ExpectedStdoutMsg: `Checking cluster: test/test
 ✔ Cluster is available
 ✔ Cluster is fully replicated
 ✔ Cluster is reconciled
 ✔ ProcessGroups are all in ready condition`,
-					AutoFix:   false,
-					HasErrors: true,
+					AutoFix:        false,
+					IgnoreRemovals: true,
+					HasErrors:      true,
 				}),
 			Entry("Container is in terminated state",
 				testCase{
@@ -294,13 +303,14 @@ var _ = Describe("[plugin] analyze cluster", func() {
 						},
 					}, nil),
 					ExpectedErrMsg: "✖ Pod test/instance-1 has an unready container: foundationdb",
-					ExpectedStdouMsg: `Checking cluster: test/test
+					ExpectedStdoutMsg: `Checking cluster: test/test
 ✔ Cluster is available
 ✔ Cluster is fully replicated
 ✔ Cluster is reconciled
 ✔ ProcessGroups are all in ready condition`,
-					AutoFix:   false,
-					HasErrors: true,
+					AutoFix:        false,
+					HasErrors:      true,
+					IgnoreRemovals: true,
 				}),
 			Entry("Container is in ready state",
 				testCase{
@@ -317,14 +327,15 @@ var _ = Describe("[plugin] analyze cluster", func() {
 						},
 					}, nil),
 					ExpectedErrMsg: "",
-					ExpectedStdouMsg: `Checking cluster: test/test
+					ExpectedStdoutMsg: `Checking cluster: test/test
 ✔ Cluster is available
 ✔ Cluster is fully replicated
 ✔ Cluster is reconciled
 ✔ ProcessGroups are all in ready condition
 ✔ Pods are all running and available`,
-					AutoFix:   false,
-					HasErrors: false,
+					AutoFix:        false,
+					HasErrors:      false,
+					IgnoreRemovals: true,
 				}),
 			Entry("Pod is stuck in terminating",
 				testCase{
@@ -336,13 +347,14 @@ var _ = Describe("[plugin] analyze cluster", func() {
 						ContainerStatuses: []corev1.ContainerStatus{},
 					}, &metav1.Time{Time: time.Now().Add(-1 * time.Hour)}),
 					ExpectedErrMsg: fmt.Sprintf("✖ Pod test/instance-1 has been stuck in terminating since %s", time.Unix(time.Now().Add(-1*time.Hour).Unix(), 0).String()),
-					ExpectedStdouMsg: `Checking cluster: test/test
+					ExpectedStdoutMsg: `Checking cluster: test/test
 ✔ Cluster is available
 ✔ Cluster is fully replicated
 ✔ Cluster is reconciled
 ✔ ProcessGroups are all in ready condition`,
-					AutoFix:   false,
-					HasErrors: true,
+					AutoFix:        false,
+					HasErrors:      true,
+					IgnoreRemovals: true,
 				}),
 			Entry("Pod is stuck in terminating and marked for removal",
 				testCase{
@@ -369,14 +381,50 @@ var _ = Describe("[plugin] analyze cluster", func() {
 						},
 					}, &metav1.Time{Time: time.Now().Add(-1 * time.Hour)}),
 					ExpectedErrMsg: "⚠ ProcessGroup: instance-1 is marked for removal, excluded state: true",
-					ExpectedStdouMsg: `Checking cluster: test/test
+					ExpectedStdoutMsg: `Checking cluster: test/test
 ✔ Cluster is available
 ✔ Cluster is fully replicated
 ✔ Cluster is reconciled
 ✔ ProcessGroups are all in ready condition
 ✔ Pods are all running and available`,
-					AutoFix:   false,
-					HasErrors: true,
+					AutoFix:        false,
+					HasErrors:      true,
+					IgnoreRemovals: false,
+				}),
+			Entry("Pod is stuck in terminating and marked for removal and removals are ignored",
+				testCase{
+					cluster: getCluster(clusterName, namespace, true, true, true, 0, []*fdbv1beta2.ProcessGroupStatus{
+						{
+							ProcessGroupID:     "instance-1",
+							RemovalTimestamp:   &metav1.Time{Time: time.Now()},
+							ExclusionTimestamp: &metav1.Time{Time: time.Now()},
+						},
+					}),
+					podList: getPodList(clusterName, namespace, corev1.PodStatus{
+						Phase: corev1.PodRunning,
+						ContainerStatuses: []corev1.ContainerStatus{
+							{
+								Name:  "test",
+								Ready: false,
+								State: corev1.ContainerState{
+									Terminated: &corev1.ContainerStateTerminated{
+										ExitCode:   127,
+										FinishedAt: metav1.Time{Time: time.Now().Add(-1 * time.Hour)},
+									},
+								},
+							},
+						},
+					}, &metav1.Time{Time: time.Now().Add(-1 * time.Hour)}),
+					ExpectedErrMsg: "⚠ Ignored 1 process groups marked for removal",
+					ExpectedStdoutMsg: `Checking cluster: test/test
+✔ Cluster is available
+✔ Cluster is fully replicated
+✔ Cluster is reconciled
+✔ ProcessGroups are all in ready condition
+✔ Pods are all running and available`,
+					AutoFix:        false,
+					HasErrors:      true,
+					IgnoreRemovals: true,
 				}),
 			Entry("Missing Pods",
 				testCase{
@@ -385,14 +433,15 @@ var _ = Describe("[plugin] analyze cluster", func() {
 					}),
 					podList:        &corev1.PodList{},
 					ExpectedErrMsg: "✖ Found no Pods for this cluster",
-					ExpectedStdouMsg: `Checking cluster: test/test
+					ExpectedStdoutMsg: `Checking cluster: test/test
 ✔ Cluster is available
 ✔ Cluster is fully replicated
 ✔ Cluster is reconciled
 ✔ ProcessGroups are all in ready condition
 ✔ Pods are all running and available`,
-					AutoFix:   false,
-					HasErrors: true,
+					AutoFix:        false,
+					HasErrors:      true,
+					IgnoreRemovals: true,
 				}),
 			Entry("Missing entry in status",
 				testCase{
@@ -401,13 +450,14 @@ var _ = Describe("[plugin] analyze cluster", func() {
 						Phase: corev1.PodRunning,
 					}, nil),
 					ExpectedErrMsg: "✖ Pod test/instance-1 with the ID instance-1 is not part of the cluster spec status",
-					ExpectedStdouMsg: `Checking cluster: test/test
+					ExpectedStdoutMsg: `Checking cluster: test/test
 ✔ Cluster is available
 ✔ Cluster is fully replicated
 ✔ Cluster is reconciled
 ✔ ProcessGroups are all in ready condition`,
-					AutoFix:   false,
-					HasErrors: true,
+					AutoFix:        false,
+					HasErrors:      true,
+					IgnoreRemovals: true,
 				}),
 			Entry("ProcessGroup has a two conditions and we ignore one.",
 				testCase{
@@ -423,8 +473,8 @@ var _ = Describe("[plugin] analyze cluster", func() {
 					podList: getPodList(clusterName, namespace, corev1.PodStatus{
 						Phase: corev1.PodRunning,
 					}, nil),
-					ExpectedErrMsg: fmt.Sprintf("✖ ProcessGroup: instance-1 has the following condition: MissingProcesses since %s\n⚠ ignored 1 conditions", time.Unix(time.Now().Unix(), 0).String()),
-					ExpectedStdouMsg: `Checking cluster: test/test
+					ExpectedErrMsg: fmt.Sprintf("✖ ProcessGroup: instance-1 has the following condition: MissingProcesses since %s\n⚠ Ignored 1 conditions", time.Unix(time.Now().Unix(), 0).String()),
+					ExpectedStdoutMsg: `Checking cluster: test/test
 ✔ Cluster is available
 ✔ Cluster is fully replicated
 ✔ Cluster is reconciled
@@ -432,6 +482,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 					AutoFix:           false,
 					HasErrors:         true,
 					IgnoredConditions: []string{string(fdbv1beta2.IncorrectPodSpec)},
+					IgnoreRemovals:    true,
 				}),
 			// TODO: test cases for auto-fix
 		)