Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1182 (2022-04-27 14:49:14)

Check exclusion status of fully excluded and in progress exclusions in two steps
# Description

The idea here is to run the exclusion check in two steps:

1. Check all processes that are already fully excluded (don't serve any roles), this is only an additional safety check and shouldn't be required actually.
2. Check all exclusions that are "in progress"

The benefit of this approach is that a slow exclusion doesn't block the removal of other process groups that are already fully excluded.

## Type of change

*Please select one of the options below.*

- New feature (non-breaking change which adds functionality)

## Discussion

-

## Testing

Unit tests + locally.

## Documentation

Like commented here: https://github.com/apple/foundationdb/issues/6854#issuecomment-1110608724 it would actually be nice if FDB would offer this functionality. if I get some spare time I might look into creating this in the c bindings. 

## Follow-up

*Are there any follow-up issues that we should pursue in the future?*

*Does this introduce new defaults that we should re-evaluate in the future?*



Patch:
diff --git a/.goreleaser.yaml b/.goreleaser.yaml
index 33bb1930..b03f6a80 100644
--- a/.goreleaser.yaml
+++ b/.goreleaser.yaml
@@ -3,8 +3,6 @@ before:
     - go mod tidy
 builds:
   - id: kubectl-fdb
-    env:
-      - CGO_ENABLED=0
     goos:
       - linux
       - windows
diff --git a/Makefile b/Makefile
index 461d0d99..74b37f6d 100644
--- a/Makefile
+++ b/Makefile
@@ -189,13 +189,13 @@ config/samples/client.yaml: config/tests/client/*.yaml
 bin/po-docgen: cmd/po-docgen/*.go
 	go build -o bin/po-docgen cmd/po-docgen/main.go  cmd/po-docgen/api.go
 
-CLUSTER_DOCS_INPUT=api/v1beta2/foundationdbcluster_types.go api/v1beta2/foundationdb_custom_parameter.go api/v1beta2/foundationdb_database_configuration.go api/v1beta2/foundationdb_process_class.go
+CLUSTER_DOCS_INPUT=api/v1beta2/foundationdbcluster_types.go api/v1beta2/foundationdb_custom_parameter.go api/v1beta2/foundationdb_database_configuration.go api/v1beta2/foundationdb_process_class.go api/v1beta2/image_config.go
 
 docs/cluster_spec.md: bin/po-docgen $(CLUSTER_DOCS_INPUT)
 	bin/po-docgen api $(CLUSTER_DOCS_INPUT) > $@
 
 docs/backup_spec.md: bin/po-docgen api/v1beta2/foundationdbbackup_types.go
-	bin/po-docgen api api/v1beta2/foundationdbbackup_types.go api/v1beta2/foundationdb_custom_parameter.go > $@
+	bin/po-docgen api api/v1beta2/foundationdbbackup_types.go api/v1beta2/foundationdb_custom_parameter.go api/v1beta2/image_config.go > $@
 
 docs/restore_spec.md: bin/po-docgen api/v1beta2/foundationdbrestore_types.go
 	bin/po-docgen api api/v1beta2/foundationdbrestore_types.go api/v1beta2/foundationdb_custom_parameter.go > $@
diff --git a/api/v1beta2/foundationdb_database_configuration.go b/api/v1beta2/foundationdb_database_configuration.go
index ed6fe9a1..6b8865cd 100644
--- a/api/v1beta2/foundationdb_database_configuration.go
+++ b/api/v1beta2/foundationdb_database_configuration.go
@@ -307,6 +307,7 @@ func (configuration *DatabaseConfiguration) GetRoleCountsWithDefaults(version Ve
 			counts.LogRouters = -1
 		}
 	}
+
 	return *counts
 }
 
@@ -552,7 +553,7 @@ func (configuration DatabaseConfiguration) getRegionPriorities() map[string]int
 // to 0
 func (configuration DatabaseConfiguration) AreSeparatedProxiesConfigured() bool {
 	counts := configuration.RoleCounts
-	return counts.Proxies == 0 && (counts.GrvProxies > 0 || counts.CommitProxies > 0)
+	return counts.GrvProxies > 0 || counts.CommitProxies > 0
 }
 
 // GetProxiesString returns a string that contains the correct fdbcli
diff --git a/api/v1beta2/foundationdbbackup_types.go b/api/v1beta2/foundationdbbackup_types.go
index 82b74ac2..e9bb212c 100644
--- a/api/v1beta2/foundationdbbackup_types.go
+++ b/api/v1beta2/foundationdbbackup_types.go
@@ -89,10 +89,15 @@ type FoundationDBBackupSpec struct {
 	// You have to ensure that the specified version in the Spec is compatible
 	// with the given version in your custom image.
 	// +kubebuilder:default:=false
+	// Deprecated: use ImageConfigs instead.
 	AllowTagOverride *bool `json:"allowTagOverride,omitempty"`
 
 	// This is the configuration of the target blobstore for this backup.
 	BlobStoreConfiguration *BlobStoreConfiguration `json:"blobStoreConfiguration,omitempty"`
+
+	// ImageConfigs allows customizing the image that we use for
+	// a container.
+	ImageConfigs []ImageConfig `json:"imageConfigs,omitempty"`
 }
 
 // FoundationDBBackupStatus describes the current status of the backup for a cluster.
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index 0b05e35d..f8644fcf 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -944,7 +944,7 @@ func (cluster *FoundationDBCluster) GetProcessSettings(processClass ProcessClass
 // UsableRegions is less than or equal to 1.
 func (cluster *FoundationDBCluster) GetRoleCountsWithDefaults() RoleCounts {
 	// We can ignore the error here since the version will be validated in an earlier step.
-	version, _ := ParseFdbVersion(cluster.Spec.Version)
+	version, _ := ParseFdbVersion(cluster.GetRunningVersion())
 	return cluster.Spec.DatabaseConfiguration.GetRoleCountsWithDefaults(version, cluster.DesiredFaultTolerance())
 }
 
@@ -1055,7 +1055,7 @@ func (cluster *FoundationDBCluster) GetProcessCountsWithDefaults() (ProcessCount
 		primaryStatelessCount += cluster.calculateProcessCountFromRole(1, processCounts.Ratekeeper) +
 			cluster.calculateProcessCountFromRole(1, processCounts.DataDistributor)
 
-		fdbVersion, err := ParseFdbVersion(cluster.Spec.Version)
+		fdbVersion, err := ParseFdbVersion(cluster.GetRunningVersion())
 		if err != nil {
 			return *processCounts, err
 		}
@@ -1072,6 +1072,7 @@ func (cluster *FoundationDBCluster) GetProcessCountsWithDefaults() (ProcessCount
 			cluster.calculateProcessCountFromRole(roleCounts.LogRouters),
 		)
 	}
+
 	return *processCounts, nil
 }
 
@@ -1411,63 +1412,10 @@ type ContainerOverrides struct {
 	ImageConfigs []ImageConfig `json:"imageConfigs,omitempty"`
 }
 
-// ImageConfig provides a policy for customizing an image.
-//
-// When multiple image configs are provided, they will be merged into a single
-// config that will be used to define the final image. For each field, we select
-// the value from the first entry in the config list that defines a value for
-// that field, and matches the version of FoundationDB the image is for. Any
-// config that specifies a different version than the one under consideration
-// will be ignored for the purposes of defining that image.
-type ImageConfig struct {
-	// Version is the version of FoundationDB this policy applies to. If this is
-	// blank, the policy applies to all FDB versions.
-	Version string `json:"version,omitempty"`
-
-	// BaseImage specifies the part of the image before the tag.
-	BaseImage string `json:"baseImage,omitempty"`
-
-	// Tag specifies a full image tag.
-	Tag string `json:"tag,omitempty"`
-
-	// TagSuffix specifies a suffix that will be added after the version to form
-	// the full tag.
-	TagSuffix string `json:"tagSuffix,omitempty"`
-}
-
-// SelectImageConfig selects image configs that apply to a version of FDB and
-// merges them into a single config.
-func SelectImageConfig(allConfigs []ImageConfig, versionString string) ImageConfig {
-	config := ImageConfig{Version: versionString}
-	for _, nextConfig := range allConfigs {
-		if nextConfig.Version != "" && nextConfig.Version != versionString {
-			continue
-		}
-		if config.BaseImage == "" {
-			config.BaseImage = nextConfig.BaseImage
-		}
-		if config.Tag == "" {
-			config.Tag = nextConfig.Tag
-		}
-		if config.TagSuffix == "" {
-			config.TagSuffix = nextConfig.TagSuffix
-		}
-	}
-	return config
-}
-
-// Image generates an image using a config.
-func (config ImageConfig) Image() string {
-	if config.Tag == "" {
-		return fmt.Sprintf("%s:%s%s", config.BaseImage, config.Version, config.TagSuffix)
-	}
-	return fmt.Sprintf("%s:%s", config.BaseImage, config.Tag)
-}
-
 // DesiredDatabaseConfiguration builds the database configuration for the
 // cluster based on its spec.
 func (cluster *FoundationDBCluster) DesiredDatabaseConfiguration() DatabaseConfiguration {
-	configuration := cluster.Spec.DatabaseConfiguration.NormalizeConfigurationWithSeparatedProxies(cluster.Spec.Version, cluster.Spec.DatabaseConfiguration.AreSeparatedProxiesConfigured())
+	configuration := cluster.Spec.DatabaseConfiguration.NormalizeConfigurationWithSeparatedProxies(cluster.GetRunningVersion(), cluster.Spec.DatabaseConfiguration.AreSeparatedProxiesConfigured())
 	configuration.RoleCounts = cluster.GetRoleCountsWithDefaults()
 	configuration.RoleCounts.Storage = 0
 
@@ -2026,3 +1974,16 @@ func (cluster *FoundationDBCluster) AddProcessGroupsToRemovalWithoutExclusionLis
 		cluster.Spec.ProcessGroupsToRemoveWithoutExclusion = append(cluster.Spec.ProcessGroupsToRemoveWithoutExclusion, processGroupID)
 	}
 }
+
+// GetRunningVersion returns the running version of the cluster defined in the cluster status or if not defined the version
+// defined in the cluster spec.
+func (cluster *FoundationDBCluster) GetRunningVersion() string {
+	// We have to use the running version here otherwise if the operator gets killed during an upgrade it will try to apply
+	// the grv and commit proxies.
+	versionString := cluster.Status.RunningVersion
+	if versionString == "" {
+		return cluster.Spec.Version
+	}
+
+	return versionString
+}
diff --git a/api/v1beta2/image_config.go b/api/v1beta2/image_config.go
new file mode 100644
index 00000000..bf582b3e
--- /dev/null
+++ b/api/v1beta2/image_config.go
@@ -0,0 +1,72 @@
+/*
+Copyright 2020-2022 FoundationDB project authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta2
+
+import "fmt"
+
+// ImageConfig provides a policy for customizing an image.
+//
+// When multiple image configs are provided, they will be merged into a single
+// config that will be used to define the final image. For each field, we select
+// the value from the first entry in the config list that defines a value for
+// that field, and matches the version of FoundationDB the image is for. Any
+// config that specifies a different version than the one under consideration
+// will be ignored for the purposes of defining that image.
+type ImageConfig struct {
+	// Version is the version of FoundationDB this policy applies to. If this is
+	// blank, the policy applies to all FDB versions.
+	Version string `json:"version,omitempty"`
+
+	// BaseImage specifies the part of the image before the tag.
+	BaseImage string `json:"baseImage,omitempty"`
+
+	// Tag specifies a full image tag.
+	Tag string `json:"tag,omitempty"`
+
+	// TagSuffix specifies a suffix that will be added after the version to form
+	// the full tag.
+	TagSuffix string `json:"tagSuffix,omitempty"`
+}
+
+// SelectImageConfig selects image configs that apply to a version of FDB and
+// merges them into a single config.
+func SelectImageConfig(allConfigs []ImageConfig, versionString string) ImageConfig {
+	config := ImageConfig{Version: versionString}
+	for _, nextConfig := range allConfigs {
+		if nextConfig.Version != "" && nextConfig.Version != versionString {
+			continue
+		}
+		if config.BaseImage == "" {
+			config.BaseImage = nextConfig.BaseImage
+		}
+		if config.Tag == "" {
+			config.Tag = nextConfig.Tag
+		}
+		if config.TagSuffix == "" {
+			config.TagSuffix = nextConfig.TagSuffix
+		}
+	}
+	return config
+}
+
+// Image generates an image using a config.
+func (config ImageConfig) Image() string {
+	if config.Tag == "" {
+		return fmt.Sprintf("%s:%s%s", config.BaseImage, config.Version, config.TagSuffix)
+	}
+	return fmt.Sprintf("%s:%s", config.BaseImage, config.Tag)
+}
diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go
index 3e7a7e21..880fc6d6 100644
--- a/api/v1beta2/zz_generated.deepcopy.go
+++ b/api/v1beta2/zz_generated.deepcopy.go
@@ -364,6 +364,11 @@ func (in *FoundationDBBackupSpec) DeepCopyInto(out *FoundationDBBackupSpec) {
 		*out = new(BlobStoreConfiguration)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.ImageConfigs != nil {
+		in, out := &in.ImageConfigs, &out.ImageConfigs
+		*out = make([]ImageConfig, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBBackupSpec.
diff --git a/config/crd/bases/apps.foundationdb.org_foundationdbbackups.yaml b/config/crd/bases/apps.foundationdb.org_foundationdbbackups.yaml
index e20fcfdf..460e5ee7 100644
--- a/config/crd/bases/apps.foundationdb.org_foundationdbbackups.yaml
+++ b/config/crd/bases/apps.foundationdb.org_foundationdbbackups.yaml
@@ -3197,6 +3197,19 @@ spec:
                   type: string
                 maxItems: 100
                 type: array
+              imageConfigs:
+                items:
+                  properties:
+                    baseImage:
+                      type: string
+                    tag:
+                      type: string
+                    tagSuffix:
+                      type: string
+                    version:
+                      type: string
+                  type: object
+                type: array
               podTemplateSpec:
                 properties:
                   metadata:
diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go
index 7c5e5b65..44b39aef 100644
--- a/controllers/cluster_controller.go
+++ b/controllers/cluster_controller.go
@@ -28,6 +28,7 @@ import (
 	"math"
 	"regexp"
 	"sort"
+	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -59,14 +60,18 @@ type FoundationDBClusterReconciler struct {
 	PodClientProvider      func(*fdbv1beta2.FoundationDBCluster, *corev1.Pod) (podclient.FdbPodClient, error)
 	DatabaseClientProvider DatabaseClientProvider
 	DeprecationOptions     internal.DeprecationOptions
+	GetTimeout             time.Duration
+	PostTimeout            time.Duration
 }
 
 // NewFoundationDBClusterReconciler creates a new FoundationDBClusterReconciler with defaults.
 func NewFoundationDBClusterReconciler(podLifecycleManager podmanager.PodLifecycleManager) *FoundationDBClusterReconciler {
-	return &FoundationDBClusterReconciler{
+	r := &FoundationDBClusterReconciler{
 		PodLifecycleManager: podLifecycleManager,
-		PodClientProvider:   newFdbPodClient,
 	}
+	r.PodClientProvider = r.newFdbPodClient
+
+	return r
 }
 
 // +kubebuilder:rbac:groups=apps.foundationdb.org,resources=foundationdbclusters,verbs=get;list;watch;create;update;patch;delete
@@ -309,6 +314,7 @@ func (r *FoundationDBClusterReconciler) getPodClient(cluster *fdbv1beta2.Foundat
 		return nil, fmt.Sprintf("Process group in cluster %s/%s does not have pod defined", cluster.Namespace, cluster.Name)
 	}
 
+	// TODO how to pass this down?
 	podClient, err := r.PodClientProvider(cluster, pod)
 	if err != nil {
 		return nil, err.Error()
@@ -731,8 +737,8 @@ func checkCoordinatorValidity(cluster *fdbv1beta2.FoundationDBCluster, status *f
 }
 
 // newFdbPodClient builds a client for working with an FDB Pod
-func newFdbPodClient(cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod) (podclient.FdbPodClient, error) {
-	return internal.NewFdbPodClient(cluster, pod, log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "pod", pod.Name))
+func (r *FoundationDBClusterReconciler) newFdbPodClient(cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod) (podclient.FdbPodClient, error) {
+	return internal.NewFdbPodClient(cluster, pod, log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "pod", pod.Name), r.GetTimeout, r.PostTimeout)
 }
 
 func (r *FoundationDBClusterReconciler) getCoordinatorSet(cluster *fdbv1beta2.FoundationDBCluster) (map[string]struct{}, error) {
diff --git a/controllers/update_backup_status.go b/controllers/update_backup_status.go
index 07fdfed3..63794c64 100644
--- a/controllers/update_backup_status.go
+++ b/controllers/update_backup_status.go
@@ -22,9 +22,9 @@ package controllers
 
 import (
 	"context"
-	"reflect"
 
 	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
+	"k8s.io/apimachinery/pkg/api/equality"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	appsv1 "k8s.io/api/apps/v1"
@@ -62,7 +62,7 @@ func (s updateBackupStatus) reconcile(ctx context.Context, r *FoundationDBBackup
 		annotationChange := mergeAnnotations(&backupDeployment.ObjectMeta, desiredBackupDeployment.ObjectMeta)
 
 		metadataMatch := !annotationChange &&
-			reflect.DeepEqual(backupDeployment.ObjectMeta.Labels, desiredBackupDeployment.ObjectMeta.Labels)
+			equality.Semantic.DeepEqual(backupDeployment.ObjectMeta.Labels, desiredBackupDeployment.ObjectMeta.Labels)
 
 		status.DeploymentConfigured = generationsMatch && metadataMatch
 
@@ -103,7 +103,7 @@ func (s updateBackupStatus) reconcile(ctx context.Context, r *FoundationDBBackup
 		return &requeue{curError: err}
 	}
 
-	if !reflect.DeepEqual(backup.Status, *originalStatus) {
+	if !equality.Semantic.DeepEqual(backup.Status, *originalStatus) {
 		err = r.Status().Update(ctx, backup)
 		if err != nil {
 			log.Error(err, "Error updating backup status", "namespace", backup.Namespace, "backup", backup.Name)
diff --git a/controllers/update_pods.go b/controllers/update_pods.go
index 51e5b980..df7f6cb8 100644
--- a/controllers/update_pods.go
+++ b/controllers/update_pods.go
@@ -76,7 +76,7 @@ func (updatePods) reconcile(ctx context.Context, r *FoundationDBClusterReconcile
 		}
 
 		if shouldRequeueDueToTerminatingPod(pod, cluster, processGroup.ProcessGroupID) {
-			return &requeue{message: "Cluster has pod that is pending deletion", delay: podSchedulingDelayDuration}
+			return &requeue{message: "Cluster has pod that is pending deletion", delay: podSchedulingDelayDuration, delayedRequeue: true}
 		}
 
 		_, idNum, err := podmanager.ParseProcessGroupID(processGroup.ProcessGroupID)
diff --git a/docs/backup_spec.md b/docs/backup_spec.md
index 4a1b6ce2..7af087cc 100644
--- a/docs/backup_spec.md
+++ b/docs/backup_spec.md
@@ -14,6 +14,7 @@ This Document documents the types introduced by the FoundationDB Operator to be
 * [FoundationDBBackupStatusBackupDetails](#foundationdbbackupstatusbackupdetails)
 * [FoundationDBLiveBackupStatus](#foundationdblivebackupstatus)
 * [FoundationDBLiveBackupStatusState](#foundationdblivebackupstatusstate)
+* [ImageConfig](#imageconfig)
 
 ## BackupGenerationStatus
 
@@ -86,8 +87,9 @@ FoundationDBBackupSpec describes the desired state of the backup for a cluster.
 | backupDeploymentMetadata | BackupDeploymentMetadata allows customizing labels and annotations on the deployment for the backup agents. | *[metav1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#objectmeta-v1-meta) | false |
 | podTemplateSpec | PodTemplateSpec allows customizing the pod template for the backup agents. | *[corev1.PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#podtemplatespec-v1-core) | false |
 | customParameters | CustomParameters defines additional parameters to pass to the backup agents. | FoundationDBCustomParameters | false |
-| allowTagOverride | This setting defines if a user provided image can have it's own tag rather than getting the provided version appended. You have to ensure that the specified version in the Spec is compatible with the given version in your custom image. | *bool | false |
+| allowTagOverride | This setting defines if a user provided image can have it's own tag rather than getting the provided version appended. You have to ensure that the specified version in the Spec is compatible with the given version in your custom image. **Deprecated: use ImageConfigs instead.** | *bool | false |
 | blobStoreConfiguration | This is the configuration of the target blobstore for this backup. | *[BlobStoreConfiguration](#blobstoreconfiguration) | false |
+| imageConfigs | ImageConfigs allows customizing the image that we use for a container. | [][ImageConfig](#imageconfig) | false |
 
 [Back to TOC](#table-of-contents)
 
@@ -151,3 +153,16 @@ URLParameter defines a single URL parameter to pass to the blobstore.
 FoundationDBCustomParameter defines a single custom knob
 
 [Back to TOC](#table-of-contents)
+
+## ImageConfig
+
+ImageConfig provides a policy for customizing an image.  When multiple image configs are provided, they will be merged into a single config that will be used to define the final image. For each field, we select the value from the first entry in the config list that defines a value for that field, and matches the version of FoundationDB the image is for. Any config that specifies a different version than the one under consideration will be ignored for the purposes of defining that image.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| version | Version is the version of FoundationDB this policy applies to. If this is blank, the policy applies to all FDB versions. | string | false |
+| baseImage | BaseImage specifies the part of the image before the tag. | string | false |
+| tag | Tag specifies a full image tag. | string | false |
+| tagSuffix | TagSuffix specifies a suffix that will be added after the version to form the full tag. | string | false |
+
+[Back to TOC](#table-of-contents)
diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
index 42d023e3..5a3d1e0d 100644
--- a/docs/cluster_spec.md
+++ b/docs/cluster_spec.md
@@ -18,7 +18,6 @@ This Document documents the types introduced by the FoundationDB Operator to be
 * [FoundationDBClusterList](#foundationdbclusterlist)
 * [FoundationDBClusterSpec](#foundationdbclusterspec)
 * [FoundationDBClusterStatus](#foundationdbclusterstatus)
-* [ImageConfig](#imageconfig)
 * [LabelConfig](#labelconfig)
 * [LockDenyListEntry](#lockdenylistentry)
 * [LockOptions](#lockoptions)
@@ -34,6 +33,7 @@ This Document documents the types introduced by the FoundationDB Operator to be
 * [Region](#region)
 * [RoleCounts](#rolecounts)
 * [VersionFlags](#versionflags)
+* [ImageConfig](#imageconfig)
 
 ## AutomaticReplacementOptions
 
@@ -253,19 +253,6 @@ FoundationDBClusterStatus defines the observed state of FoundationDBCluster
 
 [Back to TOC](#table-of-contents)
 
-## ImageConfig
-
-ImageConfig provides a policy for customizing an image.  When multiple image configs are provided, they will be merged into a single config that will be used to define the final image. For each field, we select the value from the first entry in the config list that defines a value for that field, and matches the version of FoundationDB the image is for. Any config that specifies a different version than the one under consideration will be ignored for the purposes of defining that image.
-
-| Field | Description | Scheme | Required |
-| ----- | ----------- | ------ | -------- |
-| version | Version is the version of FoundationDB this policy applies to. If this is blank, the policy applies to all FDB versions. | string | false |
-| baseImage | BaseImage specifies the part of the image before the tag. | string | false |
-| tag | Tag specifies a full image tag. | string | false |
-| tagSuffix | TagSuffix specifies a suffix that will be added after the version to form the full tag. | string | false |
-
-[Back to TOC](#table-of-contents)
-
 ## ImageType
 
 ImageType defines a single kind of images used in the cluster.
@@ -526,3 +513,16 @@ VersionFlags defines internal flags for new features in the database.
 ProcessClass models the class of a pod
 
 [Back to TOC](#table-of-contents)
+
+## ImageConfig
+
+ImageConfig provides a policy for customizing an image.  When multiple image configs are provided, they will be merged into a single config that will be used to define the final image. For each field, we select the value from the first entry in the config list that defines a value for that field, and matches the version of FoundationDB the image is for. Any config that specifies a different version than the one under consideration will be ignored for the purposes of defining that image.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| version | Version is the version of FoundationDB this policy applies to. If this is blank, the policy applies to all FDB versions. | string | false |
+| baseImage | BaseImage specifies the part of the image before the tag. | string | false |
+| tag | Tag specifies a full image tag. | string | false |
+| tagSuffix | TagSuffix specifies a suffix that will be added after the version to form the full tag. | string | false |
+
+[Back to TOC](#table-of-contents)
diff --git a/fdbclient/admin_client.go b/fdbclient/admin_client.go
index f9ba4c61..a551e101 100644
--- a/fdbclient/admin_client.go
+++ b/fdbclient/admin_client.go
@@ -330,19 +330,31 @@ func (client *cliAdminClient) GetExclusions() ([]fdbv1beta2.ProcessAddress, erro
 	return exclusions, nil
 }
 
+// exclusionStatus represents the current status of processes that should be excluded.
+// This can include processes that are currently in the progress of being excluded (data movement),
+// processes that are fully excluded and don't serve any roles and processes that are not marked for
+// exclusion.
+type exclusionStatus struct {
+	// inProgress contains all addresses that are excluded in the cluster and the exclude command can be used to verify if it's safe to remove this address.
+	inProgress []fdbv1beta2.ProcessAddress
+	// fullyExcluded contains all addresses that are excluded and don't have any roles assigned, this is a sign that the process is "fully" excluded and safe to remove.
+	fullyExcluded []fdbv1beta2.ProcessAddress
+	// notExcluded contains all addresses that are part of the input list and are not marked as excluded in the cluster, those addresses are not safe to remove.
+	notExcluded []fdbv1beta2.ProcessAddress
+}
+
 // getRemainingAndExcludedFromStatus checks which processes of the input address list are excluded in the cluster and which are not.
-// The first list return all addresses that are excluded in the cluster and the exclude command can be used to verify if it's safe to remove this address.
-// The second list contains all addresses that are part of the input list and are not marked as excluded in the cluster, those addresses are not safe to remove.
-func getRemainingAndExcludedFromStatus(status *fdbv1beta2.FoundationDBStatus, addresses []fdbv1beta2.ProcessAddress) ([]fdbv1beta2.ProcessAddress, []fdbv1beta2.ProcessAddress) {
-	notExcluded := map[string]fdbv1beta2.None{}
+func getRemainingAndExcludedFromStatus(status *fdbv1beta2.FoundationDBStatus, addresses []fdbv1beta2.ProcessAddress) exclusionStatus {
+	notExcludedAddresses := map[string]fdbv1beta2.None{}
+	fullyExcludedAddresses := map[string]fdbv1beta2.None{}
 	visitedAddresses := map[string]fdbv1beta2.None{}
 	for _, addr := range addresses {
-		notExcluded[addr.MachineAddress()] = fdbv1beta2.None{}
+		notExcludedAddresses[addr.MachineAddress()] = fdbv1beta2.None{}
 	}
 
 	// Check in the status output which processes are already marked for exclusion in the cluster
 	for _, process := range status.Cluster.Processes {
-		if _, ok := notExcluded[process.Address.MachineAddress()]; !ok {
+		if _, ok := notExcludedAddresses[process.Address.MachineAddress()]; !ok {
 			continue
 		}
 
@@ -351,32 +363,38 @@ func getRemainingAndExcludedFromStatus(status *fdbv1beta2.FoundationDBStatus, ad
 			continue
 		}
 
-		delete(notExcluded, process.Address.MachineAddress())
+		if len(process.Roles) == 0 {
+			fullyExcludedAddresses[process.Address.MachineAddress()] = fdbv1beta2.None{}
+		}
+
+		delete(notExcludedAddresses, process.Address.MachineAddress())
 	}
 
-	var remaining, excludeCheckAddr []fdbv1beta2.ProcessAddress
-	if len(notExcluded) > 0 {
-		excludeCheckAddr = make([]fdbv1beta2.ProcessAddress, 0, len(addresses)-len(notExcluded))
-		remaining = make([]fdbv1beta2.ProcessAddress, 0, len(notExcluded))
+	exclusions := exclusionStatus{
+		inProgress:    make([]fdbv1beta2.ProcessAddress, 0, len(addresses)-len(notExcludedAddresses)-len(fullyExcludedAddresses)),
+		fullyExcluded: make([]fdbv1beta2.ProcessAddress, 0, len(fullyExcludedAddresses)),
+		notExcluded:   make([]fdbv1beta2.ProcessAddress, 0, len(notExcludedAddresses)),
+	}
 
-		for _, addr := range addresses {
-			// Those addresses are not excluded, so it's not safe to start the exclude command to check
-			// if they are fully excluded. If we didn't visit that address (absent in the cluster status) we assume
-			// it's safe to run the exclude command against it.
-			_, visited := visitedAddresses[addr.MachineAddress()]
-			if _, ok := notExcluded[addr.MachineAddress()]; ok && visited {
-				remaining = append(remaining, addr)
-				continue
-			}
+	for _, addr := range addresses {
+		// Those addresses are not excluded, so it's not safe to start the exclude command to check
+		// if they are fully excluded. If we didn't visit that address (absent in the cluster status) we assume
+		// it's safe to run the exclude command against it.
+		_, visited := visitedAddresses[addr.MachineAddress()]
+		if _, ok := notExcludedAddresses[addr.MachineAddress()]; ok && visited {
+			exclusions.notExcluded = append(exclusions.notExcluded, addr)
+			continue
+		}
 
-			excludeCheckAddr = append(excludeCheckAddr, addr)
+		if _, ok := fullyExcludedAddresses[addr.MachineAddress()]; ok {
+			exclusions.fullyExcluded = append(exclusions.fullyExcluded, addr)
+			continue
 		}
 
-		return excludeCheckAddr, remaining
+		exclusions.inProgress = append(exclusions.inProgress, addr)
 	}
 
-	// All addresses are excluded
-	return addresses, nil
+	return exclusions
 }
 
 // CanSafelyRemove checks whether it is safe to remove processes from the
@@ -395,41 +413,62 @@ func (client *cliAdminClient) CanSafelyRemove(addresses []fdbv1beta2.ProcessAddr
 		return nil, err
 	}
 
-	markedAsExcluded, remaining := getRemainingAndExcludedFromStatus(status, addresses)
+	exclusions := getRemainingAndExcludedFromStatus(status, addresses)
 	client.log.Info("Filtering excluded processes",
 		"namespace", client.Cluster.Namespace,
 		"cluster", client.Cluster.Name,
-		"markedAsExcluded", markedAsExcluded,
-		"notExcludedAddresses", remaining)
+		"inProgress", exclusions.inProgress,
+		"fullyExcluded", exclusions.fullyExcluded,
+		"notExcluded", exclusions.notExcluded)
 
 	if version.HasNonBlockingExcludes(client.Cluster.GetUseNonBlockingExcludes()) {
 		output, err := client.runCommand(cliCommand{command: fmt.Sprintf(
 			"exclude no_wait %s",
-			fdbv1beta2.ProcessAddressesString(markedAsExcluded, " "),
+			fdbv1beta2.ProcessAddressesString(exclusions.inProgress, " "),
 		)})
 		if err != nil {
 			return nil, err
 		}
 		exclusionResults := parseExclusionOutput(output)
-		client.log.Info("Checking exclusion results", "namespace", client.Cluster.Namespace, "cluster", client.Cluster.Name, "addresses", markedAsExcluded, "results", exclusionResults)
-		for _, address := range markedAsExcluded {
+		client.log.Info("Checking exclusion results", "namespace", client.Cluster.Namespace, "cluster", client.Cluster.Name, "addresses", exclusions.inProgress, "results", exclusionResults)
+		for _, address := range exclusions.inProgress {
 			if exclusionResults[address.String()] != "Success" && exclusionResults[address.String()] != "Missing" {
-				remaining = append(remaining, address)
+				exclusions.notExcluded = append(exclusions.notExcluded, address)
 			}
 		}
 
-		return remaining, nil
+		return exclusions.notExcluded, nil
 	}
+
+	// We expect that this command always run without a timeout error since all processes should be fully excluded.
+	// We run this only as an additional safety check.
 	_, err = client.runCommand(cliCommand{command: fmt.Sprintf(
 		"exclude %s",
-		fdbv1beta2.ProcessAddressesString(markedAsExcluded, " "),
+		fdbv1beta2.ProcessAddressesString(exclusions.fullyExcluded, " "),
 	)})
 
 	if err != nil {
 		return nil, err
 	}
 
-	return remaining, nil
+	_, err = client.runCommand(cliCommand{command: fmt.Sprintf(
+		"exclude %s",
+		fdbv1beta2.ProcessAddressesString(exclusions.inProgress, " "),
+	)})
+
+	// When we hit a timeout error here we know that at least one of the inProgress is still not fully excluded for safety
+	// we just return the whole slice and don't do any further distinction. We have to return all addresses that are not excluded
+	// and are still in progress, but we don't want to return an error to block further actions on the successfully excluded
+	// addresses.
+	if err != nil {
+		if internal.IsTimeoutError(err) {
+			return append(exclusions.notExcluded, exclusions.inProgress...), nil
+		}
+
+		return nil, err
+	}
+
+	return exclusions.notExcluded, nil
 }
 
 // parseExclusionOutput extracts the exclusion status for each address from
diff --git a/internal/pod_client.go b/internal/pod_client.go
index 2fec4d7a..0ea4a9a5 100644
--- a/internal/pod_client.go
+++ b/internal/pod_client.go
@@ -90,6 +90,12 @@ type realFdbPodSidecarClient struct {
 
 	// logger is used to add common fields to log messages.
 	logger logr.Logger
+
+	// getTimeout defines the timeout for get requests
+	getTimeout time.Duration
+
+	// postTimeout defines the timeout for post requests
+	postTimeout time.Duration
 }
 
 // realPodSidecarClient provides a client for use in real environments, using
@@ -106,7 +112,7 @@ type realFdbPodAnnotationClient struct {
 }
 
 // NewFdbPodClient builds a client for working with an FDB Pod
-func NewFdbPodClient(cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod, log logr.Logger) (podclient.FdbPodClient, error) {
+func NewFdbPodClient(cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod, log logr.Logger, getTimeout time.Duration, postTimeout time.Duration) (podclient.FdbPodClient, error) {
 	if GetImageType(pod) == FDBImageTypeUnified {
 		return &realFdbPodAnnotationClient{Cluster: cluster, Pod: pod, logger: log}, nil
 	}
@@ -152,7 +158,7 @@ func NewFdbPodClient(cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod, l
 		tlsConfig.RootCAs = certPool
 	}
 
-	return &realFdbPodSidecarClient{Cluster: cluster, Pod: pod, useTLS: useTLS, tlsConfig: tlsConfig, logger: log}, nil
+	return &realFdbPodSidecarClient{Cluster: cluster, Pod: pod, useTLS: useTLS, tlsConfig: tlsConfig, logger: log, getTimeout: getTimeout, postTimeout: postTimeout}, nil
 }
 
 // getListenIP gets the IP address that a pod listens on.
@@ -186,12 +192,10 @@ func (client *realFdbPodSidecarClient) makeRequest(method string, path string) (
 	url := fmt.Sprintf("%s://%s:8080/%s", protocol, client.getListenIP(), path)
 	switch method {
 	case http.MethodGet:
-		// We assume that a get request should be relative fast.
-		retryClient.HTTPClient.Timeout = 5 * time.Second
+		retryClient.HTTPClient.Timeout = client.getTimeout
 		resp, err = retryClient.Get(url)
 	case http.MethodPost:
-		// A post request could take a little bit longer since we copy sometimes files.
-		retryClient.HTTPClient.Timeout = 10 * time.Second
+		retryClient.HTTPClient.Timeout = client.postTimeout
 		resp, err = retryClient.Post(url, "application/json", strings.NewReader(""))
 	default:
 		return "", fmt.Errorf("unknown HTTP method %s", method)
diff --git a/internal/pod_models.go b/internal/pod_models.go
index fbf999b9..8ebb46a3 100644
--- a/internal/pod_models.go
+++ b/internal/pod_models.go
@@ -182,10 +182,7 @@ func GetPodSpec(cluster *fdbv1beta2.FoundationDBCluster, processClass fdbv1beta2
 
 	podName, processGroupID := GetProcessGroupID(cluster, processClass, idNum)
 
-	versionString := cluster.Status.RunningVersion
-	if versionString == "" {
-		versionString = cluster.Spec.Version
-	}
+	versionString := cluster.GetRunningVersion()
 
 	image, err := GetImage(mainContainer.Image, cluster.Spec.MainContainer.ImageConfigs, versionString)
 	if err != nil {
@@ -825,7 +822,11 @@ func GetBackupDeployment(backup *fdbv1beta2.FoundationDBBackup) (*appsv1.Deploym
 		podTemplate.Spec.Containers = containers
 	}
 
-	image, err := GetImage(mainContainer.Image, []fdbv1beta2.ImageConfig{{BaseImage: "foundationdb/foundationdb"}}, backup.Spec.Version)
+	if backup.Spec.ImageConfigs == nil {
+		backup.Spec.ImageConfigs = []fdbv1beta2.ImageConfig{{BaseImage: "foundationdb/foundationdb"}}
+	}
+
+	image, err := GetImage(mainContainer.Image, backup.Spec.ImageConfigs, backup.Spec.Version)
 	if err != nil {
 		return nil, err
 	}
diff --git a/setup/setup.go b/setup/setup.go
index 5d05671c..45d25559 100644
--- a/setup/setup.go
+++ b/setup/setup.go
@@ -66,6 +66,8 @@ type Options struct {
 	PrintVersion            bool
 	LabelSelector           string
 	WatchNamespace          string
+	GetTimeout              time.Duration
+	PostTimeout             time.Duration
 }
 
 // BindFlags will parse the given flagset for the operator option flags
@@ -90,6 +92,8 @@ func (o *Options) BindFlags(fs *flag.FlagSet) {
 	fs.BoolVar(&o.PrintVersion, "version", false, "Prints the version of the operator and exits.")
 	fs.StringVar(&o.LabelSelector, "label-selector", "", "Defines a label-selector that will be used to select resources.")
 	fs.StringVar(&o.WatchNamespace, "watch-namespace", os.Getenv("WATCH_NAMESPACE"), "Defines which namespace the operator should watch")
+	fs.DurationVar(&o.GetTimeout, "get-timeout", 5*time.Second, "http timeout for get requests to the FDB sidecar")
+	fs.DurationVar(&o.PostTimeout, "post-timeout", 10*time.Second, "http timeout for post requests to the FDB sidecar")
 }
 
 // StartManager will start the FoundationDB operator manager.
@@ -173,6 +177,8 @@ func StartManager(
 		clusterReconciler.Recorder = mgr.GetEventRecorderFor("foundationdbcluster-controller")
 		clusterReconciler.DeprecationOptions = operatorOpts.DeprecationOptions
 		clusterReconciler.DatabaseClientProvider = fdbclient.NewDatabaseClientProvider(logger)
+		clusterReconciler.GetTimeout = operatorOpts.GetTimeout
+		clusterReconciler.PostTimeout = operatorOpts.PostTimeout
 		clusterReconciler.Log = logr.WithName("controllers").WithName("FoundationDBCluster")
 
 		if err := clusterReconciler.SetupWithManager(mgr, operatorOpts.MaxConcurrentReconciles, *labelSelector, watchedObjects...); err != nil {


Test patch:
diff --git a/api/v1beta2/foundationdbcluster_types_test.go b/api/v1beta2/foundationdbcluster_types_test.go
index e6aa3458..c2d3d808 100644
--- a/api/v1beta2/foundationdbcluster_types_test.go
+++ b/api/v1beta2/foundationdbcluster_types_test.go
@@ -424,15 +424,19 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 				Log:       5,
 				Stateless: 9,
 			}))
+		})
 
-			cluster.Spec.Version = "7.1.0-rc2"
-			counts, err = cluster.GetProcessCountsWithDefaults()
-			Expect(err).NotTo(HaveOccurred())
-			Expect(counts).To(Equal(ProcessCounts{
-				Storage:   5,
-				Log:       5,
-				Stateless: 9,
-			}))
+		When("using a version that supports grv and commit proxies", func() {
+			It("should return the default process counts", func() {
+				cluster.Spec.Version = "7.1.0"
+				counts, err := cluster.GetProcessCountsWithDefaults()
+				Expect(err).NotTo(HaveOccurred())
+				Expect(counts).To(Equal(ProcessCounts{
+					Storage:   5,
+					Log:       4,
+					Stateless: 22,
+				}))
+			})
 		})
 
 		It("should return the default process counts when proxies are unset", func() {
@@ -983,12 +987,14 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 				RoleCounts: RoleCounts{
 					Logs:          5,
 					Proxies:       1,
-					CommitProxies: 2,
+					CommitProxies: 4,
 					GrvProxies:    2,
 				},
 			}
-			Expect(configuration.AreSeparatedProxiesConfigured()).To(BeFalse())
+			// This check is not version dependent
+			Expect(configuration.AreSeparatedProxiesConfigured()).To(BeTrue())
 			Expect(configuration.GetConfigurationString("6.3.24")).To(Equal("double ssd usable_regions=1 logs=5 resolvers=0 log_routers=0 remote_logs=0 proxies=1 regions=[]"))
+			Expect(configuration.GetConfigurationString("7.1.0")).To(Equal("double ssd usable_regions=1 logs=5 resolvers=0 log_routers=0 remote_logs=0 commit_proxies=4 grv_proxies=2 regions=[]"))
 
 			configuration.Regions = []Region{{
 				DataCenters: []DataCenter{{
@@ -1004,8 +1010,8 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 			configuration.VersionFlags.LogSpill = 3
 			Expect(configuration.GetConfigurationString("6.3.24")).To(Equal("double ssd usable_regions=1 logs=5 resolvers=0 log_routers=0 remote_logs=0 proxies=1 log_spill:=3 regions=[]"))
 
-			Expect(configuration.GetConfigurationString("7.0.0")).To(Equal("double ssd usable_regions=1 logs=5 resolvers=0 log_routers=0 remote_logs=0 proxies=1 log_spill:=3 regions=[]"))
-			Expect(configuration.GetConfigurationString("7.1.0-rc1")).To(Equal("double ssd usable_regions=1 logs=5 resolvers=0 log_routers=0 remote_logs=0 proxies=1 log_spill:=3 regions=[]"))
+			Expect(configuration.GetConfigurationString("7.0.0")).To(Equal("double ssd usable_regions=1 logs=5 resolvers=0 log_routers=0 remote_logs=0 commit_proxies=4 grv_proxies=2 log_spill:=3 regions=[]"))
+			Expect(configuration.GetConfigurationString("7.1.0-rc1")).To(Equal("double ssd usable_regions=1 logs=5 resolvers=0 log_routers=0 remote_logs=0 commit_proxies=4 grv_proxies=2 log_spill:=3 regions=[]"))
 		})
 
 		When("CommitProxies and GrvProxies are not configured", func() {
@@ -3953,77 +3959,6 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 		)
 	})
 
-	When("merging image configs", func() {
-		It("applies chooses the first value for each field", func() {
-			configs := []ImageConfig{
-				{
-					BaseImage: "foundationdb/foundationdb",
-					Version:   Versions.Default.String(),
-				},
-				{
-					BaseImage: "foundationdb/foundationdb-slim",
-					Version:   Versions.Default.String(),
-					Tag:       "abcdef",
-					TagSuffix: "-1",
-				},
-			}
-
-			finalConfig := SelectImageConfig(configs, Versions.Default.String())
-			Expect(finalConfig).To(Equal(ImageConfig{
-				BaseImage: "foundationdb/foundationdb",
-				Version:   Versions.Default.String(),
-				Tag:       "abcdef",
-				TagSuffix: "-1",
-			}))
-		})
-
-		It("ignores configs that are for different versions", func() {
-			configs := []ImageConfig{
-				{
-					BaseImage: "foundationdb/foundationdb",
-					Version:   Versions.Default.String(),
-				},
-				{
-					Version: Versions.NextMajorVersion.String(),
-					Tag:     "abcdef",
-				},
-				{
-					TagSuffix: "-1",
-				},
-			}
-
-			finalConfig := SelectImageConfig(configs, Versions.Default.String())
-			Expect(finalConfig).To(Equal(ImageConfig{
-				BaseImage: "foundationdb/foundationdb",
-				Version:   Versions.Default.String(),
-				TagSuffix: "-1",
-			}))
-		})
-	})
-
-	When("building image names", func() {
-		It("applies the fields", func() {
-			config := ImageConfig{
-				BaseImage: "foundationdb/foundationdb-kubernetes-sidecar",
-				Version:   Versions.Default.String(),
-				TagSuffix: "-2",
-			}
-			image := config.Image()
-			Expect(image).To(Equal(fmt.Sprintf("foundationdb/foundationdb-kubernetes-sidecar:%s-2", Versions.Default)))
-		})
-
-		It("uses the tag to override the version and tag suffix", func() {
-			config := ImageConfig{
-				BaseImage: "foundationdb/foundationdb-kubernetes-sidecar",
-				Version:   Versions.Default.String(),
-				Tag:       "abcdef",
-				TagSuffix: "-2",
-			}
-			image := config.Image()
-			Expect(image).To(Equal("foundationdb/foundationdb-kubernetes-sidecar:abcdef"))
-		})
-	})
-
 	When("checking if the process group needs a replacement", func() {
 		var processGroup *ProcessGroupStatus
 		var needsReplacement bool
diff --git a/api/v1beta2/image_config_test.go b/api/v1beta2/image_config_test.go
new file mode 100644
index 00000000..b6adf120
--- /dev/null
+++ b/api/v1beta2/image_config_test.go
@@ -0,0 +1,81 @@
+package v1beta2
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("[api] ImageConfig", func() {
+	When("merging image configs", func() {
+		It("applies chooses the first value for each field", func() {
+			configs := []ImageConfig{
+				{
+					BaseImage: "foundationdb/foundationdb",
+					Version:   Versions.Default.String(),
+				},
+				{
+					BaseImage: "foundationdb/foundationdb-slim",
+					Version:   Versions.Default.String(),
+					Tag:       "abcdef",
+					TagSuffix: "-1",
+				},
+			}
+
+			finalConfig := SelectImageConfig(configs, Versions.Default.String())
+			Expect(finalConfig).To(Equal(ImageConfig{
+				BaseImage: "foundationdb/foundationdb",
+				Version:   Versions.Default.String(),
+				Tag:       "abcdef",
+				TagSuffix: "-1",
+			}))
+		})
+
+		It("ignores configs that are for different versions", func() {
+			configs := []ImageConfig{
+				{
+					BaseImage: "foundationdb/foundationdb",
+					Version:   Versions.Default.String(),
+				},
+				{
+					Version: Versions.NextMajorVersion.String(),
+					Tag:     "abcdef",
+				},
+				{
+					TagSuffix: "-1",
+				},
+			}
+
+			finalConfig := SelectImageConfig(configs, Versions.Default.String())
+			Expect(finalConfig).To(Equal(ImageConfig{
+				BaseImage: "foundationdb/foundationdb",
+				Version:   Versions.Default.String(),
+				TagSuffix: "-1",
+			}))
+		})
+	})
+
+	When("building image names", func() {
+		It("applies the fields", func() {
+			config := ImageConfig{
+				BaseImage: "foundationdb/foundationdb-kubernetes-sidecar",
+				Version:   Versions.Default.String(),
+				TagSuffix: "-2",
+			}
+			image := config.Image()
+			Expect(image).To(Equal(fmt.Sprintf("foundationdb/foundationdb-kubernetes-sidecar:%s-2", Versions.Default)))
+		})
+
+		It("uses the tag to override the version and tag suffix", func() {
+			config := ImageConfig{
+				BaseImage: "foundationdb/foundationdb-kubernetes-sidecar",
+				Version:   Versions.Default.String(),
+				Tag:       "abcdef",
+				TagSuffix: "-2",
+			}
+			image := config.Image()
+			Expect(image).To(Equal("foundationdb/foundationdb-kubernetes-sidecar:abcdef"))
+		})
+	})
+})
diff --git a/controllers/remove_process_groups_test.go b/controllers/remove_process_groups_test.go
index 4466b9e4..cc024259 100644
--- a/controllers/remove_process_groups_test.go
+++ b/controllers/remove_process_groups_test.go
@@ -175,6 +175,11 @@ var _ = Describe("remove_process_groups", func() {
 			var secondRemovedProcessGroup *fdbv1beta2.ProcessGroupStatus
 
 			BeforeEach(func() {
+				// To allow multiple process groups to be removed we have to use the update mode all
+				cluster.Spec.AutomationOptions.RemovalMode = fdbv1beta2.PodUpdateModeAll
+				err := k8sClient.Update(context.TODO(), cluster)
+				Expect(err).NotTo(HaveOccurred())
+
 				initialCnt = len(cluster.Status.ProcessGroups)
 				secondRemovedProcessGroup = cluster.Status.ProcessGroups[1]
 				marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, secondRemovedProcessGroup.ProcessGroupID, secondRemovedProcessGroup.ProcessClass, removedProcessGroup.Addresses[0])
diff --git a/fdbclient/admin_client_test.go b/fdbclient/admin_client_test.go
index 28c8a0f1..0ffd1788 100644
--- a/fdbclient/admin_client_test.go
+++ b/fdbclient/admin_client_test.go
@@ -105,139 +105,76 @@ var _ = Describe("admin_client_test", func() {
 		addr3 := fdbv1beta2.NewProcessAddress(net.ParseIP("127.0.0.3"), "", 0, nil)
 		addr4 := fdbv1beta2.NewProcessAddress(net.ParseIP("127.0.0.4"), "", 0, nil)
 		addr5 := fdbv1beta2.NewProcessAddress(net.ParseIP("127.0.0.5"), "", 0, nil)
-
-		DescribeTable("fetching the excluded and ramining processes from the status",
-			func(status *fdbv1beta2.FoundationDBStatus, addresses []fdbv1beta2.ProcessAddress, expectedExcluded []fdbv1beta2.ProcessAddress, expectedRemaining []fdbv1beta2.ProcessAddress) {
-				excluded, remaining := getRemainingAndExcludedFromStatus(status, addresses)
-				Expect(expectedExcluded).To(ContainElements(excluded))
-				Expect(len(expectedExcluded)).To(BeNumerically("==", len(excluded)))
-				Expect(expectedRemaining).To(ContainElements(remaining))
-				Expect(len(expectedRemaining)).To(BeNumerically("==", len(remaining)))
-			},
-			Entry("with an empty input address slice",
-				&fdbv1beta2.FoundationDBStatus{
-					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
-						Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
-							"1": {
-								Address:  addr1,
-								Excluded: true,
-							},
-							"2": {
-								Address: addr2,
-							},
-							"3": {
-								Address: addr3,
-							},
-							"4": {
-								Address:  addr4,
-								Excluded: true,
+		status := &fdbv1beta2.FoundationDBStatus{
+			Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+				Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
+					"1": {
+						Address:  addr1,
+						Excluded: true,
+					},
+					"2": {
+						Address: addr2,
+					},
+					"3": {
+						Address: addr3,
+					},
+					"4": {
+						Address:  addr4,
+						Excluded: true,
+						Roles: []fdbv1beta2.FoundationDBStatusProcessRoleInfo{
+							{
+								Role: "tester",
 							},
 						},
 					},
 				},
+			},
+		}
+
+		DescribeTable("fetching the excluded and remaining processes from the status",
+			func(status *fdbv1beta2.FoundationDBStatus, addresses []fdbv1beta2.ProcessAddress, expectedExcluded []fdbv1beta2.ProcessAddress, expectedRemaining []fdbv1beta2.ProcessAddress, expectedFullyExcluded []fdbv1beta2.ProcessAddress) {
+				exclusions := getRemainingAndExcludedFromStatus(status, addresses)
+				Expect(expectedExcluded).To(ContainElements(exclusions.inProgress))
+				Expect(len(expectedExcluded)).To(BeNumerically("==", len(exclusions.inProgress)))
+				Expect(expectedRemaining).To(ContainElements(exclusions.notExcluded))
+				Expect(len(expectedRemaining)).To(BeNumerically("==", len(exclusions.notExcluded)))
+				Expect(expectedFullyExcluded).To(ContainElements(exclusions.fullyExcluded))
+				Expect(len(expectedFullyExcluded)).To(BeNumerically("==", len(exclusions.fullyExcluded)))
+			},
+			Entry("with an empty input address slice",
+				status,
 				[]fdbv1beta2.ProcessAddress{},
 				nil,
 				nil,
+				nil,
 			),
 			Entry("when the process is excluded",
-				&fdbv1beta2.FoundationDBStatus{
-					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
-						Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
-							"1": {
-								Address:  addr1,
-								Excluded: true,
-							},
-							"2": {
-								Address: addr2,
-							},
-							"3": {
-								Address: addr3,
-							},
-							"4": {
-								Address:  addr4,
-								Excluded: true,
-							},
-						},
-					},
-				},
+				status,
 				[]fdbv1beta2.ProcessAddress{addr4},
 				[]fdbv1beta2.ProcessAddress{addr4},
 				nil,
+				nil,
 			),
 			Entry("when the process is not excluded",
-				&fdbv1beta2.FoundationDBStatus{
-					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
-						Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
-							"1": {
-								Address:  addr1,
-								Excluded: true,
-							},
-							"2": {
-								Address: addr2,
-							},
-							"3": {
-								Address: addr3,
-							},
-							"4": {
-								Address:  addr4,
-								Excluded: true,
-							},
-						},
-					},
-				},
+				status,
 				[]fdbv1beta2.ProcessAddress{addr3},
 				nil,
 				[]fdbv1beta2.ProcessAddress{addr3},
+				nil,
 			),
-			Entry("when some processes are excludeded and some not",
-				&fdbv1beta2.FoundationDBStatus{
-					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
-						Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
-							"1": {
-								Address:  addr1,
-								Excluded: true,
-							},
-							"2": {
-								Address: addr2,
-							},
-							"3": {
-								Address: addr3,
-							},
-							"4": {
-								Address:  addr4,
-								Excluded: true,
-							},
-						},
-					},
-				},
+			Entry("when some processes are excluded and some not",
+				status,
 				[]fdbv1beta2.ProcessAddress{addr1, addr2, addr3, addr4},
-				[]fdbv1beta2.ProcessAddress{addr1, addr4},
+				[]fdbv1beta2.ProcessAddress{addr4},
 				[]fdbv1beta2.ProcessAddress{addr2, addr3},
+				[]fdbv1beta2.ProcessAddress{addr1},
 			),
 			Entry("when a process is missing",
-				&fdbv1beta2.FoundationDBStatus{
-					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
-						Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
-							"1": {
-								Address:  addr1,
-								Excluded: true,
-							},
-							"2": {
-								Address: addr2,
-							},
-							"3": {
-								Address: addr3,
-							},
-							"4": {
-								Address:  addr4,
-								Excluded: true,
-							},
-						},
-					},
-				},
+				status,
 				[]fdbv1beta2.ProcessAddress{addr5},
 				[]fdbv1beta2.ProcessAddress{addr5},
 				[]fdbv1beta2.ProcessAddress{},
+				nil,
 			),
 		)
 	})
diff --git a/internal/pod_models_test.go b/internal/pod_models_test.go
index a3370bbe..d88d5bc5 100644
--- a/internal/pod_models_test.go
+++ b/internal/pod_models_test.go
@@ -2947,6 +2947,19 @@ var _ = Describe("pod_models", func() {
 				Expect(deployment.Spec.Template.Spec.Containers[0].Args).To(ContainElement("--customParameter=1337"))
 			})
 		})
+
+		When("defining an image config", func() {
+			BeforeEach(func() {
+				backup.Spec.ImageConfigs = []fdbv1beta2.ImageConfig{{BaseImage: "foundationdb/foundationdb", Tag: "dev"}}
+				deployment, err = GetBackupDeployment(backup)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(deployment).NotTo(BeNil())
+			})
+
+			It("should set the image based on the image configs", func() {
+				Expect(deployment.Spec.Template.Spec.Containers[0].Image).To(Equal("foundationdb/foundationdb:dev"))
+			})
+		})
 	})
 
 	Context("Get image for container", func() {