Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1191 (2022-04-28 16:21:55)

For versions that support grv and commit use grv and commit proxy count if defined
# Description

Fixes: https://github.com/FoundationDB/fdb-kubernetes-operator/issues/1179 this change makes upgrades easier since all three proxies can be defined before upgrading and after the upgrade the more precise definition (grv and commit) of proxies will be used.

## Type of change

*Please select one of the options below.*

- Bug fix (non-breaking change which fixes an issue)

## Discussion

-

## Testing

Local + unit test

## Documentation

I'll add that in a separate PR to document the upgrade process in general.

## Follow-up

-


foundationdb-ci: 
### AWS CodeBuild CI Report for Linux CentOS 7

* CodeBuild project: fdb-kubernetes-operator-pr
* Commit ID: b0db1cb70ea82db0f30af2c5fa9e752b8020a262
* Result: SUCCEEDED
* Error: `N/A`
* [Build Logs](https://d1e0l78xbkh3xa.cloudfront.net/build-logs/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/cec5ba42-3092-43ce-8e4e-8bfc1ca99893.gz) (available for 30 days)


Patch:
diff --git a/.goreleaser.yaml b/.goreleaser.yaml
index 33bb1930..b03f6a80 100644
--- a/.goreleaser.yaml
+++ b/.goreleaser.yaml
@@ -3,8 +3,6 @@ before:
     - go mod tidy
 builds:
   - id: kubectl-fdb
-    env:
-      - CGO_ENABLED=0
     goos:
       - linux
       - windows
diff --git a/Makefile b/Makefile
index 461d0d99..74b37f6d 100644
--- a/Makefile
+++ b/Makefile
@@ -189,13 +189,13 @@ config/samples/client.yaml: config/tests/client/*.yaml
 bin/po-docgen: cmd/po-docgen/*.go
 	go build -o bin/po-docgen cmd/po-docgen/main.go  cmd/po-docgen/api.go
 
-CLUSTER_DOCS_INPUT=api/v1beta2/foundationdbcluster_types.go api/v1beta2/foundationdb_custom_parameter.go api/v1beta2/foundationdb_database_configuration.go api/v1beta2/foundationdb_process_class.go
+CLUSTER_DOCS_INPUT=api/v1beta2/foundationdbcluster_types.go api/v1beta2/foundationdb_custom_parameter.go api/v1beta2/foundationdb_database_configuration.go api/v1beta2/foundationdb_process_class.go api/v1beta2/image_config.go
 
 docs/cluster_spec.md: bin/po-docgen $(CLUSTER_DOCS_INPUT)
 	bin/po-docgen api $(CLUSTER_DOCS_INPUT) > $@
 
 docs/backup_spec.md: bin/po-docgen api/v1beta2/foundationdbbackup_types.go
-	bin/po-docgen api api/v1beta2/foundationdbbackup_types.go api/v1beta2/foundationdb_custom_parameter.go > $@
+	bin/po-docgen api api/v1beta2/foundationdbbackup_types.go api/v1beta2/foundationdb_custom_parameter.go api/v1beta2/image_config.go > $@
 
 docs/restore_spec.md: bin/po-docgen api/v1beta2/foundationdbrestore_types.go
 	bin/po-docgen api api/v1beta2/foundationdbrestore_types.go api/v1beta2/foundationdb_custom_parameter.go > $@
diff --git a/api/v1beta2/foundationdb_database_configuration.go b/api/v1beta2/foundationdb_database_configuration.go
index ed6fe9a1..6b8865cd 100644
--- a/api/v1beta2/foundationdb_database_configuration.go
+++ b/api/v1beta2/foundationdb_database_configuration.go
@@ -307,6 +307,7 @@ func (configuration *DatabaseConfiguration) GetRoleCountsWithDefaults(version Ve
 			counts.LogRouters = -1
 		}
 	}
+
 	return *counts
 }
 
@@ -552,7 +553,7 @@ func (configuration DatabaseConfiguration) getRegionPriorities() map[string]int
 // to 0
 func (configuration DatabaseConfiguration) AreSeparatedProxiesConfigured() bool {
 	counts := configuration.RoleCounts
-	return counts.Proxies == 0 && (counts.GrvProxies > 0 || counts.CommitProxies > 0)
+	return counts.GrvProxies > 0 || counts.CommitProxies > 0
 }
 
 // GetProxiesString returns a string that contains the correct fdbcli
diff --git a/api/v1beta2/foundationdbbackup_types.go b/api/v1beta2/foundationdbbackup_types.go
index 82b74ac2..e9bb212c 100644
--- a/api/v1beta2/foundationdbbackup_types.go
+++ b/api/v1beta2/foundationdbbackup_types.go
@@ -89,10 +89,15 @@ type FoundationDBBackupSpec struct {
 	// You have to ensure that the specified version in the Spec is compatible
 	// with the given version in your custom image.
 	// +kubebuilder:default:=false
+	// Deprecated: use ImageConfigs instead.
 	AllowTagOverride *bool `json:"allowTagOverride,omitempty"`
 
 	// This is the configuration of the target blobstore for this backup.
 	BlobStoreConfiguration *BlobStoreConfiguration `json:"blobStoreConfiguration,omitempty"`
+
+	// ImageConfigs allows customizing the image that we use for
+	// a container.
+	ImageConfigs []ImageConfig `json:"imageConfigs,omitempty"`
 }
 
 // FoundationDBBackupStatus describes the current status of the backup for a cluster.
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index 0b05e35d..f8644fcf 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -944,7 +944,7 @@ func (cluster *FoundationDBCluster) GetProcessSettings(processClass ProcessClass
 // UsableRegions is less than or equal to 1.
 func (cluster *FoundationDBCluster) GetRoleCountsWithDefaults() RoleCounts {
 	// We can ignore the error here since the version will be validated in an earlier step.
-	version, _ := ParseFdbVersion(cluster.Spec.Version)
+	version, _ := ParseFdbVersion(cluster.GetRunningVersion())
 	return cluster.Spec.DatabaseConfiguration.GetRoleCountsWithDefaults(version, cluster.DesiredFaultTolerance())
 }
 
@@ -1055,7 +1055,7 @@ func (cluster *FoundationDBCluster) GetProcessCountsWithDefaults() (ProcessCount
 		primaryStatelessCount += cluster.calculateProcessCountFromRole(1, processCounts.Ratekeeper) +
 			cluster.calculateProcessCountFromRole(1, processCounts.DataDistributor)
 
-		fdbVersion, err := ParseFdbVersion(cluster.Spec.Version)
+		fdbVersion, err := ParseFdbVersion(cluster.GetRunningVersion())
 		if err != nil {
 			return *processCounts, err
 		}
@@ -1072,6 +1072,7 @@ func (cluster *FoundationDBCluster) GetProcessCountsWithDefaults() (ProcessCount
 			cluster.calculateProcessCountFromRole(roleCounts.LogRouters),
 		)
 	}
+
 	return *processCounts, nil
 }
 
@@ -1411,63 +1412,10 @@ type ContainerOverrides struct {
 	ImageConfigs []ImageConfig `json:"imageConfigs,omitempty"`
 }
 
-// ImageConfig provides a policy for customizing an image.
-//
-// When multiple image configs are provided, they will be merged into a single
-// config that will be used to define the final image. For each field, we select
-// the value from the first entry in the config list that defines a value for
-// that field, and matches the version of FoundationDB the image is for. Any
-// config that specifies a different version than the one under consideration
-// will be ignored for the purposes of defining that image.
-type ImageConfig struct {
-	// Version is the version of FoundationDB this policy applies to. If this is
-	// blank, the policy applies to all FDB versions.
-	Version string `json:"version,omitempty"`
-
-	// BaseImage specifies the part of the image before the tag.
-	BaseImage string `json:"baseImage,omitempty"`
-
-	// Tag specifies a full image tag.
-	Tag string `json:"tag,omitempty"`
-
-	// TagSuffix specifies a suffix that will be added after the version to form
-	// the full tag.
-	TagSuffix string `json:"tagSuffix,omitempty"`
-}
-
-// SelectImageConfig selects image configs that apply to a version of FDB and
-// merges them into a single config.
-func SelectImageConfig(allConfigs []ImageConfig, versionString string) ImageConfig {
-	config := ImageConfig{Version: versionString}
-	for _, nextConfig := range allConfigs {
-		if nextConfig.Version != "" && nextConfig.Version != versionString {
-			continue
-		}
-		if config.BaseImage == "" {
-			config.BaseImage = nextConfig.BaseImage
-		}
-		if config.Tag == "" {
-			config.Tag = nextConfig.Tag
-		}
-		if config.TagSuffix == "" {
-			config.TagSuffix = nextConfig.TagSuffix
-		}
-	}
-	return config
-}
-
-// Image generates an image using a config.
-func (config ImageConfig) Image() string {
-	if config.Tag == "" {
-		return fmt.Sprintf("%s:%s%s", config.BaseImage, config.Version, config.TagSuffix)
-	}
-	return fmt.Sprintf("%s:%s", config.BaseImage, config.Tag)
-}
-
 // DesiredDatabaseConfiguration builds the database configuration for the
 // cluster based on its spec.
 func (cluster *FoundationDBCluster) DesiredDatabaseConfiguration() DatabaseConfiguration {
-	configuration := cluster.Spec.DatabaseConfiguration.NormalizeConfigurationWithSeparatedProxies(cluster.Spec.Version, cluster.Spec.DatabaseConfiguration.AreSeparatedProxiesConfigured())
+	configuration := cluster.Spec.DatabaseConfiguration.NormalizeConfigurationWithSeparatedProxies(cluster.GetRunningVersion(), cluster.Spec.DatabaseConfiguration.AreSeparatedProxiesConfigured())
 	configuration.RoleCounts = cluster.GetRoleCountsWithDefaults()
 	configuration.RoleCounts.Storage = 0
 
@@ -2026,3 +1974,16 @@ func (cluster *FoundationDBCluster) AddProcessGroupsToRemovalWithoutExclusionLis
 		cluster.Spec.ProcessGroupsToRemoveWithoutExclusion = append(cluster.Spec.ProcessGroupsToRemoveWithoutExclusion, processGroupID)
 	}
 }
+
+// GetRunningVersion returns the running version of the cluster defined in the cluster status or if not defined the version
+// defined in the cluster spec.
+func (cluster *FoundationDBCluster) GetRunningVersion() string {
+	// We have to use the running version here otherwise if the operator gets killed during an upgrade it will try to apply
+	// the grv and commit proxies.
+	versionString := cluster.Status.RunningVersion
+	if versionString == "" {
+		return cluster.Spec.Version
+	}
+
+	return versionString
+}
diff --git a/api/v1beta2/image_config.go b/api/v1beta2/image_config.go
new file mode 100644
index 00000000..bf582b3e
--- /dev/null
+++ b/api/v1beta2/image_config.go
@@ -0,0 +1,72 @@
+/*
+Copyright 2020-2022 FoundationDB project authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta2
+
+import "fmt"
+
+// ImageConfig provides a policy for customizing an image.
+//
+// When multiple image configs are provided, they will be merged into a single
+// config that will be used to define the final image. For each field, we select
+// the value from the first entry in the config list that defines a value for
+// that field, and matches the version of FoundationDB the image is for. Any
+// config that specifies a different version than the one under consideration
+// will be ignored for the purposes of defining that image.
+type ImageConfig struct {
+	// Version is the version of FoundationDB this policy applies to. If this is
+	// blank, the policy applies to all FDB versions.
+	Version string `json:"version,omitempty"`
+
+	// BaseImage specifies the part of the image before the tag.
+	BaseImage string `json:"baseImage,omitempty"`
+
+	// Tag specifies a full image tag.
+	Tag string `json:"tag,omitempty"`
+
+	// TagSuffix specifies a suffix that will be added after the version to form
+	// the full tag.
+	TagSuffix string `json:"tagSuffix,omitempty"`
+}
+
+// SelectImageConfig selects image configs that apply to a version of FDB and
+// merges them into a single config.
+func SelectImageConfig(allConfigs []ImageConfig, versionString string) ImageConfig {
+	config := ImageConfig{Version: versionString}
+	for _, nextConfig := range allConfigs {
+		if nextConfig.Version != "" && nextConfig.Version != versionString {
+			continue
+		}
+		if config.BaseImage == "" {
+			config.BaseImage = nextConfig.BaseImage
+		}
+		if config.Tag == "" {
+			config.Tag = nextConfig.Tag
+		}
+		if config.TagSuffix == "" {
+			config.TagSuffix = nextConfig.TagSuffix
+		}
+	}
+	return config
+}
+
+// Image generates an image using a config.
+func (config ImageConfig) Image() string {
+	if config.Tag == "" {
+		return fmt.Sprintf("%s:%s%s", config.BaseImage, config.Version, config.TagSuffix)
+	}
+	return fmt.Sprintf("%s:%s", config.BaseImage, config.Tag)
+}
diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go
index 3e7a7e21..880fc6d6 100644
--- a/api/v1beta2/zz_generated.deepcopy.go
+++ b/api/v1beta2/zz_generated.deepcopy.go
@@ -364,6 +364,11 @@ func (in *FoundationDBBackupSpec) DeepCopyInto(out *FoundationDBBackupSpec) {
 		*out = new(BlobStoreConfiguration)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.ImageConfigs != nil {
+		in, out := &in.ImageConfigs, &out.ImageConfigs
+		*out = make([]ImageConfig, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBBackupSpec.
diff --git a/config/crd/bases/apps.foundationdb.org_foundationdbbackups.yaml b/config/crd/bases/apps.foundationdb.org_foundationdbbackups.yaml
index e20fcfdf..460e5ee7 100644
--- a/config/crd/bases/apps.foundationdb.org_foundationdbbackups.yaml
+++ b/config/crd/bases/apps.foundationdb.org_foundationdbbackups.yaml
@@ -3197,6 +3197,19 @@ spec:
                   type: string
                 maxItems: 100
                 type: array
+              imageConfigs:
+                items:
+                  properties:
+                    baseImage:
+                      type: string
+                    tag:
+                      type: string
+                    tagSuffix:
+                      type: string
+                    version:
+                      type: string
+                  type: object
+                type: array
               podTemplateSpec:
                 properties:
                   metadata:
diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go
index 7c5e5b65..44b39aef 100644
--- a/controllers/cluster_controller.go
+++ b/controllers/cluster_controller.go
@@ -28,6 +28,7 @@ import (
 	"math"
 	"regexp"
 	"sort"
+	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -59,14 +60,18 @@ type FoundationDBClusterReconciler struct {
 	PodClientProvider      func(*fdbv1beta2.FoundationDBCluster, *corev1.Pod) (podclient.FdbPodClient, error)
 	DatabaseClientProvider DatabaseClientProvider
 	DeprecationOptions     internal.DeprecationOptions
+	GetTimeout             time.Duration
+	PostTimeout            time.Duration
 }
 
 // NewFoundationDBClusterReconciler creates a new FoundationDBClusterReconciler with defaults.
 func NewFoundationDBClusterReconciler(podLifecycleManager podmanager.PodLifecycleManager) *FoundationDBClusterReconciler {
-	return &FoundationDBClusterReconciler{
+	r := &FoundationDBClusterReconciler{
 		PodLifecycleManager: podLifecycleManager,
-		PodClientProvider:   newFdbPodClient,
 	}
+	r.PodClientProvider = r.newFdbPodClient
+
+	return r
 }
 
 // +kubebuilder:rbac:groups=apps.foundationdb.org,resources=foundationdbclusters,verbs=get;list;watch;create;update;patch;delete
@@ -309,6 +314,7 @@ func (r *FoundationDBClusterReconciler) getPodClient(cluster *fdbv1beta2.Foundat
 		return nil, fmt.Sprintf("Process group in cluster %s/%s does not have pod defined", cluster.Namespace, cluster.Name)
 	}
 
+	// TODO how to pass this down?
 	podClient, err := r.PodClientProvider(cluster, pod)
 	if err != nil {
 		return nil, err.Error()
@@ -731,8 +737,8 @@ func checkCoordinatorValidity(cluster *fdbv1beta2.FoundationDBCluster, status *f
 }
 
 // newFdbPodClient builds a client for working with an FDB Pod
-func newFdbPodClient(cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod) (podclient.FdbPodClient, error) {
-	return internal.NewFdbPodClient(cluster, pod, log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "pod", pod.Name))
+func (r *FoundationDBClusterReconciler) newFdbPodClient(cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod) (podclient.FdbPodClient, error) {
+	return internal.NewFdbPodClient(cluster, pod, log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "pod", pod.Name), r.GetTimeout, r.PostTimeout)
 }
 
 func (r *FoundationDBClusterReconciler) getCoordinatorSet(cluster *fdbv1beta2.FoundationDBCluster) (map[string]struct{}, error) {
diff --git a/controllers/update_backup_status.go b/controllers/update_backup_status.go
index 07fdfed3..63794c64 100644
--- a/controllers/update_backup_status.go
+++ b/controllers/update_backup_status.go
@@ -22,9 +22,9 @@ package controllers
 
 import (
 	"context"
-	"reflect"
 
 	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
+	"k8s.io/apimachinery/pkg/api/equality"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	appsv1 "k8s.io/api/apps/v1"
@@ -62,7 +62,7 @@ func (s updateBackupStatus) reconcile(ctx context.Context, r *FoundationDBBackup
 		annotationChange := mergeAnnotations(&backupDeployment.ObjectMeta, desiredBackupDeployment.ObjectMeta)
 
 		metadataMatch := !annotationChange &&
-			reflect.DeepEqual(backupDeployment.ObjectMeta.Labels, desiredBackupDeployment.ObjectMeta.Labels)
+			equality.Semantic.DeepEqual(backupDeployment.ObjectMeta.Labels, desiredBackupDeployment.ObjectMeta.Labels)
 
 		status.DeploymentConfigured = generationsMatch && metadataMatch
 
@@ -103,7 +103,7 @@ func (s updateBackupStatus) reconcile(ctx context.Context, r *FoundationDBBackup
 		return &requeue{curError: err}
 	}
 
-	if !reflect.DeepEqual(backup.Status, *originalStatus) {
+	if !equality.Semantic.DeepEqual(backup.Status, *originalStatus) {
 		err = r.Status().Update(ctx, backup)
 		if err != nil {
 			log.Error(err, "Error updating backup status", "namespace", backup.Namespace, "backup", backup.Name)
diff --git a/docs/backup_spec.md b/docs/backup_spec.md
index 4a1b6ce2..7af087cc 100644
--- a/docs/backup_spec.md
+++ b/docs/backup_spec.md
@@ -14,6 +14,7 @@ This Document documents the types introduced by the FoundationDB Operator to be
 * [FoundationDBBackupStatusBackupDetails](#foundationdbbackupstatusbackupdetails)
 * [FoundationDBLiveBackupStatus](#foundationdblivebackupstatus)
 * [FoundationDBLiveBackupStatusState](#foundationdblivebackupstatusstate)
+* [ImageConfig](#imageconfig)
 
 ## BackupGenerationStatus
 
@@ -86,8 +87,9 @@ FoundationDBBackupSpec describes the desired state of the backup for a cluster.
 | backupDeploymentMetadata | BackupDeploymentMetadata allows customizing labels and annotations on the deployment for the backup agents. | *[metav1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#objectmeta-v1-meta) | false |
 | podTemplateSpec | PodTemplateSpec allows customizing the pod template for the backup agents. | *[corev1.PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#podtemplatespec-v1-core) | false |
 | customParameters | CustomParameters defines additional parameters to pass to the backup agents. | FoundationDBCustomParameters | false |
-| allowTagOverride | This setting defines if a user provided image can have it's own tag rather than getting the provided version appended. You have to ensure that the specified version in the Spec is compatible with the given version in your custom image. | *bool | false |
+| allowTagOverride | This setting defines if a user provided image can have it's own tag rather than getting the provided version appended. You have to ensure that the specified version in the Spec is compatible with the given version in your custom image. **Deprecated: use ImageConfigs instead.** | *bool | false |
 | blobStoreConfiguration | This is the configuration of the target blobstore for this backup. | *[BlobStoreConfiguration](#blobstoreconfiguration) | false |
+| imageConfigs | ImageConfigs allows customizing the image that we use for a container. | [][ImageConfig](#imageconfig) | false |
 
 [Back to TOC](#table-of-contents)
 
@@ -151,3 +153,16 @@ URLParameter defines a single URL parameter to pass to the blobstore.
 FoundationDBCustomParameter defines a single custom knob
 
 [Back to TOC](#table-of-contents)
+
+## ImageConfig
+
+ImageConfig provides a policy for customizing an image.  When multiple image configs are provided, they will be merged into a single config that will be used to define the final image. For each field, we select the value from the first entry in the config list that defines a value for that field, and matches the version of FoundationDB the image is for. Any config that specifies a different version than the one under consideration will be ignored for the purposes of defining that image.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| version | Version is the version of FoundationDB this policy applies to. If this is blank, the policy applies to all FDB versions. | string | false |
+| baseImage | BaseImage specifies the part of the image before the tag. | string | false |
+| tag | Tag specifies a full image tag. | string | false |
+| tagSuffix | TagSuffix specifies a suffix that will be added after the version to form the full tag. | string | false |
+
+[Back to TOC](#table-of-contents)
diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
index 42d023e3..5a3d1e0d 100644
--- a/docs/cluster_spec.md
+++ b/docs/cluster_spec.md
@@ -18,7 +18,6 @@ This Document documents the types introduced by the FoundationDB Operator to be
 * [FoundationDBClusterList](#foundationdbclusterlist)
 * [FoundationDBClusterSpec](#foundationdbclusterspec)
 * [FoundationDBClusterStatus](#foundationdbclusterstatus)
-* [ImageConfig](#imageconfig)
 * [LabelConfig](#labelconfig)
 * [LockDenyListEntry](#lockdenylistentry)
 * [LockOptions](#lockoptions)
@@ -34,6 +33,7 @@ This Document documents the types introduced by the FoundationDB Operator to be
 * [Region](#region)
 * [RoleCounts](#rolecounts)
 * [VersionFlags](#versionflags)
+* [ImageConfig](#imageconfig)
 
 ## AutomaticReplacementOptions
 
@@ -253,19 +253,6 @@ FoundationDBClusterStatus defines the observed state of FoundationDBCluster
 
 [Back to TOC](#table-of-contents)
 
-## ImageConfig
-
-ImageConfig provides a policy for customizing an image.  When multiple image configs are provided, they will be merged into a single config that will be used to define the final image. For each field, we select the value from the first entry in the config list that defines a value for that field, and matches the version of FoundationDB the image is for. Any config that specifies a different version than the one under consideration will be ignored for the purposes of defining that image.
-
-| Field | Description | Scheme | Required |
-| ----- | ----------- | ------ | -------- |
-| version | Version is the version of FoundationDB this policy applies to. If this is blank, the policy applies to all FDB versions. | string | false |
-| baseImage | BaseImage specifies the part of the image before the tag. | string | false |
-| tag | Tag specifies a full image tag. | string | false |
-| tagSuffix | TagSuffix specifies a suffix that will be added after the version to form the full tag. | string | false |
-
-[Back to TOC](#table-of-contents)
-
 ## ImageType
 
 ImageType defines a single kind of images used in the cluster.
@@ -526,3 +513,16 @@ VersionFlags defines internal flags for new features in the database.
 ProcessClass models the class of a pod
 
 [Back to TOC](#table-of-contents)
+
+## ImageConfig
+
+ImageConfig provides a policy for customizing an image.  When multiple image configs are provided, they will be merged into a single config that will be used to define the final image. For each field, we select the value from the first entry in the config list that defines a value for that field, and matches the version of FoundationDB the image is for. Any config that specifies a different version than the one under consideration will be ignored for the purposes of defining that image.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| version | Version is the version of FoundationDB this policy applies to. If this is blank, the policy applies to all FDB versions. | string | false |
+| baseImage | BaseImage specifies the part of the image before the tag. | string | false |
+| tag | Tag specifies a full image tag. | string | false |
+| tagSuffix | TagSuffix specifies a suffix that will be added after the version to form the full tag. | string | false |
+
+[Back to TOC](#table-of-contents)
diff --git a/internal/pod_client.go b/internal/pod_client.go
index 2fec4d7a..0ea4a9a5 100644
--- a/internal/pod_client.go
+++ b/internal/pod_client.go
@@ -90,6 +90,12 @@ type realFdbPodSidecarClient struct {
 
 	// logger is used to add common fields to log messages.
 	logger logr.Logger
+
+	// getTimeout defines the timeout for get requests
+	getTimeout time.Duration
+
+	// postTimeout defines the timeout for post requests
+	postTimeout time.Duration
 }
 
 // realPodSidecarClient provides a client for use in real environments, using
@@ -106,7 +112,7 @@ type realFdbPodAnnotationClient struct {
 }
 
 // NewFdbPodClient builds a client for working with an FDB Pod
-func NewFdbPodClient(cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod, log logr.Logger) (podclient.FdbPodClient, error) {
+func NewFdbPodClient(cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod, log logr.Logger, getTimeout time.Duration, postTimeout time.Duration) (podclient.FdbPodClient, error) {
 	if GetImageType(pod) == FDBImageTypeUnified {
 		return &realFdbPodAnnotationClient{Cluster: cluster, Pod: pod, logger: log}, nil
 	}
@@ -152,7 +158,7 @@ func NewFdbPodClient(cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod, l
 		tlsConfig.RootCAs = certPool
 	}
 
-	return &realFdbPodSidecarClient{Cluster: cluster, Pod: pod, useTLS: useTLS, tlsConfig: tlsConfig, logger: log}, nil
+	return &realFdbPodSidecarClient{Cluster: cluster, Pod: pod, useTLS: useTLS, tlsConfig: tlsConfig, logger: log, getTimeout: getTimeout, postTimeout: postTimeout}, nil
 }
 
 // getListenIP gets the IP address that a pod listens on.
@@ -186,12 +192,10 @@ func (client *realFdbPodSidecarClient) makeRequest(method string, path string) (
 	url := fmt.Sprintf("%s://%s:8080/%s", protocol, client.getListenIP(), path)
 	switch method {
 	case http.MethodGet:
-		// We assume that a get request should be relative fast.
-		retryClient.HTTPClient.Timeout = 5 * time.Second
+		retryClient.HTTPClient.Timeout = client.getTimeout
 		resp, err = retryClient.Get(url)
 	case http.MethodPost:
-		// A post request could take a little bit longer since we copy sometimes files.
-		retryClient.HTTPClient.Timeout = 10 * time.Second
+		retryClient.HTTPClient.Timeout = client.postTimeout
 		resp, err = retryClient.Post(url, "application/json", strings.NewReader(""))
 	default:
 		return "", fmt.Errorf("unknown HTTP method %s", method)
diff --git a/internal/pod_models.go b/internal/pod_models.go
index fbf999b9..8ebb46a3 100644
--- a/internal/pod_models.go
+++ b/internal/pod_models.go
@@ -182,10 +182,7 @@ func GetPodSpec(cluster *fdbv1beta2.FoundationDBCluster, processClass fdbv1beta2
 
 	podName, processGroupID := GetProcessGroupID(cluster, processClass, idNum)
 
-	versionString := cluster.Status.RunningVersion
-	if versionString == "" {
-		versionString = cluster.Spec.Version
-	}
+	versionString := cluster.GetRunningVersion()
 
 	image, err := GetImage(mainContainer.Image, cluster.Spec.MainContainer.ImageConfigs, versionString)
 	if err != nil {
@@ -825,7 +822,11 @@ func GetBackupDeployment(backup *fdbv1beta2.FoundationDBBackup) (*appsv1.Deploym
 		podTemplate.Spec.Containers = containers
 	}
 
-	image, err := GetImage(mainContainer.Image, []fdbv1beta2.ImageConfig{{BaseImage: "foundationdb/foundationdb"}}, backup.Spec.Version)
+	if backup.Spec.ImageConfigs == nil {
+		backup.Spec.ImageConfigs = []fdbv1beta2.ImageConfig{{BaseImage: "foundationdb/foundationdb"}}
+	}
+
+	image, err := GetImage(mainContainer.Image, backup.Spec.ImageConfigs, backup.Spec.Version)
 	if err != nil {
 		return nil, err
 	}
diff --git a/setup/setup.go b/setup/setup.go
index 5d05671c..45d25559 100644
--- a/setup/setup.go
+++ b/setup/setup.go
@@ -66,6 +66,8 @@ type Options struct {
 	PrintVersion            bool
 	LabelSelector           string
 	WatchNamespace          string
+	GetTimeout              time.Duration
+	PostTimeout             time.Duration
 }
 
 // BindFlags will parse the given flagset for the operator option flags
@@ -90,6 +92,8 @@ func (o *Options) BindFlags(fs *flag.FlagSet) {
 	fs.BoolVar(&o.PrintVersion, "version", false, "Prints the version of the operator and exits.")
 	fs.StringVar(&o.LabelSelector, "label-selector", "", "Defines a label-selector that will be used to select resources.")
 	fs.StringVar(&o.WatchNamespace, "watch-namespace", os.Getenv("WATCH_NAMESPACE"), "Defines which namespace the operator should watch")
+	fs.DurationVar(&o.GetTimeout, "get-timeout", 5*time.Second, "http timeout for get requests to the FDB sidecar")
+	fs.DurationVar(&o.PostTimeout, "post-timeout", 10*time.Second, "http timeout for post requests to the FDB sidecar")
 }
 
 // StartManager will start the FoundationDB operator manager.
@@ -173,6 +177,8 @@ func StartManager(
 		clusterReconciler.Recorder = mgr.GetEventRecorderFor("foundationdbcluster-controller")
 		clusterReconciler.DeprecationOptions = operatorOpts.DeprecationOptions
 		clusterReconciler.DatabaseClientProvider = fdbclient.NewDatabaseClientProvider(logger)
+		clusterReconciler.GetTimeout = operatorOpts.GetTimeout
+		clusterReconciler.PostTimeout = operatorOpts.PostTimeout
 		clusterReconciler.Log = logr.WithName("controllers").WithName("FoundationDBCluster")
 
 		if err := clusterReconciler.SetupWithManager(mgr, operatorOpts.MaxConcurrentReconciles, *labelSelector, watchedObjects...); err != nil {


Test patch:
diff --git a/api/v1beta2/foundationdbcluster_types_test.go b/api/v1beta2/foundationdbcluster_types_test.go
index e6aa3458..c2d3d808 100644
--- a/api/v1beta2/foundationdbcluster_types_test.go
+++ b/api/v1beta2/foundationdbcluster_types_test.go
@@ -424,15 +424,19 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 				Log:       5,
 				Stateless: 9,
 			}))
+		})
 
-			cluster.Spec.Version = "7.1.0-rc2"
-			counts, err = cluster.GetProcessCountsWithDefaults()
-			Expect(err).NotTo(HaveOccurred())
-			Expect(counts).To(Equal(ProcessCounts{
-				Storage:   5,
-				Log:       5,
-				Stateless: 9,
-			}))
+		When("using a version that supports grv and commit proxies", func() {
+			It("should return the default process counts", func() {
+				cluster.Spec.Version = "7.1.0"
+				counts, err := cluster.GetProcessCountsWithDefaults()
+				Expect(err).NotTo(HaveOccurred())
+				Expect(counts).To(Equal(ProcessCounts{
+					Storage:   5,
+					Log:       4,
+					Stateless: 22,
+				}))
+			})
 		})
 
 		It("should return the default process counts when proxies are unset", func() {
@@ -983,12 +987,14 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 				RoleCounts: RoleCounts{
 					Logs:          5,
 					Proxies:       1,
-					CommitProxies: 2,
+					CommitProxies: 4,
 					GrvProxies:    2,
 				},
 			}
-			Expect(configuration.AreSeparatedProxiesConfigured()).To(BeFalse())
+			// This check is not version dependent
+			Expect(configuration.AreSeparatedProxiesConfigured()).To(BeTrue())
 			Expect(configuration.GetConfigurationString("6.3.24")).To(Equal("double ssd usable_regions=1 logs=5 resolvers=0 log_routers=0 remote_logs=0 proxies=1 regions=[]"))
+			Expect(configuration.GetConfigurationString("7.1.0")).To(Equal("double ssd usable_regions=1 logs=5 resolvers=0 log_routers=0 remote_logs=0 commit_proxies=4 grv_proxies=2 regions=[]"))
 
 			configuration.Regions = []Region{{
 				DataCenters: []DataCenter{{
@@ -1004,8 +1010,8 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 			configuration.VersionFlags.LogSpill = 3
 			Expect(configuration.GetConfigurationString("6.3.24")).To(Equal("double ssd usable_regions=1 logs=5 resolvers=0 log_routers=0 remote_logs=0 proxies=1 log_spill:=3 regions=[]"))
 
-			Expect(configuration.GetConfigurationString("7.0.0")).To(Equal("double ssd usable_regions=1 logs=5 resolvers=0 log_routers=0 remote_logs=0 proxies=1 log_spill:=3 regions=[]"))
-			Expect(configuration.GetConfigurationString("7.1.0-rc1")).To(Equal("double ssd usable_regions=1 logs=5 resolvers=0 log_routers=0 remote_logs=0 proxies=1 log_spill:=3 regions=[]"))
+			Expect(configuration.GetConfigurationString("7.0.0")).To(Equal("double ssd usable_regions=1 logs=5 resolvers=0 log_routers=0 remote_logs=0 commit_proxies=4 grv_proxies=2 log_spill:=3 regions=[]"))
+			Expect(configuration.GetConfigurationString("7.1.0-rc1")).To(Equal("double ssd usable_regions=1 logs=5 resolvers=0 log_routers=0 remote_logs=0 commit_proxies=4 grv_proxies=2 log_spill:=3 regions=[]"))
 		})
 
 		When("CommitProxies and GrvProxies are not configured", func() {
@@ -3953,77 +3959,6 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 		)
 	})
 
-	When("merging image configs", func() {
-		It("applies chooses the first value for each field", func() {
-			configs := []ImageConfig{
-				{
-					BaseImage: "foundationdb/foundationdb",
-					Version:   Versions.Default.String(),
-				},
-				{
-					BaseImage: "foundationdb/foundationdb-slim",
-					Version:   Versions.Default.String(),
-					Tag:       "abcdef",
-					TagSuffix: "-1",
-				},
-			}
-
-			finalConfig := SelectImageConfig(configs, Versions.Default.String())
-			Expect(finalConfig).To(Equal(ImageConfig{
-				BaseImage: "foundationdb/foundationdb",
-				Version:   Versions.Default.String(),
-				Tag:       "abcdef",
-				TagSuffix: "-1",
-			}))
-		})
-
-		It("ignores configs that are for different versions", func() {
-			configs := []ImageConfig{
-				{
-					BaseImage: "foundationdb/foundationdb",
-					Version:   Versions.Default.String(),
-				},
-				{
-					Version: Versions.NextMajorVersion.String(),
-					Tag:     "abcdef",
-				},
-				{
-					TagSuffix: "-1",
-				},
-			}
-
-			finalConfig := SelectImageConfig(configs, Versions.Default.String())
-			Expect(finalConfig).To(Equal(ImageConfig{
-				BaseImage: "foundationdb/foundationdb",
-				Version:   Versions.Default.String(),
-				TagSuffix: "-1",
-			}))
-		})
-	})
-
-	When("building image names", func() {
-		It("applies the fields", func() {
-			config := ImageConfig{
-				BaseImage: "foundationdb/foundationdb-kubernetes-sidecar",
-				Version:   Versions.Default.String(),
-				TagSuffix: "-2",
-			}
-			image := config.Image()
-			Expect(image).To(Equal(fmt.Sprintf("foundationdb/foundationdb-kubernetes-sidecar:%s-2", Versions.Default)))
-		})
-
-		It("uses the tag to override the version and tag suffix", func() {
-			config := ImageConfig{
-				BaseImage: "foundationdb/foundationdb-kubernetes-sidecar",
-				Version:   Versions.Default.String(),
-				Tag:       "abcdef",
-				TagSuffix: "-2",
-			}
-			image := config.Image()
-			Expect(image).To(Equal("foundationdb/foundationdb-kubernetes-sidecar:abcdef"))
-		})
-	})
-
 	When("checking if the process group needs a replacement", func() {
 		var processGroup *ProcessGroupStatus
 		var needsReplacement bool
diff --git a/api/v1beta2/image_config_test.go b/api/v1beta2/image_config_test.go
new file mode 100644
index 00000000..b6adf120
--- /dev/null
+++ b/api/v1beta2/image_config_test.go
@@ -0,0 +1,81 @@
+package v1beta2
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("[api] ImageConfig", func() {
+	When("merging image configs", func() {
+		It("applies chooses the first value for each field", func() {
+			configs := []ImageConfig{
+				{
+					BaseImage: "foundationdb/foundationdb",
+					Version:   Versions.Default.String(),
+				},
+				{
+					BaseImage: "foundationdb/foundationdb-slim",
+					Version:   Versions.Default.String(),
+					Tag:       "abcdef",
+					TagSuffix: "-1",
+				},
+			}
+
+			finalConfig := SelectImageConfig(configs, Versions.Default.String())
+			Expect(finalConfig).To(Equal(ImageConfig{
+				BaseImage: "foundationdb/foundationdb",
+				Version:   Versions.Default.String(),
+				Tag:       "abcdef",
+				TagSuffix: "-1",
+			}))
+		})
+
+		It("ignores configs that are for different versions", func() {
+			configs := []ImageConfig{
+				{
+					BaseImage: "foundationdb/foundationdb",
+					Version:   Versions.Default.String(),
+				},
+				{
+					Version: Versions.NextMajorVersion.String(),
+					Tag:     "abcdef",
+				},
+				{
+					TagSuffix: "-1",
+				},
+			}
+
+			finalConfig := SelectImageConfig(configs, Versions.Default.String())
+			Expect(finalConfig).To(Equal(ImageConfig{
+				BaseImage: "foundationdb/foundationdb",
+				Version:   Versions.Default.String(),
+				TagSuffix: "-1",
+			}))
+		})
+	})
+
+	When("building image names", func() {
+		It("applies the fields", func() {
+			config := ImageConfig{
+				BaseImage: "foundationdb/foundationdb-kubernetes-sidecar",
+				Version:   Versions.Default.String(),
+				TagSuffix: "-2",
+			}
+			image := config.Image()
+			Expect(image).To(Equal(fmt.Sprintf("foundationdb/foundationdb-kubernetes-sidecar:%s-2", Versions.Default)))
+		})
+
+		It("uses the tag to override the version and tag suffix", func() {
+			config := ImageConfig{
+				BaseImage: "foundationdb/foundationdb-kubernetes-sidecar",
+				Version:   Versions.Default.String(),
+				Tag:       "abcdef",
+				TagSuffix: "-2",
+			}
+			image := config.Image()
+			Expect(image).To(Equal("foundationdb/foundationdb-kubernetes-sidecar:abcdef"))
+		})
+	})
+})
diff --git a/controllers/remove_process_groups_test.go b/controllers/remove_process_groups_test.go
index 4466b9e4..cc024259 100644
--- a/controllers/remove_process_groups_test.go
+++ b/controllers/remove_process_groups_test.go
@@ -175,6 +175,11 @@ var _ = Describe("remove_process_groups", func() {
 			var secondRemovedProcessGroup *fdbv1beta2.ProcessGroupStatus
 
 			BeforeEach(func() {
+				// To allow multiple process groups to be removed we have to use the update mode all
+				cluster.Spec.AutomationOptions.RemovalMode = fdbv1beta2.PodUpdateModeAll
+				err := k8sClient.Update(context.TODO(), cluster)
+				Expect(err).NotTo(HaveOccurred())
+
 				initialCnt = len(cluster.Status.ProcessGroups)
 				secondRemovedProcessGroup = cluster.Status.ProcessGroups[1]
 				marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, secondRemovedProcessGroup.ProcessGroupID, secondRemovedProcessGroup.ProcessClass, removedProcessGroup.Addresses[0])
diff --git a/internal/pod_models_test.go b/internal/pod_models_test.go
index a3370bbe..d88d5bc5 100644
--- a/internal/pod_models_test.go
+++ b/internal/pod_models_test.go
@@ -2947,6 +2947,19 @@ var _ = Describe("pod_models", func() {
 				Expect(deployment.Spec.Template.Spec.Containers[0].Args).To(ContainElement("--customParameter=1337"))
 			})
 		})
+
+		When("defining an image config", func() {
+			BeforeEach(func() {
+				backup.Spec.ImageConfigs = []fdbv1beta2.ImageConfig{{BaseImage: "foundationdb/foundationdb", Tag: "dev"}}
+				deployment, err = GetBackupDeployment(backup)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(deployment).NotTo(BeNil())
+			})
+
+			It("should set the image based on the image configs", func() {
+				Expect(deployment.Spec.Template.Spec.Containers[0].Image).To(Equal("foundationdb/foundationdb:dev"))
+			})
+		})
 	})
 
 	Context("Get image for container", func() {