Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1879 (2023-11-09 12:07:23)

Make sure the e2e test suite only runs locality based e2e tests if the provided versions support it
# Description

If a FDB versions doesn't support the locality based exclusions, we should just skip this check.

## Type of change

*Please select one of the options below.*

- Bug fix (non-breaking change which fixes an issue)

## Discussion

-

## Testing

CI will run e2e tests.

## Documentation

-

## Follow-up

-



Patch:
diff --git a/api/v1beta2/foundationdb_database_configuration.go b/api/v1beta2/foundationdb_database_configuration.go
index a40dd1d8..60076d5a 100644
--- a/api/v1beta2/foundationdb_database_configuration.go
+++ b/api/v1beta2/foundationdb_database_configuration.go
@@ -234,12 +234,12 @@ func (configuration *DatabaseConfiguration) CountUniqueDataCenters() int {
 // This will fill in defaults of -1 for some fields that have a default
 // of 0, and will ensure that the region configuration is ordered
 // consistently.
-func (configuration DatabaseConfiguration) NormalizeConfigurationWithSeparatedProxies(version string) DatabaseConfiguration {
+func (configuration DatabaseConfiguration) NormalizeConfigurationWithSeparatedProxies(version string, areSeparatedProxiesConfigured bool) DatabaseConfiguration {
 	result := configuration.NormalizeConfiguration()
 
 	parsedVersion, _ := ParseFdbVersion(version)
 	if parsedVersion.HasSeparatedProxies() {
-		if !configuration.AreSeparatedProxiesConfigured() {
+		if !areSeparatedProxiesConfigured {
 			result.GrvProxies = 0
 			result.CommitProxies = 0
 		} else {
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index d0a2244c..e9a1c0ff 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -1815,7 +1815,7 @@ type ContainerOverrides struct {
 // DesiredDatabaseConfiguration builds the database configuration for the
 // cluster based on its spec.
 func (cluster *FoundationDBCluster) DesiredDatabaseConfiguration() DatabaseConfiguration {
-	configuration := cluster.Spec.DatabaseConfiguration.NormalizeConfigurationWithSeparatedProxies(cluster.GetRunningVersion())
+	configuration := cluster.Spec.DatabaseConfiguration.NormalizeConfigurationWithSeparatedProxies(cluster.GetRunningVersion(), cluster.Spec.DatabaseConfiguration.AreSeparatedProxiesConfigured())
 	configuration.RoleCounts = cluster.GetRoleCountsWithDefaults()
 	configuration.RoleCounts.Storage = 0
 
diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go
index 5ef48034..58d83098 100644
--- a/controllers/cluster_controller.go
+++ b/controllers/cluster_controller.go
@@ -476,9 +476,14 @@ func (r *FoundationDBClusterReconciler) getStatusFromClusterOrDummyStatus(logger
 	if err != nil {
 		return nil, err
 	}
-
 	defer adminClient.Close()
 
+	// If the cluster is not yet configured, we can reduce the timeout to make sure the initial reconcile steps
+	// are faster.
+	if !cluster.Status.Configured {
+		adminClient.SetTimeout(10 * time.Second)
+	}
+
 	status, err := adminClient.GetStatus()
 	if err == nil {
 		return status, nil
diff --git a/controllers/update_database_configuration.go b/controllers/update_database_configuration.go
index e93c993d..e65e5d0b 100644
--- a/controllers/update_database_configuration.go
+++ b/controllers/update_database_configuration.go
@@ -65,7 +65,7 @@ func (u updateDatabaseConfiguration) reconcile(ctx context.Context, r *Foundatio
 
 	desiredConfiguration := cluster.DesiredDatabaseConfiguration()
 	desiredConfiguration.RoleCounts.Storage = 0
-	currentConfiguration := status.Cluster.DatabaseConfiguration.NormalizeConfigurationWithSeparatedProxies(cluster.Spec.Version)
+	currentConfiguration := status.Cluster.DatabaseConfiguration.NormalizeConfigurationWithSeparatedProxies(cluster.Spec.Version, cluster.Spec.DatabaseConfiguration.AreSeparatedProxiesConfigured())
 	// We have to reset the excluded servers here otherwise we will trigger a reconfiguration if one or more servers
 	// are excluded.
 	currentConfiguration.ExcludedServers = nil
diff --git a/controllers/update_pod_config.go b/controllers/update_pod_config.go
index d5ec0514..cc27583f 100644
--- a/controllers/update_pod_config.go
+++ b/controllers/update_pod_config.go
@@ -71,6 +71,11 @@ func (updatePodConfig) reconcile(ctx context.Context, r *FoundationDBClusterReco
 			continue
 		}
 
+		if !pod.DeletionTimestamp.IsZero() {
+			logger.V(1).Info("ignore Pod that has a deletion timestamp for config updates")
+			continue
+		}
+
 		serverPerPod, err := internal.GetServersPerPodForPod(pod, processGroup.ProcessClass)
 		if err != nil {
 			curLogger.Error(err, "Error when receiving storage server per Pod")
diff --git a/controllers/update_status.go b/controllers/update_status.go
index 047d5191..e361e9bd 100644
--- a/controllers/update_status.go
+++ b/controllers/update_status.go
@@ -99,7 +99,7 @@ func (updateStatus) reconcile(ctx context.Context, r *FoundationDBClusterReconci
 	// about the current database configuration, leading to a wrong signal that the database configuration must be changed as
 	// the configuration will be overwritten with the default values.
 	if databaseStatus.Client.DatabaseStatus.Available {
-		clusterStatus.DatabaseConfiguration = databaseStatus.Cluster.DatabaseConfiguration.NormalizeConfigurationWithSeparatedProxies(cluster.Spec.Version)
+		clusterStatus.DatabaseConfiguration = databaseStatus.Cluster.DatabaseConfiguration.NormalizeConfigurationWithSeparatedProxies(cluster.Spec.Version, cluster.Spec.DatabaseConfiguration.AreSeparatedProxiesConfigured())
 		// Removing excluded servers as we don't want them during comparison.
 		clusterStatus.DatabaseConfiguration.ExcludedServers = nil
 		cluster.ClearMissingVersionFlags(&clusterStatus.DatabaseConfiguration)
@@ -318,6 +318,12 @@ func tryConnectionOptions(logger logr.Logger, cluster *fdbv1beta2.FoundationDBCl
 			return originalConnectionString, clientErr
 		}
 
+		// If the cluster is not yet configured, we can reduce the timeout to make sure the initial reconcile steps
+		// are faster.
+		if !cluster.Status.Configured {
+			adminClient.SetTimeout(10 * time.Second)
+		}
+
 		var activeConnectionString string
 		activeConnectionString, err = adminClient.GetConnectionString()
 
diff --git a/fdbclient/admin_client.go b/fdbclient/admin_client.go
index 01a84775..43778ef1 100644
--- a/fdbclient/admin_client.go
+++ b/fdbclient/admin_client.go
@@ -89,6 +89,9 @@ type cliAdminClient struct {
 	// fdbLibClient is an interface to interact with a FDB cluster over the FDB client libraries. In the real fdb lib client
 	// we will issue the actual requests against FDB. In the mock runner we will return predefined output.
 	fdbLibClient fdbLibClient
+
+	// timeout defines the timeout that should be used for interacting with FDB.
+	timeout time.Duration
 }
 
 // NewCliAdminClient generates an Admin client for a cluster
@@ -275,7 +278,7 @@ func (client *cliAdminClient) runCommand(command cliCommand) (string, error) {
 func (client *cliAdminClient) getStatusFromCli() (*fdbv1beta2.FoundationDBStatus, error) {
 	// Always use the max timeout here. Otherwise we will retry multiple times with an increasing timeout. As the
 	// timeout is only the upper bound using directly the max timeout reduces the calls to a single call.
-	output, err := client.runCommand(cliCommand{command: "status json", timeout: MaxCliTimeout})
+	output, err := client.runCommand(cliCommand{command: "status json", timeout: client.getTimeout()})
 	if err != nil {
 		return nil, err
 	}
@@ -313,8 +316,9 @@ func (client *cliAdminClient) GetStatus() (*fdbv1beta2.FoundationDBStatus, error
 	adminClientMutex.Lock()
 	defer adminClientMutex.Unlock()
 
+	startTime := time.Now()
 	// This will call directly the database and fetch the status information from the system key space.
-	status, err := getStatusFromDB(client.fdbLibClient, client.log, MaxCliTimeout)
+	status, err := getStatusFromDB(client.fdbLibClient, client.log, client.getTimeout())
 	// There is a limitation in the multi version client if the cluster is only partially upgraded e.g. because not
 	// all fdbserver processes are restarted, then the multi version client sometimes picks the wrong version
 	// to connect to the cluster. This will result in an empty status only reporting the unreachable coordinators.
@@ -326,7 +330,7 @@ func (client *cliAdminClient) GetStatus() (*fdbv1beta2.FoundationDBStatus, error
 		status, err = client.getStatus()
 	}
 
-	client.log.V(1).Info("Completed GetStatus() call", "error", err, "status", status)
+	client.log.V(1).Info("Completed GetStatus() call", "error", err, "status", status, "duration", time.Since(startTime).String())
 
 	return status, err
 }
@@ -348,7 +352,7 @@ func (client *cliAdminClient) ConfigureDatabase(configuration fdbv1beta2.Databas
 
 // GetMaintenanceZone gets current maintenance zone, if any. Returns empty string if maintenance mode is off
 func (client *cliAdminClient) GetMaintenanceZone() (string, error) {
-	mode, err := client.fdbLibClient.getValueFromDBUsingKey("\xff/maintenance", DefaultCLITimeout)
+	mode, err := client.fdbLibClient.getValueFromDBUsingKey("\xff/maintenance", client.getTimeout())
 	if err != nil {
 		return "", err
 	}
@@ -394,7 +398,7 @@ func (client *cliAdminClient) ExcludeProcesses(addresses []fdbv1beta2.ProcessAdd
 
 	excludeCommand.WriteString(fdbv1beta2.ProcessAddressesString(addresses, " "))
 
-	_, err = client.runCommand(cliCommand{command: excludeCommand.String(), timeout: MaxCliTimeout})
+	_, err = client.runCommand(cliCommand{command: excludeCommand.String(), timeout: client.getTimeout()})
 
 	return err
 }
@@ -445,7 +449,7 @@ func (client *cliAdminClient) KillProcesses(addresses []fdbv1beta2.ProcessAddres
 		fdbv1beta2.ProcessAddressesStringWithoutFlags(addresses, " "),
 	)
 	// Run the kill command once with the max timeout to reduce the risk of multiple recoveries happening.
-	_, err := client.runCommand(cliCommand{command: killCommand, timeout: MaxCliTimeout})
+	_, err := client.runCommand(cliCommand{command: killCommand, timeout: client.getTimeout()})
 
 	return err
 }
@@ -488,7 +492,7 @@ func cleanConnectionStringOutput(input string) string {
 func (client *cliAdminClient) GetConnectionString() (string, error) {
 	// This will call directly the database and fetch the connection string
 	// from the system key space.
-	outputBytes, err := getConnectionStringFromDB(client.fdbLibClient, MaxCliTimeout)
+	outputBytes, err := getConnectionStringFromDB(client.fdbLibClient, client.getTimeout())
 
 	if err != nil {
 		return "", err
@@ -694,3 +698,17 @@ func (client *cliAdminClient) WithValues(keysAndValues ...interface{}) {
 	}
 	cmdRunner.log = newLogger
 }
+
+// SetTimeout will overwrite the default timeout for interacting the FDB cluster.
+func (client *cliAdminClient) SetTimeout(timeout time.Duration) {
+	client.timeout = timeout
+}
+
+// getTimeout will return the timeout that is specified for the admin client or otherwise the MaxCliTimeout.
+func (client *cliAdminClient) getTimeout() time.Duration {
+	if client.timeout == 0 {
+		return MaxCliTimeout
+	}
+
+	return client.timeout
+}
diff --git a/fdbclient/common.go b/fdbclient/common.go
index 6ecec8b1..ddd1a906 100644
--- a/fdbclient/common.go
+++ b/fdbclient/common.go
@@ -40,7 +40,7 @@ import (
 var DefaultCLITimeout = 10 * time.Second
 
 // MaxCliTimeout is the maximum CLI timeout that will be used for requests that might be slower to respond.
-var MaxCliTimeout = 60 * time.Second
+var MaxCliTimeout = 40 * time.Second
 
 const (
 	defaultTransactionTimeout = 5 * time.Second
diff --git a/pkg/fdbadminclient/admin_client.go b/pkg/fdbadminclient/admin_client.go
index 8d92b116..5e22701e 100644
--- a/pkg/fdbadminclient/admin_client.go
+++ b/pkg/fdbadminclient/admin_client.go
@@ -22,6 +22,7 @@ package fdbadminclient
 
 import (
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	"time"
 )
 
 // AdminClient describes an interface for running administrative commands on a
@@ -115,4 +116,7 @@ type AdminClient interface {
 	// WithValues will update the logger used by the current AdminClient to contain the provided key value pairs. The provided
 	// arguments must be even.
 	WithValues(keysAndValues ...interface{})
+
+	// SetTimeout will overwrite the default timeout for interacting the FDB cluster.
+	SetTimeout(timeout time.Duration)
 }
diff --git a/pkg/fdbadminclient/mock/admin_client_mock.go b/pkg/fdbadminclient/mock/admin_client_mock.go
index 53eab762..b2bdcfa6 100644
--- a/pkg/fdbadminclient/mock/admin_client_mock.go
+++ b/pkg/fdbadminclient/mock/admin_client_mock.go
@@ -1039,3 +1039,6 @@ func (client *AdminClient) GetWorstDurabilityLag() (fdbv1beta2.FoundationDBStatu
 // WithValues will update the logger used by the current AdminClient to contain the provided key value pairs. The provided
 // arguments must be even.
 func (client *AdminClient) WithValues(_ ...interface{}) {}
+
+// SetTimeout will overwrite the default timeout for interacting the FDB cluster.
+func (client *AdminClient) SetTimeout(_ time.Duration) {}
diff --git a/setup/setup.go b/setup/setup.go
index 7abedd99..4963dc1d 100644
--- a/setup/setup.go
+++ b/setup/setup.go
@@ -91,7 +91,7 @@ func (o *Options) BindFlags(fs *flag.FlagSet) {
 	)
 	fs.StringVar(&o.LogFile, "log-file", "", "The path to a file to write logs to.")
 	fs.IntVar(&o.CliTimeout, "cli-timeout", 10, "The timeout to use for CLI commands in seconds.")
-	fs.IntVar(&o.MaxCliTimeout, "max-cli-timeout", 60, "The maximum timeout to use for CLI commands in seconds. This timeout is used for CLI requests that are known to be potentially slow like get status or exclude.")
+	fs.IntVar(&o.MaxCliTimeout, "max-cli-timeout", 40, "The maximum timeout to use for CLI commands in seconds. This timeout is used for CLI requests that are known to be potentially slow like get status or exclude.")
 	fs.IntVar(&o.MaxConcurrentReconciles, "max-concurrent-reconciles", 1, "Defines the maximum number of concurrent reconciles for all controllers.")
 	fs.BoolVar(&o.CleanUpOldLogFile, "cleanup-old-cli-logs", true, "Defines if the operator should delete old fdbcli log files.")
 	fs.DurationVar(&o.LogFileMinAge, "log-file-min-age", 5*time.Minute, "Defines the minimum age of fdbcli log files before removing when \"--cleanup-old-cli-logs\" is set.")


Test patch:
diff --git a/api/v1beta2/foundationdbcluster_types_test.go b/api/v1beta2/foundationdbcluster_types_test.go
index bca0a5d5..aa747e2a 100644
--- a/api/v1beta2/foundationdbcluster_types_test.go
+++ b/api/v1beta2/foundationdbcluster_types_test.go
@@ -2534,7 +2534,7 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 			It("should set the correct value (-1) for log routers", func() {
 				spec := DatabaseConfiguration{}
 				spec.RemoteLogs = 9
-				normalized := spec.NormalizeConfigurationWithSeparatedProxies(version)
+				normalized := spec.NormalizeConfigurationWithSeparatedProxies(version, false)
 				Expect(normalized.LogRouters).To(Equal(-1))
 				Expect(normalized.RemoteLogs).To(Equal(9))
 			})
@@ -2582,7 +2582,7 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 						},
 					},
 				}
-				normalized := spec.NormalizeConfigurationWithSeparatedProxies(version)
+				normalized := spec.NormalizeConfigurationWithSeparatedProxies(version, false)
 				Expect(normalized.Regions).To(Equal([]Region{
 					{
 						DataCenters: []DataCenter{
@@ -2666,7 +2666,7 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 						},
 					},
 				}
-				normalized := spec.NormalizeConfigurationWithSeparatedProxies(version)
+				normalized := spec.NormalizeConfigurationWithSeparatedProxies(version, false)
 				Expect(normalized.Regions).To(Equal([]Region{
 					{
 						DataCenters: []DataCenter{
diff --git a/controllers/update_pod_config_test.go b/controllers/update_pod_config_test.go
index 1c2a8eb4..1dd6ebf7 100644
--- a/controllers/update_pod_config_test.go
+++ b/controllers/update_pod_config_test.go
@@ -34,43 +34,63 @@ import (
 
 var _ = Describe("updatePodConfig", func() {
 	var cluster *fdbv1beta2.FoundationDBCluster
-	var requeue *requeue
+	var req *requeue
 	var err error
-	var pods []*corev1.Pod
+	var pod *corev1.Pod
 
 	BeforeEach(func() {
 		cluster = internal.CreateDefaultCluster()
-		err = setupClusterForTest(cluster)
-		Expect(err).NotTo(HaveOccurred())
+		Expect(setupClusterForTest(cluster)).NotTo(HaveOccurred())
 
-		pods, err = clusterReconciler.PodLifecycleManager.GetPods(context.TODO(), clusterReconciler, cluster, internal.GetSinglePodListOptions(cluster, "storage-1")...)
+		processGroup := fdbv1beta2.FindProcessGroupByID(cluster.Status.ProcessGroups, "storage-1")
+		Expect(processGroup).NotTo(BeNil())
+		pod, err = clusterReconciler.PodLifecycleManager.GetPod(context.TODO(), clusterReconciler, cluster, processGroup.GetPodName(cluster))
 		Expect(err).NotTo(HaveOccurred())
 
-		for _, container := range pods[0].Spec.Containers {
-			pods[0].Status.ContainerStatuses = append(pods[0].Status.ContainerStatuses, corev1.ContainerStatus{Ready: true, Name: container.Name})
+		for _, container := range pod.Spec.Containers {
+			pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{Ready: true, Name: container.Name})
 		}
 	})
 
 	JustBeforeEach(func() {
-		requeue = updatePodConfig{}.reconcile(context.TODO(), clusterReconciler, cluster, nil, globalControllerLogger)
+		req = updatePodConfig{}.reconcile(context.TODO(), clusterReconciler, cluster, nil, globalControllerLogger)
 		Expect(err).NotTo(HaveOccurred())
 	})
 
 	Context("with a reconciled cluster", func() {
 		It("should not requeue", func() {
 			Expect(err).NotTo(HaveOccurred())
-			Expect(requeue).To(BeNil())
+			Expect(req).To(BeNil())
 		})
 	})
 
 	When("a Pod is stuck in Pending", func() {
 		BeforeEach(func() {
-			pods[0].Status.Phase = corev1.PodPending
+			pod.Status.Phase = corev1.PodPending
+		})
+
+		AfterEach(func() {
+			pod.Status.Phase = corev1.PodRunning
+		})
+
+		It("should not requeue", func() {
+			Expect(err).NotTo(HaveOccurred())
+			Expect(req).To(BeNil())
+		})
+	})
+
+	When("a Pod is stuck in terminating", func() {
+		BeforeEach(func() {
+			Expect(k8sClient.MockStuckTermination(pod, true)).NotTo(HaveOccurred())
+		})
+
+		AfterEach(func() {
+			Expect(k8sClient.MockStuckTermination(pod, false)).NotTo(HaveOccurred())
 		})
 
 		It("should not requeue", func() {
 			Expect(err).NotTo(HaveOccurred())
-			Expect(requeue).To(BeNil())
+			Expect(req).To(BeNil())
 		})
 	})
 })
diff --git a/e2e/fixtures/pods.go b/e2e/fixtures/pods.go
index 265debe3..f7de0302 100644
--- a/e2e/fixtures/pods.go
+++ b/e2e/fixtures/pods.go
@@ -21,8 +21,10 @@
 package fixtures
 
 import (
-	ctx "context"
+	"context"
 	"fmt"
+	"github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/api/equality"
 	"log"
 	"math/rand"
 	"time"
@@ -72,10 +74,25 @@ func RandomPickOnePod(input []corev1.Pod) corev1.Pod {
 }
 
 // SetFinalizerForPod will set the provided finalizer slice for the Pods
-func (factory *Factory) SetFinalizerForPod(pod *corev1.Pod, finalizers []string) error {
-	patch := client.MergeFrom(pod.DeepCopy())
-	pod.SetFinalizers(finalizers)
-	return factory.GetControllerRuntimeClient().Patch(ctx.Background(), pod, patch)
+func (factory *Factory) SetFinalizerForPod(pod *corev1.Pod, finalizers []string) {
+	if pod == nil {
+		return
+	}
+
+	controllerClient := factory.GetControllerRuntimeClient()
+	gomega.Eventually(func(g gomega.Gomega) bool {
+		fetchedPod := &corev1.Pod{}
+		g.Expect(controllerClient.Get(context.Background(), client.ObjectKeyFromObject(pod), fetchedPod)).NotTo(gomega.HaveOccurred())
+
+		if !equality.Semantic.DeepEqual(finalizers, fetchedPod.Finalizers) {
+			fetchedPod.SetFinalizers(finalizers)
+			g.Expect(controllerClient.Update(context.Background(), fetchedPod)).NotTo(gomega.HaveOccurred())
+		}
+
+		g.Expect(fetchedPod.Finalizers).To(gomega.ConsistOf(finalizers))
+
+		return true
+	}).WithTimeout(1 * time.Minute).WithPolling(1 * time.Second).Should(gomega.BeTrue())
 }
 
 // GetProcessClass returns the Process class of this Pod.
diff --git a/e2e/test_operator/operator_test.go b/e2e/test_operator/operator_test.go
index f08f7a16..7c59f813 100644
--- a/e2e/test_operator/operator_test.go
+++ b/e2e/test_operator/operator_test.go
@@ -374,7 +374,7 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 		})
 
 		BeforeEach(func() {
-			useLocalitiesForExclusion = pointer.BoolDeref(fdbCluster.GetCluster().Spec.AutomationOptions.UseLocalitiesForExclusion, false)
+			useLocalitiesForExclusion = fdbCluster.GetCluster().UseLocalitiesForExclusion()
 		})
 
 		AfterEach(func() {
@@ -383,6 +383,7 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 			spec := fdbCluster.GetCluster().Spec.DeepCopy()
 			spec.AutomationOptions.UseLocalitiesForExclusion = pointer.Bool(useLocalitiesForExclusion)
 			fdbCluster.UpdateClusterSpecWithSpec(spec)
+			Expect(fdbCluster.GetCluster().UseLocalitiesForExclusion()).To(Equal(useLocalitiesForExclusion))
 		})
 
 		When("IP addresses are used for exclusion", func() {
@@ -399,9 +400,19 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 
 		When("localities are used for exclusion", func() {
 			BeforeEach(func() {
-				spec := fdbCluster.GetCluster().Spec.DeepCopy()
+				cluster := fdbCluster.GetCluster()
+
+				fdbVersion, err := fdbv1beta2.ParseFdbVersion(cluster.GetRunningVersion())
+				Expect(err).NotTo(HaveOccurred())
+
+				if !fdbVersion.SupportsLocalityBasedExclusions() {
+					Skip("provided FDB version: " + cluster.GetRunningVersion() + " doesn't support locality based exclusions")
+				}
+
+				spec := cluster.Spec.DeepCopy()
 				spec.AutomationOptions.UseLocalitiesForExclusion = pointer.Bool(true)
 				fdbCluster.UpdateClusterSpecWithSpec(spec)
+				Expect(fdbCluster.GetCluster().UseLocalitiesForExclusion()).To(BeTrue())
 			})
 
 			It("should remove the targeted Pod", func() {
@@ -924,23 +935,17 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 		BeforeEach(func() {
 			podsBeforeReplacement = fdbCluster.GetPodsNames()
 			replacePod = fixtures.ChooseRandomPod(fdbCluster.GetPods())
-			Expect(factory.SetFinalizerForPod(replacePod, []string{"foundationdb.org/test"})).ShouldNot(HaveOccurred())
+			factory.SetFinalizerForPod(replacePod, []string{"foundationdb.org/test"})
 			fdbCluster.ReplacePod(*replacePod, true)
 		})
 
 		AfterEach(func() {
-			Expect(factory.SetFinalizerForPod(replacePod, []string{})).ShouldNot(HaveOccurred())
+			factory.SetFinalizerForPod(replacePod, []string{})
 			Expect(fdbCluster.ClearProcessGroupsToRemove()).NotTo(HaveOccurred())
 		})
 
 		It("should replace the Pod stuck in Terminating state", func() {
-			// The `replacePod` still exists as a
-			// Terminating Pod because it has a finalizer
-			// set to it and thus it's not deleted
-			// yet. Moreover, `GetPods()` returns pods even
-			// in terminating state since it's status.phase is
-			// still Running (although the containers have exited).
-			Expect(len(fdbCluster.GetPodsNames())).Should(Equal(len(podsBeforeReplacement) + 1))
+			Expect(fdbCluster.GetPodsNames()).Should(Or(HaveLen(len(podsBeforeReplacement)+1), HaveLen(len(podsBeforeReplacement))))
 		})
 	})
 
@@ -1697,4 +1702,48 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 			}).WithTimeout(10 * time.Minute).WithPolling(5 * time.Second).Should(BeNumerically("==", 1))
 		})
 	})
+
+	When("using proxies instead of grv and commit proxies", func() {
+		var originalRoleCounts fdbv1beta2.RoleCounts
+
+		BeforeEach(func() {
+			spec := fdbCluster.GetCluster().Spec.DeepCopy()
+			originalRoleCounts = spec.DatabaseConfiguration.RoleCounts
+			spec.DatabaseConfiguration.RoleCounts.Proxies = spec.DatabaseConfiguration.RoleCounts.GrvProxies + spec.DatabaseConfiguration.RoleCounts.CommitProxies
+			// Reset the more specific GRV and Commit proxy count.
+			spec.DatabaseConfiguration.RoleCounts.GrvProxies = 0
+			spec.DatabaseConfiguration.RoleCounts.CommitProxies = 0
+
+			fmt.Println("Original:", fixtures.ToJSON(originalRoleCounts), "new roleCounts:", fixtures.ToJSON(spec.DatabaseConfiguration.RoleCounts))
+			fdbCluster.UpdateClusterSpecWithSpec(spec)
+			Expect(fdbCluster.WaitForReconciliation()).NotTo(HaveOccurred())
+		})
+
+		AfterEach(func() {
+			spec := fdbCluster.GetCluster().Spec.DeepCopy()
+			spec.DatabaseConfiguration.RoleCounts = originalRoleCounts
+			fdbCluster.UpdateClusterSpecWithSpec(spec)
+			Expect(fdbCluster.WaitForReconciliation()).NotTo(HaveOccurred())
+		})
+
+		It("should configure the database to run with GRV and commit proxies but keep the proxies in the status field of the FoundationDB resource", func() {
+			// Make sure the FoundationDB configured GRV and commit proxies.
+			status := fdbCluster.GetStatus()
+			Expect(status.Cluster.DatabaseConfiguration.RoleCounts.CommitProxies).To(BeNumerically(">", 0))
+			Expect(status.Cluster.DatabaseConfiguration.RoleCounts.GrvProxies).To(BeNumerically(">", 0))
+			Expect(status.Cluster.DatabaseConfiguration.RoleCounts.Proxies).NotTo(BeZero())
+
+			fmt.Println("Status configuration:", fixtures.ToJSON(status.Cluster.DatabaseConfiguration))
+			// Verify the FoundationDB Cluster resource
+			cluster := fdbCluster.GetCluster()
+			// Make sure that the spec of the FoundationDB resource only has proxies defined.
+			Expect(cluster.Spec.DatabaseConfiguration.RoleCounts.GrvProxies).To(BeZero())
+			Expect(cluster.Spec.DatabaseConfiguration.RoleCounts.CommitProxies).To(BeZero())
+			Expect(cluster.Spec.DatabaseConfiguration.RoleCounts.Proxies).NotTo(BeZero())
+			// Make sure that the status of the FoundationDB resource only has proxies defined.
+			Expect(cluster.Status.DatabaseConfiguration.RoleCounts.GrvProxies).To(BeZero())
+			Expect(cluster.Status.DatabaseConfiguration.RoleCounts.CommitProxies).To(BeZero())
+			Expect(cluster.Status.DatabaseConfiguration.RoleCounts.Proxies).NotTo(BeZero())
+		})
+	})
 })
diff --git a/e2e/test_operator_upgrades/operator_upgrades_test.go b/e2e/test_operator_upgrades/operator_upgrades_test.go
index 95514438..437260a3 100644
--- a/e2e/test_operator_upgrades/operator_upgrades_test.go
+++ b/e2e/test_operator_upgrades/operator_upgrades_test.go
@@ -541,7 +541,7 @@ var _ = Describe("Operator Upgrades", Label("e2e", "pr"), func() {
 			processGroupMarkedForRemoval := fixtures.GetProcessGroupID(podMarkedForRemoval)
 			log.Println("picked Pod", podMarkedForRemoval.Name, "to be marked for removal")
 			// Set a finalizer for this Pod to make sure the Pod object cannot be garbage collected
-			Expect(factory.SetFinalizerForPod(&podMarkedForRemoval, []string{"foundationdb.org/test"})).ShouldNot(HaveOccurred())
+			factory.SetFinalizerForPod(&podMarkedForRemoval, []string{"foundationdb.org/test"})
 			// Don't wait for reconciliation as the cluster will never reconcile.
 			fdbCluster.ReplacePod(podMarkedForRemoval, false)
 			// Make sure the process group is marked for removal
@@ -570,7 +570,7 @@ var _ = Describe("Operator Upgrades", Label("e2e", "pr"), func() {
 			Expect(fdbCluster.WaitForReconciliation(fixtures.SoftReconcileOption(true))).NotTo(HaveOccurred())
 
 			// Make sure we remove the finalizer to not block the clean up.
-			Expect(factory.SetFinalizerForPod(&podMarkedForRemoval, []string{})).ShouldNot(HaveOccurred())
+			factory.SetFinalizerForPod(&podMarkedForRemoval, []string{})
 
 			// Make sure the cluster has no data loss.
 			fdbCluster.EnsureTeamTrackersHaveMinReplicas()
diff --git a/e2e/test_operator_upgrades_variations/operator_upgrades_variations_test.go b/e2e/test_operator_upgrades_variations/operator_upgrades_variations_test.go
index a9717d70..ff80e60b 100644
--- a/e2e/test_operator_upgrades_variations/operator_upgrades_variations_test.go
+++ b/e2e/test_operator_upgrades_variations/operator_upgrades_variations_test.go
@@ -233,6 +233,13 @@ var _ = Describe("Operator Upgrades", Label("e2e", "pr"), func() {
 	DescribeTable(
 		"with locality based exclusions",
 		func(beforeVersion string, targetVersion string) {
+			fdbVersion, err := fdbv1beta2.ParseFdbVersion(beforeVersion)
+			Expect(err).NotTo(HaveOccurred())
+
+			if !fdbVersion.SupportsLocalityBasedExclusions() {
+				Skip("provided FDB version: " + beforeVersion + " doesn't support locality based exclusions")
+			}
+
 			performUpgrade(testConfig{
 				beforeVersion: beforeVersion,
 				targetVersion: targetVersion,