Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1209 (2022-05-10 08:14:08)

Update build version of the operator to 7.1.4
# Description

Update the primary library/build version of the operator to `7.1.4`

## Type of change

*Please select one of the options below.*

- New feature (non-breaking change which adds functionality)

## Discussion

-

## Testing

Local + unit tests.

## Documentation

-

## Follow-up

-


foundationdb-ci: 
### AWS CodeBuild CI Report for Linux CentOS 7

* CodeBuild project: fdb-kubernetes-operator-pr
* Commit ID: 3885523008be85097a0be121c68b226aabc63c44
* Result: SUCCEEDED
* Error: `N/A`
* [Build Logs](https://d1e0l78xbkh3xa.cloudfront.net/build-logs/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/3919ef0c-6a09-45b8-8ac2-b38787392890.gz) (available for 30 days)

johscheuer: I created locally a cluster and changed it to use the DNS support and the cluster comes up with out any issues:

```bash
Process performance details:
  10.42.0.38:4501        (  3% cpu; 10% machine; 0.000 Gbps;  1% disk IO; 0.2 GB / 8.0 GB RAM  )
  10.42.0.42:4501        (  5% cpu; 10% machine; 0.000 Gbps;  2% disk IO; 0.2 GB / 8.0 GB RAM  )
  10.42.0.43:4501        (  7% cpu; 10% machine; 0.000 Gbps;  2% disk IO; 0.2 GB / 8.0 GB RAM  )
  10.42.0.44:4501        (  5% cpu; 10% machine; 0.000 Gbps;  3% disk IO; 0.3 GB / 8.0 GB RAM  )
  10.42.0.45:4501        (  5% cpu;  9% machine; 0.000 Gbps;  2% disk IO; 0.2 GB / 8.0 GB RAM  )
  10.42.0.46:4501        (  3% cpu; 10% machine; 0.000 Gbps;  1% disk IO; 0.2 GB / 8.0 GB RAM  )
  10.42.0.47:4501        (  4% cpu; 10% machine; 0.000 Gbps;  2% disk IO; 0.2 GB / 8.0 GB RAM  )
  10.42.0.49:4501        (  9% cpu; 10% machine; 0.000 Gbps;  1% disk IO; 0.2 GB / 8.0 GB RAM  )

Coordination servers:
  test-cluster-log-1.test-cluster.default.svc.cluster.local:4501  (reachable)
  test-cluster-log-2.test-cluster.default.svc.cluster.local:4501  (reachable)
  test-cluster-log-3.test-cluster.default.svc.cluster.local:4501  (reachable)
```

Patch:
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index bcbc24bc..45378969 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -6,7 +6,7 @@ on:
   pull_request:
 
 env:
-  FDB_VER: "6.3.23"
+  FDB_VER: "7.1.4"
 
 jobs:
   lint-go:
diff --git a/Dockerfile b/Dockerfile
index 05f82272..381245cc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,7 +2,7 @@
 FROM docker.io/library/golang:1.17.8 as builder
 
 # Install FDB this version is only required to compile the fdb operator
-ARG FDB_VERSION=6.3.23
+ARG FDB_VERSION=7.1.4
 ARG FDB_WEBSITE=https://github.com/apple/foundationdb/releases/download
 ARG TAG="latest"
 
diff --git a/Makefile b/Makefile
index 74b37f6d..00ee3574 100644
--- a/Makefile
+++ b/Makefile
@@ -37,6 +37,7 @@ GORELEASER_PKG=github.com/goreleaser/goreleaser@v1.6.3
 GORELEASER=$(GOBIN)/goreleaser
 BUILD_DEPS?=
 BUILDER?="docker"
+KUBECTL_ARGS?=
 
 define godep
 BUILD_DEPS+=$(1)
@@ -113,16 +114,16 @@ run: generate manifests
 
 # Install CRDs into a cluster
 install: manifests
-	kustomize build config/crd | kubectl apply -f -
+	kustomize build config/crd | kubectl $(KUBECTL_ARGS) apply -f -
 
 # Uninstall CRDs from a cluster
 uninstall: manifests
-	kustomize build config/crd | kubectl delete -f -
+	kustomize build config/crd | kubectl $(KUBECTL_ARGS) delete -f -
 
 # Deploy controller in the configured Kubernetes cluster in ~/.kube/config
 deploy: install manifests
 	cd config/development && kustomize edit set image foundationdb/fdb-kubernetes-operator=${IMG}
-	kustomize build config/development | kubectl apply -f -
+	kustomize build config/development | kubectl $(KUBECTL_ARGS) apply -f -
 
 # Generate manifests e.g. CRD, RBAC etc.
 manifests: ${MANIFESTS}
@@ -167,7 +168,7 @@ container-push:
 rebuild-operator: container-build deploy bounce
 
 bounce:
-	kubectl delete pod -l app=fdb-kubernetes-operator-controller-manager
+	kubectl $(KUBECTL_ARGS) delete pod -l app=fdb-kubernetes-operator-controller-manager
 
 samples: ${SAMPLES}
 
diff --git a/api/v1beta2/foundationdb_database_configuration.go b/api/v1beta2/foundationdb_database_configuration.go
index 6b8865cd..fad8239a 100644
--- a/api/v1beta2/foundationdb_database_configuration.go
+++ b/api/v1beta2/foundationdb_database_configuration.go
@@ -49,6 +49,9 @@ type DatabaseConfiguration struct {
 	// Regions defines the regions that the database can replicate in.
 	Regions []Region `json:"regions,omitempty"`
 
+	// ExcludedServers defines the list  of excluded servers form the database.
+	ExcludedServers []ExcludedServers `json:"excluded_servers,omitempty"`
+
 	// RoleCounts defines how many processes the database should recruit for
 	// each role.
 	RoleCounts `json:""`
@@ -70,6 +73,15 @@ type Region struct {
 	SatelliteRedundancyMode RedundancyMode `json:"satellite_redundancy_mode,omitempty"`
 }
 
+// ExcludedServers represents the excluded servers in the database configuration
+type ExcludedServers struct {
+	// The Address of the excluded server.
+	Address string `json:"address,omitempty"`
+
+	// The Locality of the excluded server.
+	Locality string `json:"locality,omitempty"`
+}
+
 // DataCenter represents a data center in the region configuration
 type DataCenter struct {
 	// The ID of the data center. This must match the dcid locality field.
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index acd6f729..542f9234 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -307,6 +307,11 @@ type ProcessGroupStatus struct {
 	ProcessGroupConditions []*ProcessGroupCondition `json:"processGroupConditions,omitempty"`
 }
 
+// GetExclusionString returns the exclusion string
+func (processGroupStatus *ProcessGroupStatus) GetExclusionString() string {
+	return fmt.Sprintf("locality_instance_id:%s", processGroupStatus.ProcessGroupID)
+}
+
 // IsExcluded returns if a process group is excluded
 func (processGroupStatus *ProcessGroupStatus) IsExcluded() bool {
 	return (processGroupStatus.ExclusionTimestamp != nil && !processGroupStatus.ExclusionTimestamp.IsZero()) || processGroupStatus.ExclusionSkipped
@@ -810,6 +815,10 @@ type FoundationDBClusterAutomationOptions struct {
 	// The default is false.
 	UseNonBlockingExcludes *bool `json:"useNonBlockingExcludes,omitempty"`
 
+	// UseLocalitiesForExclusion defines whether the exclusions are done using localities instead of IP addresses.
+	// The default is false.
+	UseLocalitiesForExclusion *bool `json:"useLocalitiesForExclusion,omitempty"`
+
 	// IgnoreTerminatingPodsSeconds defines how long a Pod has to be in the Terminating Phase before
 	// we ignore it during reconciliation. This prevents Pod that are stuck in Terminating to block
 	// further reconciliation.
@@ -1754,6 +1763,17 @@ func (cluster *FoundationDBCluster) GetUseNonBlockingExcludes() bool {
 	return *cluster.Spec.AutomationOptions.UseNonBlockingExcludes
 }
 
+// UseLocalitiesForExclusion returns the value of UseLocalitiesForExclusion or false if unset.
+func (cluster *FoundationDBCluster) UseLocalitiesForExclusion() bool {
+	fdbVersion, err := ParseFdbVersion(cluster.Spec.Version)
+	if err != nil {
+		// Fall back to use exclusions with IP if we can't parse the version.
+		// This should never happen since the version is validated in earlier steps.
+		return false
+	}
+	return fdbVersion.IsAtLeast(Versions.NextMajorVersion) && pointer.BoolDeref(cluster.Spec.AutomationOptions.UseLocalitiesForExclusion, false)
+}
+
 // GetProcessClassLabel provides the label that this cluster is using for the
 // process class when identifying resources.
 func (cluster *FoundationDBCluster) GetProcessClassLabel() string {
diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go
index 83214d09..9e213845 100644
--- a/api/v1beta2/zz_generated.deepcopy.go
+++ b/api/v1beta2/zz_generated.deepcopy.go
@@ -238,6 +238,11 @@ func (in *DatabaseConfiguration) DeepCopyInto(out *DatabaseConfiguration) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.ExcludedServers != nil {
+		in, out := &in.ExcludedServers, &out.ExcludedServers
+		*out = make([]ExcludedServers, len(*in))
+		copy(*out, *in)
+	}
 	out.RoleCounts = in.RoleCounts
 	out.VersionFlags = in.VersionFlags
 }
@@ -252,6 +257,21 @@ func (in *DatabaseConfiguration) DeepCopy() *DatabaseConfiguration {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ExcludedServers) DeepCopyInto(out *ExcludedServers) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExcludedServers.
+func (in *ExcludedServers) DeepCopy() *ExcludedServers {
+	if in == nil {
+		return nil
+	}
+	out := new(ExcludedServers)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *FaultTolerance) DeepCopyInto(out *FaultTolerance) {
 	*out = *in
@@ -460,6 +480,11 @@ func (in *FoundationDBClusterAutomationOptions) DeepCopyInto(out *FoundationDBCl
 		*out = new(bool)
 		**out = **in
 	}
+	if in.UseLocalitiesForExclusion != nil {
+		in, out := &in.UseLocalitiesForExclusion, &out.UseLocalitiesForExclusion
+		*out = new(bool)
+		**out = **in
+	}
 	if in.IgnoreTerminatingPodsSeconds != nil {
 		in, out := &in.IgnoreTerminatingPodsSeconds, &out.IgnoreTerminatingPodsSeconds
 		*out = new(int)
diff --git a/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml b/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
index 5f833a19..5b030ef4 100644
--- a/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
+++ b/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
@@ -9353,6 +9353,8 @@ spec:
                         minimum: 0
                         type: integer
                     type: object
+                  useLocalitiesForExclusion:
+                    type: boolean
                   useNonBlockingExcludes:
                     type: boolean
                   waitBetweenRemovalsSeconds:
@@ -9425,6 +9427,15 @@ spec:
                 properties:
                   commit_proxies:
                     type: integer
+                  excluded_servers:
+                    items:
+                      properties:
+                        address:
+                          type: string
+                        locality:
+                          type: string
+                      type: object
+                    type: array
                   grv_proxies:
                     type: integer
                   log_routers:
@@ -12819,6 +12830,15 @@ spec:
                 properties:
                   commit_proxies:
                     type: integer
+                  excluded_servers:
+                    items:
+                      properties:
+                        address:
+                          type: string
+                        locality:
+                          type: string
+                      type: object
+                    type: array
                   grv_proxies:
                     type: integer
                   log_routers:
diff --git a/controllers/exclude_processes.go b/controllers/exclude_processes.go
index 1ee34db0..7769f2a8 100644
--- a/controllers/exclude_processes.go
+++ b/controllers/exclude_processes.go
@@ -55,56 +55,38 @@ func (e excludeProcesses) reconcile(_ context.Context, r *FoundationDBClusterRec
 		}
 	}
 
-	addresses := make([]fdbv1beta2.ProcessAddress, 0, removalCount)
-	processClassesToExclude := make(map[fdbv1beta2.ProcessClass]fdbv1beta2.None)
+	var fdbProcessesToExclude []fdbv1beta2.ProcessAddress
+	var processClassesToExclude map[fdbv1beta2.ProcessClass]fdbv1beta2.None
 	if removalCount > 0 {
 		exclusions, err := adminClient.GetExclusions()
 		if err != nil {
 			return &requeue{curError: err}
 		}
-
 		log.Info("current exclusions", "ex", exclusions)
-		currentExclusionMap := make(map[string]fdbv1beta2.None, len(exclusions))
-		for _, address := range exclusions {
-			currentExclusionMap[address.String()] = fdbv1beta2.None{}
-		}
-
-		for _, processGroup := range cluster.Status.ProcessGroups {
-			for _, address := range processGroup.Addresses {
-				// Already excluded, so we don't have to exclude it again
-				if _, ok := currentExclusionMap[address]; ok {
-					continue
-				}
-
-				if processGroup.IsMarkedForRemoval() && !processGroup.IsExcluded() {
-					addresses = append(addresses, fdbv1beta2.ProcessAddress{IPAddress: net.ParseIP(address)})
-					processClassesToExclude[processGroup.ProcessClass] = fdbv1beta2.None{}
-				}
-			}
-		}
+		fdbProcessesToExclude, processClassesToExclude = getProcessesToExclude(exclusions, cluster, removalCount)
 	}
 
-	if len(addresses) > 0 {
+	if len(fdbProcessesToExclude) > 0 {
 		for processClass := range processClassesToExclude {
 			canExclude, missingProcesses := canExcludeNewProcesses(cluster, processClass)
 			if !canExclude {
 				// We want to delay the requeue so that the operator can do some other tasks
 				// before retrying.
 				return &requeue{
-					message:        fmt.Sprintf("Waiting for missing processes: %v. Addresses to exclude: %v", missingProcesses, addresses),
+					message:        fmt.Sprintf("Waiting for missing processes: %v. Addresses to exclude: %v", missingProcesses, fdbProcessesToExclude),
 					delayedRequeue: true,
 				}
 			}
 		}
 
-		hasLock, err := r.takeLock(cluster, fmt.Sprintf("excluding processes: %v", addresses))
+		hasLock, err := r.takeLock(cluster, fmt.Sprintf("excluding processes: %v", fdbProcessesToExclude))
 		if !hasLock {
 			return &requeue{curError: err}
 		}
 
-		r.Recorder.Event(cluster, corev1.EventTypeNormal, "ExcludingProcesses", fmt.Sprintf("Excluding %v", addresses))
+		r.Recorder.Event(cluster, corev1.EventTypeNormal, "ExcludingProcesses", fmt.Sprintf("Excluding %v", fdbProcessesToExclude))
 
-		err = adminClient.ExcludeProcesses(addresses)
+		err = adminClient.ExcludeProcesses(fdbProcessesToExclude)
 		if err != nil {
 			// If we run into a timeout error don't requeue directly
 			// to allow the operator to take some other tasks.
@@ -124,6 +106,44 @@ func (e excludeProcesses) reconcile(_ context.Context, r *FoundationDBClusterRec
 	return nil
 }
 
+func getProcessesToExclude(exclusions []fdbv1beta2.ProcessAddress, cluster *fdbv1beta2.FoundationDBCluster, removalCount int) ([]fdbv1beta2.ProcessAddress, map[fdbv1beta2.ProcessClass]fdbv1beta2.None) {
+	processClassesToExclude := make(map[fdbv1beta2.ProcessClass]fdbv1beta2.None)
+	fdbProcessesToExclude := make([]fdbv1beta2.ProcessAddress, 0, removalCount)
+
+	currentExclusionMap := make(map[string]fdbv1beta2.None, len(exclusions))
+	for _, exclusion := range exclusions {
+		currentExclusionMap[exclusion.String()] = fdbv1beta2.None{}
+	}
+
+	for _, processGroup := range cluster.Status.ProcessGroups {
+		// Process already excluded using locality, so we don't have to exclude it again
+		if _, ok := currentExclusionMap[processGroup.GetExclusionString()]; ok {
+			continue
+		}
+
+		// We are excluding process here using the locality field. It might be possible that the process was already excluded using IP before
+		// but for the sake of consistency it is better to exclude process using locality as well.
+		if cluster.UseLocalitiesForExclusion() && processGroup.IsMarkedForRemoval() && !processGroup.IsExcluded() {
+			fdbProcessesToExclude = append(fdbProcessesToExclude, fdbv1beta2.ProcessAddress{StringAddress: processGroup.GetExclusionString()})
+			processClassesToExclude[processGroup.ProcessClass] = fdbv1beta2.None{}
+			continue
+		}
+
+		for _, address := range processGroup.Addresses {
+			// Already excluded, so we don't have to exclude it again
+			if _, ok := currentExclusionMap[address]; ok {
+				continue
+			}
+
+			if processGroup.IsMarkedForRemoval() && !processGroup.IsExcluded() {
+				fdbProcessesToExclude = append(fdbProcessesToExclude, fdbv1beta2.ProcessAddress{IPAddress: net.ParseIP(address)})
+				processClassesToExclude[processGroup.ProcessClass] = fdbv1beta2.None{}
+			}
+		}
+	}
+	return fdbProcessesToExclude, processClassesToExclude
+}
+
 func canExcludeNewProcesses(cluster *fdbv1beta2.FoundationDBCluster, processClass fdbv1beta2.ProcessClass) (bool, []string) {
 	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "excludeProcesses")
 
diff --git a/controllers/remove_process_groups.go b/controllers/remove_process_groups.go
index 62d8ac69..c54e76c3 100644
--- a/controllers/remove_process_groups.go
+++ b/controllers/remove_process_groups.go
@@ -241,32 +241,17 @@ func includeProcessGroup(ctx context.Context, r *FoundationDBClusterReconciler,
 	}
 	defer adminClient.Close()
 
-	addresses := make([]fdbv1beta2.ProcessAddress, 0)
+	var fdbProcessesToInclude []fdbv1beta2.ProcessAddress
+	var processGroups []*fdbv1beta2.ProcessGroupStatus
+	fdbProcessesToInclude, processGroups = getProcessesToInclude(cluster, removedProcessGroups)
+	if len(fdbProcessesToInclude) > 0 {
+		r.Recorder.Event(cluster, corev1.EventTypeNormal, "IncludingProcesses", fmt.Sprintf("Including removed processes: %v", fdbProcessesToInclude))
 
-	hasStatusUpdate := false
-
-	processGroups := make([]*fdbv1beta2.ProcessGroupStatus, 0, len(cluster.Status.ProcessGroups))
-	for _, processGroup := range cluster.Status.ProcessGroups {
-		if processGroup.IsMarkedForRemoval() && removedProcessGroups[processGroup.ProcessGroupID] {
-			for _, pAddr := range processGroup.Addresses {
-				addresses = append(addresses, fdbv1beta2.ProcessAddress{IPAddress: net.ParseIP(pAddr)})
-			}
-			hasStatusUpdate = true
-		} else {
-			processGroups = append(processGroups, processGroup)
-		}
-	}
-
-	if len(addresses) > 0 {
-		r.Recorder.Event(cluster, corev1.EventTypeNormal, "IncludingProcesses", fmt.Sprintf("Including removed processes: %v", addresses))
-
-		err = adminClient.IncludeProcesses(addresses)
+		err = adminClient.IncludeProcesses(fdbProcessesToInclude)
 		if err != nil {
 			return err
 		}
-	}
 
-	if hasStatusUpdate {
 		cluster.Status.ProcessGroups = processGroups
 		err := r.Status().Update(ctx, cluster)
 		if err != nil {
@@ -277,6 +262,26 @@ func includeProcessGroup(ctx context.Context, r *FoundationDBClusterReconciler,
 	return nil
 }
 
+func getProcessesToInclude(cluster *fdbv1beta2.FoundationDBCluster, removedProcessGroups map[string]bool) ([]fdbv1beta2.ProcessAddress, []*fdbv1beta2.ProcessGroupStatus) {
+	fdbProcessesToInclude := make([]fdbv1beta2.ProcessAddress, 0)
+	// TODO (09harsh): don't allocate extra space for processGroups, #1196
+	processGroups := make([]*fdbv1beta2.ProcessGroupStatus, 0, len(cluster.Status.ProcessGroups))
+
+	for _, processGroup := range cluster.Status.ProcessGroups {
+		if processGroup.IsMarkedForRemoval() && removedProcessGroups[processGroup.ProcessGroupID] {
+			if cluster.UseLocalitiesForExclusion() {
+				fdbProcessesToInclude = append(fdbProcessesToInclude, fdbv1beta2.ProcessAddress{StringAddress: processGroup.GetExclusionString()})
+			}
+			for _, pAddr := range processGroup.Addresses {
+				fdbProcessesToInclude = append(fdbProcessesToInclude, fdbv1beta2.ProcessAddress{IPAddress: net.ParseIP(pAddr)})
+			}
+		} else {
+			processGroups = append(processGroups, processGroup)
+		}
+	}
+	return fdbProcessesToInclude, processGroups
+}
+
 func (r *FoundationDBClusterReconciler) getProcessGroupsToRemove(cluster *fdbv1beta2.FoundationDBCluster, remainingMap map[string]bool) (bool, bool, []*fdbv1beta2.ProcessGroupStatus) {
 	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "removeProcessGroups")
 	var cordSet map[string]struct{}
diff --git a/controllers/update_status.go b/controllers/update_status.go
index 139bbdf9..1c0d8cab 100644
--- a/controllers/update_status.go
+++ b/controllers/update_status.go
@@ -105,6 +105,8 @@ func (updateStatus) reconcile(ctx context.Context, r *FoundationDBClusterReconci
 
 	status.HasListenIPsForAllPods = cluster.NeedsExplicitListenAddress()
 	status.DatabaseConfiguration = databaseStatus.Cluster.DatabaseConfiguration.NormalizeConfigurationWithSeparatedProxies(cluster.Spec.Version, cluster.Spec.DatabaseConfiguration.AreSeparatedProxiesConfigured())
+	// Removing excluded servers as we don't want them during comparison.
+	status.DatabaseConfiguration.ExcludedServers = nil
 	cluster.ClearMissingVersionFlags(&status.DatabaseConfiguration)
 	status.Configured = cluster.Status.Configured || (databaseStatus.Client.DatabaseStatus.Available && databaseStatus.Cluster.Layers.Error != "configurationMissing")
 
diff --git a/distrolesss.dockerfile b/distrolesss.dockerfile
index b83bf94d..bbfa2148 100644
--- a/distrolesss.dockerfile
+++ b/distrolesss.dockerfile
@@ -2,7 +2,7 @@
 FROM docker.io/library/golang:1.17.8 as builder
 
 # Install FDB this version is only required to compile the fdb operator
-ARG FDB_VERSION=6.3.23
+ARG FDB_VERSION=7.1.4
 ARG FDB_WEBSITE=https://github.com/apple/foundationdb/releases/download
 ARG TAG="latest"
 
diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
index 5a3d1e0d..1d2affa1 100644
--- a/docs/cluster_spec.md
+++ b/docs/cluster_spec.md
@@ -29,6 +29,7 @@ This Document documents the types introduced by the FoundationDB Operator to be
 * [RoutingConfig](#routingconfig)
 * [DataCenter](#datacenter)
 * [DatabaseConfiguration](#databaseconfiguration)
+* [ExcludedServers](#excludedservers)
 * [ProcessCounts](#processcounts)
 * [Region](#region)
 * [RoleCounts](#rolecounts)
@@ -155,6 +156,7 @@ FoundationDBClusterAutomationOptions provides flags for enabling or disabling op
 | replacements | Replacements contains options for automatically replacing failed processes. | [AutomaticReplacementOptions](#automaticreplacementoptions) | false |
 | ignorePendingPodsDuration | IgnorePendingPodsDuration defines how long a Pod has to be in the Pending Phase before ignore it during reconciliation. This prevents Pod that are stuck in Pending to block further reconciliation. | time.Duration | false |
 | useNonBlockingExcludes | UseNonBlockingExcludes defines whether the operator is allowed to use non blocking exclude commands. The default is false. | *bool | false |
+| useLocalitiesForExclusion | UseLocalitiesForExclusion defines whether the exclusions are done using localities instead of IP addresses. The default is false. | *bool | false |
 | ignoreTerminatingPodsSeconds | IgnoreTerminatingPodsSeconds defines how long a Pod has to be in the Terminating Phase before we ignore it during reconciliation. This prevents Pod that are stuck in Terminating to block further reconciliation. | *int | false |
 | maxConcurrentReplacements | MaxConcurrentReplacements defines how many process groups can be concurrently replaced if they are misconfigured. If the value will be set to 0 this will block replacements and these misconfigured Pods must be replaced manually or by another process. For each reconcile loop the operator calculates the maximum number of possible replacements by taken this value as the upper limit and removes all ongoing replacements that have not finished. Which means if the value is set to 5 and we have 4 ongoing replacements (process groups marked with remove but not excluded) the operator is allowed to replace on further process group. | *int | false |
 | deletionMode | DeletionMode defines the deletion mode for this cluster. This can be PodUpdateModeNone, PodUpdateModeAll, PodUpdateModeZone or PodUpdateModeProcessGroup. The DeletionMode defines how Pods are deleted in order to update them or when they are removed. | [PodUpdateMode](#podupdatemode) | false |
@@ -423,11 +425,23 @@ DatabaseConfiguration represents the configuration of the database
 | storage_engine | StorageEngine defines the storage engine the database uses. | [StorageEngine](#storageengine) | false |
 | usable_regions | UsableRegions defines how many regions the database should store data in. | int | false |
 | regions | Regions defines the regions that the database can replicate in. | [][Region](#region) | false |
+| excluded_servers | ExcludedServers defines the list  of excluded servers form the database. | [][ExcludedServers](#excludedservers) | false |
 | RoleCounts | RoleCounts defines how many processes the database should recruit for each role. | [RoleCounts](#rolecounts) | true |
 | VersionFlags | VersionFlags defines internal flags for testing new features in the database. | [VersionFlags](#versionflags) | true |
 
 [Back to TOC](#table-of-contents)
 
+## ExcludedServers
+
+ExcludedServers represents the excluded servers in the database configuration
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| address | The Address of the excluded server. | string | false |
+| locality | The Locality of the excluded server. | string | false |
+
+[Back to TOC](#table-of-contents)
+
 ## ProcessCounts
 
 ProcessCounts represents the number of processes we have for each valid process class.  If one of the counts in the spec is set to 0, we will infer the process count for that class from the role counts. If one of the counts in the spec is set to -1, we will not create any processes for that class. See GetProcessCountsWithDefaults for more information on the rules for inferring process counts.
diff --git a/fdbclient/admin_client.go b/fdbclient/admin_client.go
index a551e101..fa52864c 100644
--- a/fdbclient/admin_client.go
+++ b/fdbclient/admin_client.go
@@ -60,7 +60,6 @@ func parseMaxCommandOutput() int {
 	return result
 }
 
-var exclusionLinePattern = regexp.MustCompile("(?m)^ +(.*)$")
 var protocolVersionRegex = regexp.MustCompile(`(?m)^protocol (\w+)$`)
 
 // cliAdminClient provides an implementation of the admin interface using the
@@ -311,20 +310,21 @@ func (client *cliAdminClient) IncludeProcesses(addresses []fdbv1beta2.ProcessAdd
 // GetExclusions gets a list of the addresses currently excluded from the
 // database.
 func (client *cliAdminClient) GetExclusions() ([]fdbv1beta2.ProcessAddress, error) {
-	output, err := client.runCommand(cliCommand{command: "exclude"})
+	status, err := client.GetStatus()
 	if err != nil {
 		return nil, err
 	}
-	lines := strings.Split(output, "\n")
-	exclusions := make([]fdbv1beta2.ProcessAddress, 0, len(lines))
-	for _, line := range lines {
-		exclusionMatch := exclusionLinePattern.FindStringSubmatch(line)
-		if exclusionMatch != nil {
-			pAddr, err := fdbv1beta2.ParseProcessAddress(exclusionMatch[1])
+	excludedServers := status.Cluster.DatabaseConfiguration.ExcludedServers
+	exclusions := make([]fdbv1beta2.ProcessAddress, 0, len(excludedServers))
+	for _, excludedServer := range excludedServers {
+		if excludedServer.Address != "" {
+			pAddr, err := fdbv1beta2.ParseProcessAddress(excludedServer.Address)
 			if err != nil {
 				return nil, err
 			}
 			exclusions = append(exclusions, pAddr)
+		} else {
+			exclusions = append(exclusions, fdbv1beta2.ProcessAddress{StringAddress: excludedServer.Locality})
 		}
 	}
 	return exclusions, nil
diff --git a/internal/removals/remove.go b/internal/removals/remove.go
index f8961c71..5f42b115 100644
--- a/internal/removals/remove.go
+++ b/internal/removals/remove.go
@@ -120,7 +120,6 @@ func GetZonedRemovals(status *fdbv1beta2.FoundationDBStatus, processGroupsToRemo
 // GetRemainingMap returns a map that indicates if a process group is fully excluded in the cluster.
 func GetRemainingMap(logger logr.Logger, adminClient fdbadminclient.AdminClient, cluster *fdbv1beta2.FoundationDBCluster) (map[string]bool, error) {
 	var err error
-
 	addresses := make([]fdbv1beta2.ProcessAddress, 0, len(cluster.Status.ProcessGroups))
 	for _, processGroup := range cluster.Status.ProcessGroups {
 		if !processGroup.IsMarkedForRemoval() || processGroup.IsExcluded() {
@@ -132,6 +131,9 @@ func GetRemainingMap(logger logr.Logger, adminClient fdbadminclient.AdminClient,
 			continue
 		}
 
+		if cluster.UseLocalitiesForExclusion() {
+			addresses = append(addresses, fdbv1beta2.ProcessAddress{StringAddress: processGroup.GetExclusionString()})
+		}
 		for _, pAddr := range processGroup.Addresses {
 			addresses = append(addresses, fdbv1beta2.ProcessAddress{IPAddress: net.ParseIP(pAddr)})
 		}


Test patch:
diff --git a/api/v1beta2/foundationdb_status_test.go b/api/v1beta2/foundationdb_status_test.go
index 0b570e95..9b4598ed 100644
--- a/api/v1beta2/foundationdb_status_test.go
+++ b/api/v1beta2/foundationdb_status_test.go
@@ -75,12 +75,13 @@ var _ = Describe("FoundationDBStatus", func() {
 						MaxZoneFailuresWithoutLosingData:         1,
 					},
 					DatabaseConfiguration: DatabaseConfiguration{
-						RedundancyMode: RedundancyModeDouble,
-						StorageEngine:  StorageEngineSSD2,
-						UsableRegions:  1,
-						Regions:        nil,
-						RoleCounts:     RoleCounts{Storage: 0, Logs: 3, Proxies: 3, Resolvers: 1, LogRouters: 0, RemoteLogs: 0},
-						VersionFlags:   VersionFlags{LogSpill: 2},
+						RedundancyMode:  RedundancyModeDouble,
+						StorageEngine:   StorageEngineSSD2,
+						UsableRegions:   1,
+						Regions:         nil,
+						ExcludedServers: make([]ExcludedServers, 0),
+						RoleCounts:      RoleCounts{Storage: 0, Logs: 3, Proxies: 3, Resolvers: 1, LogRouters: 0, RemoteLogs: 0},
+						VersionFlags:    VersionFlags{LogSpill: 2},
 					},
 					Processes: map[string]FoundationDBStatusProcessInfo{
 						"b9c25278c0fa207bc2a73bda2300d0a9": {
@@ -459,12 +460,13 @@ var _ = Describe("FoundationDBStatus", func() {
 	When("parsing the status json with a 7.1.0-rc1 cluster", func() {
 		status := FoundationDBStatusClusterInfo{
 			DatabaseConfiguration: DatabaseConfiguration{
-				RedundancyMode: "double",
-				StorageEngine:  StorageEngineSSD2,
-				UsableRegions:  1,
-				Regions:        nil,
-				RoleCounts:     RoleCounts{Storage: 0, Logs: 3, Proxies: 3, CommitProxies: 2, GrvProxies: 1, Resolvers: 1, LogRouters: -1, RemoteLogs: -1},
-				VersionFlags:   VersionFlags{LogSpill: 2, LogVersion: 0},
+				RedundancyMode:  "double",
+				StorageEngine:   StorageEngineSSD2,
+				UsableRegions:   1,
+				Regions:         nil,
+				ExcludedServers: make([]ExcludedServers, 0),
+				RoleCounts:      RoleCounts{Storage: 0, Logs: 3, Proxies: 3, CommitProxies: 2, GrvProxies: 1, Resolvers: 1, LogRouters: -1, RemoteLogs: -1},
+				VersionFlags:    VersionFlags{LogSpill: 2, LogVersion: 0},
 			},
 			Processes: map[string]FoundationDBStatusProcessInfo{
 				"eb48ada3a682e86363f06aa89e1041fa": {
diff --git a/controllers/exclude_processes_test.go b/controllers/exclude_processes_test.go
index e796f346..2e384ca3 100644
--- a/controllers/exclude_processes_test.go
+++ b/controllers/exclude_processes_test.go
@@ -22,6 +22,10 @@ package controllers
 
 import (
 	"context"
+	"fmt"
+	"net"
+
+	"k8s.io/utils/pointer"
 
 	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
 
@@ -34,21 +38,21 @@ var _ = Describe("exclude_processes", func() {
 	var cluster *fdbv1beta2.FoundationDBCluster
 	var err error
 
-	BeforeEach(func() {
-		cluster = internal.CreateDefaultCluster()
-		err = k8sClient.Create(context.TODO(), cluster)
-		Expect(err).NotTo(HaveOccurred())
+	Describe("canExcludeNewProcesses", func() {
+		BeforeEach(func() {
+			cluster = internal.CreateDefaultCluster()
+			err = k8sClient.Create(context.TODO(), cluster)
+			Expect(err).NotTo(HaveOccurred())
 
-		result, err := reconcileCluster(cluster)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(result.Requeue).To(BeFalse())
+			result, err := reconcileCluster(cluster)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result.Requeue).To(BeFalse())
 
-		generation, err := reloadCluster(cluster)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(generation).To(Equal(int64(1)))
-	})
+			generation, err := reloadCluster(cluster)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(generation).To(Equal(int64(1)))
+		})
 
-	Describe("canExcludeNewProcesses", func() {
 		Context("with a small cluster", func() {
 			When("all processes are healthy", func() {
 				It("should allow the exclusion", func() {
@@ -134,6 +138,218 @@ var _ = Describe("exclude_processes", func() {
 			})
 		})
 	})
+
+	When("validating getProcessesToExclude", func() {
+		var exclusions []fdbv1beta2.ProcessAddress
+
+		BeforeEach(func() {
+			cluster = &fdbv1beta2.FoundationDBCluster{
+				Status: fdbv1beta2.FoundationDBClusterStatus{
+					ProcessGroups: []*fdbv1beta2.ProcessGroupStatus{
+						{ProcessGroupID: "storage-1", ProcessClass: "storage", Addresses: []string{"1.1.1.1"}},
+						{ProcessGroupID: "storage-2", ProcessClass: "storage", Addresses: []string{"1.1.1.2"}},
+						{ProcessGroupID: "storage-3", ProcessClass: "storage", Addresses: []string{"1.1.1.3"}},
+						{ProcessGroupID: "stateless-1", ProcessClass: "stateless", Addresses: []string{"1.1.1.4"}},
+						{ProcessGroupID: "stateless-2", ProcessClass: "stateless", Addresses: []string{"1.1.1.5"}},
+						{ProcessGroupID: "stateless-3", ProcessClass: "stateless", Addresses: []string{"1.1.1.6"}},
+						{ProcessGroupID: "stateless-4", ProcessClass: "stateless", Addresses: []string{"1.1.1.7"}},
+						{ProcessGroupID: "stateless-5", ProcessClass: "stateless", Addresses: []string{"1.1.1.8"}},
+						{ProcessGroupID: "stateless-6", ProcessClass: "stateless", Addresses: []string{"1.1.1.9"}},
+						{ProcessGroupID: "stateless-7", ProcessClass: "stateless", Addresses: []string{"1.1.2.1"}},
+						{ProcessGroupID: "stateless-8", ProcessClass: "stateless", Addresses: []string{"1.1.2.2"}},
+						{ProcessGroupID: "stateless-9", ProcessClass: "stateless", Addresses: []string{"1.1.2.3"}},
+						{ProcessGroupID: "log-1", ProcessClass: "log", Addresses: []string{"1.1.2.4"}},
+						{ProcessGroupID: "log-2", ProcessClass: "log", Addresses: []string{"1.1.2.5"}},
+						{ProcessGroupID: "log-3", ProcessClass: "log", Addresses: []string{"1.1.2.6"}},
+						{ProcessGroupID: "log-4", ProcessClass: "log", Addresses: []string{"1.1.2.7"}},
+					},
+				},
+				Spec: fdbv1beta2.FoundationDBClusterSpec{
+					AutomationOptions: fdbv1beta2.FoundationDBClusterAutomationOptions{
+						UseLocalitiesForExclusion: pointer.Bool(true),
+					},
+				},
+			}
+			exclusions = []fdbv1beta2.ProcessAddress{}
+		})
+
+		Context("cluster doesn't supports locality based exclusions", func() {
+			BeforeEach(func() {
+				cluster.Spec.Version = fdbv1beta2.Versions.Default.String()
+			})
+
+			When("there are no exclusions", func() {
+				It("should not exclude anything", func() {
+					fdbProcessesToExclude, processClassesToExclude := getProcessesToExclude(exclusions, cluster, 0)
+					Expect(len(processClassesToExclude)).To(Equal(0))
+					Expect(len(fdbProcessesToExclude)).To(Equal(0))
+				})
+			})
+
+			When("excluding one process", func() {
+				BeforeEach(func() {
+					processGroup := cluster.Status.ProcessGroups[0]
+					Expect(processGroup.ProcessGroupID).To(Equal("storage-1"))
+					processGroup.MarkForRemoval()
+					cluster.Status.ProcessGroups[0] = processGroup
+				})
+
+				It("should report the excluded process", func() {
+					fdbProcessesToExclude, processClassesToExclude := getProcessesToExclude(exclusions, cluster, 0)
+					Expect(len(processClassesToExclude)).To(Equal(1))
+					Expect(processClassesToExclude).To(Equal(map[fdbv1beta2.ProcessClass]fdbv1beta2.None{fdbv1beta2.ProcessClassStorage: {}}))
+					Expect(len(fdbProcessesToExclude)).To(Equal(1))
+					Expect(fdbv1beta2.ProcessAddressesString(fdbProcessesToExclude, " ")).To(Equal("1.1.1.1"))
+				})
+			})
+
+			When("excluding two process", func() {
+				BeforeEach(func() {
+					processGroup1 := cluster.Status.ProcessGroups[0]
+					Expect(processGroup1.ProcessGroupID).To(Equal("storage-1"))
+					processGroup1.MarkForRemoval()
+					cluster.Status.ProcessGroups[0] = processGroup1
+
+					processGroup2 := cluster.Status.ProcessGroups[1]
+					Expect(processGroup2.ProcessGroupID).To(Equal("storage-2"))
+					processGroup2.MarkForRemoval()
+					cluster.Status.ProcessGroups[1] = processGroup2
+				})
+
+				It("should report the excluded process", func() {
+					fdbProcessesToExclude, processClassesToExclude := getProcessesToExclude(exclusions, cluster, 0)
+					Expect(len(processClassesToExclude)).To(Equal(1))
+					Expect(processClassesToExclude).To(Equal(map[fdbv1beta2.ProcessClass]fdbv1beta2.None{fdbv1beta2.ProcessClassStorage: {}}))
+					Expect(len(fdbProcessesToExclude)).To(Equal(2))
+					Expect(fdbv1beta2.ProcessAddressesString(fdbProcessesToExclude, " ")).To(Equal("1.1.1.1 1.1.1.2"))
+				})
+			})
+
+			When("excluding two process with one already excluded", func() {
+				BeforeEach(func() {
+					processGroup1 := cluster.Status.ProcessGroups[0]
+					Expect(processGroup1.ProcessGroupID).To(Equal("storage-1"))
+					processGroup1.MarkForRemoval()
+					cluster.Status.ProcessGroups[0] = processGroup1
+
+					processGroup2 := cluster.Status.ProcessGroups[1]
+					Expect(processGroup2.ProcessGroupID).To(Equal("storage-2"))
+					processGroup2.MarkForRemoval()
+					cluster.Status.ProcessGroups[1] = processGroup2
+
+					exclusions = append(exclusions, fdbv1beta2.ProcessAddress{IPAddress: net.ParseIP(processGroup2.Addresses[0])})
+				})
+
+				It("should report the excluded process", func() {
+					fdbProcessesToExclude, processClassesToExclude := getProcessesToExclude(exclusions, cluster, 1)
+					Expect(len(processClassesToExclude)).To(Equal(1))
+					Expect(processClassesToExclude).To(Equal(map[fdbv1beta2.ProcessClass]fdbv1beta2.None{fdbv1beta2.ProcessClassStorage: {}}))
+					Expect(len(fdbProcessesToExclude)).To(Equal(1))
+					Expect(fdbv1beta2.ProcessAddressesString(fdbProcessesToExclude, " ")).To(Equal("1.1.1.1"))
+				})
+			})
+		})
+
+		Context("cluster supports locality based exclusions", func() {
+			BeforeEach(func() {
+				cluster.Spec.Version = fdbv1beta2.Versions.NextMajorVersion.String()
+			})
+
+			When("there are no exclusions", func() {
+				It("should not exclude anything", func() {
+					fdbProcessesToExclude, processClassesToExclude := getProcessesToExclude(exclusions, cluster, 0)
+					Expect(len(processClassesToExclude)).To(Equal(0))
+					Expect(len(fdbProcessesToExclude)).To(Equal(0))
+				})
+			})
+
+			When("excluding one process", func() {
+				BeforeEach(func() {
+					processGroup := cluster.Status.ProcessGroups[0]
+					Expect(processGroup.ProcessGroupID).To(Equal("storage-1"))
+					processGroup.MarkForRemoval()
+					cluster.Status.ProcessGroups[0] = processGroup
+				})
+
+				It("should report the excluded process", func() {
+					fdbProcessesToExclude, processClassesToExclude := getProcessesToExclude(exclusions, cluster, 0)
+					Expect(len(processClassesToExclude)).To(Equal(1))
+					Expect(processClassesToExclude).To(Equal(map[fdbv1beta2.ProcessClass]fdbv1beta2.None{fdbv1beta2.ProcessClassStorage: {}}))
+					Expect(len(fdbProcessesToExclude)).To(Equal(1))
+					Expect(fdbv1beta2.ProcessAddressesString(fdbProcessesToExclude, " ")).To(Equal(cluster.Status.ProcessGroups[0].GetExclusionString()))
+				})
+			})
+
+			When("excluding two process", func() {
+				BeforeEach(func() {
+					processGroup1 := cluster.Status.ProcessGroups[0]
+					Expect(processGroup1.ProcessGroupID).To(Equal("storage-1"))
+					processGroup1.MarkForRemoval()
+					cluster.Status.ProcessGroups[0] = processGroup1
+
+					processGroup2 := cluster.Status.ProcessGroups[1]
+					Expect(processGroup2.ProcessGroupID).To(Equal("storage-2"))
+					processGroup2.MarkForRemoval()
+					cluster.Status.ProcessGroups[1] = processGroup2
+				})
+
+				It("should report the excluded process", func() {
+					fdbProcessesToExclude, processClassesToExclude := getProcessesToExclude(exclusions, cluster, 0)
+					Expect(len(processClassesToExclude)).To(Equal(1))
+					Expect(processClassesToExclude).To(Equal(map[fdbv1beta2.ProcessClass]fdbv1beta2.None{fdbv1beta2.ProcessClassStorage: {}}))
+					Expect(len(fdbProcessesToExclude)).To(Equal(2))
+					Expect(fdbv1beta2.ProcessAddressesString(fdbProcessesToExclude, " ")).To(Equal(fmt.Sprintf("%s %s", cluster.Status.ProcessGroups[0].GetExclusionString(), cluster.Status.ProcessGroups[1].GetExclusionString())))
+				})
+			})
+
+			When("excluding two process with one already excluded using IP", func() {
+				BeforeEach(func() {
+					processGroup1 := cluster.Status.ProcessGroups[0]
+					Expect(processGroup1.ProcessGroupID).To(Equal("storage-1"))
+					processGroup1.MarkForRemoval()
+					cluster.Status.ProcessGroups[0] = processGroup1
+
+					processGroup2 := cluster.Status.ProcessGroups[1]
+					Expect(processGroup2.ProcessGroupID).To(Equal("storage-2"))
+					processGroup2.MarkForRemoval()
+					cluster.Status.ProcessGroups[1] = processGroup2
+
+					exclusions = append(exclusions, fdbv1beta2.ProcessAddress{IPAddress: net.ParseIP(processGroup2.Addresses[0])})
+				})
+
+				It("should report the excluded process", func() {
+					fdbProcessesToExclude, processClassesToExclude := getProcessesToExclude(exclusions, cluster, 1)
+					Expect(len(processClassesToExclude)).To(Equal(1))
+					Expect(processClassesToExclude).To(Equal(map[fdbv1beta2.ProcessClass]fdbv1beta2.None{fdbv1beta2.ProcessClassStorage: {}}))
+					Expect(len(fdbProcessesToExclude)).To(Equal(2))
+					Expect(fdbv1beta2.ProcessAddressesString(fdbProcessesToExclude, " ")).To(Equal(fmt.Sprintf("%s %s", cluster.Status.ProcessGroups[0].GetExclusionString(), cluster.Status.ProcessGroups[1].GetExclusionString())))
+				})
+			})
+
+			When("excluding two process with one already excluded using locality", func() {
+				BeforeEach(func() {
+					processGroup1 := cluster.Status.ProcessGroups[0]
+					Expect(processGroup1.ProcessGroupID).To(Equal("storage-1"))
+					processGroup1.MarkForRemoval()
+					cluster.Status.ProcessGroups[0] = processGroup1
+
+					processGroup2 := cluster.Status.ProcessGroups[1]
+					Expect(processGroup2.ProcessGroupID).To(Equal("storage-2"))
+					processGroup2.MarkForRemoval()
+					cluster.Status.ProcessGroups[1] = processGroup2
+
+					exclusions = append(exclusions, fdbv1beta2.ProcessAddress{StringAddress: processGroup2.GetExclusionString()})
+				})
+				It("should report the excluded process", func() {
+					fdbProcessesToExclude, processClassesToExclude := getProcessesToExclude(exclusions, cluster, 1)
+					Expect(len(processClassesToExclude)).To(Equal(1))
+					Expect(processClassesToExclude).To(Equal(map[fdbv1beta2.ProcessClass]fdbv1beta2.None{fdbv1beta2.ProcessClassStorage: {}}))
+					Expect(len(fdbProcessesToExclude)).To(Equal(1))
+					Expect(fdbv1beta2.ProcessAddressesString(fdbProcessesToExclude, " ")).To(Equal(cluster.Status.ProcessGroups[0].GetExclusionString()))
+				})
+			})
+		})
+	})
 })
 
 func createMissingProcesses(cluster *fdbv1beta2.FoundationDBCluster, count int, processClass fdbv1beta2.ProcessClass) {
diff --git a/controllers/remove_process_groups_test.go b/controllers/remove_process_groups_test.go
index cc024259..1b5cfee5 100644
--- a/controllers/remove_process_groups_test.go
+++ b/controllers/remove_process_groups_test.go
@@ -22,6 +22,7 @@ package controllers
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	"k8s.io/utils/pointer"
@@ -38,249 +39,350 @@ var _ = Describe("remove_process_groups", func() {
 	var cluster *fdbv1beta2.FoundationDBCluster
 	var result *requeue
 
-	BeforeEach(func() {
-		cluster = internal.CreateDefaultCluster()
-		err := k8sClient.Create(context.TODO(), cluster)
-		Expect(err).NotTo(HaveOccurred())
+	Context("validating process removal", func() {
+		BeforeEach(func() {
+			cluster = internal.CreateDefaultCluster()
+			err := k8sClient.Create(context.TODO(), cluster)
+			Expect(err).NotTo(HaveOccurred())
 
-		result, err := reconcileCluster(cluster)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(result.Requeue).To(BeFalse())
+			result, err := reconcileCluster(cluster)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result.Requeue).To(BeFalse())
 
-		generation, err := reloadCluster(cluster)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(generation).To(Equal(int64(1)))
-	})
+			generation, err := reloadCluster(cluster)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(generation).To(Equal(int64(1)))
+		})
 
-	JustBeforeEach(func() {
-		result = removeProcessGroups{}.reconcile(context.TODO(), clusterReconciler, cluster)
-	})
+		JustBeforeEach(func() {
+			result = removeProcessGroups{}.reconcile(context.TODO(), clusterReconciler, cluster)
+		})
 
-	When("trying to remove a coordinator", func() {
-		coordinatorIP := "1.1.1.1"
-		coordinatorID := "storage-1"
+		When("trying to remove a coordinator", func() {
+			coordinatorIP := "1.1.1.1"
+			coordinatorID := "storage-1"
 
-		BeforeEach(func() {
-			marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, coordinatorID, fdbv1beta2.ProcessClassStorage, coordinatorIP)
-			Expect(marked).To(BeTrue())
-			Expect(processGroup).To(BeNil())
-		})
+			BeforeEach(func() {
+				marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, coordinatorID, fdbv1beta2.ProcessClassStorage, coordinatorIP)
+				Expect(marked).To(BeTrue())
+				Expect(processGroup).To(BeNil())
+			})
 
-		It("should not remove the coordinator", func() {
-			Expect(result).NotTo(BeNil())
-			Expect(result.message).To(Equal("Reconciliation needs to exclude more processes"))
-		})
+			It("should not remove the coordinator", func() {
+				Expect(result).NotTo(BeNil())
+				Expect(result.message).To(Equal("Reconciliation needs to exclude more processes"))
+			})
 
-		It("should exclude the coordinator from the process group list to remove", func() {
-			remaining := map[string]bool{
-				coordinatorIP: false,
-			}
+			It("should exclude the coordinator from the process group list to remove", func() {
+				remaining := map[string]bool{
+					coordinatorIP: false,
+				}
 
-			allExcluded, newExclusions, processes := clusterReconciler.getProcessGroupsToRemove(cluster, remaining)
-			Expect(allExcluded).To(BeFalse())
-			Expect(processes).To(BeEmpty())
-			Expect(newExclusions).To(BeFalse())
+				allExcluded, newExclusions, processes := clusterReconciler.getProcessGroupsToRemove(cluster, remaining)
+				Expect(allExcluded).To(BeFalse())
+				Expect(processes).To(BeEmpty())
+				Expect(newExclusions).To(BeFalse())
+			})
 		})
-	})
 
-	When("removing a process group", func() {
-		var removedProcessGroup *fdbv1beta2.ProcessGroupStatus
+		When("removing a process group", func() {
+			var removedProcessGroup *fdbv1beta2.ProcessGroupStatus
 
-		BeforeEach(func() {
-			removedProcessGroup = cluster.Status.ProcessGroups[0]
-			marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, removedProcessGroup.ProcessGroupID, removedProcessGroup.ProcessClass, removedProcessGroup.Addresses[0])
-			Expect(marked).To(BeTrue())
-			Expect(processGroup).To(BeNil())
-			// Exclude the process group
-			adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
-			Expect(err).NotTo(HaveOccurred())
-			adminClient.ExcludedAddresses = removedProcessGroup.Addresses
-		})
+			BeforeEach(func() {
+				removedProcessGroup = cluster.Status.ProcessGroups[0]
+				marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, removedProcessGroup.ProcessGroupID, removedProcessGroup.ProcessClass, removedProcessGroup.Addresses[0])
+				Expect(marked).To(BeTrue())
+				Expect(processGroup).To(BeNil())
+				// Exclude the process group
+				adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
+				Expect(err).NotTo(HaveOccurred())
+				adminClient.ExcludedAddresses = removedProcessGroup.Addresses
+			})
 
-		When("using the default setting of EnforceFullReplicationForDeletion", func() {
-			When("the cluster is fully replicated", func() {
-				It("should successfully remove that process group", func() {
-					Expect(result).To(BeNil())
-					// Ensure resources are deleted
-					removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
-					Expect(err).To(BeNil())
-					Expect(removed).To(BeTrue())
-					Expect(include).To(BeTrue())
+			When("using the default setting of EnforceFullReplicationForDeletion", func() {
+				When("the cluster is fully replicated", func() {
+					It("should successfully remove that process group", func() {
+						Expect(result).To(BeNil())
+						// Ensure resources are deleted
+						removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+						Expect(err).To(BeNil())
+						Expect(removed).To(BeTrue())
+						Expect(include).To(BeTrue())
+					})
 				})
-			})
 
-			When("the cluster has degraded availability fault tolerance", func() {
-				BeforeEach(func() {
-					adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
-					Expect(err).NotTo(HaveOccurred())
-					adminClient.maxZoneFailuresWithoutLosingAvailability = pointer.Int(0)
+				When("the cluster has degraded availability fault tolerance", func() {
+					BeforeEach(func() {
+						adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
+						Expect(err).NotTo(HaveOccurred())
+						adminClient.maxZoneFailuresWithoutLosingAvailability = pointer.Int(0)
+					})
+
+					It("should not remove that process group", func() {
+						Expect(result).NotTo(BeNil())
+						Expect(result.message).To(Equal("Removals cannot proceed because cluster has degraded fault tolerance"))
+						// Ensure resources are not deleted
+						removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+						Expect(err).To(BeNil())
+						Expect(removed).To(BeFalse())
+						Expect(include).To(BeFalse())
+					})
 				})
 
-				It("should not remove that process group", func() {
-					Expect(result).NotTo(BeNil())
-					Expect(result.message).To(Equal("Removals cannot proceed because cluster has degraded fault tolerance"))
-					// Ensure resources are not deleted
-					removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
-					Expect(err).To(BeNil())
-					Expect(removed).To(BeFalse())
-					Expect(include).To(BeFalse())
+				When("the cluster has degraded data fault tolerance", func() {
+					BeforeEach(func() {
+						adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
+						Expect(err).NotTo(HaveOccurred())
+						adminClient.maxZoneFailuresWithoutLosingData = pointer.Int(0)
+					})
+
+					It("should not remove that process group", func() {
+						Expect(result).NotTo(BeNil())
+						Expect(result.message).To(Equal("Removals cannot proceed because cluster has degraded fault tolerance"))
+						// Ensure resources are not deleted
+						removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+						Expect(err).To(BeNil())
+						Expect(removed).To(BeFalse())
+						Expect(include).To(BeFalse())
+					})
+				})
+
+				When("the cluster is not available", func() {
+					BeforeEach(func() {
+						adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
+						Expect(err).NotTo(HaveOccurred())
+						adminClient.frozenStatus = &fdbv1beta2.FoundationDBStatus{
+							Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+								DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+									Available: false,
+								},
+							},
+						}
+					})
+
+					It("should not remove that process group", func() {
+						Expect(result).NotTo(BeNil())
+						Expect(result.message).To(Equal("Removals cannot proceed because cluster has degraded fault tolerance"))
+						// Ensure resources are not deleted
+						removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+						Expect(err).To(BeNil())
+						Expect(removed).To(BeFalse())
+						Expect(include).To(BeFalse())
+					})
 				})
 			})
 
-			When("the cluster has degraded data fault tolerance", func() {
+			When("Removing multiple process groups", func() {
+				var initialCnt int
+				var secondRemovedProcessGroup *fdbv1beta2.ProcessGroupStatus
+
 				BeforeEach(func() {
+					// To allow multiple process groups to be removed we have to use the update mode all
+					cluster.Spec.AutomationOptions.RemovalMode = fdbv1beta2.PodUpdateModeAll
+					err := k8sClient.Update(context.TODO(), cluster)
+					Expect(err).NotTo(HaveOccurred())
+
+					initialCnt = len(cluster.Status.ProcessGroups)
+					secondRemovedProcessGroup = cluster.Status.ProcessGroups[1]
+					marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, secondRemovedProcessGroup.ProcessGroupID, secondRemovedProcessGroup.ProcessClass, removedProcessGroup.Addresses[0])
+					Expect(marked).To(BeTrue())
+					Expect(processGroup).To(BeNil())
+					// Exclude the process group
 					adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
 					Expect(err).NotTo(HaveOccurred())
-					adminClient.maxZoneFailuresWithoutLosingData = pointer.Int(0)
+					adminClient.ExcludedAddresses = append(adminClient.ExcludedAddresses, secondRemovedProcessGroup.Addresses...)
 				})
 
-				It("should not remove that process group", func() {
-					Expect(result).NotTo(BeNil())
-					Expect(result.message).To(Equal("Removals cannot proceed because cluster has degraded fault tolerance"))
-					// Ensure resources are not deleted
+				It("should remove only one process group", func() {
+					Expect(result).To(BeNil())
+					Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 1))
+					// Ensure resources are deleted
 					removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
 					Expect(err).To(BeNil())
+					Expect(removed).To(BeTrue())
+					Expect(include).To(BeTrue())
+					// Ensure resources are not deleted
+					removed, include, err = confirmRemoval(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+					Expect(err).To(BeNil())
 					Expect(removed).To(BeFalse())
 					Expect(include).To(BeFalse())
 				})
-			})
 
-			When("the cluster is not available", func() {
-				BeforeEach(func() {
-					adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
-					Expect(err).NotTo(HaveOccurred())
-					adminClient.frozenStatus = &fdbv1beta2.FoundationDBStatus{
-						Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
-							DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
-								Available: false,
-							},
-						},
-					}
+				When("a process group is marked as terminating and all resources are removed it should be removed", func() {
+					BeforeEach(func() {
+						secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+						err := removeProcessGroup(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+						Expect(err).NotTo(HaveOccurred())
+						// Sleep here to prevent some timeing issues.
+						time.Sleep(10 * time.Microsecond)
+					})
+
+					It("should remove the process group and the terminated process group", func() {
+						Expect(result).To(BeNil())
+						Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
+						// Ensure resources are deleted
+						removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+						Expect(err).To(BeNil())
+						Expect(removed).To(BeTrue())
+						Expect(include).To(BeTrue())
+						// Ensure resources are deleted
+						removed, include, err = confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+						Expect(err).To(BeNil())
+						Expect(removed).To(BeTrue())
+						Expect(include).To(BeTrue())
+					})
 				})
 
-				It("should not remove that process group", func() {
-					Expect(result).NotTo(BeNil())
-					Expect(result.message).To(Equal("Removals cannot proceed because cluster has degraded fault tolerance"))
-					// Ensure resources are not deleted
-					removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
-					Expect(err).To(BeNil())
-					Expect(removed).To(BeFalse())
-					Expect(include).To(BeFalse())
+				When("a process group is marked as terminating and the resources are not removed", func() {
+					BeforeEach(func() {
+						secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+						// Sleep here to prevent some timeing issues.
+						time.Sleep(10 * time.Microsecond)
+					})
+
+					It("should remove the process group and the terminated process group", func() {
+						Expect(result).To(BeNil())
+						Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
+						// Ensure resources are deleted
+						removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+						Expect(err).To(BeNil())
+						Expect(removed).To(BeTrue())
+						Expect(include).To(BeTrue())
+						// Ensure resources are deleted
+						removed, include, err = confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+						Expect(err).To(BeNil())
+						Expect(removed).To(BeTrue())
+						Expect(include).To(BeTrue())
+					})
+				})
+
+				When("a process group is marked as terminating and not fully removed", func() {
+					BeforeEach(func() {
+						secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+						// Set the wait time to the default value
+						cluster.Spec.AutomationOptions.WaitBetweenRemovalsSeconds = pointer.Int(60)
+					})
+
+					It("should remove only one process group", func() {
+						Expect(result).NotTo(BeNil())
+						Expect(result.message).To(HavePrefix("not allowed to remove process groups, waiting:"))
+						Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 0))
+						// Ensure resources are not deleted
+						removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+						Expect(err).To(BeNil())
+						Expect(removed).To(BeFalse())
+						Expect(include).To(BeFalse())
+						// Ensure resources are deleted
+						removed, include, err = confirmRemoval(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+						Expect(err).To(BeNil())
+						Expect(removed).To(BeFalse())
+						Expect(include).To(BeFalse())
+					})
 				})
 			})
 		})
 
-		When("Removing multiple process groups", func() {
-			var initialCnt int
-			var secondRemovedProcessGroup *fdbv1beta2.ProcessGroupStatus
+		AfterEach(func() {
+			k8sClient.Clear()
+		})
+	})
 
-			BeforeEach(func() {
-				// To allow multiple process groups to be removed we have to use the update mode all
-				cluster.Spec.AutomationOptions.RemovalMode = fdbv1beta2.PodUpdateModeAll
-				err := k8sClient.Update(context.TODO(), cluster)
-				Expect(err).NotTo(HaveOccurred())
+	Context("validating getProcessesToInclude", func() {
+		var removedProcessGroups map[string]bool
 
-				initialCnt = len(cluster.Status.ProcessGroups)
-				secondRemovedProcessGroup = cluster.Status.ProcessGroups[1]
-				marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, secondRemovedProcessGroup.ProcessGroupID, secondRemovedProcessGroup.ProcessClass, removedProcessGroup.Addresses[0])
-				Expect(marked).To(BeTrue())
-				Expect(processGroup).To(BeNil())
-				// Exclude the process group
-				adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
-				Expect(err).NotTo(HaveOccurred())
-				adminClient.ExcludedAddresses = append(adminClient.ExcludedAddresses, secondRemovedProcessGroup.Addresses...)
+		BeforeEach(func() {
+			cluster = &fdbv1beta2.FoundationDBCluster{
+				Status: fdbv1beta2.FoundationDBClusterStatus{
+					ProcessGroups: []*fdbv1beta2.ProcessGroupStatus{
+						{ProcessGroupID: "storage-1", ProcessClass: "storage", Addresses: []string{"1.1.1.1"}},
+						{ProcessGroupID: "storage-2", ProcessClass: "storage", Addresses: []string{"1.1.1.2"}},
+						{ProcessGroupID: "storage-3", ProcessClass: "storage", Addresses: []string{"1.1.1.3"}},
+						{ProcessGroupID: "stateless-1", ProcessClass: "stateless", Addresses: []string{"1.1.1.4"}},
+						{ProcessGroupID: "stateless-2", ProcessClass: "stateless", Addresses: []string{"1.1.1.5"}},
+						{ProcessGroupID: "stateless-3", ProcessClass: "stateless", Addresses: []string{"1.1.1.6"}},
+						{ProcessGroupID: "stateless-4", ProcessClass: "stateless", Addresses: []string{"1.1.1.7"}},
+						{ProcessGroupID: "stateless-5", ProcessClass: "stateless", Addresses: []string{"1.1.1.8"}},
+						{ProcessGroupID: "stateless-6", ProcessClass: "stateless", Addresses: []string{"1.1.1.9"}},
+						{ProcessGroupID: "stateless-7", ProcessClass: "stateless", Addresses: []string{"1.1.2.1"}},
+						{ProcessGroupID: "stateless-8", ProcessClass: "stateless", Addresses: []string{"1.1.2.2"}},
+						{ProcessGroupID: "stateless-9", ProcessClass: "stateless", Addresses: []string{"1.1.2.3"}},
+						{ProcessGroupID: "log-1", ProcessClass: "log", Addresses: []string{"1.1.2.4"}},
+						{ProcessGroupID: "log-2", ProcessClass: "log", Addresses: []string{"1.1.2.5"}},
+						{ProcessGroupID: "log-3", ProcessClass: "log", Addresses: []string{"1.1.2.6"}},
+						{ProcessGroupID: "log-4", ProcessClass: "log", Addresses: []string{"1.1.2.7"}},
+					},
+				},
+				Spec: fdbv1beta2.FoundationDBClusterSpec{
+					AutomationOptions: fdbv1beta2.FoundationDBClusterAutomationOptions{
+						UseLocalitiesForExclusion: pointer.Bool(true),
+					},
+				},
+			}
+			removedProcessGroups = make(map[string]bool)
+		})
+
+		Context("cluster doesn't support inclusions using locality", func() {
+			BeforeEach(func() {
+				cluster.Spec.Version = fdbv1beta2.Versions.Default.String()
 			})
 
-			It("should remove only one process group", func() {
-				Expect(result).To(BeNil())
-				Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 1))
-				// Ensure resources are deleted
-				removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
-				Expect(err).To(BeNil())
-				Expect(removed).To(BeTrue())
-				Expect(include).To(BeTrue())
-				// Ensure resources are not deleted
-				removed, include, err = confirmRemoval(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
-				Expect(err).To(BeNil())
-				Expect(removed).To(BeFalse())
-				Expect(include).To(BeFalse())
+			When("including no process", func() {
+				It("should not include any process", func() {
+					fdbProcessesToInclude, processGroups := getProcessesToInclude(cluster, removedProcessGroups)
+					Expect(len(fdbProcessesToInclude)).To(Equal(0))
+					Expect(len(processGroups)).To(Equal(16))
+				})
 			})
 
-			When("a process group is marked as terminating and all resources are removed it should be removed", func() {
+			When("including one process", func() {
 				BeforeEach(func() {
-					secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
-					err := removeProcessGroup(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
-					Expect(err).NotTo(HaveOccurred())
-					// Sleep here to prevent some timeing issues.
-					time.Sleep(10 * time.Microsecond)
+					processGroup := cluster.Status.ProcessGroups[0]
+					Expect(processGroup.ProcessGroupID).To(Equal("storage-1"))
+					processGroup.MarkForRemoval()
+					cluster.Status.ProcessGroups[0] = processGroup
+
+					removedProcessGroups[processGroup.ProcessGroupID] = true
 				})
 
-				It("should remove the process group and the terminated process group", func() {
-					Expect(result).To(BeNil())
-					Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
-					// Ensure resources are deleted
-					removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
-					Expect(err).To(BeNil())
-					Expect(removed).To(BeTrue())
-					Expect(include).To(BeTrue())
-					// Ensure resources are deleted
-					removed, include, err = confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
-					Expect(err).To(BeNil())
-					Expect(removed).To(BeTrue())
-					Expect(include).To(BeTrue())
+				It("should include one process", func() {
+					fdbProcessesToInclude, processGroups := getProcessesToInclude(cluster, removedProcessGroups)
+					Expect(len(fdbProcessesToInclude)).To(Equal(1))
+					Expect(fdbv1beta2.ProcessAddressesString(fdbProcessesToInclude, " ")).To(Equal("1.1.1.1"))
+					Expect(len(processGroups)).To(Equal(15))
 				})
 			})
+		})
 
-			When("a process group is marked as terminating and the resources are not removed", func() {
-				BeforeEach(func() {
-					secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
-					// Sleep here to prevent some timeing issues.
-					time.Sleep(10 * time.Microsecond)
-				})
+		Context("cluster support inclusions using locality", func() {
+			BeforeEach(func() {
+				cluster.Spec.Version = fdbv1beta2.Versions.NextMajorVersion.String()
+			})
 
-				It("should remove the process group and the terminated process group", func() {
-					Expect(result).To(BeNil())
-					Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
-					// Ensure resources are deleted
-					removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
-					Expect(err).To(BeNil())
-					Expect(removed).To(BeTrue())
-					Expect(include).To(BeTrue())
-					// Ensure resources are deleted
-					removed, include, err = confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
-					Expect(err).To(BeNil())
-					Expect(removed).To(BeTrue())
-					Expect(include).To(BeTrue())
+			When("including no process", func() {
+				It("should not include any process", func() {
+					fdbProcessesToInclude, processGroups := getProcessesToInclude(cluster, removedProcessGroups)
+					Expect(len(fdbProcessesToInclude)).To(Equal(0))
+					Expect(len(processGroups)).To(Equal(16))
 				})
 			})
 
-			When("a process group is marked as terminating and not fully removed", func() {
+			When("including one process", func() {
 				BeforeEach(func() {
-					secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
-					// Set the wait time to the default value
-					cluster.Spec.AutomationOptions.WaitBetweenRemovalsSeconds = pointer.Int(60)
+					processGroup := cluster.Status.ProcessGroups[0]
+					Expect(processGroup.ProcessGroupID).To(Equal("storage-1"))
+					processGroup.MarkForRemoval()
+					cluster.Status.ProcessGroups[0] = processGroup
+
+					removedProcessGroups[processGroup.ProcessGroupID] = true
 				})
 
-				It("should remove only one process group", func() {
-					Expect(result).NotTo(BeNil())
-					Expect(result.message).To(HavePrefix("not allowed to remove process groups, waiting:"))
-					Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 0))
-					// Ensure resources are not deleted
-					removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
-					Expect(err).To(BeNil())
-					Expect(removed).To(BeFalse())
-					Expect(include).To(BeFalse())
-					// Ensure resources are deleted
-					removed, include, err = confirmRemoval(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
-					Expect(err).To(BeNil())
-					Expect(removed).To(BeFalse())
-					Expect(include).To(BeFalse())
+				It("should include one process", func() {
+					fdbProcessesToInclude, processGroups := getProcessesToInclude(cluster, removedProcessGroups)
+					Expect(len(fdbProcessesToInclude)).To(Equal(2))
+					Expect(fdbv1beta2.ProcessAddressesString(fdbProcessesToInclude, " ")).To(Equal(fmt.Sprintf("%s %s", cluster.Status.ProcessGroups[0].GetExclusionString(), cluster.Status.ProcessGroups[0].Addresses[0])))
+					Expect(len(processGroups)).To(Equal(15))
 				})
 			})
 		})
 	})
-
-	AfterEach(func() {
-		k8sClient.Clear()
-	})
 })