Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1573 (2023-04-06 10:06:25)

Add kind setup script for e2e tests
# Description

Add a script to setup a kind cluster for e2e tests. Those can be used to spin up a kind cluster in CI to run our e2e tests against a Kind cluster.

## Type of change

*Please select one of the options below.*

- New feature (non-breaking change which adds functionality)

## Discussion

-

## Testing

I ran those scripts locally.

## Documentation

Updated the e2e documentation.

## Follow-up

Change CI tests to make use of this new script.


foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: 2e45a6ea6b27102b71490760ed4219eed622f050
* Duration 4:10:47
* Result: :x: **FAILED**
* Error: `Error while executing command: if $fail_test; then exit 1; fi. Reason: exit status 1`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/f9fa3cc7-6e19-44b5-b7e7-875551b8118d.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/f9fa3cc7-6e19-44b5-b7e7-875551b8118d/workspace.zip) zip file of the working directory (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: 4c39e392bc327cd23faab0b72a393b5733a5c0b6
* Duration 2:48:15
* Result: :white_check_mark: **SUCCEEDED**
* Error: `N/A`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/cd392e0e-ad3b-4229-b025-07d2972336ca.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/cd392e0e-ad3b-4229-b025-07d2972336ca/workspace.zip) zip file of the working directory (available for 30 days)


Patch:
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 5f90820f..f67462e3 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -18,7 +18,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.19.5
+          go-version: 1.19.8
       - uses: actions/cache@v2
         with:
           path: ~/go/pkg/mod
@@ -44,7 +44,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.19.5
+        go-version: 1.19.8
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
       with:
@@ -65,13 +65,15 @@ jobs:
         curl -L --fail "https://github.com/apple/foundationdb/releases/download/${FDB_VER}/foundationdb-clients_${FDB_VER}-1_amd64.deb" -o fdb.deb
         curl -Lo kind https://kind.sigs.k8s.io/dl/${KIND_VER}/kind-linux-amd64
     - name: Install dependencies
+      env:
+        KUBE_VERSION: ${{ matrix.kubever }}
       run: |
         # Install dependencies either from cache or freshly downloaded
         sudo dpkg -i fdb.deb
-        # Install Kind and start a local Kubernetes cluster
+        # Install Kind and start a local Kind cluster.
         chmod +x kind
         sudo mv kind /usr/local/bin/kind
-        ./e2e/setup-local-fdb-cluster.sh yes ${{ matrix.kubever }}
+        ./e2e/scripts/start_kind_cluster.sh
     #  https://github.com/goreleaser/goreleaser/issues/1311
     - name: Get current semver tag
       run: echo "::set-output name=CURRENT_TAG::$(git describe --tags --match "v*" --abbrev=0)"
@@ -95,6 +97,10 @@ jobs:
         # Ensure we can upgrade the CRD with the current changes
         make install
         kubectl wait --for condition="established" crd --all
+        # Add additional buffer time because we do not have a strong signal on
+        # when the CRD updates are complete.
+        # TODO: Find a way to get a stronger signal on this.
+        sleep 60
     - name: Ensure samples are in a valid format
       run: |
         kubectl apply -f ./config/samples --dry-run=server
@@ -105,7 +111,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.19.5
+        go-version: 1.19.8
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
       with:
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 3d8c1d0e..571e9cde 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -43,7 +43,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.19.5
+          go-version: 1.19.8
       #  https://github.com/goreleaser/goreleaser/issues/1311
       - name: Get current semver tag
         run: echo "::set-output name=CURRENT_TAG::$(git describe --tags --match "v*" --abbrev=0)"
diff --git a/.gitignore b/.gitignore
index d386a41b..3d3611c0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -41,3 +41,6 @@ config/development/kustomization.yaml
 tmp/
 
 dist/
+
+# Ignore any temporary created kube config files
+*.kubeconfig
diff --git a/Dockerfile b/Dockerfile
index 1690d5ea..ebdb8e69 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,7 +1,7 @@
 ARG BASE_IMAGE=docker.io/debian:bullseye
 
 # Build the manager binary
-FROM docker.io/library/golang:1.19.5 as builder
+FROM docker.io/library/golang:1.19.8 as builder
 
 # Install FDB this version is only required to compile the fdb operator
 ARG FDB_VERSION=6.2.29
diff --git a/api/v1beta2/foundationdb_status.go b/api/v1beta2/foundationdb_status.go
index ed72130e..5bfd512e 100644
--- a/api/v1beta2/foundationdb_status.go
+++ b/api/v1beta2/foundationdb_status.go
@@ -263,7 +263,7 @@ type FoundationDBStatusConnectedClient struct {
 	Address string `json:"address,omitempty"`
 
 	// LogGroup provides the trace log group the client has set.
-	LogGroup string `json:"log_group,omitempty"`
+	LogGroup LogGroup `json:"log_group,omitempty"`
 }
 
 // Description returns a string description of the a connected client.
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index 209a3feb..da0561b3 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -924,9 +924,14 @@ type FoundationDBClusterAutomationOptions struct {
 
 	// IgnoreLogGroupsForUpgrade defines the list of LogGroups that should be ignored during fdb version upgrade.
 	// +kubebuilder:validation:MaxItems=10
-	IgnoreLogGroupsForUpgrade []string `json:"ignoreLogGroupsForUpgrade,omitempty"`
+	IgnoreLogGroupsForUpgrade []LogGroup `json:"ignoreLogGroupsForUpgrade,omitempty"`
 }
 
+// LogGroup represents a LogGroup used by a FoundationDB process to log trace events. The LogGroup can be used to filter
+// clients during an upgrade.
+// +kubebuilder:validation:MaxLength=256
+type LogGroup string
+
 // MaintenanceModeOptions controls options for placing zones in maintenance mode.
 type MaintenanceModeOptions struct {
 	// UseMaintenanceModeChecker defines whether the operator is allowed to use maintenance mode before updating pods.
@@ -1718,6 +1723,12 @@ type RoutingConfig struct {
 	// latest stable version of FoundationDB.
 	UseDNSInClusterFile *bool `json:"useDNSInClusterFile,omitempty"`
 
+	// DefineDNSLocalityFields determines whether to define pod DNS names on pod
+	// specs and provide them in the locality arguments to fdbserver.
+	//
+	// This is ignored if UseDNSInCluster is true.
+	DefineDNSLocalityFields *bool `json:"defineDNSLocalityFields,omitempty"`
+
 	// DNSDomain defines the cluster domain used in a DNS name generated for a
 	// service.
 	// The default is `cluster.local`.
@@ -1774,6 +1785,13 @@ type BuggifyConfig struct {
 	// processes. IgnoreDuringRestart does not support the wildcard option to ignore all of this specific cluster processes.
 	// +kubebuilder:validation:MaxItems=1000
 	IgnoreDuringRestart []ProcessGroupID `json:"ignoreDuringRestart,omitempty"`
+
+	// BlockRemoval defines a list of process group IDs that will not be removed, even if they are marked for removal.
+	// The operator will trigger the exclusion but the removal of the resources will be blocked until they are removed
+	// from this list. This setting can be used to simulate cases where a process group is marked for removal but the
+	// resources are not yet removed.
+	// +kubebuilder:validation:MaxItems=1000
+	BlockRemoval []ProcessGroupID `json:"blockRemoval,omitempty"`
 }
 
 // LabelConfig allows customizing labels used by the operator.
@@ -1988,7 +2006,7 @@ const (
 // NeedsHeadlessService determines whether we need to create a headless service
 // for this cluster.
 func (cluster *FoundationDBCluster) NeedsHeadlessService() bool {
-	return cluster.UseDNSInClusterFile() || pointer.BoolDeref(cluster.Spec.Routing.HeadlessService, false)
+	return cluster.DefineDNSLocalityFields() || pointer.BoolDeref(cluster.Spec.Routing.HeadlessService, false)
 }
 
 // UseDNSInClusterFile determines whether we need to use DNS entries in the
@@ -1997,6 +2015,12 @@ func (cluster *FoundationDBCluster) UseDNSInClusterFile() bool {
 	return pointer.BoolDeref(cluster.Spec.Routing.UseDNSInClusterFile, false)
 }
 
+// DefineDNSLocalityFields determines whether we need to put DNS entries in the
+// pod spec and process locality.
+func (cluster *FoundationDBCluster) DefineDNSLocalityFields() bool {
+	return pointer.BoolDeref(cluster.Spec.Routing.DefineDNSLocalityFields, false) || cluster.UseDNSInClusterFile()
+}
+
 // GetDNSDomain gets the domain used when forming DNS names generated for a
 // service.
 func (cluster *FoundationDBCluster) GetDNSDomain() string {
diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go
index 5f7b4d38..33682424 100644
--- a/api/v1beta2/zz_generated.deepcopy.go
+++ b/api/v1beta2/zz_generated.deepcopy.go
@@ -118,6 +118,11 @@ func (in *BuggifyConfig) DeepCopyInto(out *BuggifyConfig) {
 		*out = make([]ProcessGroupID, len(*in))
 		copy(*out, *in)
 	}
+	if in.BlockRemoval != nil {
+		in, out := &in.BlockRemoval, &out.BlockRemoval
+		*out = make([]ProcessGroupID, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BuggifyConfig.
@@ -550,7 +555,7 @@ func (in *FoundationDBClusterAutomationOptions) DeepCopyInto(out *FoundationDBCl
 	in.MaintenanceModeOptions.DeepCopyInto(&out.MaintenanceModeOptions)
 	if in.IgnoreLogGroupsForUpgrade != nil {
 		in, out := &in.IgnoreLogGroupsForUpgrade, &out.IgnoreLogGroupsForUpgrade
-		*out = make([]string, len(*in))
+		*out = make([]LogGroup, len(*in))
 		copy(*out, *in)
 	}
 }
@@ -1648,6 +1653,11 @@ func (in *RoutingConfig) DeepCopyInto(out *RoutingConfig) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.DefineDNSLocalityFields != nil {
+		in, out := &in.DefineDNSLocalityFields, &out.DefineDNSLocalityFields
+		*out = new(bool)
+		**out = **in
+	}
 	if in.DNSDomain != nil {
 		in, out := &in.DNSDomain, &out.DNSDomain
 		*out = new(string)
diff --git a/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml b/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
index 766f9b55..5f6bdc72 100644
--- a/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
+++ b/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
@@ -9877,6 +9877,7 @@ spec:
                     type: integer
                   ignoreLogGroupsForUpgrade:
                     items:
+                      maxLength: 256
                       type: string
                     maxItems: 10
                     type: array
@@ -9936,6 +9937,12 @@ spec:
                 type: object
               buggify:
                 properties:
+                  blockRemoval:
+                    items:
+                      maxLength: 63
+                      type: string
+                    maxItems: 1000
+                    type: array
                   crashLoop:
                     items:
                       maxLength: 63
@@ -13572,6 +13579,8 @@ spec:
                 type: boolean
               routing:
                 properties:
+                  defineDNSLocalityFields:
+                    type: boolean
                   dnsDomain:
                     maxLength: 253
                     minLength: 1
diff --git a/config/samples/backup.yaml b/config/samples/backup.yaml
index c2238ba6..4d7b0ff8 100644
--- a/config/samples/backup.yaml
+++ b/config/samples/backup.yaml
@@ -118,7 +118,7 @@ spec:
             requests:
               storage: 16G
   routing:
-    headlessService: true
+    defineDNSLocalityFields: true
   sidecarContainer:
     enableLivenessProbe: true
     enableReadinessProbe: false
diff --git a/config/samples/client.yaml b/config/samples/client.yaml
index de2e0f01..d0d69bbd 100644
--- a/config/samples/client.yaml
+++ b/config/samples/client.yaml
@@ -144,7 +144,7 @@ spec:
             requests:
               storage: 16G
   routing:
-    headlessService: true
+    defineDNSLocalityFields: true
   sidecarContainer:
     enableLivenessProbe: true
     enableReadinessProbe: false
diff --git a/config/samples/cluster.yaml b/config/samples/cluster.yaml
index 35b78934..ee18c58b 100644
--- a/config/samples/cluster.yaml
+++ b/config/samples/cluster.yaml
@@ -61,7 +61,7 @@ spec:
             requests:
               storage: 16G
   routing:
-    headlessService: true
+    defineDNSLocalityFields: true
   sidecarContainer:
     enableLivenessProbe: true
     enableReadinessProbe: false
diff --git a/config/samples/restore.yaml b/config/samples/restore.yaml
index 172afa4d..9b16cd4b 100644
--- a/config/samples/restore.yaml
+++ b/config/samples/restore.yaml
@@ -119,7 +119,7 @@ spec:
             requests:
               storage: 16G
   routing:
-    headlessService: true
+    defineDNSLocalityFields: true
   sidecarContainer:
     enableLivenessProbe: true
     enableReadinessProbe: false
diff --git a/controllers/bounce_processes.go b/controllers/bounce_processes.go
index 3e6d38dc..0be00ff9 100644
--- a/controllers/bounce_processes.go
+++ b/controllers/bounce_processes.go
@@ -23,6 +23,7 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"github.com/FoundationDB/fdb-kubernetes-operator/internal/buggify"
 	"time"
 
 	"github.com/FoundationDB/fdb-kubernetes-operator/internal/restarts"
@@ -133,7 +134,7 @@ func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReco
 		}
 	}
 
-	filteredAddresses, removedAddresses := filterIgnoredProcessGroups(cluster, addresses)
+	filteredAddresses, removedAddresses := buggify.FilterIgnoredProcessGroups(cluster, addresses)
 	if removedAddresses {
 		addresses = filteredAddresses
 	}
@@ -285,41 +286,3 @@ func getAddressesForUpgrade(logger logr.Logger, r *FoundationDBClusterReconciler
 
 	return addresses, nil
 }
-
-// filterIgnoredProcessGroups removes all addresses from the addresses slice that are associated with a process group that should be ignored
-// during a restart.
-func filterIgnoredProcessGroups(cluster *fdbv1beta2.FoundationDBCluster, addresses []fdbv1beta2.ProcessAddress) ([]fdbv1beta2.ProcessAddress, bool) {
-	if len(cluster.Spec.Buggify.IgnoreDuringRestart) == 0 {
-		return addresses, false
-	}
-
-	ignoredIDs := make(map[fdbv1beta2.ProcessGroupID]fdbv1beta2.None, len(cluster.Spec.Buggify.IgnoreDuringRestart))
-	ignoredAddresses := make(map[string]fdbv1beta2.None, len(cluster.Spec.Buggify.IgnoreDuringRestart))
-
-	for _, id := range cluster.Spec.Buggify.IgnoreDuringRestart {
-		ignoredIDs[id] = fdbv1beta2.None{}
-	}
-
-	for _, processGroup := range cluster.Status.ProcessGroups {
-		if _, ok := ignoredIDs[processGroup.ProcessGroupID]; !ok {
-			continue
-		}
-
-		for _, address := range processGroup.Addresses {
-			ignoredAddresses[address] = fdbv1beta2.None{}
-		}
-	}
-
-	filteredAddresses := make([]fdbv1beta2.ProcessAddress, 0, len(addresses)-len(ignoredAddresses))
-	removedAddresses := false
-	for _, address := range addresses {
-		if _, ok := ignoredAddresses[address.MachineAddress()]; ok {
-			removedAddresses = true
-			continue
-		}
-
-		filteredAddresses = append(filteredAddresses, address)
-	}
-
-	return filteredAddresses, removedAddresses
-}
diff --git a/controllers/check_client_compatibility.go b/controllers/check_client_compatibility.go
index 91e87f43..12b04cce 100644
--- a/controllers/check_client_compatibility.go
+++ b/controllers/check_client_compatibility.go
@@ -79,7 +79,7 @@ func (c checkClientCompatibility) reconcile(_ context.Context, r *FoundationDBCl
 		return &requeue{curError: err}
 	}
 
-	ignoredLogGroups := make(map[string]fdbv1beta2.None)
+	ignoredLogGroups := make(map[fdbv1beta2.LogGroup]fdbv1beta2.None)
 	for _, logGroup := range cluster.Spec.AutomationOptions.IgnoreLogGroupsForUpgrade {
 		ignoredLogGroups[logGroup] = fdbv1beta2.None{}
 	}
@@ -98,7 +98,7 @@ func (c checkClientCompatibility) reconcile(_ context.Context, r *FoundationDBCl
 	return nil
 }
 
-func getUnsupportedClients(supportedVersions []fdbv1beta2.FoundationDBStatusSupportedVersion, protocolVersion string, ignoredLogGroups map[string]fdbv1beta2.None) []string {
+func getUnsupportedClients(supportedVersions []fdbv1beta2.FoundationDBStatusSupportedVersion, protocolVersion string, ignoredLogGroups map[fdbv1beta2.LogGroup]fdbv1beta2.None) []string {
 	var unsupportedClients []string
 	for _, versionInfo := range supportedVersions {
 		if versionInfo.ProtocolVersion == "Unknown" {
diff --git a/controllers/remove_process_groups.go b/controllers/remove_process_groups.go
index 5c544d3c..48b2660c 100644
--- a/controllers/remove_process_groups.go
+++ b/controllers/remove_process_groups.go
@@ -23,6 +23,7 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"github.com/FoundationDB/fdb-kubernetes-operator/internal/buggify"
 	"net"
 	"time"
 
@@ -71,6 +72,13 @@ func (u removeProcessGroups) reconcile(ctx context.Context, r *FoundationDBClust
 		}
 	}
 
+	// Ensure we only remove process groups that are not blocked to be removed by the buggify config.
+	processGroupsToRemove = buggify.FilterBlockedRemovals(cluster, processGroupsToRemove)
+	// If all of the process groups are filtered out we can stop doing the next steps.
+	if len(processGroupsToRemove) == 0 {
+		return nil
+	}
+
 	status, err := adminClient.GetStatus()
 	if err != nil {
 		return &requeue{curError: err}
diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
index a84de5c2..ab9f58b6 100644
--- a/docs/cluster_spec.md
+++ b/docs/cluster_spec.md
@@ -62,6 +62,7 @@ BuggifyConfig provides options for injecting faults into a cluster for testing.
 | crashLoopContainers | CrashLoopContainers defines a list of process group IDs and containers that should be put into a crash looping state. | [][CrashLoopContainerObject](#crashloopcontainerobject) | false |
 | emptyMonitorConf | EmptyMonitorConf instructs the operator to update all of the fdbmonitor.conf files to have zero fdbserver processes configured. | bool | false |
 | ignoreDuringRestart | IgnoreDuringRestart instructs the operator to ignore the provided process groups IDs during the restart command. This can be useful to simulate cases where the kill command is not restarting all processes. IgnoreDuringRestart does not support the wildcard option to ignore all of this specific cluster processes. | [][ProcessGroupID](#processgroupid) | false |
+| blockRemoval | BlockRemoval defines a list of process group IDs that will not be removed, even if they are marked for removal. The operator will trigger the exclusion but the removal of the resources will be blocked until they are removed from this list. This setting can be used to simulate cases where a process group is marked for removal but the resources are not yet removed. | [][ProcessGroupID](#processgroupid) | false |
 
 [Back to TOC](#table-of-contents)
 
@@ -183,7 +184,7 @@ FoundationDBClusterAutomationOptions provides flags for enabling or disabling op
 | podUpdateStrategy | PodUpdateStrategy defines how Pod spec changes are rolled out either by replacing Pods or by deleting Pods. The default for this is ReplaceTransactionSystem. | [PodUpdateStrategy](#podupdatestrategy) | false |
 | useManagementAPI | UseManagementAPI defines if the operator should make use of the management API instead of using fdbcli to interact with the FoundationDB cluster. | *bool | false |
 | maintenanceModeOptions | MaintenanceModeOptions contains options for maintenance mode related settings. | [MaintenanceModeOptions](#maintenancemodeoptions) | false |
-| ignoreLogGroupsForUpgrade | IgnoreLogGroupsForUpgrade defines the list of LogGroups that should be ignored during fdb version upgrade. | []string | false |
+| ignoreLogGroupsForUpgrade | IgnoreLogGroupsForUpgrade defines the list of LogGroups that should be ignored during fdb version upgrade. | [][LogGroup](#loggroup) | false |
 
 [Back to TOC](#table-of-contents)
 
@@ -333,6 +334,12 @@ LockSystemStatus provides a summary of the status of the locking system.
 
 [Back to TOC](#table-of-contents)
 
+## LogGroup
+
+LogGroup represents a LogGroup used by a FoundationDB process to log trace events. The LogGroup can be used to filter clients during an upgrade.
+
+[Back to TOC](#table-of-contents)
+
 ## MaintenanceModeInfo
 
 MaintenanceModeInfo contains information regarding the zone and process groups that are put into maintenance mode by the operator
@@ -446,6 +453,7 @@ RoutingConfig allows configuring routing to our pods, and services that sit in f
 | publicIPSource | PublicIPSource specifies what source a process should use to get its public IPs.  This supports the values `pod` and `service`. | *[PublicIPSource](#publicipsource) | false |
 | podIPFamily | PodIPFamily tells the pod which family of IP addresses to use. You can use 4 to represent IPv4, and 6 to represent IPv6. This feature is only supported in FDB 7.0 or later, and requires dual-stack support in your Kubernetes environment. | *int | false |
 | useDNSInClusterFile | UseDNSInClusterFile determines whether to use DNS names rather than IP addresses to identify coordinators in the cluster file. NOTE: This is an experimental feature, and is not supported in the latest stable version of FoundationDB. | *bool | false |
+| defineDNSLocalityFields | DefineDNSLocalityFields determines whether to define pod DNS names on pod specs and provide them in the locality arguments to fdbserver.  This is ignored if UseDNSInCluster is true. | *bool | false |
 | dnsDomain | DNSDomain defines the cluster domain used in a DNS name generated for a service. The default is `cluster.local`. | *string | false |
 
 [Back to TOC](#table-of-contents)
diff --git a/docs/manual/fault_domains.md b/docs/manual/fault_domains.md
index c8d70157..b2843b74 100644
--- a/docs/manual/fault_domains.md
+++ b/docs/manual/fault_domains.md
@@ -315,7 +315,6 @@ FoundationDB clusters that are spread across different DC's or Kubernetes cluste
 The reason behind this is that the coordinator selection is a global process and different `coordinatorSelection` of the `FoundationDBCluster` resources can lead to an undefined behaviour or in the worst case flapping coordinators.
 There are plans to support this feature in the future.
 
-
 ## Next
 
 You can continue on to the [next section](tls.md) or go back to the [table of contents](index.md).
diff --git a/docs/manual/operations.md b/docs/manual/operations.md
index fec7af0e..da973e92 100644
--- a/docs/manual/operations.md
+++ b/docs/manual/operations.md
@@ -1,5 +1,20 @@
 # Common Operations
 
+## Common localities
+
+The operator sets per default a predefined set of [localities](https://apple.github.io/foundationdb/configuration.html#fdbserver-section) based on the FoundationDBCluster configuration.
+The following localities will be set by the operator:
+
+- `--locality_instance_id`: This will get the value from `FDB_INSTANCE_ID` and will represent the process group ID, e.g. `storage-1`.
+- `--locality_process_id`: This will only be set if `storageServersPerPod` is set to a value larger than `1`. The format will be `$FDB_INSTANCE_ID` with the process counter as suffix and a `-` as a separator, e.g. `storage-1-1`.
+- `--locality_machineid`: The value will be set depending on the fault domain key. For `foundationdb.org/none`, this will be the Pod's name, for all other cases this will be the node name on which the pod is running.
+- `--locality_zoneid`: The value will be set depending on the fault domain key. For `foundationdb.org/none`, this will be the Pod's name, otherwise this will be the node name per default where the Pod is running. If `ValueFrom` is defined in the fault domain this value will be used. If `foundationdb.org/kubernetes-cluster` is specified as fault domain key the predefined `value` will be used.
+- `--locality_dcid`: This value will be set to the value defined in `cluster.Spec.DataCenter`, if this value is not set the locality will not be set. This locality is used for FoundationDB deployments in multiple datacenters/Kubernetes clusters.
+- `--locality_data_hall`: This value will be set to the value defined in `cluster.Spec.DataHall`, if this value is not set the locality will not be set. Currently this locality doesn't have any affect, but will be used in the future for `three_data_hall` replication.
+- `--locality_dns_name`: This value will only be set if `cluster.Spec.Routing.DefineDNSLocalityFields` is set to true. The value will be set to the `FDB_DNS_NAME` environment variable, which is set by the operator.
+
+The operator uses the `locality_instance_id` to identify the process from the [machine-readable status](https://apple.github.io/foundationdb/mr-status.html) and match it to the according process group managed by the operator.
+
 ## Replacing a Process
 
 If you delete a pod, the operator will automatically create a new pod to replace it. If there is a volume available for re-use, we will create a new pod to match that volume. This means that in general you can replace a bad process just by deleting the pod. This may not be desirable in all situations, as it creates a loss of fault tolerance until the replacement pod is created. This also requires that the original volume be available, which may not be possible in some failure scenarios.
diff --git a/docs/manual/technical_design.md b/docs/manual/technical_design.md
index 18633109..8eea9cf0 100644
--- a/docs/manual/technical_design.md
+++ b/docs/manual/technical_design.md
@@ -223,9 +223,14 @@ The `ChooseRemovals` subreconciler flags processes for removal when the current
 
 ### ExcludeProcesses
 
-The `ExcludeProcesses` subreconciler runs an `exclude` command in `fdbcli` for any process group that is marked for removal and is not already being excluded. The `exclude` command tells FoundationDB that a process should not serve any roles, and that any data on that process should be moved to other processes. This exclusion can take a long time, but this subreconciler does not wait for exclusion to complete.
+The `ExcludeProcesses` subreconciler runs an `exclude` command in `fdbcli` for any process group that is marked for removal and is not already being excluded.
+The `exclude` command tells FoundationDB that a process should not serve any roles, and that any data on that process should be moved to other processes.
+This exclusion can take a long time, but this subreconciler does not wait for exclusion to complete.
 
-If there are processes that are not reporting to the cluster and are not marked for removal, this subreconciler will not run any exclusion commands. This is designed to prevent the operator from triggering exclusions before the replacement processes are available. In the case where there are multiple processes that are failing, this can cause reconciliation to get stuck. You can work around this by telling the operator to replace all of the failing processes.
+If there are processes that are not reporting to the cluster and are not marked for removal, this subreconciler will not run any exclusion commands.
+This is designed to prevent the operator from triggering exclusions before the replacement processes are available.
+In the case where there are multiple processes that are failing, this can cause reconciliation to get stuck.
+You can work around this by telling the operator to replace all of the failing processes.
 
 This action requires a lock.
 
@@ -395,6 +400,32 @@ When the operator checks the status of the cluster, it needs to check if the pro
 
 All of the flows above go through the `foundationdb` container. The `foundationdb-kubernetes-sidecar` container is only used in the upgrade flow. The sidecar container runs the same image as the main container, but with a different set of arguments to tell it to run in sidecar mode. During the upgrade, the operator upgrades the sidecar to the new version of FDB while leaving the main container at the old version. The sidecar compares the version of FoundationDB that it is running against the main container version, which is provided in its start command. If these versions are the same, the sidecar will do nothing. If they are different, it will copy the FDB binaries from its own image into a volume that it shares with the main container. The main container will receive the desired version of FDB as part of its configuration file. When the main container sees a version of FDB that is different from the one it is running, it will look for the FDB binaries in the directory it shares with the sidecar. If it finds those new binaries, it will load the new configuration and run the binaries from that directory. If these binaries are missing, fdb-kubernetes-monitor will reject the new configuration. Once the new configuration is accepted by all of the pods, the operator will restart the processes so they start running with the new binaries. Once the new version is running, the operator will perform a rolling bounce to update the main container to the new FDB version.
 
+## Interaction Between the Operator and the FoundationDB cluster
+
+The operator will use the [machine-readable status](https://apple.github.io/foundationdb/mr-status.html) of FoundationDB to observe the current state of the FoundationDB cluster.
+Based on the machine-readable status the operator can issue commands to reconcile to the desired state.
+
+### Exclusions in the operator
+
+The operator uses the [exclude](https://apple.github.io/foundationdb/command-line-interface.html#exclude) command to make sure it's safe [to remove a Pod from the cluster](https://apple.github.io/foundationdb/administration.html#removing-machines-from-a-cluster).
+The exclusion is handled in the [exclusion subreconciler](#excludeprocesses) and will make sure that at least a minimum number of processes are up and running before issuing the `exclude` command.
+When the operator observes that some processes must be excluded it tries to exclude them all at once as the exclusion command can trigger a recovery in the FoundationDB cluster.
+A process marked as `excluded` in the machine-readable status means that the FoundationDB cluster will not consider this specific process as an eligible process for the transaction system and all data will be moved away from this process.
+The operator will also make sure to select new coordinators if at least one coordinator is marked as excluded, this is handled in the [change coordinators subreconciler](#changecoordinators).
+
+In order to be able to verify if it's safe to remove the resources of a process the operator will check the roles of the excluded processes.
+If a process is serving no roles and is marked as excluded, it's safe to remove the resources of this process.
+If a process has at least one role, it's not safe to remove this process.
+If a process is missing in the machine-readable status the operator will issue an additional `exclude` command for those missing processes to ensure they are not serving any log or storage roles.
+
+The current default for the operator is to use the Pod IP for the exclusion command, if a Pod get's deleted and recreated it could get a new IP address and the operator has to issue a new exclude command for the new IP address.
+To workaround this FoundationDB added support for locality based exclusions in 7.0 and the operator supports this by setting [useLocalitiesForExclusion](https://github.com/FoundationDB/fdb-kubernetes-operator/blob/main/docs/cluster_spec.md#foundationdbclusterautomationoptions) in the FoundationDBCluster spec.
+
+**Warning** the locality-based exclusions are not well tested in our e2e test setup ye.
+locality based exclusion
+
+The operator is not able to use the `failed` option for exclusions.
+
 ## Next
 
 You can continue on to the [next section](upgrades.md) or go back to the [table of contents](index.md).
diff --git a/docs/manual/upgrades.md b/docs/manual/upgrades.md
index cb35e979..42cf9fdd 100644
--- a/docs/manual/upgrades.md
+++ b/docs/manual/upgrades.md
@@ -148,6 +148,29 @@ We discussed in [Process restarted during staging Phase](#process-restarted-duri
 For version incompatible upgrades the operator has no way to differentiate between processes not reporting to the cluster and processes running in the incompatible version, both will be missing from the processes list.
 The only difference is that the incompatible processes might be reported in the `incompatible_connections` field.
 
+#### Processes are not able to join the FDB cluster
+
+We've seen cases where a subset of `fdbserver` were not able to join the FDB cluster.
+This can happen due to a race condition where a new set of coordinators is chosen.
+The affected process types are stateless and transaction or log processes.
+The newly created Pods for the transaction system will have the condition `MissingProcesses` and if you exec into one of those Pods you will see the `fdbserver` process running.
+When you check the logs from `fdbmonitor` you will see the following logs:
+
+```bash
+$ cat /var/log/fdb-trace-logs/fdbmonitor-*
+Severity="10" LogGroup="default" Process="fdbmonitor": Updated configuration for fdbserver.1
+Severity="40" LogGroup="default" Process="fdbserver.1": Warning: FDBD has not joined the cluster after 5 seconds.
+Severity="40" LogGroup="default" Process="fdbserver.1":   Check configuration and availability using the 'status' command with the fdbcli
+```
+
+You can resolve this issue by restarting all those processes with the kubectl plugin (or manually):
+
+```bash
+kubectl fdb -n aci-fdb-prod restart -c cluster --process-condition=MissingProcesses
+```
+
+There is a fix on FoundationDB side that should prevent this behaviour in the future: https://github.com/apple/foundationdb/pull/9814.
+
 ### How we test upgrades in the operator
 
 For every PR against the operator repository we run a set of tests in addition to that we are running those tests on a regular basis with different versions to ensure we test major, minor and patch upgrades.
diff --git a/docs/manual/warnings.md b/docs/manual/warnings.md
index 5fb7fcf7..baadd8f6 100644
--- a/docs/manual/warnings.md
+++ b/docs/manual/warnings.md
@@ -25,7 +25,7 @@ The `podTemplate` field allows you to customize nearly every part of the pods th
 * The pod will always have a container called `foundationdb`, a container called `foundationdb-kubernetes-sidecar`, and an init container called `foundationdb-kubernetes-init`. If you do not define containers with these names, the operator will add them. If you define containers with these names, the operator will modify them to add the necessary fields and default values.
 * You cannot define a command or arguments for the `foundationdb` container.
 * The image version for the built-in containers will be set by the operator. If you define a custom image, the operator will add a tag to the end with the image version the operator needs.
-* You cannot directly set the affinity for the pod.
+* You can set affinities on the Pod level, but depending on the `fault domain key` the operator will add at least one `PodAntiAffinity` to try to spread the Pods across multiple failure domains.
 * The pod will always have volumes named `data`, `dynamic-conf`, `config-map`, and `fdb-trace-logs`, which will be defined by the operator. You cannot define custom volumes with these names.
 * The `foundationdb` container will always have volume mounts with the names `data`, `dynamic-conf`, and `fdb-trace-logs`, which will be defined by the operator. You cannot define volume mounts with these names.
 * The `foundationdb-kubernetes-sidecar` and `foundationdb-kubernetes-init` containers will always have volume mounts with the names `config-map` and `dynamic-conf`, which will be defined by the operator. You cannot define volume mounts with these names.
diff --git a/fdbclient/admin_client.go b/fdbclient/admin_client.go
index f823a098..4abab7d3 100644
--- a/fdbclient/admin_client.go
+++ b/fdbclient/admin_client.go
@@ -110,6 +110,16 @@ func NewCliAdminClient(cluster *fdbv1beta2.FoundationDBCluster, _ client.Client,
 	}, nil
 }
 
+// getMaxTimeout returns the maximum timeout, this is either the default of 40 seconds or if the provided default timeout
+// is higher it will be the default cli timeout.
+func (client *cliAdminClient) getMaxTimeout() time.Duration {
+	if DefaultCLITimeout > 40*time.Second {
+		return DefaultCLITimeout
+	}
+
+	return 40 * time.Second
+}
+
 // cliCommand describes a command that we are running against FDB.
 type cliCommand struct {
 	// binary is the binary to run.
@@ -268,7 +278,6 @@ func (client *cliAdminClient) runCommand(command cliCommand) (string, error) {
 
 // runCommandWithBackoff is a wrapper around runCommand which allows retrying commands if they hit a timeout.
 func (client *cliAdminClient) runCommandWithBackoff(command string) (string, error) {
-	maxTimeout := 40 * time.Second
 	currentTimeout := DefaultCLITimeout
 
 	var rawResult string
@@ -278,7 +287,7 @@ func (client *cliAdminClient) runCommandWithBackoff(command string) (string, err
 	// it with the default timeout of 10s we will try it 3 times with the following timeouts: 10s - 20s - 40s. We have
 	// seen that during upgrades of version incompatible version, when not all coordinators are properly restarted that
 	// the response time will be increased.
-	for currentTimeout <= maxTimeout {
+	for currentTimeout <= client.getMaxTimeout() {
 		rawResult, err = client.runCommand(cliCommand{command: command, timeout: currentTimeout})
 		if err == nil {
 			break
@@ -429,7 +438,7 @@ func (client *cliAdminClient) ExcludeProcesses(addresses []fdbv1beta2.ProcessAdd
 
 	excludeCommand.WriteString(fdbv1beta2.ProcessAddressesString(addresses, " "))
 
-	_, err = client.runCommandWithBackoff(excludeCommand.String())
+	_, err = client.runCommand(cliCommand{command: excludeCommand.String(), timeout: client.getMaxTimeout()})
 
 	return err
 }
@@ -511,7 +520,6 @@ func getRemainingAndExcludedFromStatus(status *fdbv1beta2.FoundationDBStatus, ad
 		delete(notExcludedAddresses, process.Address.MachineAddress())
 	}
 
-	// TODO double check the capacity we reserve here.
 	exclusions := exclusionStatus{
 		inProgress:      make([]fdbv1beta2.ProcessAddress, 0, len(addresses)-len(notExcludedAddresses)-len(fullyExcludedAddresses)),
 		fullyExcluded:   make([]fdbv1beta2.ProcessAddress, 0, len(fullyExcludedAddresses)),
diff --git a/internal/buggify/buggify.go b/internal/buggify/buggify.go
new file mode 100644
index 00000000..b7491803
--- /dev/null
+++ b/internal/buggify/buggify.go
@@ -0,0 +1,86 @@
+/*
+ * buggify.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package buggify
+
+import fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+
+// FilterBlockedRemovals will remove all matching process groups from the processGroupsToRemove slice that are defined in
+// the BlockRemoval.
+func FilterBlockedRemovals(cluster *fdbv1beta2.FoundationDBCluster, processGroupsToRemove []*fdbv1beta2.ProcessGroupStatus) []*fdbv1beta2.ProcessGroupStatus {
+	if len(cluster.Spec.Buggify.BlockRemoval) == 0 {
+		return processGroupsToRemove
+	}
+
+	// processGroupsToRemove
+	blockedProcessGroups := make(map[fdbv1beta2.ProcessGroupID]fdbv1beta2.None, len(cluster.Spec.Buggify.BlockRemoval))
+	for _, blockedProcessGroup := range cluster.Spec.Buggify.BlockRemoval {
+		blockedProcessGroups[blockedProcessGroup] = fdbv1beta2.None{}
+	}
+
+	filteredList := make([]*fdbv1beta2.ProcessGroupStatus, 0, len(processGroupsToRemove))
+	for _, processGroup := range processGroupsToRemove {
+		if _, ok := blockedProcessGroups[processGroup.ProcessGroupID]; ok {
+			continue
+		}
+
+		filteredList = append(filteredList, processGroup)
+	}
+
+	return filteredList
+}
+
+// FilterIgnoredProcessGroups removes all addresses from the addresses slice that are associated with a process group that should be ignored
+// during a restart.
+func FilterIgnoredProcessGroups(cluster *fdbv1beta2.FoundationDBCluster, addresses []fdbv1beta2.ProcessAddress) ([]fdbv1beta2.ProcessAddress, bool) {
+	if len(cluster.Spec.Buggify.IgnoreDuringRestart) == 0 {
+		return addresses, false
+	}
+
+	ignoredIDs := make(map[fdbv1beta2.ProcessGroupID]fdbv1beta2.None, len(cluster.Spec.Buggify.IgnoreDuringRestart))
+	ignoredAddresses := make(map[string]fdbv1beta2.None, len(cluster.Spec.Buggify.IgnoreDuringRestart))
+
+	for _, id := range cluster.Spec.Buggify.IgnoreDuringRestart {
+		ignoredIDs[id] = fdbv1beta2.None{}
+	}
+
+	for _, processGroup := range cluster.Status.ProcessGroups {
+		if _, ok := ignoredIDs[processGroup.ProcessGroupID]; !ok {
+			continue
+		}
+
+		for _, address := range processGroup.Addresses {
+			ignoredAddresses[address] = fdbv1beta2.None{}
+		}
+	}
+
+	filteredAddresses := make([]fdbv1beta2.ProcessAddress, 0, len(addresses)-len(ignoredAddresses))
+	removedAddresses := false
+	for _, address := range addresses {
+		if _, ok := ignoredAddresses[address.MachineAddress()]; ok {
+			removedAddresses = true
+			continue
+		}
+
+		filteredAddresses = append(filteredAddresses, address)
+	}
+
+	return filteredAddresses, removedAddresses
+}
diff --git a/internal/monitor_conf.go b/internal/monitor_conf.go
index 141486d6..a221467c 100644
--- a/internal/monitor_conf.go
+++ b/internal/monitor_conf.go
@@ -261,7 +261,7 @@ func GetMonitorProcessConfiguration(cluster *fdbv1beta2.FoundationDBCluster, pro
 		configuration.Arguments = append(configuration.Arguments, monitorapi.Argument{Value: fmt.Sprintf("--locality_data_hall=%s", cluster.Spec.DataHall)})
 	}
 
-	if cluster.UseDNSInClusterFile() {
+	if cluster.DefineDNSLocalityFields() {
 		configuration.Arguments = append(configuration.Arguments, monitorapi.Argument{ArgumentType: monitorapi.ConcatenateArgumentType, Values: []monitorapi.Argument{
 			{Value: "--locality_dns_name="},
 			{ArgumentType: monitorapi.EnvironmentArgumentType, Source: "FDB_DNS_NAME"},
diff --git a/internal/pod_models.go b/internal/pod_models.go
index 257ed248..884d27a9 100644
--- a/internal/pod_models.go
+++ b/internal/pod_models.go
@@ -560,7 +560,7 @@ func configureSidecarContainer(container *corev1.Container, initMode bool, proce
 
 		sidecarEnv = append(sidecarEnv, getEnvForMonitorConfigSubstitution(cluster, processGroupID)...)
 
-		if cluster.UseDNSInClusterFile() {
+		if cluster.DefineDNSLocalityFields() {
 			sidecarArgs = append(sidecarArgs, "--substitute-variable", "FDB_DNS_NAME")
 			sidecarEnv = append(sidecarEnv, corev1.EnvVar{Name: "FDB_DNS_NAME", Value: GetPodDNSName(cluster, podName)})
 		}
@@ -657,7 +657,7 @@ func configureSidecarContainer(container *corev1.Container, initMode bool, proce
 
 // getEnvForMonitorConfigSubstitution provides the environment variables that
 // are used for substituting variables into the monitor config.
-func getEnvForMonitorConfigSubstitution(cluster *fdbv1beta2.FoundationDBCluster, instanceID fdbv1beta2.ProcessGroupID) []corev1.EnvVar {
+func getEnvForMonitorConfigSubstitution(cluster *fdbv1beta2.FoundationDBCluster, processGroupID fdbv1beta2.ProcessGroupID) []corev1.EnvVar {
 	env := make([]corev1.EnvVar, 0)
 
 	publicIPSource := cluster.Spec.Routing.PublicIPSource
@@ -724,7 +724,7 @@ func getEnvForMonitorConfigSubstitution(cluster *fdbv1beta2.FoundationDBCluster,
 		}
 	}
 
-	env = append(env, corev1.EnvVar{Name: "FDB_INSTANCE_ID", Value: string(instanceID)})
+	env = append(env, corev1.EnvVar{Name: "FDB_INSTANCE_ID", Value: string(processGroupID)})
 
 	return env
 }
diff --git a/kubectl-fdb/Readme.md b/kubectl-fdb/Readme.md
index 83c68153..68cb50df 100644
--- a/kubectl-fdb/Readme.md
+++ b/kubectl-fdb/Readme.md
@@ -2,6 +2,13 @@
 
 ## Installation
 
+Make sure you have `sha256sum` and `jq` installed otherwise run the following `brew` commands:
+
+```bash
+brew install coreutils # required for sha256sum
+brew install jq # required for jq
+```
+
 The `kubectl fdb plugin` is released as a binary as part of our release process.
 For the latest version take a look at the [release page](https://github.com/FoundationDB/fdb-kubernetes-operator/releases).
 
@@ -10,7 +17,7 @@ Install from release:
 ```bash
 pushd $TMPDIR
 OS=$(uname)
-ARCH=$(uname -mu)
+ARCH=$(uname -m)
 VERSION="$(curl -s "https://api.github.com/repos/FoundationDB/fdb-kubernetes-operator/releases/latest" | jq -r '.tag_name')"
 curl -sLO "https://github.com/FoundationDB/fdb-kubernetes-operator/releases/download/${VERSION}/checksums.txt"
 curl -sLO "https://github.com/FoundationDB/fdb-kubernetes-operator/releases/download/${VERSION}/kubectl-fdb_${VERSION}_${OS}_${ARCH}"
diff --git a/kubectl-fdb/cmd/k8s_client.go b/kubectl-fdb/cmd/k8s_client.go
index 2bd82e8d..0d62b54e 100644
--- a/kubectl-fdb/cmd/k8s_client.go
+++ b/kubectl-fdb/cmd/k8s_client.go
@@ -103,7 +103,7 @@ func getNodes(kubeClient client.Client, nodeSelector map[string]string) ([]strin
 		return []string{}, err
 	}
 
-	nodes := make([]string, len(nodesList.Items))
+	nodes := make([]string, 0, len(nodesList.Items))
 
 	for _, node := range nodesList.Items {
 		nodes = append(nodes, node.Name)
diff --git a/pkg/fdbadminclient/mock/admin_client_mock.go b/pkg/fdbadminclient/mock/admin_client_mock.go
index b31d3921..e79b7400 100644
--- a/pkg/fdbadminclient/mock/admin_client_mock.go
+++ b/pkg/fdbadminclient/mock/admin_client_mock.go
@@ -317,7 +317,7 @@ func (client *AdminClient) GetStatus() (*fdbv1beta2.FoundationDBStatus, error) {
 			for _, address := range addresses {
 				protocolClients = append(protocolClients, fdbv1beta2.FoundationDBStatusConnectedClient{
 					Address:  address,
-					LogGroup: client.Cluster.Name,
+					LogGroup: fdbv1beta2.LogGroup(client.Cluster.Name),
 				})
 			}
 


Test patch:
diff --git a/api/v1beta2/foundationdbcluster_types_test.go b/api/v1beta2/foundationdbcluster_types_test.go
index 637d7e91..1616880e 100644
--- a/api/v1beta2/foundationdbcluster_types_test.go
+++ b/api/v1beta2/foundationdbcluster_types_test.go
@@ -4300,11 +4300,17 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 				Expect(cluster.NeedsHeadlessService()).To(BeTrue())
 			})
 
-			It("can be overridden by the DNS setting", func() {
+			It("can be overridden by the DNS in cluster file setting", func() {
 				cluster.Spec.Routing.HeadlessService = pointer.Bool(false)
 				cluster.Spec.Routing.UseDNSInClusterFile = pointer.Bool(true)
 				Expect(cluster.NeedsHeadlessService()).To(BeTrue())
 			})
+
+			It("can be overridden by the DNS in locality setting", func() {
+				cluster.Spec.Routing.HeadlessService = pointer.Bool(false)
+				cluster.Spec.Routing.DefineDNSLocalityFields = pointer.Bool(true)
+				Expect(cluster.NeedsHeadlessService()).To(BeTrue())
+			})
 		})
 
 		When("checking whether we use DNS in the cluster file", func() {
@@ -4316,6 +4322,21 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 			})
 		})
 
+		When("checking whether we use DNS in the locality fields", func() {
+			It("respects the value in the flag", func() {
+				Expect(cluster.DefineDNSLocalityFields()).To(BeFalse())
+
+				cluster.Spec.Routing.DefineDNSLocalityFields = pointer.Bool(true)
+				Expect(cluster.DefineDNSLocalityFields()).To(BeTrue())
+			})
+
+			It("can be overridden by the DNS in cluster file setting", func() {
+				cluster.Spec.Routing.DefineDNSLocalityFields = pointer.Bool(false)
+				cluster.Spec.Routing.UseDNSInClusterFile = pointer.Bool(true)
+				Expect(cluster.DefineDNSLocalityFields()).To(BeTrue())
+			})
+		})
+
 		When("getting the DNS domain", func() {
 			It("allows overrides in the spec", func() {
 				Expect(cluster.GetDNSDomain()).To(Equal("cluster.local"))
diff --git a/config/tests/base/cluster.yaml b/config/tests/base/cluster.yaml
index 4a585c13..e116dfe4 100644
--- a/config/tests/base/cluster.yaml
+++ b/config/tests/base/cluster.yaml
@@ -9,7 +9,7 @@ spec:
     # cluster.
     key: foundationdb.org/none
   routing:
-    headlessService: true
+    defineDNSLocalityFields: true
   processCounts:
     # Disable stateless processes to save resources.
     stateless: -1
diff --git a/config/tests/dns/dns.yaml b/config/tests/dns/dns.yaml
index 355e24a1..e1124be9 100644
--- a/config/tests/dns/dns.yaml
+++ b/config/tests/dns/dns.yaml
@@ -2,4 +2,4 @@
   path: "/spec/routing/useDNSInClusterFile"
   value: true
 - op: remove
-  path: "/spec/routing/headlessService"
+  path: "/spec/routing/defineDNSLocalityFields"
diff --git a/controllers/admin_client_test.go b/controllers/admin_client_test.go
index e6e79171..bfc896d0 100644
--- a/controllers/admin_client_test.go
+++ b/controllers/admin_client_test.go
@@ -172,7 +172,7 @@ var _ = Describe("admin_client_test", func() {
 
 		Context("with the DNS names enabled", func() {
 			BeforeEach(func() {
-				cluster.Spec.Routing.UseDNSInClusterFile = pointer.Bool(true)
+				cluster.Spec.Routing.DefineDNSLocalityFields = pointer.Bool(true)
 				err = k8sClient.Update(context.TODO(), cluster)
 				Expect(err).NotTo(HaveOccurred())
 			})
diff --git a/controllers/bounce_processes_test.go b/controllers/bounce_processes_test.go
index 045212d4..e2ffe85a 100644
--- a/controllers/bounce_processes_test.go
+++ b/controllers/bounce_processes_test.go
@@ -23,6 +23,7 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"github.com/FoundationDB/fdb-kubernetes-operator/internal/buggify"
 	"time"
 
 	"github.com/FoundationDB/fdb-kubernetes-operator/pkg/fdbadminclient/mock"
@@ -515,7 +516,7 @@ var _ = Describe("bounceProcesses", func() {
 					processAddresses = append(processAddresses, process.Address)
 				}
 
-				filteredAddresses, removed = filterIgnoredProcessGroups(cluster, processAddresses)
+				filteredAddresses, removed = buggify.FilterIgnoredProcessGroups(cluster, processAddresses)
 			})
 
 			It("should filter the ignored address", func() {
@@ -544,7 +545,7 @@ var _ = Describe("bounceProcesses", func() {
 					processAddresses = append(processAddresses, process.Address)
 				}
 
-				filteredAddresses, removed = filterIgnoredProcessGroups(cluster, processAddresses)
+				filteredAddresses, removed = buggify.FilterIgnoredProcessGroups(cluster, processAddresses)
 			})
 
 			It("should filter the ignored address", func() {
diff --git a/controllers/change_coordinators_test.go b/controllers/change_coordinators_test.go
index bf528031..94bf3383 100644
--- a/controllers/change_coordinators_test.go
+++ b/controllers/change_coordinators_test.go
@@ -642,7 +642,6 @@ var _ = Describe("Change coordinators", func() {
 			})
 
 			When("the Pods do not have DNS names", func() {
-
 				It("should not requeue", func() {
 					Expect(requeue).To(BeNil())
 				})
diff --git a/controllers/check_client_compatibility_test.go b/controllers/check_client_compatibility_test.go
index 69a4152f..5723948a 100644
--- a/controllers/check_client_compatibility_test.go
+++ b/controllers/check_client_compatibility_test.go
@@ -209,7 +209,7 @@ var supportedVersion = []fdbv1beta2.FoundationDBStatusSupportedVersion{
 var _ = Describe("check client compatibility", func() {
 	When("getting the list of unsupported clients from the cluster status json", func() {
 		type testCase struct {
-			ignoredLogGroups           map[string]fdbv1beta2.None
+			ignoredLogGroups           map[fdbv1beta2.LogGroup]fdbv1beta2.None
 			expectedUnsupportedClients []string
 		}
 
@@ -236,7 +236,7 @@ var _ = Describe("check client compatibility", func() {
 				}),
 			Entry("with non empty ignoreProcessGroups map.",
 				testCase{
-					ignoredLogGroups: map[string]fdbv1beta2.None{"sample-cluster-client": {}},
+					ignoredLogGroups: map[fdbv1beta2.LogGroup]fdbv1beta2.None{"sample-cluster-client": {}},
 					expectedUnsupportedClients: []string{
 						"10.1.18.249:34874 (fdb-kubernetes-operator)",
 						"10.1.18.249:35022 (fdb-kubernetes-operator)",
diff --git a/controllers/cluster_controller_test.go b/controllers/cluster_controller_test.go
index 47066f72..07f47347 100644
--- a/controllers/cluster_controller_test.go
+++ b/controllers/cluster_controller_test.go
@@ -2692,7 +2692,7 @@ var _ = Describe("cluster_controller", func() {
 			Context("patch upgrade from 7.1 to 7.2 with non empty ignoreLogGroups", func() {
 				BeforeEach(func() {
 					cluster.Spec.Version = "7.2.0"
-					cluster.Spec.AutomationOptions.IgnoreLogGroupsForUpgrade = []string{cluster.Name}
+					cluster.Spec.AutomationOptions.IgnoreLogGroupsForUpgrade = []fdbv1beta2.LogGroup{fdbv1beta2.LogGroup(cluster.Name)}
 					Expect(k8sClient.Update(context.TODO(), cluster)).NotTo(HaveOccurred())
 					generationGap = 2
 				})
diff --git a/e2e/Makefile b/e2e/Makefile
index 2f1f8755..e90da55e 100644
--- a/e2e/Makefile
+++ b/e2e/Makefile
@@ -9,7 +9,7 @@ TIMEOUT?=168h
 
 NAMESPACE?=
 CONTEXT?=
-FDB_VERSION?=7.1.27
+FDB_VERSION?=7.1.31
 PREVIOUS_FDB_VERSION?=6.3.25
 ## Expectation is that you are running standard build image which generates both regular and debug (Symbols) images.
 FDB_IMAGE?=foundationdb/foundationdb:$(FDB_VERSION)
@@ -39,7 +39,20 @@ clean:
 # This target can be used to delete namespaces that were created during testing. If a namespace is provided the user and
 # the namespace (as a prefix) must match.
 destroy-my-namespaces:
-	./remove_namespaces ${TEST_USERNAME} ${NAMESPACE}
+	./scripts/remove_namespaces ${TEST_USERNAME} ${NAMESPACE}
+
+# This target can be used to create a kind cluster that can be used for e2e testing.
+kind-setup:
+	@CHAOS_NAMESPACE=$(CHAOS_NAMESPACE) FDB_VERSION=$(FDB_VERSION) REGISTRY=$(REGISTRY) $(shell pwd)/scripts/setup_e2e.sh
+
+# This target will remove the kind cluster.
+kind-destroy:
+	@kind delete clusters e2e-tests
+
+# This target will build the operator image and updates the image in the kind clusters.
+kind-update-operator:
+	@BUILD_PLATFORM="linux/amd64" make -C "$(shell pwd)/.." container-build
+	@kind load docker-image "fdb-kubernetes-operator:latest" --name "e2e-tests"
 
 # No one in the team can figure out how to get ginkgo's test launcher to properly
 # manage concurrency, or control which tests are run in a predictable manner.
diff --git a/e2e/README.md b/e2e/README.md
index c3fdca05..f3b16934 100644
--- a/e2e/README.md
+++ b/e2e/README.md
@@ -48,41 +48,22 @@ If those tests are running on a cluster that has no chaos-mesh installed, you ca
 ### Running e2e tests in kind
 
 [kind](https://kind.sigs.k8s.io) provides an easy way to run a local Kubernetes cluster.
+The following steps assume that `kind` is already installed.
 
 ```bash
-kind create cluster
-# This command assumes to be executed from the project root.
-make container-build
-# Push the image into the kind cluster.
-kind load docker-image "fdb-kubernetes-operator:latest"
+make -C e2e kind-setup
 ```
 
-Before you run the e2e tests you have to ensure that the latest CRDs for the operator are installed:
+This will call the [setup_e2e.sh](./scripts/setup_e2e.sh) script to setup `kind` and install chaos-mesh.
+After testing you can run the following command to remove the kind cluster:
 
 ```bash
-# This command should be executed from the project root
-kubectl apply -f ./config/crd/bases/
+make -C e2e kind-destroy
 ```
 
-If you want to run all tests, including tests that inject chaos you have to install [chaos mesh](https://chaos-mesh.org):
+If you want to iterate over different builds of the operator, you don't have to recreate the kind cluster multiple times.
+You just can rebuild the operator image and push the new image inside the kind cluster:
 
 ```bash
-chaos_mesh_version="2.5.0"
-
-kubectl create ns chaos-testing || true
-helm repo add chaos-mesh https://charts.chaos-mesh.org
-helm repo update
-
-# The configuration below is tested on a local Kind installation and might be different for the target Kubernetes cluster.
-helm upgrade -i chaos-mesh chaos-mesh/chaos-mesh \
-    --namespace chaos-testing \
-    --set dashboard.securityMode=false \
-    --set chaosDaemon.socketPath=/run/containerd/containerd.sock \
-    --set chaosDaemon.runtime=containerd \
-    --version "${chaos_mesh_version}"
-
-# Check if the Pods are running
-kubectl wait --for=condition=ready pods --namespace chaos-testing -l app.kubernetes.io/instance=chaos-mesh
+make -C e2e kind-update-operator
 ```
-
-The actual installation steps might be different based on your Kubernetes cluster.
diff --git a/e2e/fixtures/fdb_cluster.go b/e2e/fixtures/fdb_cluster.go
index b47ce6c6..1339318d 100644
--- a/e2e/fixtures/fdb_cluster.go
+++ b/e2e/fixtures/fdb_cluster.go
@@ -1119,3 +1119,9 @@ func (fdbCluster *FdbCluster) UpdateContainerImage(pod *corev1.Pod, containerNam
 
 	gomega.Expect(fdbCluster.factory.GetControllerRuntimeClient().Update(ctx.Background(), pod)).NotTo(gomega.HaveOccurred())
 }
+
+// SetBuggifyBlockRemoval will set the provided list of process group IDs to be blocked for removal.
+func (fdbCluster *FdbCluster) SetBuggifyBlockRemoval(blockRemovals []fdbv1beta2.ProcessGroupID) {
+	fdbCluster.cluster.Spec.Buggify.BlockRemoval = blockRemovals
+	fdbCluster.UpdateClusterSpec()
+}
diff --git a/e2e/fixtures/fdb_cluster_specs.go b/e2e/fixtures/fdb_cluster_specs.go
index bf848836..250ee5b1 100644
--- a/e2e/fixtures/fdb_cluster_specs.go
+++ b/e2e/fixtures/fdb_cluster_specs.go
@@ -67,7 +67,7 @@ func (factory *Factory) createFDBClusterSpec(
 				},
 				// Allow the operator to remove all Pods that are excluded and marked for deletion to remove at once.
 				RemovalMode: fdbv1beta2.PodUpdateModeAll,
-				IgnoreLogGroupsForUpgrade: []string{
+				IgnoreLogGroupsForUpgrade: []fdbv1beta2.LogGroup{
 					"fdb-kubernetes-operator",
 				},
 			},
diff --git a/e2e/remove_namespaces b/e2e/scripts/remove_namespaces
similarity index 100%
rename from e2e/remove_namespaces
rename to e2e/scripts/remove_namespaces
diff --git a/e2e/scripts/setup-local-fdb-cluster.sh b/e2e/scripts/setup-local-fdb-cluster.sh
new file mode 100755
index 00000000..8c68b666
--- /dev/null
+++ b/e2e/scripts/setup-local-fdb-cluster.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# This source file is part of the FoundationDB open source project
+#
+# Copyright 2023 Apple Inc. and the FoundationDB project authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script works out of the box for Linux environments (tested on Ubuntu 16.04.5 LTS), after installing dependent software (kind, Docker, and kustomize);
+# Running in other environments may involve non-trivial debugging effort.
+
+# Set up local FDB non-HA cluster on a 4-node kind k8s cluster
+# This only works on x86 machines as FDB doesn't provide arm64 Linux binaries yet.
+set -eo errexit
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)
+cd "${SCRIPT_DIR}"
+
+# Kubernetes version for the Kind clusters
+KUBE_VERSION=${KUBE_VERSION:-"v1.24.7"}
+# Kind cluster name
+CLUSTER=${CLUSTER:-"local-cluster"}
+
+# Create the Kind cluster with the specified Kubernetes version.
+CLUSTER=${CLUSTER} KUBE_VERSION=${KUBE_VERSION} ${SCRIPT_DIR}/start_kind_cluster.sh
+
+kubectl config use-context "kind-${CLUSTER}"
+
+echo "===Start building operator"
+${SCRIPT_DIR}/../../config/test-certs/generate_secrets.bash
+BUILD_PLATFORM="linux/amd64" make -C "${SCRIPT_DIR}/../.." rebuild-operator
+
+echo "===Load operator image to kind cluster"
+kind load docker-image "fdb-kubernetes-operator:latest" --name "${CLUSTER}"
+
+echo "===Creating a FDB cluster with the FDB operator"
+kubectl apply -k "${SCRIPT_DIR}/../../config/tests/base"
+
+echo "===Done==="
+kubectl get fdb
+echo "Waiting for creating FDB Pods..."
+sleep 10;
+kubectl wait --for=condition=ready pod -l foundationdb.org/fdb-cluster-name=test-cluster
+kubectl get fdb
diff --git a/e2e/scripts/setup_e2e.sh b/e2e/scripts/setup_e2e.sh
new file mode 100755
index 00000000..8c8a7b92
--- /dev/null
+++ b/e2e/scripts/setup_e2e.sh
@@ -0,0 +1,86 @@
+
+#!/usr/bin/env bash
+
+# This source file is part of the FoundationDB open source project
+#
+# Copyright 2023 Apple Inc. and the FoundationDB project authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script works out of the box for Linux environments (tested on Ubuntu 16.04.5 LTS), after installing dependent software (kind, Docker, and kustomize);
+# Running in other environments may involve non-trivial debugging effort.
+set -eo errexit
+
+function get_image_name() {
+  if [[ -z ${REGISTRY} ]]
+  then
+    echo "${1}"
+  else
+    echo "${REGISTRY}/${1}"
+  fi
+}
+
+function preload_foundationdb_images() {
+  kind load docker-image "${2}" --name "${1}"
+  kind load docker-image "${3}" --name "${1}"
+}
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)
+cd "${SCRIPT_DIR}"
+
+# Kubernetes version for the Kind clusters
+KUBE_VERSION=${KUBE_VERSION:-"v1.24.7"}
+# Defines the FDB version that should be preloaded into the Kind cluster
+FDB_VERSION=${FDB_VERSION:-"7.1.25"}
+# Defines the registry to pull the images from.
+REGISTRY=${REGISTRY:-""}
+# Name for the new Kind cluster
+CLUSTER=${CLUSTER:-"e2e-tests"}
+# Chaos Mesh version that should be used to install chaos-mesh
+CHAOS_MESH_VERSION=${CHAOS_MESH_VERSION:-"2.5.0"}
+# Defines the namespace to install Chaos Mesh to.
+CHAOS_NAMESPACE=${CHAOS_NAMESPACE:-"chaos-testing"}
+
+# Create the Kind cluster with the specified Kubernetes version.
+CLUSTER=${CLUSTER} KUBE_VERSION=${KUBE_VERSION} ${SCRIPT_DIR}/start_kind_cluster.sh
+
+echo "Preload FoundationDB images for version ${FDB_VERSION}"
+fdb_image=$(get_image_name "foundationdb/foundationdb:${FDB_VERSION}")
+fdb_sidecar_image=$(get_image_name "foundationdb/foundationdb-kubernetes-sidecar:${FDB_VERSION}-1")
+docker pull "${fdb_image}"
+docker pull "${fdb_sidecar_image}"
+
+preload_foundationdb_images "${CLUSTER}" "${fdb_image}" "${fdb_sidecar_image}"
+
+echo "Build the operator image from the current revision"
+BUILD_PLATFORM="linux/amd64" make -C "${SCRIPT_DIR}/../.." container-build
+kind load docker-image "fdb-kubernetes-operator:latest" --name "${CLUSTER}"
+
+echo "Install the CRDs in the Kind cluster"
+kubectl apply -f "${SCRIPT_DIR}/../../config/crd/bases/"
+
+echo "Install the chaos-mesh in the Kind cluster"
+kubectl create ns "${CHAOS_NAMESPACE}" || true
+helm repo add chaos-mesh https://charts.chaos-mesh.org
+helm repo update
+
+# The configuration below is tested on a local Kind installation and might be different for the target Kubernetes cluster.
+helm upgrade -i chaos-mesh chaos-mesh/chaos-mesh \
+    --namespace "${CHAOS_NAMESPACE}" \
+    --set dashboard.securityMode=false \
+    --set chaosDaemon.socketPath=/run/containerd/containerd.sock \
+    --set chaosDaemon.runtime=containerd \
+    --version "${CHAOS_MESH_VERSION}"
+
+# Check if the Pods are running
+kubectl wait --for=condition=ready pods --namespace chaos-testing -l app.kubernetes.io/instance=chaos-mesh
diff --git a/e2e/scripts/start_kind_cluster.sh b/e2e/scripts/start_kind_cluster.sh
new file mode 100755
index 00000000..c59db50b
--- /dev/null
+++ b/e2e/scripts/start_kind_cluster.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# This source file is part of the FoundationDB open source project
+#
+# Copyright 2023 Apple Inc. and the FoundationDB project authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -eo errexit
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)
+cd "${SCRIPT_DIR}"
+
+# Kubernetes version for the Kind clusters
+KUBE_VERSION=${KUBE_VERSION:-"v1.24.7"}
+# Name for the new Kind cluster
+CLUSTER=${CLUSTER:-"ci-test"}
+
+# Create the Kind cluster with the specified Kubernetes version.
+cat <<EOF | kind create cluster --name "${CLUSTER}" --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  image: kindest/node:${KUBE_VERSION}
+- role: worker
+  image: kindest/node:${KUBE_VERSION}
+- role: worker
+  image: kindest/node:${KUBE_VERSION}
+- role: worker
+  image: kindest/node:${KUBE_VERSION}
+EOF
diff --git a/e2e/setup-local-fdb-cluster.sh b/e2e/setup-local-fdb-cluster.sh
deleted file mode 100755
index 788668eb..00000000
--- a/e2e/setup-local-fdb-cluster.sh
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/env bash
-# Set up local FDB non-HA cluster on a 4-node kind k8s cluster
-# This only works on x86 machines as FDB doesn't provide arm64 Linux binaries yet
-# Assumptions: (1) There is at most one kind cluster in the env; (2) kubectl is pointing to the kind cluster
-
-if [ "$#" -le 1 ]; then
-read -p "create kind cluster? (enter yes or no): " createKindCluster
-read -p "enter k8s node version? (e.g., v1.23.0): " version
-else
-    createKindCluster=$1
-    version=${2}
-fi
-
-cluster=${cluster:-"local-cluster"}
-
-if [ "${createKindCluster}" = "yes" ]; then
-    echo "===Start creating k8s cluster on kind"
-    cat <<EOF | kind create cluster --name ${cluster} --config=-
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-nodes:
-- role: control-plane
-  image: kindest/node:${version}
-- role: worker
-  image: kindest/node:${version}
-- role: worker
-  image: kindest/node:${version}
-- role: worker
-  image: kindest/node:${version}
-EOF
-
-else
-    echo "===Skip creating k8s cluster on kind"
-    # TODO: make sure kind is using the local-cluster context
-    kind get clusters
-fi
-
-echo "===Start building operator"
-echo "---We should be at the repo's root directory: "
-pwd
-
-# TODO use script dir here
-./config/test-certs/generate_secrets.bash
-make rebuild-operator
-
-echo "===Load operator image to kind cluster"
-kind load docker-image "fdb-kubernetes-operator:latest" --name ${cluster}
-
-echo "===Creating a FDB cluster with the FDB operator"
-kubectl apply -k ./config/tests/base
-
-echo "===Done==="
-
-# TODO: Make sure kubectl context is pointing to the kind cluster
-kubectl get fdb
-echo "Waiting for creating FDB Pods..."
-sleep 10;
-kubectl wait --for=condition=ready pod -l foundationdb.org/fdb-cluster-name=test-cluster
-kubectl get fdb
diff --git a/e2e/test_operator/operator_test.go b/e2e/test_operator/operator_test.go
index 6091cc2c..d4ccb204 100644
--- a/e2e/test_operator/operator_test.go
+++ b/e2e/test_operator/operator_test.go
@@ -788,7 +788,7 @@ var _ = Describe("Operator", Label("e2e"), func() {
 			pickedPod = fixtures.ChooseRandomPod(fdbCluster.GetStatelessPods())
 			log.Println("Selected Pod:", pickedPod.Name, " to be skipped during the restart")
 			fdbCluster.SetIgnoreDuringRestart(
-				[]fdbv1beta2.ProcessGroupID{fdbv1beta2.ProcessGroupID(pickedPod.Labels[fdbCluster.GetCachedCluster().GetProcessGroupIDLabel()])},
+				[]fdbv1beta2.ProcessGroupID{fixtures.GetProcessGroupID(*pickedPod)},
 			)
 
 			Expect(
@@ -810,7 +810,7 @@ var _ = Describe("Operator", Label("e2e"), func() {
 		})
 
 		It("should not restart the process on the ignore list", func() {
-			processGroupID := fdbv1beta2.ProcessGroupID(pickedPod.Labels[fdbCluster.GetCachedCluster().GetProcessGroupIDLabel()])
+			processGroupID := fixtures.GetProcessGroupID(*pickedPod)
 
 			// Ensure that the process group has the condition IncorrectCommandLine and is kept in that state for 1 minute.
 			Eventually(func() bool {
@@ -828,4 +828,43 @@ var _ = Describe("Operator", Label("e2e"), func() {
 			}).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).MustPassRepeatedly(12).Should(BeTrue())
 		})
 	})
+
+	// TODO (johscheuer): enable this test once the CRD is updated in out CI cluster.
+	PWhen("a process group is set to be blocked for removal", func() {
+		var podMarkedForRemoval corev1.Pod
+		var processGroupID fdbv1beta2.ProcessGroupID
+
+		BeforeEach(func() {
+			initialPods := fdbCluster.GetStatelessPods()
+			podMarkedForRemoval = fixtures.RandomPickOnePod(initialPods.Items)
+			log.Println("Setting Pod", podMarkedForRemoval.Name, "to be blocked to be removed and mark it for removal.")
+			processGroupID = fixtures.GetProcessGroupID(podMarkedForRemoval)
+			fdbCluster.SetBuggifyBlockRemoval([]fdbv1beta2.ProcessGroupID{processGroupID})
+			fdbCluster.ReplacePod(podMarkedForRemoval, false)
+		})
+
+		AfterEach(func() {
+			fdbCluster.SetBuggifyBlockRemoval(nil)
+			fdbCluster.EnsurePodIsDeleted(podMarkedForRemoval.Name)
+		})
+
+		It("should exclude the Pod but not remove the resources", func() {
+			Eventually(func() bool {
+				cluster := fdbCluster.GetCluster()
+
+				for _, processGroup := range cluster.Status.ProcessGroups {
+					if processGroup.ProcessGroupID != processGroupID {
+						continue
+					}
+
+					return !processGroup.ExclusionTimestamp.IsZero() && !processGroup.RemovalTimestamp.IsZero()
+				}
+
+				return false
+			}).WithTimeout(5 * time.Minute).WithPolling(15 * time.Second).ShouldNot(BeTrue())
+			Consistently(func() *corev1.Pod {
+				return fdbCluster.GetPod(podMarkedForRemoval.Name)
+			}).WithTimeout(1 * time.Minute).WithPolling(15 * time.Second).ShouldNot(BeNil())
+		})
+	})
 })
diff --git a/internal/buggify/buggify_test.go b/internal/buggify/buggify_test.go
new file mode 100644
index 00000000..20a394ac
--- /dev/null
+++ b/internal/buggify/buggify_test.go
@@ -0,0 +1,161 @@
+/*
+ * buggify_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package buggify
+
+import (
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Buggify", func() {
+	DescribeTable("filtering blocked removals", func(cluster *fdbv1beta2.FoundationDBCluster, initialProcessGroupsToRemove []*fdbv1beta2.ProcessGroupStatus, expected []*fdbv1beta2.ProcessGroupStatus) {
+		Expect(FilterBlockedRemovals(cluster, initialProcessGroupsToRemove)).To(ConsistOf(expected))
+	},
+		Entry("No process group is defined to be filtered",
+			&fdbv1beta2.FoundationDBCluster{},
+			nil,
+			nil,
+		),
+		Entry("One process group is defined to be filtered",
+			&fdbv1beta2.FoundationDBCluster{
+				Spec: fdbv1beta2.FoundationDBClusterSpec{
+					Buggify: fdbv1beta2.BuggifyConfig{
+						BlockRemoval: []fdbv1beta2.ProcessGroupID{
+							"storage-1",
+						},
+					},
+				},
+			},
+			[]*fdbv1beta2.ProcessGroupStatus{
+				{
+					ProcessGroupID: "storage-1",
+				},
+				{
+					ProcessGroupID: "storage-2",
+				},
+				{
+					ProcessGroupID: "storage-3",
+				},
+			},
+			[]*fdbv1beta2.ProcessGroupStatus{
+				{
+					ProcessGroupID: "storage-2",
+				},
+				{
+					ProcessGroupID: "storage-3",
+				},
+			},
+		),
+		Entry("No matching process group is defined to be filtered",
+			&fdbv1beta2.FoundationDBCluster{
+				Spec: fdbv1beta2.FoundationDBClusterSpec{
+					Buggify: fdbv1beta2.BuggifyConfig{
+						BlockRemoval: []fdbv1beta2.ProcessGroupID{
+							"storage-42",
+						},
+					},
+				},
+			},
+			[]*fdbv1beta2.ProcessGroupStatus{
+				{
+					ProcessGroupID: "storage-1",
+				},
+				{
+					ProcessGroupID: "storage-2",
+				},
+				{
+					ProcessGroupID: "storage-3",
+				},
+			},
+			[]*fdbv1beta2.ProcessGroupStatus{
+				{
+					ProcessGroupID: "storage-1",
+				},
+				{
+					ProcessGroupID: "storage-2",
+				},
+				{
+					ProcessGroupID: "storage-3",
+				},
+			},
+		),
+		Entry("Multiple process groups are defined to be filtered",
+			&fdbv1beta2.FoundationDBCluster{
+				Spec: fdbv1beta2.FoundationDBClusterSpec{
+					Buggify: fdbv1beta2.BuggifyConfig{
+						BlockRemoval: []fdbv1beta2.ProcessGroupID{
+							"storage-1",
+							"storage-3",
+						},
+					},
+				},
+			},
+			[]*fdbv1beta2.ProcessGroupStatus{
+				{
+					ProcessGroupID: "storage-1",
+				},
+				{
+					ProcessGroupID: "storage-2",
+				},
+				{
+					ProcessGroupID: "storage-3",
+				},
+			},
+			[]*fdbv1beta2.ProcessGroupStatus{
+				{
+					ProcessGroupID: "storage-2",
+				},
+			},
+		),
+		Entry("One matching process group and one non-matching process group is defined to be filtered",
+			&fdbv1beta2.FoundationDBCluster{
+				Spec: fdbv1beta2.FoundationDBClusterSpec{
+					Buggify: fdbv1beta2.BuggifyConfig{
+						BlockRemoval: []fdbv1beta2.ProcessGroupID{
+							"storage-1",
+							"storage-42",
+						},
+					},
+				},
+			},
+			[]*fdbv1beta2.ProcessGroupStatus{
+				{
+					ProcessGroupID: "storage-1",
+				},
+				{
+					ProcessGroupID: "storage-2",
+				},
+				{
+					ProcessGroupID: "storage-3",
+				},
+			},
+			[]*fdbv1beta2.ProcessGroupStatus{
+				{
+					ProcessGroupID: "storage-2",
+				},
+				{
+					ProcessGroupID: "storage-3",
+				},
+			},
+		),
+	)
+})
diff --git a/internal/buggify/suite_test.go b/internal/buggify/suite_test.go
new file mode 100644
index 00000000..7c8fa964
--- /dev/null
+++ b/internal/buggify/suite_test.go
@@ -0,0 +1,33 @@
+/*
+ * suite_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package buggify
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestCmd(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "FDB buggify")
+}
diff --git a/internal/monitor_conf_test.go b/internal/monitor_conf_test.go
index e232a751..4af58ae5 100644
--- a/internal/monitor_conf_test.go
+++ b/internal/monitor_conf_test.go
@@ -790,6 +790,35 @@ var _ = Describe("monitor_conf", func() {
 			})
 		})
 
+		Context("with DNS names in locality fields", func() {
+			BeforeEach(func() {
+				cluster.Spec.Routing.DefineDNSLocalityFields = pointer.Bool(true)
+				conf, err = GetMonitorConf(cluster, fdbv1beta2.ProcessClassStorage, nil, cluster.GetStorageServersPerPod())
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			It("should generate the storage conf", func() {
+				Expect(conf).To(Equal(strings.Join([]string{
+					"[general]",
+					"kill_on_configuration_change = false",
+					"restart_delay = 60",
+					"[fdbserver.1]",
+					"command = $BINARY_DIR/fdbserver",
+					"cluster_file = /var/fdb/data/fdb.cluster",
+					"seed_cluster_file = /var/dynamic-conf/fdb.cluster",
+					"public_address = $FDB_PUBLIC_IP:4501",
+					"class = storage",
+					"logdir = /var/log/fdb-trace-logs",
+					"loggroup = " + cluster.Name,
+					"datadir = /var/fdb/data",
+					"locality_instance_id = $FDB_INSTANCE_ID",
+					"locality_machineid = $FDB_MACHINE_ID",
+					"locality_zoneid = $FDB_ZONE_ID",
+					"locality_dns_name = $FDB_DNS_NAME",
+				}, "\n")))
+			})
+		})
+
 		Context("with a basic storage instance with multiple storage servers per Pod", func() {
 			BeforeEach(func() {
 				cluster.Spec.StorageServersPerPod = 2
diff --git a/internal/pod_models_test.go b/internal/pod_models_test.go
index 836c9264..eb2a0530 100644
--- a/internal/pod_models_test.go
+++ b/internal/pod_models_test.go
@@ -924,6 +924,90 @@ var _ = Describe("pod_models", func() {
 			})
 		})
 
+		When("enabling DNS in the locality fields", func() {
+			BeforeEach(func() {
+				cluster.Spec.Routing.DefineDNSLocalityFields = pointer.Bool(true)
+				spec, err = GetPodSpec(cluster, fdbv1beta2.ProcessClassStorage, 1)
+			})
+
+			It("should set an additional environment variable on the init container", func() {
+				Expect(len(spec.InitContainers)).To(Equal(1))
+				initContainer := spec.InitContainers[0]
+				Expect(initContainer.Name).To(Equal("foundationdb-kubernetes-init"))
+				Expect(initContainer.Args).To(Equal([]string{
+					"--copy-file",
+					"fdb.cluster",
+					"--input-monitor-conf",
+					"fdbmonitor.conf",
+					"--copy-binary",
+					"fdbserver",
+					"--copy-binary",
+					"fdbcli",
+					"--main-container-version",
+					"6.2.21",
+					"--substitute-variable",
+					"FDB_POD_IP",
+					"--substitute-variable",
+					"FDB_DNS_NAME",
+					"--init-mode",
+				}))
+				Expect(initContainer.Env).To(Equal([]corev1.EnvVar{
+					{Name: "FDB_PUBLIC_IP", ValueFrom: &corev1.EnvVarSource{
+						FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"},
+					}},
+					{Name: "FDB_POD_IP", ValueFrom: &corev1.EnvVarSource{
+						FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"},
+					}},
+					{Name: "FDB_MACHINE_ID", ValueFrom: &corev1.EnvVarSource{
+						FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"},
+					}},
+					{Name: "FDB_ZONE_ID", ValueFrom: &corev1.EnvVarSource{
+						FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"},
+					}},
+					{Name: "FDB_INSTANCE_ID", Value: "storage-1"},
+					{Name: "FDB_DNS_NAME", Value: "operator-test-1-storage-1.operator-test-1.my-ns.svc.cluster.local"},
+				}))
+			})
+
+			It("should set an additional environment variable on the sidecar container", func() {
+				sidecarContainer := spec.Containers[1]
+				Expect(sidecarContainer.Name).To(Equal("foundationdb-kubernetes-sidecar"))
+				Expect(sidecarContainer.Args).To(Equal([]string{
+					"--copy-file",
+					"fdb.cluster",
+					"--input-monitor-conf",
+					"fdbmonitor.conf",
+					"--copy-binary",
+					"fdbserver",
+					"--copy-binary",
+					"fdbcli",
+					"--main-container-version",
+					"6.2.21",
+					"--substitute-variable",
+					"FDB_POD_IP",
+					"--substitute-variable",
+					"FDB_DNS_NAME",
+				}))
+				Expect(sidecarContainer.Env).To(Equal([]corev1.EnvVar{
+					{Name: "FDB_PUBLIC_IP", ValueFrom: &corev1.EnvVarSource{
+						FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"},
+					}},
+					{Name: "FDB_POD_IP", ValueFrom: &corev1.EnvVarSource{
+						FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"},
+					}},
+					{Name: "FDB_MACHINE_ID", ValueFrom: &corev1.EnvVarSource{
+						FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"},
+					}},
+					{Name: "FDB_ZONE_ID", ValueFrom: &corev1.EnvVarSource{
+						FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"},
+					}},
+					{Name: "FDB_INSTANCE_ID", Value: "storage-1"},
+					{Name: "FDB_DNS_NAME", Value: "operator-test-1-storage-1.operator-test-1.my-ns.svc.cluster.local"},
+					{Name: "FDB_TLS_VERIFY_PEERS", Value: ""},
+				}))
+			})
+		})
+
 		When("having a predefined node affinity rules", func() {
 			BeforeEach(func() {
 				affinity := &corev1.Affinity{