Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1245 (2022-06-15 17:48:51)

Add coordinator as stateful process
# Description

Fixes: https://github.com/FoundationDB/fdb-kubernetes-operator/issues/1244

## Type of change

*Please select one of the options below.*

- Bug fix (non-breaking change which fixes an issue)

## Discussion

-

## Testing

Local testing.

## Documentation

-

## Follow-up

-



Patch:
diff --git a/.gitignore b/.gitignore
index 7fd15474..d386a41b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,6 +35,7 @@ foundationdb-kubernetes-sidecar/website
 
 config/default/manager_image_patch.yaml
 config/default/manager_image_patch.yaml-e
+config/development/kustomization.yaml
 
 # General temp directory
 tmp/
diff --git a/Makefile b/Makefile
index 00ee3574..8865e8f2 100644
--- a/Makefile
+++ b/Makefile
@@ -2,6 +2,8 @@
 
 # Image URL to use all building/pushing image targets
 IMG ?= fdb-kubernetes-operator:latest
+SIDECAR_IMG ?=
+REMOTE_BUILD ?= 0
 CRD_OPTIONS ?= "crd:maxDescLen=0,crdVersions=v1,generateEmbeddedObjectMeta=true"
 
 ifneq "$(FDB_WEBSITE)" ""
@@ -67,7 +69,7 @@ endif
 
 all: deps generate fmt vet manager snapshot manifests samples documentation test_if_changed
 
-.PHONY: clean all manager samples documentation run install uninstall deploy manifests fmt vet generate container-build container-push rebuild-operator bounce lint
+.PHONY: clean all manager samples documentation run install uninstall deploy manifests fmt vet generate container-build container-push container-push-if-remote rebuild-operator bounce lint
 
 deps: $(BUILD_DEPS)
 
@@ -78,6 +80,7 @@ clean:
 	rm -r bin
 	rm -f $(SAMPLES)
 	rm -f config/rbac/role.yaml
+	rm -f config/development/kustomization.yaml
 	rm -rf ./dist/*
 	find . -name "cover.out" -delete
 
@@ -120,9 +123,20 @@ install: manifests
 uninstall: manifests
 	kustomize build config/crd | kubectl $(KUBECTL_ARGS) delete -f -
 
-# Deploy controller in the configured Kubernetes cluster in ~/.kube/config
-deploy: install manifests
+# Apply config to the local development environment based on environment
+# variables.
+config/development/kustomization.yaml: config/development/kustomization.yaml.sample
+	cp config/development/kustomization.yaml.sample config/development/kustomization.yaml
 	cd config/development && kustomize edit set image foundationdb/fdb-kubernetes-operator=${IMG}
+ifneq "$(SIDECAR_IMG)" ""
+	cd config/development && kustomize edit set image foundationdb/foundationdb-kubernetes-sidecar=${SIDECAR_IMG}
+endif
+ifeq "$(REMOTE_BUILD)" "1"
+	cd config/development && kustomize edit add patch --path=remote_build.yaml
+endif
+
+# Deploy controller in the configured Kubernetes cluster in ~/.kube/config
+deploy: install manifests config/development/kustomization.yaml
 	kustomize build config/development | kubectl $(KUBECTL_ARGS) apply -f -
 
 # Generate manifests e.g. CRD, RBAC etc.
@@ -164,8 +178,14 @@ container-build: test_if_changed
 container-push:
 	$(BUILDER) push ${IMG}
 
+# Push the container image
+container-push-if-remote:
+ifeq "$(REMOTE_BUILD)" "1"
+	$(BUILDER) push ${IMG}
+endif
+
 # Rebuilds, deploys, and bounces the operator
-rebuild-operator: container-build deploy bounce
+rebuild-operator: container-build container-push-if-remote deploy bounce
 
 bounce:
 	kubectl $(KUBECTL_ARGS) delete pod -l app=fdb-kubernetes-operator-controller-manager
diff --git a/README.md b/README.md
index 55298e35..0710876c 100644
--- a/README.md
+++ b/README.md
@@ -63,7 +63,7 @@ To get this controller running in a local Kubernetes cluster:
 1. Run `kubectl apply -k ./config/tests/base`
    to create a new FoundationDB cluster with the operator.
 
-### Running locally with nerdctl
+### Running Locally with nerdctl
 
 Instead of Docker you can also use [nerdctl](https://github.com/containerd/nerdctl) to build and push your images.
 In order to use a different image builder than docker you can use the env variable `BUILDER`:
@@ -76,6 +76,15 @@ export BUILDER='nerdctl -n k8s.io'
 You can test your setup with `SKIP_TEST=1 make container-build` which will build the image locally.
 After the command successfully finished you can verify with `nerdctl -n k8s.io images fdb-kubernetes-operator:latest` that the image is available.
 
+### Customizing Your Build
+
+The makefile supports environment variables that allow you to customize your build. You can use these to push to custom docker repos and deployment platforms.
+
+* `IMG`: This specifies the image that gets built for the operator.
+* `SIDECAR_IMG`: This specifies the image for the foundationdb-kubernetes-sidecar process used in init containers for the operator. This does not change the images used for the FoundationDB clusters, which are specified in the cluster spec.
+* `REMOTE_BUILD`: This can be set to 1 to indicate that you are running the operator in a remote environment, rather than on your local development machine. This will activate a [remote build](config/development/remote_build.yaml) patch, which changes the image pull policy in the operator's pod spec. Setting this also tells the Makefile to push images as part of the `rebuild-operator` command.
+* `FDB_WEBSITE`: This specifies the base path for the website used to download FDB client packages in the docker builds. You can use this to download custom binaries from your own host, provided that your path structure matches the paths expected in the [Dockerfile](Dockerfile).
+
 ### Known Limitations
 
 1. Support for backups in the operator is still in development, and there are significant missing features.
diff --git a/api/v1beta2/foundationdb_process_class.go b/api/v1beta2/foundationdb_process_class.go
index 16ac8952..d17bafa9 100644
--- a/api/v1beta2/foundationdb_process_class.go
+++ b/api/v1beta2/foundationdb_process_class.go
@@ -38,11 +38,13 @@ const (
 	ProcessClassClusterController ProcessClass = "cluster_controller"
 	// ProcessClassTest model for FDB class test
 	ProcessClassTest ProcessClass = "test"
+	// ProcessClassCoordinator model for FDB class coordinator
+	ProcessClassCoordinator ProcessClass = "coordinator"
 )
 
 // IsStateful determines whether a process class should store data.
 func (pClass ProcessClass) IsStateful() bool {
-	return pClass == ProcessClassStorage || pClass == ProcessClassLog || pClass == ProcessClassTransaction
+	return pClass == ProcessClassStorage || pClass == ProcessClassLog || pClass == ProcessClassTransaction || pClass == ProcessClassCoordinator
 }
 
 // IsTransaction determines whether a process class is part of the transaction system.
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index e1fb5013..8d71567d 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -1870,17 +1870,6 @@ func (cluster *FoundationDBCluster) NeedsReplacement(processGroup *ProcessGroupS
 		return false
 	}
 
-	// See: https://github.com/FoundationDB/fdb-kubernetes-operator/issues/1201 when a process group is set into crash loop
-	// we don't want to replace it (otherwise the newly created process group won't have that setting).
-	crashLoop, crashLoopAll := cluster.GetCrashLoopProcessGroups()
-	if crashLoopAll {
-		return false
-	}
-
-	if _, ok := crashLoop[processGroup.ProcessGroupID]; ok {
-		return false
-	}
-
 	if cluster.Spec.AutomationOptions.PodUpdateStrategy == PodUpdateStrategyReplacement {
 		return true
 	}
@@ -2035,3 +2024,32 @@ func (cluster *FoundationDBCluster) GetCrashLoopProcessGroups() (map[string]None
 
 	return crashLoopPods, false
 }
+
+// Validate checks if all settings in the cluster are valid, if not and error will be returned. If multiple issues are
+// found all of them will be returned in a single error.
+func (cluster *FoundationDBCluster) Validate() error {
+	var validations []string
+
+	// Check if the provided storage engine is valid for the defined FDB version.
+	version, err := ParseFdbVersion(cluster.Spec.Version)
+	if err != nil {
+		return err
+	}
+
+	if !version.IsStorageEngineSupported(cluster.Spec.DatabaseConfiguration.StorageEngine) {
+		validations = append(validations, fmt.Sprintf("storage engine %s is not supported on version %s", cluster.Spec.DatabaseConfiguration.StorageEngine, cluster.Spec.Version))
+	}
+
+	// Check if all coordinator processes are stateful
+	for _, selection := range cluster.Spec.CoordinatorSelection {
+		if !selection.ProcessClass.IsStateful() {
+			validations = append(validations, fmt.Sprintf("%s is not a valid process class for coordinators", selection.ProcessClass))
+		}
+	}
+
+	if len(validations) == 0 {
+		return nil
+	}
+
+	return fmt.Errorf(strings.Join(validations, ", "))
+}
diff --git a/config/deployment/manager.yaml b/config/deployment/manager.yaml
index 21bd96e0..45c93cce 100644
--- a/config/deployment/manager.yaml
+++ b/config/deployment/manager.yaml
@@ -83,7 +83,7 @@ spec:
       containers:
       - command:
         - /manager
-        image: foundationdb/fdb-kubernetes-operator:v1.5.0
+        image: foundationdb/fdb-kubernetes-operator:v1.6.0
         name: manager
         env:
           - name: WATCH_NAMESPACE
diff --git a/config/development/kustomization.yaml b/config/development/kustomization.yaml.sample
similarity index 100%
rename from config/development/kustomization.yaml
rename to config/development/kustomization.yaml.sample
diff --git a/config/development/remote_build.yaml b/config/development/remote_build.yaml
new file mode 100644
index 00000000..5e9ea970
--- /dev/null
+++ b/config/development/remote_build.yaml
@@ -0,0 +1,10 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+spec:
+  template:
+    spec:
+      containers:
+        - name: manager
+          imagePullPolicy: Always
diff --git a/config/samples/deployment.yaml b/config/samples/deployment.yaml
index 04bab1e0..836782b2 100644
--- a/config/samples/deployment.yaml
+++ b/config/samples/deployment.yaml
@@ -150,7 +150,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: foundationdb/fdb-kubernetes-operator:v1.5.0
+        image: foundationdb/fdb-kubernetes-operator:v1.6.0
         name: manager
         ports:
         - containerPort: 8080
diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go
index 4ce472bd..6fe79f6f 100644
--- a/controllers/cluster_controller.go
+++ b/controllers/cluster_controller.go
@@ -120,13 +120,10 @@ func (r *FoundationDBClusterReconciler) Reconcile(ctx context.Context, request c
 		return ctrl.Result{}, fmt.Errorf("version %s is not supported", cluster.Spec.Version)
 	}
 
-	storageEngineSupported, err := isStorageEngineSupported(cluster.Spec.Version, cluster.Spec.DatabaseConfiguration.StorageEngine)
+	err = cluster.Validate()
 	if err != nil {
-		return ctrl.Result{}, err
-	}
-	if !storageEngineSupported {
-		r.Recorder.Event(cluster, corev1.EventTypeWarning, "Storage engine not supported", fmt.Sprintf("storage engine %s is not supported on version %s", cluster.Spec.DatabaseConfiguration.StorageEngine, cluster.Spec.Version))
-		return ctrl.Result{}, fmt.Errorf("storage engine %s is not supported on version %s", cluster.Spec.DatabaseConfiguration.StorageEngine, cluster.Spec.Version)
+		r.Recorder.Event(cluster, corev1.EventTypeWarning, "ClusterSpec not valid", err.Error())
+		return ctrl.Result{}, fmt.Errorf("ClusterSpec is not valid: %w", err)
 	}
 
 	subReconcilers := []clusterSubReconciler{
@@ -134,9 +131,9 @@ func (r *FoundationDBClusterReconciler) Reconcile(ctx context.Context, request c
 		updateLockConfiguration{},
 		updateConfigMap{},
 		checkClientCompatibility{},
+		deletePodsForBuggification{},
 		replaceMisconfiguredProcessGroups{},
 		replaceFailedProcessGroups{},
-		deletePodsForBuggification{},
 		addProcessGroups{},
 		addServices{},
 		addPVCs{},
@@ -750,11 +747,3 @@ func (r *FoundationDBClusterReconciler) getCoordinatorSet(cluster *fdbv1beta2.Fo
 
 	return adminClient.GetCoordinatorSet()
 }
-
-func isStorageEngineSupported(versionString string, storageEngine fdbv1beta2.StorageEngine) (bool, error) {
-	version, err := fdbv1beta2.ParseFdbVersion(versionString)
-	if err != nil {
-		return false, err
-	}
-	return version.IsStorageEngineSupported(storageEngine), nil
-}
diff --git a/controllers/delete_pods_for_buggification.go b/controllers/delete_pods_for_buggification.go
index 26a4b1e5..9e960e21 100644
--- a/controllers/delete_pods_for_buggification.go
+++ b/controllers/delete_pods_for_buggification.go
@@ -44,10 +44,13 @@ func (d deletePodsForBuggification) reconcile(ctx context.Context, r *Foundation
 	}
 
 	podMap := internal.CreatePodMap(cluster, pods)
-	updates := make([]*corev1.Pod, 0)
-
 	crashLoopPods, crashLoopAll := cluster.GetCrashLoopProcessGroups()
+	noSchedulePods := make(map[string]fdbv1beta2.None, len(cluster.Spec.Buggify.NoSchedule))
+	for _, processGroupID := range cluster.Spec.Buggify.NoSchedule {
+		noSchedulePods[processGroupID] = fdbv1beta2.None{}
+	}
 
+	var updates []*corev1.Pod
 	for _, processGroup := range cluster.Status.ProcessGroups {
 		if processGroup.IsMarkedForRemoval() {
 			logger.V(1).Info("Ignore process group marked for removal",
@@ -80,6 +83,42 @@ func (d deletePodsForBuggification) reconcile(ctx context.Context, r *Foundation
 				"shouldCrashLoop", shouldCrashLoop,
 				"inCrashLoop", inCrashLoop)
 			updates = append(updates, pod)
+			continue
+		}
+
+		// Recreate Pods that should be in the no schedule state
+		var inNoSchedule, shouldBeNoSchedule bool
+		_, shouldBeNoSchedule = noSchedulePods[processGroup.ProcessGroupID]
+
+		// Ensure the Pod has an Affinity set before accessing it.
+		if pod.Spec.Affinity == nil ||
+			pod.Spec.Affinity.NodeAffinity == nil ||
+			pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
+			// Uf the Pod should be in the noSchedule state we have to recreate the Pod.
+			if shouldBeNoSchedule {
+				logger.Info("Deleting pod for buggification",
+					"processGroupID", processGroup.ProcessGroupID,
+					"shouldBeNoSchedule", shouldBeNoSchedule,
+					"inNoSchedule", inNoSchedule)
+				updates = append(updates, pod)
+			}
+			continue
+		}
+
+		for _, term := range pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms {
+			for _, expression := range term.MatchExpressions {
+				if expression.Key == fdbv1beta2.NodeSelectorNoScheduleLabel {
+					inNoSchedule = true
+				}
+			}
+		}
+
+		if inNoSchedule != shouldBeNoSchedule {
+			logger.Info("Deleting pod for buggification",
+				"processGroupID", processGroup.ProcessGroupID,
+				"shouldBeNoSchedule", shouldBeNoSchedule,
+				"inNoSchedule", inNoSchedule)
+			updates = append(updates, pod)
 		}
 	}
 
@@ -94,5 +133,6 @@ func (d deletePodsForBuggification) reconcile(ctx context.Context, r *Foundation
 
 		return &requeue{message: "Pods need to be recreated"}
 	}
+
 	return nil
 }
diff --git a/controllers/remove_process_groups.go b/controllers/remove_process_groups.go
index 579268e0..ca85ed6c 100644
--- a/controllers/remove_process_groups.go
+++ b/controllers/remove_process_groups.go
@@ -244,9 +244,7 @@ func includeProcessGroup(ctx context.Context, r *FoundationDBClusterReconciler,
 	}
 	defer adminClient.Close()
 
-	var fdbProcessesToInclude []fdbv1beta2.ProcessAddress
-	var processGroups []*fdbv1beta2.ProcessGroupStatus
-	fdbProcessesToInclude, processGroups = getProcessesToInclude(cluster, removedProcessGroups)
+	fdbProcessesToInclude := getProcessesToInclude(cluster, removedProcessGroups)
 	if len(fdbProcessesToInclude) > 0 {
 		r.Recorder.Event(cluster, corev1.EventTypeNormal, "IncludingProcesses", fmt.Sprintf("Including removed processes: %v", fdbProcessesToInclude))
 
@@ -255,7 +253,6 @@ func includeProcessGroup(ctx context.Context, r *FoundationDBClusterReconciler,
 			return err
 		}
 
-		cluster.Status.ProcessGroups = processGroups
 		err := r.Status().Update(ctx, cluster)
 		if err != nil {
 			return err
@@ -265,11 +262,14 @@ func includeProcessGroup(ctx context.Context, r *FoundationDBClusterReconciler,
 	return nil
 }
 
-func getProcessesToInclude(cluster *fdbv1beta2.FoundationDBCluster, removedProcessGroups map[string]bool) ([]fdbv1beta2.ProcessAddress, []*fdbv1beta2.ProcessGroupStatus) {
+func getProcessesToInclude(cluster *fdbv1beta2.FoundationDBCluster, removedProcessGroups map[string]bool) []fdbv1beta2.ProcessAddress {
 	fdbProcessesToInclude := make([]fdbv1beta2.ProcessAddress, 0)
-	// TODO (09harsh): don't allocate extra space for processGroups, #1196
-	processGroups := make([]*fdbv1beta2.ProcessGroupStatus, 0, len(cluster.Status.ProcessGroups))
 
+	if len(removedProcessGroups) == 0 {
+		return fdbProcessesToInclude
+	}
+
+	idx := 0
 	for _, processGroup := range cluster.Status.ProcessGroups {
 		if processGroup.IsMarkedForRemoval() && removedProcessGroups[processGroup.ProcessGroupID] {
 			if cluster.UseLocalitiesForExclusion() {
@@ -278,11 +278,16 @@ func getProcessesToInclude(cluster *fdbv1beta2.FoundationDBCluster, removedProce
 			for _, pAddr := range processGroup.Addresses {
 				fdbProcessesToInclude = append(fdbProcessesToInclude, fdbv1beta2.ProcessAddress{IPAddress: net.ParseIP(pAddr)})
 			}
-		} else {
-			processGroups = append(processGroups, processGroup)
+			continue
 		}
+		cluster.Status.ProcessGroups[idx] = processGroup
+		idx++
 	}
-	return fdbProcessesToInclude, processGroups
+
+	// Remove the trailing duplicates.
+	cluster.Status.ProcessGroups = cluster.Status.ProcessGroups[:idx]
+
+	return fdbProcessesToInclude
 }
 
 func (r *FoundationDBClusterReconciler) getProcessGroupsToRemove(cluster *fdbv1beta2.FoundationDBCluster, remainingMap map[string]bool) (bool, bool, []*fdbv1beta2.ProcessGroupStatus) {
diff --git a/docs/changelog/v1.6.0.md b/docs/changelog/v1.6.0.md
new file mode 100644
index 00000000..16b3365e
--- /dev/null
+++ b/docs/changelog/v1.6.0.md
@@ -0,0 +1,6 @@
+# v1.6.0
+
+* Reorder the removal of headless services to avoid a race condition
+* Add missing constraints on collection sizes in some fields in the CRD
+* Revert to an older version of the FDB bindings to match the oldest supported
+  database version.
\ No newline at end of file
diff --git a/docs/manual/fault_domains.md b/docs/manual/fault_domains.md
index 195dc024..80a668ef 100644
--- a/docs/manual/fault_domains.md
+++ b/docs/manual/fault_domains.md
@@ -246,6 +246,13 @@ In this example the processes with the class `storage` will be preferred over pr
 That means that a `log` process will only be considered a valid coordinator if there are no other `storage` processes that can be selected without hurting the fault domain requirements.
 Changing the `coordinatorSelection` can result in new coordinators e.g. if the current preferred class will be removed.
 
+The operator supports the following classes as coordinators:
+
+- `storage`
+- `log`
+- `transaction`
+- `coordinator`
+
 ### Known limitations
 
 FoundationDB clusters that are spread across different DC's or Kubernetes clusters only support the same `coordinatorSelection`.
diff --git a/helm/fdb-operator/values.yaml b/helm/fdb-operator/values.yaml
index 4e915d81..6b1f0e06 100644
--- a/helm/fdb-operator/values.yaml
+++ b/helm/fdb-operator/values.yaml
@@ -1,7 +1,7 @@
 ---
 image:
   repository: foundationdb/fdb-kubernetes-operator
-  tag: v1.5.0
+  tag: v1.6.0
   pullPolicy: IfNotPresent
 
 initContainers:


Test patch:
diff --git a/api/v1beta2/foundationdbcluster_types_test.go b/api/v1beta2/foundationdbcluster_types_test.go
index 09811eed..92d299ef 100644
--- a/api/v1beta2/foundationdbcluster_types_test.go
+++ b/api/v1beta2/foundationdbcluster_types_test.go
@@ -4433,44 +4433,6 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 				},
 				false,
 			),
-			Entry("Default update strategy log process that is in crash loop",
-				&FoundationDBCluster{
-					Spec: FoundationDBClusterSpec{
-						AutomationOptions: FoundationDBClusterAutomationOptions{
-							PodUpdateStrategy: "",
-						},
-						Buggify: BuggifyConfig{
-							CrashLoop: []string{
-								"log-1",
-							},
-						},
-					},
-				},
-				&ProcessGroupStatus{
-					ProcessGroupID: "log-1",
-					ProcessClass:   ProcessClassTransaction,
-				},
-				false,
-			),
-			Entry("Default update strategy log process with crash loop all",
-				&FoundationDBCluster{
-					Spec: FoundationDBClusterSpec{
-						AutomationOptions: FoundationDBClusterAutomationOptions{
-							PodUpdateStrategy: "",
-						},
-						Buggify: BuggifyConfig{
-							CrashLoop: []string{
-								"*",
-							},
-						},
-					},
-				},
-				&ProcessGroupStatus{
-					ProcessGroupID: "log-1",
-					ProcessClass:   ProcessClassTransaction,
-				},
-				false,
-			),
 		)
 	})
 
@@ -4551,4 +4513,119 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 			})
 		})
 	})
+
+	When("validating a cluster", func() {
+		DescribeTable("it should return if the cluster is valid",
+			func(cluster *FoundationDBCluster, expected error) {
+				if expected == nil {
+					Expect(cluster.Validate()).NotTo(HaveOccurred())
+				} else {
+					Expect(cluster.Validate()).To(Equal(expected))
+				}
+
+			},
+			Entry("valid cluster spec",
+				&FoundationDBCluster{
+					Spec: FoundationDBClusterSpec{
+						Version: "6.3.2",
+						DatabaseConfiguration: DatabaseConfiguration{
+							StorageEngine: StorageEngineSSD2,
+						},
+					},
+				},
+				nil,
+			),
+			Entry("using invalid storage engine",
+				&FoundationDBCluster{
+					Spec: FoundationDBClusterSpec{
+						Version: "6.3.2",
+						DatabaseConfiguration: DatabaseConfiguration{
+							StorageEngine: StorageEngineRocksDbV1,
+						},
+					},
+				},
+				fmt.Errorf("storage engine ssd-rocksdb-v1 is not supported on version 6.3.2"),
+			),
+			Entry("using valid storage engine",
+				&FoundationDBCluster{
+					Spec: FoundationDBClusterSpec{
+						Version: Versions.SupportsRocksDBV1.String(),
+						DatabaseConfiguration: DatabaseConfiguration{
+							StorageEngine: StorageEngineRocksDbV1,
+						},
+					},
+				},
+				nil,
+			),
+			Entry("using valid coordinator selection",
+				&FoundationDBCluster{
+					Spec: FoundationDBClusterSpec{
+						Version: Versions.SupportsRocksDBV1.String(),
+						DatabaseConfiguration: DatabaseConfiguration{
+							StorageEngine: StorageEngineRocksDbV1,
+						},
+						CoordinatorSelection: []CoordinatorSelectionSetting{
+							{
+								ProcessClass: ProcessClassStorage,
+							},
+							{
+								ProcessClass: ProcessClassLog,
+							},
+							{
+								ProcessClass: ProcessClassCoordinator,
+							},
+							{
+								ProcessClass: ProcessClassTransaction,
+							},
+						},
+					},
+				},
+				nil,
+			),
+			Entry("using invalid coordinator selection",
+				&FoundationDBCluster{
+					Spec: FoundationDBClusterSpec{
+						Version: Versions.SupportsRocksDBV1.String(),
+						DatabaseConfiguration: DatabaseConfiguration{
+							StorageEngine: StorageEngineRocksDbV1,
+						},
+						CoordinatorSelection: []CoordinatorSelectionSetting{
+							{
+								ProcessClass: ProcessClassStorage,
+							},
+							{
+								ProcessClass: ProcessClassLog,
+							},
+							{
+								ProcessClass: ProcessClassCoordinator,
+							},
+							{
+								ProcessClass: ProcessClassTransaction,
+							},
+							{
+								ProcessClass: ProcessClassStateless,
+							},
+						},
+					},
+				},
+				fmt.Errorf("stateless is not a valid process class for coordinators"),
+			),
+			Entry("multiple validations",
+				&FoundationDBCluster{
+					Spec: FoundationDBClusterSpec{
+						Version: "6.1.3",
+						DatabaseConfiguration: DatabaseConfiguration{
+							StorageEngine: StorageEngineRocksDbV1,
+						},
+						CoordinatorSelection: []CoordinatorSelectionSetting{
+							{
+								ProcessClass: ProcessClassStateless,
+							},
+						},
+					},
+				},
+				fmt.Errorf("storage engine ssd-rocksdb-v1 is not supported on version 6.1.3, stateless is not a valid process class for coordinators"),
+			),
+		)
+	})
 })
diff --git a/config/tests/dns/client.yaml b/config/tests/dns/client.yaml
index b1e9e690..08e7e6f7 100644
--- a/config/tests/dns/client.yaml
+++ b/config/tests/dns/client.yaml
@@ -17,7 +17,7 @@ spec:
       automountServiceAccountToken: false
       initContainers:
         - name: foundationdb-kubernetes-init-7-1
-          image: foundationdb/foundationdb-kubernetes-sidecar:7.1.0-rc2-1
+          image: foundationdb/foundationdb-kubernetes-sidecar:7.1.5-1
           imagePullPolicy: IfNotPresent
           args:
             - "--copy-file"
diff --git a/config/tests/dns/dns.yaml b/config/tests/dns/dns.yaml
index 922eb01e..21bf5b0e 100644
--- a/config/tests/dns/dns.yaml
+++ b/config/tests/dns/dns.yaml
@@ -1,6 +1,6 @@
 - op: add
   path: "/spec/version"
-  value: 7.1.0-rc2
+  value: 7.1.5
 - op: add
   path: "/spec/routing/useDNSInClusterFile"
   value: true
diff --git a/controllers/delete_pods_for_buggification_test.go b/controllers/delete_pods_for_buggification_test.go
index 66168c3b..f51f2546 100644
--- a/controllers/delete_pods_for_buggification_test.go
+++ b/controllers/delete_pods_for_buggification_test.go
@@ -87,25 +87,50 @@ var _ = Describe("delete_pods_for_buggification", func() {
 	})
 
 	Context("with a buggification that needs to be enabled", func() {
-		BeforeEach(func() {
-			cluster.Spec.Buggify.CrashLoop = []string{"storage-1"}
-		})
-
-		It("should requeue", func() {
-			Expect(requeue).NotTo(BeNil())
-			Expect(requeue.message).To(Equal("Pods need to be recreated"))
+		When("using crashLoop", func() {
+			BeforeEach(func() {
+				cluster.Spec.Buggify.CrashLoop = []string{"storage-1"}
+			})
+
+			It("should requeue", func() {
+				Expect(requeue).NotTo(BeNil())
+				Expect(requeue.message).To(Equal("Pods need to be recreated"))
+			})
+
+			It("should delete the pod", func() {
+				pods := &corev1.PodList{}
+				err = k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(pods.Items)).To(Equal(len(originalPods.Items) - 1))
+
+				pod := &corev1.Pod{}
+				err = k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: cluster.Namespace, Name: "operator-test-1-storage-1"}, pod)
+				Expect(err).To(HaveOccurred())
+				Expect(k8serrors.IsNotFound(err)).To(BeTrue())
+			})
 		})
 
-		It("should delete the pod", func() {
-			pods := &corev1.PodList{}
-			err = k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(len(pods.Items)).To(Equal(len(originalPods.Items) - 1))
-
-			pod := &corev1.Pod{}
-			err = k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: cluster.Namespace, Name: "operator-test-1-storage-1"}, pod)
-			Expect(err).To(HaveOccurred())
-			Expect(k8serrors.IsNotFound(err)).To(BeTrue())
+		When("using noSchedule", func() {
+			BeforeEach(func() {
+				cluster.Spec.Buggify.NoSchedule = []string{"storage-1"}
+			})
+
+			It("should requeue", func() {
+				Expect(requeue).NotTo(BeNil())
+				Expect(requeue.message).To(Equal("Pods need to be recreated"))
+			})
+
+			It("should delete the pod", func() {
+				pods := &corev1.PodList{}
+				err = k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(pods.Items)).To(Equal(len(originalPods.Items) - 1))
+
+				pod := &corev1.Pod{}
+				err = k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: cluster.Namespace, Name: "operator-test-1-storage-1"}, pod)
+				Expect(err).To(HaveOccurred())
+				Expect(k8serrors.IsNotFound(err)).To(BeTrue())
+			})
 		})
 	})
 
@@ -133,65 +158,162 @@ var _ = Describe("delete_pods_for_buggification", func() {
 	})
 
 	Context("with a buggification that needs to be disabled", func() {
-		BeforeEach(func() {
-			pod := &corev1.Pod{}
-			err = k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: cluster.Namespace, Name: "operator-test-1-storage-1"}, pod)
-			Expect(err).NotTo(HaveOccurred())
-
-			err = k8sClient.Delete(context.TODO(), pod)
-			Expect(err).NotTo(HaveOccurred())
-
-			pod.Spec.Containers[0].Args = []string{"crash-loop"}
-			err = k8sClient.Create(context.TODO(), pod)
-			Expect(err).NotTo(HaveOccurred())
+		When("using crashLoop", func() {
+			BeforeEach(func() {
+				pod := &corev1.Pod{}
+				err = k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: cluster.Namespace, Name: "operator-test-1-storage-1"}, pod)
+				Expect(err).NotTo(HaveOccurred())
+
+				err = k8sClient.Delete(context.TODO(), pod)
+				Expect(err).NotTo(HaveOccurred())
+
+				pod.Spec.Containers[0].Args = []string{"crash-loop"}
+				err = k8sClient.Create(context.TODO(), pod)
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			It("should requeue", func() {
+				Expect(requeue).NotTo(BeNil())
+				Expect(requeue.message).To(Equal("Pods need to be recreated"))
+			})
+
+			It("should delete the pod", func() {
+				pods := &corev1.PodList{}
+				err = k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(pods.Items)).To(Equal(len(originalPods.Items) - 1))
+
+				pod := &corev1.Pod{}
+				err = k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: cluster.Namespace, Name: "operator-test-1-storage-1"}, pod)
+				Expect(err).To(HaveOccurred())
+				Expect(k8serrors.IsNotFound(err)).To(BeTrue())
+			})
 		})
 
-		It("should requeue", func() {
-			Expect(requeue).NotTo(BeNil())
-			Expect(requeue.message).To(Equal("Pods need to be recreated"))
-		})
+		When("using noSchedule", func() {
+			BeforeEach(func() {
+				pod := &corev1.Pod{}
+				err = k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: cluster.Namespace, Name: "operator-test-1-storage-1"}, pod)
+				Expect(err).NotTo(HaveOccurred())
 
-		It("should delete the pod", func() {
-			pods := &corev1.PodList{}
-			err = k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(len(pods.Items)).To(Equal(len(originalPods.Items) - 1))
+				err = k8sClient.Delete(context.TODO(), pod)
+				Expect(err).NotTo(HaveOccurred())
 
-			pod := &corev1.Pod{}
-			err = k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: cluster.Namespace, Name: "operator-test-1-storage-1"}, pod)
-			Expect(err).To(HaveOccurred())
-			Expect(k8serrors.IsNotFound(err)).To(BeTrue())
+				pod.Spec.Affinity = &corev1.Affinity{
+					NodeAffinity: &corev1.NodeAffinity{
+						RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
+							NodeSelectorTerms: []corev1.NodeSelectorTerm{
+								{
+									MatchExpressions: []corev1.NodeSelectorRequirement{
+										{
+											Key: fdbv1beta2.NodeSelectorNoScheduleLabel,
+										},
+									},
+								},
+							},
+						},
+					},
+				}
+
+				err = k8sClient.Create(context.TODO(), pod)
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			It("should requeue", func() {
+				Expect(requeue).NotTo(BeNil())
+				Expect(requeue.message).To(Equal("Pods need to be recreated"))
+			})
+
+			It("should delete the pod", func() {
+				pods := &corev1.PodList{}
+				err = k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(pods.Items)).To(Equal(len(originalPods.Items) - 1))
+
+				pod := &corev1.Pod{}
+				err = k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: cluster.Namespace, Name: "operator-test-1-storage-1"}, pod)
+				Expect(err).To(HaveOccurred())
+				Expect(k8serrors.IsNotFound(err)).To(BeTrue())
+			})
 		})
 	})
 
 	Context("with a buggification that is already on", func() {
-		BeforeEach(func() {
-			pod := &corev1.Pod{}
-			err = k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: cluster.Namespace, Name: "operator-test-1-storage-1"}, pod)
-			Expect(err).NotTo(HaveOccurred())
-
-			err = k8sClient.Delete(context.TODO(), pod)
-			Expect(err).NotTo(HaveOccurred())
-
-			pod.Spec.Containers[0].Args = []string{"crash-loop"}
-			err = k8sClient.Create(context.TODO(), pod)
-			Expect(err).NotTo(HaveOccurred())
-			cluster.Spec.Buggify.CrashLoop = []string{"storage-1"}
+		When("using crashLoop", func() {
+			BeforeEach(func() {
+				pod := &corev1.Pod{}
+				err = k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: cluster.Namespace, Name: "operator-test-1-storage-1"}, pod)
+				Expect(err).NotTo(HaveOccurred())
+
+				err = k8sClient.Delete(context.TODO(), pod)
+				Expect(err).NotTo(HaveOccurred())
+
+				pod.Spec.Containers[0].Args = []string{"crash-loop"}
+				err = k8sClient.Create(context.TODO(), pod)
+				Expect(err).NotTo(HaveOccurred())
+				cluster.Spec.Buggify.CrashLoop = []string{"storage-1"}
+			})
+
+			It("should not requeue", func() {
+				Expect(requeue).To(BeNil())
+			})
+
+			It("should not delete any pods", func() {
+				pods := &corev1.PodList{}
+				err = k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(pods.Items)).To(Equal(len(originalPods.Items)))
+
+				pod := &corev1.Pod{}
+				err = k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: cluster.Namespace, Name: "operator-test-1-storage-1"}, pod)
+				Expect(err).NotTo(HaveOccurred())
+			})
 		})
 
-		It("should not requeue", func() {
-			Expect(requeue).To(BeNil())
-		})
+		When("using noSchedule", func() {
+			BeforeEach(func() {
+				pod := &corev1.Pod{}
+				err = k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: cluster.Namespace, Name: "operator-test-1-storage-1"}, pod)
+				Expect(err).NotTo(HaveOccurred())
 
-		It("should not delete any pods", func() {
-			pods := &corev1.PodList{}
-			err = k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(len(pods.Items)).To(Equal(len(originalPods.Items)))
+				err = k8sClient.Delete(context.TODO(), pod)
+				Expect(err).NotTo(HaveOccurred())
 
-			pod := &corev1.Pod{}
-			err = k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: cluster.Namespace, Name: "operator-test-1-storage-1"}, pod)
-			Expect(err).NotTo(HaveOccurred())
+				pod.Spec.Affinity = &corev1.Affinity{
+					NodeAffinity: &corev1.NodeAffinity{
+						RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
+							NodeSelectorTerms: []corev1.NodeSelectorTerm{
+								{
+									MatchExpressions: []corev1.NodeSelectorRequirement{
+										{
+											Key: fdbv1beta2.NodeSelectorNoScheduleLabel,
+										},
+									},
+								},
+							},
+						},
+					},
+				}
+
+				err = k8sClient.Create(context.TODO(), pod)
+				Expect(err).NotTo(HaveOccurred())
+				cluster.Spec.Buggify.NoSchedule = []string{"storage-1"}
+			})
+
+			It("should not requeue", func() {
+				Expect(requeue).To(BeNil())
+			})
+
+			It("should not delete any pods", func() {
+				pods := &corev1.PodList{}
+				err = k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(pods.Items)).To(Equal(len(originalPods.Items)))
+
+				pod := &corev1.Pod{}
+				err = k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: cluster.Namespace, Name: "operator-test-1-storage-1"}, pod)
+				Expect(err).NotTo(HaveOccurred())
+			})
 		})
 	})
 
diff --git a/controllers/remove_process_groups_test.go b/controllers/remove_process_groups_test.go
index 1b5cfee5..543cccfc 100644
--- a/controllers/remove_process_groups_test.go
+++ b/controllers/remove_process_groups_test.go
@@ -328,9 +328,7 @@ var _ = Describe("remove_process_groups", func() {
 
 			When("including no process", func() {
 				It("should not include any process", func() {
-					fdbProcessesToInclude, processGroups := getProcessesToInclude(cluster, removedProcessGroups)
-					Expect(len(fdbProcessesToInclude)).To(Equal(0))
-					Expect(len(processGroups)).To(Equal(16))
+					Expect(len(getProcessesToInclude(cluster, removedProcessGroups))).To(Equal(0))
 				})
 			})
 
@@ -345,10 +343,10 @@ var _ = Describe("remove_process_groups", func() {
 				})
 
 				It("should include one process", func() {
-					fdbProcessesToInclude, processGroups := getProcessesToInclude(cluster, removedProcessGroups)
+					fdbProcessesToInclude := getProcessesToInclude(cluster, removedProcessGroups)
 					Expect(len(fdbProcessesToInclude)).To(Equal(1))
 					Expect(fdbv1beta2.ProcessAddressesString(fdbProcessesToInclude, " ")).To(Equal("1.1.1.1"))
-					Expect(len(processGroups)).To(Equal(15))
+					Expect(len(cluster.Status.ProcessGroups)).To(Equal(15))
 				})
 			})
 		})
@@ -360,27 +358,29 @@ var _ = Describe("remove_process_groups", func() {
 
 			When("including no process", func() {
 				It("should not include any process", func() {
-					fdbProcessesToInclude, processGroups := getProcessesToInclude(cluster, removedProcessGroups)
+					fdbProcessesToInclude := getProcessesToInclude(cluster, removedProcessGroups)
 					Expect(len(fdbProcessesToInclude)).To(Equal(0))
-					Expect(len(processGroups)).To(Equal(16))
+					Expect(len(cluster.Status.ProcessGroups)).To(Equal(16))
 				})
 			})
 
 			When("including one process", func() {
+				var removedProcessGroup *fdbv1beta2.ProcessGroupStatus
+
 				BeforeEach(func() {
-					processGroup := cluster.Status.ProcessGroups[0]
-					Expect(processGroup.ProcessGroupID).To(Equal("storage-1"))
-					processGroup.MarkForRemoval()
-					cluster.Status.ProcessGroups[0] = processGroup
+					removedProcessGroup = cluster.Status.ProcessGroups[0]
+					Expect(removedProcessGroup.ProcessGroupID).To(Equal("storage-1"))
+					removedProcessGroup.MarkForRemoval()
+					cluster.Status.ProcessGroups[0] = removedProcessGroup
 
-					removedProcessGroups[processGroup.ProcessGroupID] = true
+					removedProcessGroups[removedProcessGroup.ProcessGroupID] = true
 				})
 
 				It("should include one process", func() {
-					fdbProcessesToInclude, processGroups := getProcessesToInclude(cluster, removedProcessGroups)
+					fdbProcessesToInclude := getProcessesToInclude(cluster, removedProcessGroups)
 					Expect(len(fdbProcessesToInclude)).To(Equal(2))
-					Expect(fdbv1beta2.ProcessAddressesString(fdbProcessesToInclude, " ")).To(Equal(fmt.Sprintf("%s %s", cluster.Status.ProcessGroups[0].GetExclusionString(), cluster.Status.ProcessGroups[0].Addresses[0])))
-					Expect(len(processGroups)).To(Equal(15))
+					Expect(fdbv1beta2.ProcessAddressesString(fdbProcessesToInclude, " ")).To(Equal(fmt.Sprintf("%s %s", removedProcessGroup.GetExclusionString(), removedProcessGroup.Addresses[0])))
+					Expect(len(cluster.Status.ProcessGroups)).To(Equal(15))
 				})
 			})
 		})