Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1924 (2024-01-23 12:42:15)

Add additional safety checks for bouncing and excluding processes
# Description

The idea of this PR is to reduce the risk of the operator bringing a cluster into a bad/worse state. The current implementation checks for cluster availability and active generations and the bounce check validates the minimum uptime. If those thresholds are crossed those actions must be performed manually. I have not added an additional safety check for automatic replacements as those will be limited by the exclusion safety check and the number of concurrent automatic replacements.

## Type of change

- New feature (non-breaking change which adds functionality)

## Discussion

There is another PR: https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1852 which will be merged after this one.

## Testing

Ran unit tests and we will run some e2e tests (CI).

## Documentation

I will update the documentation to reflect those safety checks.

## Follow-up

We can extend those safety checks in the future if we discover the need for it.

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: ab49f8f02e96ccb2f6782b7c7a9f5a78567e3a28
* Duration 0:03:07
* Result: :x: **FAILED**
* Error: `Error while executing command: IMG=${REGISTRY}/${OPERATOR_IMAGE} make container-build container-push. Reason: exit status 2`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/0a6427ec-bdf1-47cd-9c00-ebccbe664b7d.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net) zip file of the working directory (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: ab49f8f02e96ccb2f6782b7c7a9f5a78567e3a28
* Duration 1:40:20
* Result: :x: **FAILED**
* Error: `Error while executing command: if $fail_test; then exit 1; fi. Reason: exit status 1`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/df630522-ee9b-4061-a44b-e7a2fc714ac6.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/df630522-ee9b-4061-a44b-e7a2fc714ac6/workspace.zip) zip file of the working directory (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: 1c51d772a48766999f38ec9840b3b7789a6f56d8
* Duration 1:43:17
* Result: :white_check_mark: **SUCCEEDED**
* Error: `N/A`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/f2d442a2-78cd-41aa-a07b-ad8f4389b583.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/f2d442a2-78cd-41aa-a07b-ad8f4389b583/workspace.zip) zip file of the working directory (available for 30 days)


Patch:
diff --git a/api/v1beta2/foundationdb_status.go b/api/v1beta2/foundationdb_status.go
index 7a2744c3..902664a7 100644
--- a/api/v1beta2/foundationdb_status.go
+++ b/api/v1beta2/foundationdb_status.go
@@ -129,6 +129,14 @@ type FoundationDBStatusClusterInfo struct {
 
 	// Messages represents the possible messages that are part of the cluster information.
 	Messages []FoundationDBStatusMessage `json:"messages,omitempty"`
+
+	// BounceImpact represents the bounce_impact part of the machine-readable status.
+	BounceImpact FoundationDBBounceImpact `json:"bounce_impact,omitempty"`
+}
+
+// FoundationDBBounceImpact represents the bounce_impact part of the machine-readable status.
+type FoundationDBBounceImpact struct {
+	CanCleanBounce *bool `json:"can_clean_bounce,omitempty"`
 }
 
 // FaultTolerance provides information about the fault tolerance status
diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go
index 8b2c7c8a..15ffb621 100644
--- a/api/v1beta2/zz_generated.deepcopy.go
+++ b/api/v1beta2/zz_generated.deepcopy.go
@@ -483,6 +483,26 @@ func (in *FoundationDBBackupStatusBackupDetails) DeepCopy() *FoundationDBBackupS
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FoundationDBBounceImpact) DeepCopyInto(out *FoundationDBBounceImpact) {
+	*out = *in
+	if in.CanCleanBounce != nil {
+		in, out := &in.CanCleanBounce, &out.CanCleanBounce
+		*out = new(bool)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBBounceImpact.
+func (in *FoundationDBBounceImpact) DeepCopy() *FoundationDBBounceImpact {
+	if in == nil {
+		return nil
+	}
+	out := new(FoundationDBBounceImpact)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *FoundationDBCluster) DeepCopyInto(out *FoundationDBCluster) {
 	*out = *in
@@ -1056,6 +1076,7 @@ func (in *FoundationDBStatusClusterInfo) DeepCopyInto(out *FoundationDBStatusClu
 		*out = make([]FoundationDBStatusMessage, len(*in))
 		copy(*out, *in)
 	}
+	in.BounceImpact.DeepCopyInto(&out.BounceImpact)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBStatusClusterInfo.
diff --git a/controllers/bounce_processes.go b/controllers/bounce_processes.go
index 5fbc179b..6fce0f07 100644
--- a/controllers/bounce_processes.go
+++ b/controllers/bounce_processes.go
@@ -78,19 +78,25 @@ func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReco
 
 	logger.V(1).Info("processes that can be restarted", "addresses", addresses)
 
-	if minimumUptime < float64(cluster.GetMinimumUptimeSecondsForBounce()) {
-		r.Recorder.Event(cluster, corev1.EventTypeNormal, "NeedsBounce",
-			fmt.Sprintf("Spec require a bounce of some processes, but the cluster has only been up for %f seconds", minimumUptime))
+	// Check if the cluster can safely bounce processes.
+	err = fdbstatus.CanSafelyBounceProcesses(minimumUptime, float64(cluster.GetMinimumUptimeSecondsForBounce()), status)
+	if err != nil {
+		r.Recorder.Event(cluster, corev1.EventTypeNormal, "NeedsBounce", err.Error())
 		cluster.Status.Generations.NeedsBounce = cluster.ObjectMeta.Generation
 		err = r.updateOrApply(ctx, cluster)
 		if err != nil {
 			logger.Error(err, "Error updating cluster status")
 		}
 
-		// Retry after we waited the minimum uptime
+		// Retry after we waited the minimum uptime or at least 15 seconds.
+		delayTime := cluster.GetMinimumUptimeSecondsForBounce() - int(minimumUptime)
+		if delayTime < 15 {
+			delayTime = 15
+		}
+
 		return &requeue{
-			message: "Cluster needs to stabilize before bouncing",
-			delay:   time.Second * time.Duration(cluster.GetMinimumUptimeSecondsForBounce()-int(minimumUptime)),
+			message: err.Error(),
+			delay:   time.Second * time.Duration(delayTime),
 		}
 	}
 
diff --git a/controllers/exclude_processes.go b/controllers/exclude_processes.go
index fff71be5..e465f643 100644
--- a/controllers/exclude_processes.go
+++ b/controllers/exclude_processes.go
@@ -66,57 +66,65 @@ func (e excludeProcesses) reconcile(_ context.Context, r *FoundationDBClusterRec
 	logger.Info("current exclusions", "exclusions", exclusions)
 	fdbProcessesToExcludeByClass, ongoingExclusionsByClass := getProcessesToExclude(exclusions, cluster)
 
-	if len(fdbProcessesToExcludeByClass) > 0 {
-		var fdbProcessesToExclude []fdbv1beta2.ProcessAddress
-		desiredProcesses, err := cluster.GetProcessCountsWithDefaults()
-		if err != nil {
-			return &requeue{curError: err, delayedRequeue: true}
-		}
-
-		desiredProcessesMap := desiredProcesses.Map()
-		for processClass := range fdbProcessesToExcludeByClass {
-			contextLogger := logger.WithValues("processClass", processClass)
-			ongoingExclusions := ongoingExclusionsByClass[processClass]
-			processesToExclude := fdbProcessesToExcludeByClass[processClass]
+	// No processes have to be excluded we can directly return.
+	if len(fdbProcessesToExcludeByClass) == 0 {
+		return nil
+	}
 
-			allowedExclusions, missingProcesses := getAllowedExclusionsAndMissingProcesses(contextLogger, cluster, processClass, desiredProcessesMap[processClass], ongoingExclusions, r.InSimulation)
-			if allowedExclusions <= 0 {
-				contextLogger.Info("Waiting for missing processes before continuing with the exclusion", "missingProcesses", missingProcesses, "addressesToExclude", processesToExclude, "allowedExclusions", allowedExclusions, "ongoingExclusions", ongoingExclusions)
-				continue
-			}
+	// Make sure it's safe to exclude processes.
+	err = fdbstatus.CanSafelyExcludeProcesses(status)
+	if err != nil {
+		return &requeue{curError: err, delayedRequeue: true}
+	}
 
-			// If we are not able to exclude all processes at once print a log message.
-			if len(processesToExclude) > allowedExclusions {
-				contextLogger.Info("Some processes are still missing but continuing with the exclusion", "missingProcesses", missingProcesses, "addressesToExclude", processesToExclude, "allowedExclusions", allowedExclusions, "ongoingExclusions", ongoingExclusions)
-			}
+	var fdbProcessesToExclude []fdbv1beta2.ProcessAddress
+	desiredProcesses, err := cluster.GetProcessCountsWithDefaults()
+	if err != nil {
+		return &requeue{curError: err, delayedRequeue: true}
+	}
 
-			if len(processesToExclude) < allowedExclusions {
-				allowedExclusions = len(processesToExclude)
-			}
+	desiredProcessesMap := desiredProcesses.Map()
+	for processClass := range fdbProcessesToExcludeByClass {
+		contextLogger := logger.WithValues("processClass", processClass)
+		ongoingExclusions := ongoingExclusionsByClass[processClass]
+		processesToExclude := fdbProcessesToExcludeByClass[processClass]
 
-			// TODO: As a next step we could exclude transaction (log + stateless) processes together and exclude
-			// storage processes with a separate call. This would make sure that no storage checks will block
-			// the exclusion of transaction processes.
+		allowedExclusions, missingProcesses := getAllowedExclusionsAndMissingProcesses(contextLogger, cluster, processClass, desiredProcessesMap[processClass], ongoingExclusions, r.InSimulation)
+		if allowedExclusions <= 0 {
+			contextLogger.Info("Waiting for missing processes before continuing with the exclusion", "missingProcesses", missingProcesses, "addressesToExclude", processesToExclude, "allowedExclusions", allowedExclusions, "ongoingExclusions", ongoingExclusions)
+			continue
+		}
 
-			// Add as many processes as allowed to the exclusion list.
-			fdbProcessesToExclude = append(fdbProcessesToExclude, processesToExclude[:allowedExclusions]...)
+		// If we are not able to exclude all processes at once print a log message.
+		if len(processesToExclude) > allowedExclusions {
+			contextLogger.Info("Some processes are still missing but continuing with the exclusion", "missingProcesses", missingProcesses, "addressesToExclude", processesToExclude, "allowedExclusions", allowedExclusions, "ongoingExclusions", ongoingExclusions)
 		}
 
-		if len(fdbProcessesToExclude) == 0 {
-			return &requeue{
-				message:        "more exclusions needed but not allowed, have to wait for new processes to come up",
-				delayedRequeue: true,
-			}
+		if len(processesToExclude) < allowedExclusions {
+			allowedExclusions = len(processesToExclude)
 		}
 
-		r.Recorder.Event(cluster, corev1.EventTypeNormal, "ExcludingProcesses", fmt.Sprintf("Excluding %v", fdbProcessesToExclude))
+		// TODO: As a next step we could exclude transaction (log + stateless) processes together and exclude
+		// storage processes with a separate call. This would make sure that no storage checks will block
+		// the exclusion of transaction processes.
 
-		err = adminClient.ExcludeProcesses(fdbProcessesToExclude)
-		if err != nil {
-			return &requeue{curError: err, delayedRequeue: true}
+		// Add as many processes as allowed to the exclusion list.
+		fdbProcessesToExclude = append(fdbProcessesToExclude, processesToExclude[:allowedExclusions]...)
+	}
+
+	if len(fdbProcessesToExclude) == 0 {
+		return &requeue{
+			message:        "more exclusions needed but not allowed, have to wait for new processes to come up",
+			delayedRequeue: true,
 		}
 	}
 
+	r.Recorder.Event(cluster, corev1.EventTypeNormal, "ExcludingProcesses", fmt.Sprintf("Excluding %v", fdbProcessesToExclude))
+	err = adminClient.ExcludeProcesses(fdbProcessesToExclude)
+	if err != nil {
+		return &requeue{curError: err, delayedRequeue: true}
+	}
+
 	return nil
 }
 
diff --git a/pkg/fdbstatus/status_checks.go b/pkg/fdbstatus/status_checks.go
index 205469bb..10062a7a 100644
--- a/pkg/fdbstatus/status_checks.go
+++ b/pkg/fdbstatus/status_checks.go
@@ -71,22 +71,9 @@ func getRemainingAndExcludedFromStatus(logger logr.Logger, status *fdbv1beta2.Fo
 	// the cluster status information as only the latest log processes will have the log process role. If we don't check
 	// for the active generations we have the risk to remove a log process that still has mutations on it that must be
 	// popped.
-	if status.Cluster.RecoveryState.ActiveGenerations > 1 {
-		logger.Info("Skipping exclusion check as there are multiple active generations", "activeGenerations", status.Cluster.RecoveryState.ActiveGenerations)
-		return exclusionStatus{
-			inProgress:      nil,
-			fullyExcluded:   nil,
-			notExcluded:     addresses,
-			missingInStatus: nil,
-		}
-	}
-
-	// If the database is unavailable the status might contain the processes but with missing role information. In this
-	// case we should not perform any validation on the machine-readable status as this information might not be correct.
-	// We don't want to run the exclude command to check for the status of the exclusions as this might result in a recovery
-	// of the cluster or brings the cluster into a worse state.
-	if !status.Client.DatabaseStatus.Available {
-		logger.Info("Skipping exclusion check as the database is unavailable")
+	err := DefaultSafetyChecks(status, 1, "check exclusion status")
+	if err != nil {
+		logger.Info("Skipping exclusion check as there are issues with the machine-readable status", "error", err.Error())
 		return exclusionStatus{
 			inProgress:      nil,
 			fullyExcluded:   nil,
@@ -459,3 +446,51 @@ func HasDesiredFaultToleranceFromStatus(log logr.Logger, status *fdbv1beta2.Foun
 
 	return true
 }
+
+// DefaultSafetyChecks performs a set of default safety checks, e.g. it checks if the cluster is available from the
+// client perspective and it checks that there are not too many active generations.
+func DefaultSafetyChecks(status *fdbv1beta2.FoundationDBStatus, maximumActiveGenerations int, action string) error {
+	// If there are more than 10 active generations we should not allow the cluster to bounce processes as this could
+	// cause another recovery increasing the active generations. In general the active generations should be at 1 during
+	// normal operations.
+	if status.Cluster.RecoveryState.ActiveGenerations > maximumActiveGenerations {
+		return fmt.Errorf("cluster has %d active generations, but only %d active generations are allowed to safely %s", status.Cluster.RecoveryState.ActiveGenerations, maximumActiveGenerations, action)
+	}
+
+	// If the database is unavailable we shouldn't perform any action on the cluster.
+	if !status.Client.DatabaseStatus.Available {
+		return fmt.Errorf("cluster is unavailable, cannot %s", action)
+	}
+
+	return nil
+}
+
+// CanSafelyBounceProcesses returns nil when it is safe to do a bounce on the cluster or returns an error with more information
+// why it's not safe to bounce processes in the cluster.
+func CanSafelyBounceProcesses(currentUptime float64, minimumUptime float64, status *fdbv1beta2.FoundationDBStatus) error {
+	err := DefaultSafetyChecks(status, 10, "bounce processes")
+	if err != nil {
+		return err
+	}
+
+	// If the current uptime of the cluster is below the minimum uptime we should not allow to bounce processes. This is
+	// a safeguard to reduce the risk of repeated bounces in a short timeframe.
+	if currentUptime < minimumUptime {
+		return fmt.Errorf("cluster has only been up for %.2f seconds, but must be up for %.2f seconds to safely bounce", minimumUptime, minimumUptime)
+	}
+
+	// If the machine-readable status reports that a clean bounce is not possible, we shouldn't perform a bounce. This
+	// value will be false if the cluster is not fully recovered:
+	// https://github.com/apple/foundationdb/blob/7.3.29/fdbserver/Status.actor.cpp#L437-L448
+	if status.Cluster.BounceImpact.CanCleanBounce != nil && !*status.Cluster.BounceImpact.CanCleanBounce {
+		return fmt.Errorf("cannot perform a clean bounce based on cluster status, current recovery state: %s", status.Cluster.RecoveryState.Name)
+	}
+
+	return nil
+}
+
+// CanSafelyExcludeProcesses currently performs the DefaultSafetyChecks. In the future this check might be extended to
+// perform more specific checks.
+func CanSafelyExcludeProcesses(status *fdbv1beta2.FoundationDBStatus) error {
+	return DefaultSafetyChecks(status, 10, "exclude processes")
+}
diff --git a/setup/setup.go b/setup/setup.go
index be704f2b..832a8c64 100644
--- a/setup/setup.go
+++ b/setup/setup.go
@@ -25,16 +25,20 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"log"
 	"os"
 	"path"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/go-logr/logr"
 
-	"github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	"github.com/FoundationDB/fdb-kubernetes-operator/controllers"
 	"github.com/FoundationDB/fdb-kubernetes-operator/fdbclient"
 	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
@@ -160,6 +164,38 @@ func StartManager(
 	fdbclient.DefaultCLITimeout = time.Duration(operatorOpts.CliTimeout) * time.Second
 	fdbclient.MaxCliTimeout = time.Duration(operatorOpts.MaxCliTimeout) * time.Second
 
+	// Define the cache options for the client cache used by the operator. If no label selector is defined, the
+	// default cache configuration will be used.
+	cacheOptions := cache.Options{}
+	// Only if a label selector is defined we have to update the cache options.
+	if operatorOpts.LabelSelector != "" {
+		// Parse the label selector, if the label selector is not parsable panic.
+		labelSelector, parseErr := labels.Parse(operatorOpts.LabelSelector)
+		if parseErr != nil {
+			log.Fatalf("could not parse label selector: %s, got error: %s", operatorOpts.LabelSelector, parseErr)
+		}
+
+		selector := cache.ObjectSelector{
+			Label: labelSelector,
+		}
+
+		// Set the label selector for all resources that the operator manages, this should reduce the resources that
+		// are cached by the operator if a label selector is provided.s
+		cacheOptions.SelectorsByObject = map[client.Object]cache.ObjectSelector{
+			&fdbv1beta2.FoundationDBCluster{}: selector,
+			&corev1.Pod{}:                     selector,
+			&corev1.PersistentVolumeClaim{}:   selector,
+			&corev1.ConfigMap{}:               selector,
+			&corev1.Service{}:                 selector,
+			&appsv1.Deployment{}:              selector,
+		}
+
+		// Make sure we set the label selector for any additional watched objects.
+		for _, object := range watchedObjects {
+			cacheOptions.SelectorsByObject[object] = selector
+		}
+	}
+
 	options := ctrl.Options{
 		Scheme:             scheme,
 		MetricsBindAddress: operatorOpts.MetricsAddr,
@@ -169,11 +205,13 @@ func StartManager(
 		RenewDeadline:      &operatorOpts.RenewDeadline,
 		RetryPeriod:        &operatorOpts.RetryPeriod,
 		Port:               9443,
+		NewCache:           cache.BuilderWithOptions(cacheOptions),
 	}
 
 	if operatorOpts.WatchNamespace != "" {
 		options.Namespace = operatorOpts.WatchNamespace
 		setupLog.Info("Operator starting in single namespace mode", "namespace", options.Namespace)
+		cacheOptions.Namespace = operatorOpts.WatchNamespace
 	} else {
 		setupLog.Info("Operator starting in Global mode")
 	}
@@ -288,8 +326,8 @@ func moveFDBBinaries(log logr.Logger) error {
 	}
 
 	for _, binEntry := range binDir {
-		if binEntry.IsDir() && v1beta2.VersionRegex.Match([]byte(binEntry.Name())) {
-			version, err := v1beta2.ParseFdbVersion(binEntry.Name())
+		if binEntry.IsDir() && fdbv1beta2.VersionRegex.Match([]byte(binEntry.Name())) {
+			version, err := fdbv1beta2.ParseFdbVersion(binEntry.Name())
 			if err != nil {
 				return err
 			}


Test patch:
diff --git a/api/v1beta2/foundationdb_status_test.go b/api/v1beta2/foundationdb_status_test.go
index 0105b81a..56f4b669 100644
--- a/api/v1beta2/foundationdb_status_test.go
+++ b/api/v1beta2/foundationdb_status_test.go
@@ -22,6 +22,7 @@ package v1beta2
 
 import (
 	"encoding/json"
+	"k8s.io/utils/pointer"
 	"net"
 	"os"
 	"path/filepath"
@@ -906,6 +907,9 @@ var _ = Describe("FoundationDBStatus", func() {
 				SecondsSinceLastRecovered: 76.8155,
 			},
 			Generation: 2,
+			BounceImpact: FoundationDBBounceImpact{
+				CanCleanBounce: pointer.Bool(true),
+			},
 		}
 
 		It("should parse all values correctly", func() {
diff --git a/pkg/fdbstatus/status_checks_test.go b/pkg/fdbstatus/status_checks_test.go
index 74a46406..aa9a380f 100644
--- a/pkg/fdbstatus/status_checks_test.go
+++ b/pkg/fdbstatus/status_checks_test.go
@@ -21,7 +21,9 @@
 package fdbstatus
 
 import (
+	"fmt"
 	"github.com/go-logr/logr"
+	"k8s.io/utils/pointer"
 	"net"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
@@ -1730,4 +1732,279 @@ var _ = Describe("status_checks", func() {
 			},
 			false)
 	})
+
+	When("performing the default safety check.", func() {
+		DescribeTable("should return if the safety check is satisfied or not",
+			func(status *fdbv1beta2.FoundationDBStatus, maximumActiveGeneration int, expected error) {
+				err := DefaultSafetyChecks(status, maximumActiveGeneration, "test")
+				if expected == nil {
+					Expect(err).To(BeNil())
+				} else {
+					Expect(err).To(HaveOccurred())
+					Expect(err).To(Equal(expected))
+				}
+			},
+			Entry("cluster is fully reconciled",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						RecoveryState: fdbv1beta2.RecoveryState{
+							ActiveGenerations: 1,
+						},
+					},
+				},
+				1,
+				nil,
+			),
+			Entry("cluster is unavailable",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: false,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						RecoveryState: fdbv1beta2.RecoveryState{
+							ActiveGenerations: 1,
+						},
+					},
+				},
+				1,
+				fmt.Errorf("cluster is unavailable, cannot test"),
+			),
+			Entry("cluster has too many active generations",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						RecoveryState: fdbv1beta2.RecoveryState{
+							ActiveGenerations: 10,
+						},
+					},
+				},
+				1,
+				fmt.Errorf("cluster has 10 active generations, but only 1 active generations are allowed to safely test"),
+			),
+			Entry("cluster has more than one active generations",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						RecoveryState: fdbv1beta2.RecoveryState{
+							ActiveGenerations: 10,
+						},
+					},
+				},
+				10,
+				nil,
+			),
+		)
+	})
+
+	When("performing the exclude safety check.", func() {
+		DescribeTable("should return if the safety check is satisfied or not",
+			func(status *fdbv1beta2.FoundationDBStatus, expected error) {
+				err := CanSafelyExcludeProcesses(status)
+				if expected == nil {
+					Expect(err).To(BeNil())
+				} else {
+					Expect(err).To(HaveOccurred())
+					Expect(err).To(Equal(expected))
+				}
+			},
+			Entry("cluster is fully reconciled",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						RecoveryState: fdbv1beta2.RecoveryState{
+							ActiveGenerations: 1,
+						},
+					},
+				},
+				nil,
+			),
+			Entry("cluster is unavailable",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: false,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						RecoveryState: fdbv1beta2.RecoveryState{
+							ActiveGenerations: 1,
+						},
+					},
+				},
+				fmt.Errorf("cluster is unavailable, cannot exclude processes"),
+			),
+			Entry("cluster has more than one active generations",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						RecoveryState: fdbv1beta2.RecoveryState{
+							ActiveGenerations: 10,
+						},
+					},
+				},
+				nil,
+			),
+			Entry("cluster has more than one active generations",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						RecoveryState: fdbv1beta2.RecoveryState{
+							ActiveGenerations: 11,
+						},
+					},
+				},
+				fmt.Errorf("cluster has 11 active generations, but only 10 active generations are allowed to safely exclude processes"),
+			),
+		)
+	})
+
+	When("performing the bounce safety check.", func() {
+		DescribeTable("should return if the safety check is satisfied or not",
+			func(status *fdbv1beta2.FoundationDBStatus, currentUptime float64, minimumUptime float64, expected error) {
+				err := CanSafelyBounceProcesses(currentUptime, minimumUptime, status)
+				if expected == nil {
+					Expect(err).NotTo(HaveOccurred())
+				} else {
+					Expect(err).To(HaveOccurred())
+					Expect(err).To(Equal(expected))
+				}
+			},
+			Entry("cluster is fully reconciled",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						BounceImpact: fdbv1beta2.FoundationDBBounceImpact{
+							CanCleanBounce: pointer.Bool(true),
+						},
+						RecoveryState: fdbv1beta2.RecoveryState{
+							ActiveGenerations: 1,
+						},
+					},
+				},
+				42.0,
+				10.0,
+				nil,
+			),
+			Entry("cluster is unavailable",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: false,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						RecoveryState: fdbv1beta2.RecoveryState{
+							ActiveGenerations: 1,
+						},
+					},
+				},
+				42.0,
+				10.0,
+				fmt.Errorf("cluster is unavailable, cannot bounce processes"),
+			),
+			Entry("cluster has more than one active generations",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						RecoveryState: fdbv1beta2.RecoveryState{
+							ActiveGenerations: 10,
+						},
+					},
+				},
+				42.0,
+				10.0,
+				nil,
+			),
+			Entry("cluster has more than one active generations",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						RecoveryState: fdbv1beta2.RecoveryState{
+							ActiveGenerations: 11,
+						},
+					},
+				},
+				42.0,
+				10.0,
+				fmt.Errorf("cluster has 11 active generations, but only 10 active generations are allowed to safely bounce processes"),
+			),
+			Entry("cluster is not up for long enough",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						RecoveryState: fdbv1beta2.RecoveryState{
+							ActiveGenerations: 1,
+						},
+					},
+				},
+				42.0,
+				60.0,
+				fmt.Errorf("cluster has only been up for 60.00 seconds, but must be up for 60.00 seconds to safely bounce"),
+			),
+			Entry("cluster cannot clean bounce",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						BounceImpact: fdbv1beta2.FoundationDBBounceImpact{
+							CanCleanBounce: pointer.Bool(false),
+						},
+						RecoveryState: fdbv1beta2.RecoveryState{
+							ActiveGenerations: 1,
+							Name:              "exploding",
+						},
+					},
+				},
+				42.0,
+				10.0,
+				fmt.Errorf("cannot perform a clean bounce based on cluster status, current recovery state: exploding"),
+			),
+		)
+	})
 })