Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1711 (2023-06-27 18:15:21)

Replacement logic should ignore process groups that are in maintenance mode
# Description

Modify operator replacement logic to ignore process groups that are in maintenance mode.

## Type of change

*Please select one of the options below.*

- New feature (non-breaking change which adds functionality)

## Discussion


## Testing

Manual testing.

## Documentation


## Follow-up



foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: efaa4ae5098fb90287e3314050941fadb32ee0f4
* Duration 4:06:52
* Result: :x: **FAILED**
* Error: `Error while executing command: if $fail_test; then exit 1; fi. Reason: exit status 1`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/6e28bcbf-8447-4ced-a1b4-29f2a540b252.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/6e28bcbf-8447-4ced-a1b4-29f2a540b252/workspace.zip) zip file of the working directory (available for 30 days)


Patch:
diff --git a/api/v1beta2/foundationdb_status.go b/api/v1beta2/foundationdb_status.go
index 4044a14b..cfbcabf7 100644
--- a/api/v1beta2/foundationdb_status.go
+++ b/api/v1beta2/foundationdb_status.go
@@ -98,6 +98,9 @@ type FoundationDBStatusClusterInfo struct {
 	// Logs provides information about log processes running in the cluster.
 	Logs []FoundationDBStatusLogInfo `json:"logs,omitempty"`
 
+	// Qos provides information about various qos metrics of the cluster.
+	Qos FoundationDBStatusQosInfo `json:"qos,omitempty"`
+
 	// FaultTolerance provides information about the fault tolerance status
 	// of the cluster.
 	FaultTolerance FaultTolerance `json:"fault_tolerance,omitempty"`
@@ -336,6 +339,20 @@ type FoundationDBStatusLogInfo struct {
 	SatelliteLogReplicationFactor int  `json:"satellite_log_replication_factor,omitempty"`
 }
 
+// FoundationDBStatusLagInfo provides information about the lag being experienced by a storage
+// server in the cluster.
+type FoundationDBStatusLagInfo struct {
+	Seconds  float64 `json:"seconds,omitempty"`
+	Versions int64   `json:"versions,omitempty"`
+}
+
+// FoundationDBStatusQosInfo provides information about various qos metrics of the cluster.
+type FoundationDBStatusQosInfo struct {
+	LimitingDurabilityLagStorageServer FoundationDBStatusLagInfo `json:"limiting_durability_lag_storage_server,omitempty"`
+	WorstDataLagStorageServer          FoundationDBStatusLagInfo `json:"worst_data_lag_storage_server,omitempty"`
+	WorstDurabilityLagStorageServer    FoundationDBStatusLagInfo `json:"worst_durability_lag_storage_server,omitempty"`
+}
+
 // ProcessRole models the role of a pod.
 type ProcessRole string
 
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index 3bdf8c0b..f88a8a8b 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -746,6 +746,11 @@ func (processGroupStatus *ProcessGroupStatus) GetConditionTime(conditionType Pro
 	return nil
 }
 
+// IsUnderMaintenance checks if the process is in maintenance zone.
+func (processGroupStatus *ProcessGroupStatus) IsUnderMaintenance(maintenanceZone FaultDomain) bool {
+	return processGroupStatus.FaultDomain == maintenanceZone
+}
+
 // GetCondition returns the ProcessGroupStatus's ProcessGroupCondition that matches the conditionType;
 // It returns nil if the ProcessGroupStatus doesn't have a matching condition
 func (processGroupStatus *ProcessGroupStatus) GetCondition(conditionType ProcessGroupConditionType) *ProcessGroupCondition {
@@ -1042,6 +1047,7 @@ type FoundationDBClusterAutomationOptions struct {
 	MaintenanceModeOptions MaintenanceModeOptions `json:"maintenanceModeOptions,omitempty"`
 
 	// IgnoreLogGroupsForUpgrade defines the list of LogGroups that should be ignored during fdb version upgrade.
+	// The default is a list that includes "fdb-kubernetes-operator".
 	// +kubebuilder:validation:MaxItems=10
 	IgnoreLogGroupsForUpgrade []LogGroup `json:"ignoreLogGroupsForUpgrade,omitempty"`
 }
@@ -2638,3 +2644,14 @@ func (cluster *FoundationDBCluster) GetMaxZonesWithUnavailablePods() int {
 func (cluster *FoundationDBCluster) CacheDatabaseStatusForReconciliation(defaultValue bool) bool {
 	return pointer.BoolDeref(cluster.Spec.AutomationOptions.CacheDatabaseStatusForReconciliation, defaultValue)
 }
+
+// GetIgnoreLogGroupsForUpgrade will return the IgnoreLogGroupsForUpgrade, if the value is not set it will include the default `fdb-kubernetes-operator`
+// LogGroup.
+func (cluster *FoundationDBCluster) GetIgnoreLogGroupsForUpgrade() []LogGroup {
+	if len(cluster.Spec.AutomationOptions.IgnoreLogGroupsForUpgrade) > 0 {
+		return cluster.Spec.AutomationOptions.IgnoreLogGroupsForUpgrade
+	}
+
+	// Should we better read the FDB_NETWORK_OPTION_TRACE_LOG_GROUP env variable?
+	return []LogGroup{"fdb-kubernetes-operator"}
+}
diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go
index a399cabf..555fb880 100644
--- a/api/v1beta2/zz_generated.deepcopy.go
+++ b/api/v1beta2/zz_generated.deepcopy.go
@@ -1038,6 +1038,7 @@ func (in *FoundationDBStatusClusterInfo) DeepCopyInto(out *FoundationDBStatusClu
 		*out = make([]FoundationDBStatusLogInfo, len(*in))
 		copy(*out, *in)
 	}
+	out.Qos = in.Qos
 	out.FaultTolerance = in.FaultTolerance
 	if in.IncompatibleConnections != nil {
 		in, out := &in.IncompatibleConnections, &out.IncompatibleConnections
@@ -1147,6 +1148,21 @@ func (in *FoundationDBStatusDataStatistics) DeepCopy() *FoundationDBStatusDataSt
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FoundationDBStatusLagInfo) DeepCopyInto(out *FoundationDBStatusLagInfo) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBStatusLagInfo.
+func (in *FoundationDBStatusLagInfo) DeepCopy() *FoundationDBStatusLagInfo {
+	if in == nil {
+		return nil
+	}
+	out := new(FoundationDBStatusLagInfo)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *FoundationDBStatusLayerInfo) DeepCopyInto(out *FoundationDBStatusLayerInfo) {
 	*out = *in
@@ -1273,6 +1289,24 @@ func (in *FoundationDBStatusProcessRoleInfo) DeepCopy() *FoundationDBStatusProce
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FoundationDBStatusQosInfo) DeepCopyInto(out *FoundationDBStatusQosInfo) {
+	*out = *in
+	out.LimitingDurabilityLagStorageServer = in.LimitingDurabilityLagStorageServer
+	out.WorstDataLagStorageServer = in.WorstDataLagStorageServer
+	out.WorstDurabilityLagStorageServer = in.WorstDurabilityLagStorageServer
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBStatusQosInfo.
+func (in *FoundationDBStatusQosInfo) DeepCopy() *FoundationDBStatusQosInfo {
+	if in == nil {
+		return nil
+	}
+	out := new(FoundationDBStatusQosInfo)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *FoundationDBStatusSupportedVersion) DeepCopyInto(out *FoundationDBStatusSupportedVersion) {
 	*out = *in
diff --git a/controllers/check_client_compatibility.go b/controllers/check_client_compatibility.go
index 37418049..e51a95d7 100644
--- a/controllers/check_client_compatibility.go
+++ b/controllers/check_client_compatibility.go
@@ -85,7 +85,7 @@ func (c checkClientCompatibility) reconcile(ctx context.Context, r *FoundationDB
 	}
 
 	ignoredLogGroups := make(map[fdbv1beta2.LogGroup]fdbv1beta2.None)
-	for _, logGroup := range cluster.Spec.AutomationOptions.IgnoreLogGroupsForUpgrade {
+	for _, logGroup := range cluster.GetIgnoreLogGroupsForUpgrade() {
 		ignoredLogGroups[logGroup] = fdbv1beta2.None{}
 	}
 	unsupportedClients := getUnsupportedClients(status.Cluster.Clients.SupportedVersions, protocolVersion, ignoredLogGroups)
diff --git a/controllers/replace_failed_process_groups.go b/controllers/replace_failed_process_groups.go
index ff2f324a..813fe184 100644
--- a/controllers/replace_failed_process_groups.go
+++ b/controllers/replace_failed_process_groups.go
@@ -59,7 +59,7 @@ func (c replaceFailedProcessGroups) reconcile(ctx context.Context, r *Foundation
 
 	// Only replace process groups without an address, if the cluster has the desired fault tolerance and is available.
 	hasDesiredFaultTolerance := fdbstatus.HasDesiredFaultToleranceFromStatus(logger, status, cluster)
-	if replacements.ReplaceFailedProcessGroups(logger, cluster, hasDesiredFaultTolerance) {
+	if replacements.ReplaceFailedProcessGroups(logger, cluster, status, hasDesiredFaultTolerance) {
 		err := r.updateOrApply(ctx, cluster)
 		if err != nil {
 			return &requeue{curError: err}
diff --git a/docs/changelog/v1.20.0.md b/docs/changelog/v1.20.0.md
new file mode 100644
index 00000000..90b1a331
--- /dev/null
+++ b/docs/changelog/v1.20.0.md
@@ -0,0 +1,25 @@
+# v1.20.0
+
+## Changes
+
+### Operator
+
+* Correct the way how the e2e test framework checks the username [#1717](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1717)
+* Expose methods to create a cluster based on the generated spec [#1716](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1716)
+* Add a method to the e2e factory to collect logs for a Pod [#1715](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1715)
+* Document upgrades for 7.1+ and add the default log group to the ignore list [#1713](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1713)
+* Add some more logging if exclusions check is skipped [#1709](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1709)
+* Add data lag metrics from machine-readable status [#1708](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1708)
+* Run kill command once without back off to reduce risk of multiple recoveries [#1706](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1706)
+* Improve the logging for the cluster controller to make sure all fields are set [#1705](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1705)
+* Make sure we wait some time before checking the incompatible connections [#1704](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1704)
+* Update chaos-mesh and the controller runtime version [#1701](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1701)
+* Make sure mock k8s client implements controller runtime client [#1700](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1700)
+* Allow to mock an error in the mock client [#1699](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1699)
+* Fix panic in analyze command if cluster is not found [#1697](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1697)
+* Fix test case for version compatible upgrades [#1696](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1696)
+* Add prefix factory option [#1695](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1695)
+* Disable availability check for unstable network test [#1692](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1692)
+* Move status checks dedicated package [#1691](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1691)
+* Do fault tolerance checks on status object [#1689](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1689)
+* Update taint feature design doc and user manual [#1632](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1632)
diff --git a/docs/changelog/v1.20.1.md b/docs/changelog/v1.20.1.md
new file mode 100644
index 00000000..dae8e13d
--- /dev/null
+++ b/docs/changelog/v1.20.1.md
@@ -0,0 +1,7 @@
+# v1.20.1
+
+## Changes
+
+### Operator
+
+* Fix exclusion logic multiple processes [#1721](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1721)
diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
index c8018711..4a56bb58 100644
--- a/docs/cluster_spec.md
+++ b/docs/cluster_spec.md
@@ -194,7 +194,7 @@ FoundationDBClusterAutomationOptions provides flags for enabling or disabling op
 | podUpdateStrategy | PodUpdateStrategy defines how Pod spec changes are rolled out either by replacing Pods or by deleting Pods. The default for this is ReplaceTransactionSystem. | [PodUpdateStrategy](#podupdatestrategy) | false |
 | useManagementAPI | UseManagementAPI defines if the operator should make use of the management API instead of using fdbcli to interact with the FoundationDB cluster. | *bool | false |
 | maintenanceModeOptions | MaintenanceModeOptions contains options for maintenance mode related settings. | [MaintenanceModeOptions](#maintenancemodeoptions) | false |
-| ignoreLogGroupsForUpgrade | IgnoreLogGroupsForUpgrade defines the list of LogGroups that should be ignored during fdb version upgrade. | [][LogGroup](#loggroup) | false |
+| ignoreLogGroupsForUpgrade | IgnoreLogGroupsForUpgrade defines the list of LogGroups that should be ignored during fdb version upgrade. The default is a list that includes \"fdb-kubernetes-operator\". | [][LogGroup](#loggroup) | false |
 
 [Back to TOC](#table-of-contents)
 
diff --git a/docs/design/node_taint_eviction.md b/docs/design/node_taint_eviction.md
index deb80871..5481b39d 100644
--- a/docs/design/node_taint_eviction.md
+++ b/docs/design/node_taint_eviction.md
@@ -2,36 +2,33 @@
 
 ## Metadata
 
-* Authors: @johscheuer
+* Authors: @johscheuer, @xumengpanda
 * Created: 2023-02-14
-* Updated: 2023-03-08
+* Updated: 2023-05-11
 
 ## Background
 
 The operator is currently able to detect failures of Pods and replace them if required, the replacement is happening based on the [conditions that need replacement](https://github.com/FoundationDB/fdb-kubernetes-operator/blob/v1.14.0/api/v1beta2/foundationdbcluster_types.go#L65).
-One of the missing pieces in failure mitigation is to evict Pods from tainted nodes.
+One of the missing pieces in failure mitigation is to evict Pods from tainted Nodes.
 In Kubernetes it's common to use [taints](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration) to indicate potential problems with a node or planned maintenance.
-This information could be consumed by the operator to proactively replace Pods that are running on nodes with a given set of taints.
+This information could be consumed by the operator to proactively replace Pods that are running on Nodes with a given set of taints.
 
 ## General Design Goals
 
-The goal of this design is to implement a mechanism that allows the operator to replace Pods that are running on nodes with a given set of taints.
-It's expected that the operator will detect the Pods running on tainted nodes in a timely manner, either by receiving an event triggering the reconciliation loop or when running the periodic reconciliation, which will happen every 10h per default.
-The actual replacement of all Pods running on tainted nodes, depends on the data that must be replicated and the number of affected Pods.
+The goal of this design is to implement a mechanism that allows the operator to replace Pods that are running on Nodes with a given set of taints.
+It's expected that the operator will detect the Pods running on tainted Nodes in a timely manner, either by receiving an event triggering the reconciliation loop or by periodic reconciliation events that happen every 10h by default. Once the operator marks Pods on tainted Nodes as `NodeTaintReplacing`, the operator will re-use the replacement logic to replace those Pods. The actual replacement speed depends on the speed of excluding processes, the number of Pods to replace, and the allowed maximum number of inflight replacements (defined by `automationOptions.replacements.maxConcurrentReplacements`).
 
 ## Current Implementation
+The operator does not have this feature. It does not have access to list Node information. It does not consider taint-related events in Pod's condition.
 
-We don't have a current implementation of this and the operator is currently not able to access any node information.
-The closest implementation to this is the [Replace Failed ProcessGroups reconciler](https://github.com/FoundationDB/fdb-kubernetes-operator/blob/main/controllers/replace_failed_process_groups.go), which could be extended for this.
+The operator does replace failed ProcessGroups in the [Replace Failed ProcessGroups reconciler](https://github.com/FoundationDB/fdb-kubernetes-operator/blob/main/controllers/replace_failed_process_groups.go). It can extend it to support this feature.
 
 ## Proposed Design
 
-We have to make sure that this feature can be enabled and disabled, e.g. if someone wants to run the operator inside a cluster without node access.
-The `AutomaticReplacementOptions` will be extended to support the configuration of replacements for Pods that run on tainted nodes.
-The new field will be called `taints` and will contain a list, the default would be an empty list, of taint configurations.
-That configuration will include the taint key, this has to be an exact match or a wildcard `*`, and the duration this taint must be present on the node.
-The `durationInSeconds` allow to specify how sensitive the operator should be to taints.
-If a wildcard is specified as key this will be used as default value, except there is an exact match defined for a taint key.
+### Design Requirements
+**Requirement 1.** The feature can be enabled and disabled by users, because (1) when someone wants to run the operator inside a k8s cluster without Node access, they should be able to bypass this feature; (2) when we notice problems during roll-out of this feature, we need to disable it.
+
+The feature can be disabled by option `automationOptions.replacements.taintReplacementOptions = {}`. YAML example is as follows. Please note that `taintReplacementOptions` section is deleted.
 
 ```yaml
 ...
@@ -40,53 +37,56 @@ automaticReplacementOptions:
   failureDetectionTimeSeconds: 7200
   maxConcurrentReplacements: 1
   enableNodeFailureDetection: true
-  taints:
-  - key: "example.org/maintenance"
-    durationInSeconds: 7200 # Ensure the taint is present for at least 2 hours before replacing Pods on a node with this taint
-  - key: "*" # The wildcard would allow to define a catch all configuration
-    durationInSeconds: 3600 # Ensure the taint is present for at least 1 hour before replacing Pods on a node with this taint
 ```
 
-We will add a new [ProcessGroupConditionType](https://github.com/FoundationDB/fdb-kubernetes-operator/blob/main/api/v1beta2/foundationdbcluster_types.go#L673), called `RunningOnTaintedNode` to mark process groups that are running on a tainted node.
-In addition we will extend the  [conditions that need replacement](https://github.com/FoundationDB/fdb-kubernetes-operator/blob/v1.14.0/api/v1beta2/foundationdbcluster_types.go#L65) by the newly created `RunningOnTaintedNode` condition.
-Those two steps will make sure that the marked process groups will be eventually replaced by the operator.
-The minimum time until a process group that is running on a newly tainted node will be replaced is `failureDetectionTimeSeconds` (default 2h) + `durationInSeconds` from the taint configuration.
-In the example above this will be 4 hours for a node that gets tainted with `example.org/maintenance`.
-
-The [update_status](https://github.com/FoundationDB/fdb-kubernetes-operator/blob/main/controllers/update_status.go) subreconciler will be extended to mark process groups running on a tainted node with the new condition.
-This can be achieved by iterating over all Pods and make requests to the node object, one example implementation could look like this:
-
-```go
-func checkProcessGroupsRunningOnTaintedNodes(pods []*corev1.Pod, status *fdbv1beta2.FoundationDBClusterStatus) {
-    if !cluster.NodeFailureDetectionEnabled() {
-        return
-    }
-
-    nodeTaintedMap := internal.GetNodes(pods) // This will generate a map containing the node name as key and a boolean value indicating if the node is "tainted"
-    for node, _ := range nodeMap {
-        // Check if node has a taint for the minimum duration in that case we will set the value to true.
-    }
-
-    podMap := internal.CreatePodMap(cluster, pods)
-    for _, processGroup := range status.ProcessGroups {
-        pod, podExists := podMap[processGroup.ProcessGroupID]
-        if !podExists {
-            continue
-        }
-
-        tainted, nodeFound := nodeTaintedMap[pod.Spec.NodeName]
-        if !nodeFound {
-            continue
-        }
-
-        processGroup.UpdateCondition(fdbv1beta2.RunningOnTaintedNode, tainted, status.ProcessGroups, processGroup.ProcessGroupID)
-    }
-}
+**Requirement 2.** The feature allows users to configure a set of specific taint keys to detect and replace.
+
+Users can configure the taint feature in the new field `taints` inside `automaticReplacementOptions`. The `taints` field is a list of `(key, durationInSeconds)` pairs, which defines operator's behavior on the taint key.
+
+Users can use `automaticReplacementOptions.taintReplacementTimeSeconds` to control how fast operator should replace Pods on tainted Nodes. Users can configure operator to replace Pods on tainted Nodes much faster than the normal replacement by setting a smaller value of `taintReplacementTimeSeconds`.
+
+Example configuration:
+
+```yaml
+...
+automaticReplacementOptions:
+  enabled: true
+  failureDetectionTimeSeconds: 7200
+  maxConcurrentReplacements: 1
+  enableNodeFailureDetection: true
+  taintReplacementOptions:
+  - key: "example.org/maintenance"
+    durationInSeconds: 7200 # Mark Pods to be replaced after this taint key has been on the Node for at least 7200 seconds (i.e., 2 hours).
+  - key: "example.org/disconnected"
+    durationInSeconds: 1200 # Mark Pods to be replaced after this taint key has been on the Node for at least 1200 seconds (i.e., 20 mins).
+  - key: "*" # The wildcard will match all taint keys
+    durationInSeconds: 3600 # Mark Pods to be replaced after a taint key, which does not have exact match in the taints list, has been on the Node for at least 3600 seconds.
+  taintReplacementTimeSeconds: 1800 # A Pod marked to be replaced due to taints must stay in the to-be-replaced state for 1800 seconds before it is automatically replaced.
 ```
 
-The limitation of this approach would be that we only detect Pods running on tainted nodes when a reconciliation is happening (those are not coupled to node events).
-To work around this we could extend the operator and let the operator watch node events, that could be potentially a high volume of events, so we have to make sure we get the right signals.
-The controller-runtime already supports to let the operator watch resources that are not managed by itself, see [Watching Externally Managed Resources](https://book.kubebuilder.io/reference/watching-resources/externally-managed.html).
+In the above configuration, it takes at least 3000 seconds (1200 + 1800) for the operator to replace a Pod on a Node with taint key `example.com/disconnected`, because it takes `durationInSeconds`, i.e., 1200 seconds, to mark the Pod to be replaced and it takes another `taintReplacementTimeSeconds`, i.e., 1800 seconds, to start replacing the Pod.
+
+**Requirement 3.** The feature lists the taint status for process groups. This allows users to have a holistic view of which Pods in a FDB cluster are running on tainted Nodes and being replaced.
+
+### Implementation
+
+We need to make the following four types of changes:
+
+**CRD change.** We need to extend the `automaticReplacementOptions` with the new fields `taints` and `taintReplacementTimeSeconds`
+
+**Detect Node taint and change ProcessGroup condition.**  We can extend the existing `validateProcessGroup()` function in `update_status` reconciler to detect if a Pod is running on a tainted Node and mark the ProcessGroup's taint-related condition accordingly.
+
+**Status of ProcessGroup taint-related condition.** We will add two [ProcessGroupConditionType](https://github.com/FoundationDB/fdb-kubernetes-operator/blob/main/api/v1beta2/foundationdbcluster_types.go#L673):
+- `NodeTaintDetected` represents a ProcessGroup Node is tainted, but not long enough to replace it.
+- `NodeTaintReplacing` represents a ProcessGroup whose Node has been tainted for longer-than-configured time and the ProcessGroup should be replaced.
+
+**Replace ProcessGroups with `NodeTaintReplacing` condition.** We can extend the existing replacement logic to replace ProcessGroups with `NodeTaintReplacing` condition. We can do this by adding the `NodeTaintReplacing` condition to [`conditionsThatNeedReplacement` list](https://github.com/FoundationDB/fdb-kubernetes-operator/blob/d38d6bc7abf7764215976b75d581c933280389f8/api/v1beta2/foundationdbcluster_types.go#L65-L67)
+
+
+## Limitations
+The operator will only detect tainted Nodes when its reconciliation is triggered. This may increase the delay of detecting and replacing Pods on tainted Nodes.
+
+The limitation can be resolved by letting the operator watch node events. The controller-runtime already supports to let the operator watch resources that are not managed by itself, see [Watching Externally Managed Resources](https://book.kubebuilder.io/reference/watching-resources/externally-managed.html). [Tracked issue#1629](https://github.com/FoundationDB/fdb-kubernetes-operator/issues/1629)
 
 ## Related Links
 
diff --git a/docs/manual/operations.md b/docs/manual/operations.md
index da973e92..64a58352 100644
--- a/docs/manual/operations.md
+++ b/docs/manual/operations.md
@@ -74,6 +74,27 @@ spec:
   version: 7.2.4
 ```
 
+The supported versions of the operator are documented in the [compatibility](../compatibility.md) guide.
+Downgrades of patch versions are supported, but not for major or minor versions.
+
+For version upgrades from 7.1+ to another major or minor versions the client compatibility check might requires some additional configuration:
+
+```yaml
+apiVersion: apps.foundationdb.org/v1beta2
+kind: FoundationDBCluster
+metadata:
+  name: sample-cluster
+spec:
+  version: 7.2.4
+  automationOptions:
+    ignoreLogGroupsForUpgrade:
+      - fdb-kubernetes-operator
+```
+
+The default value for the log group of the operator is `fdb-kubernetes-operator` but can be changed by setting the environment variable `FDB_NETWORK_OPTION_TRACE_LOG_GROUP`.
+The operator version `v1.19.0` and never sets this value as a default and those changes are not required.
+
+
 The upgrade process is described in more detail in [upgrades](./upgrades.md).
 
 ## Renaming a Cluster
diff --git a/docs/manual/replacements_and_deletions.md b/docs/manual/replacements_and_deletions.md
index 98a06001..ff00cac3 100644
--- a/docs/manual/replacements_and_deletions.md
+++ b/docs/manual/replacements_and_deletions.md
@@ -27,7 +27,7 @@ The following changes can only be rolled out through replacement:
 The number of inflight replacements can be configured by setting `maxConcurrentReplacements`, per default the operator will replace all misconfigured process groups.
 Depending on the cluster size this can require a quota that is has double the capacity of the actual required resources.
 
-## Automatic Replacements
+## Automatic Replacements for ProcessGroups in Undesired State
 
 The operator has an option to automatically replace pods that are in a bad state. This behavior is disabled by default, but you can enable it by setting the field `automationOptions.replacements.enabled` in the cluster spec.
 This will replace any pods that meet the following criteria:
@@ -42,11 +42,62 @@ The following conditions are currently eligible for replacement:
 * `MissingPod`: This indicates a process group that doesn't have a Pod assigned.
 * `MissingPVC`: This indicates that a process group that doesn't have a PVC assigned.
 * `MissingService`: This indicates that a process group that doesn't have a Service assigned.
-* `PodPending`: This indicates that a process group where the pod is in a pending state.
+* `PodPending`: This indicates that a process group where the Pod is in a pending state.
+* `NodeTaintReplacing`: This indicates a process group where the Pod has been running on a tainted Node for at least the configured duration. If a ProcessGroup has the `NodeTaintReplacing` condition, the replacement cannot be stopped, even after the Node taint was removed.
 
 Process groups that are set into the crash loop state with the `Buggify` setting won't be replaced by the operator.
 If the `cluster.Spec.Buggify.EmptyMonitorConf` setting is active the operator won't replace any process groups.
 
+## Automatic Replacements for ProcessGroups on Tainted Nodes
+The operator has an option to automatically replace ProcessGroups where the associated Pod is running on a tainted Node. This feature is disabled by default, but can be enabled by setting `automationOptions.replacements.taintReplacementOptions`.
+
+We use three examples below to illustrate how to set up the feature.
+
+### Example Setup 1
+
+The following YAML setup lets the operator detect Pods running on Nodes with taint key `example.com/maintenance`, set the ProcessGroup' condition to `NodeTaintReplacing`, if their Nodes have been tainted for 3600 seconds, and replace the Pods after 1800 seconds.
+
+```yaml
+spec:
+    automationOptions:
+      replacements:
+        taintReplacementOptions:
+        - Key: example.com/maintenance
+          DurationInSeconds: 3600
+        taintReplacementTimeSeconds: 1800
+        enabled: true
+```
+
+If there are multiple Pods on tainted Nodes, the operator will simultaneously replace at most `automationOptions.replacements.maxConcurrentReplacements` Pods.
+
+### Example Setup 2
+
+We can enable the taint feature on all taint keys except one taint key with the following  configuration:
+```yaml
+spec:
+    automationOptions:
+      replacements:
+        taintReplacementOptions:
+        - Key: "*"
+          DurationInSeconds: 3600
+        - Key: example.com/taint-key-to-ignore
+          DurationInSeconds: 9223372036854775807
+        enabled: true
+```
+
+The operator will detect and mark all Pods on tainted Nodes with `NodeTaintDetected` condition. But the operator will ignore the taint key `example.com/taint-key-to-ignore` when it adds `NodeTaintReplacing` condition to Pods, because the key's `DurationInSeconds` is set to max of int64. For example, if a Node has only the taint key `example.com/taint-key-to-ignore`, its Pods will only be marked with  `NodeTaintDetected` condition. When the Node has another taint key, say `example.com/any-other-key`, its Pods will be added `NodeTaintReplacing` condition when the other taint key has been on the Node for 3600 seconds.
+
+### Example Setup 3
+
+We can disable the taint feature by resetting `automationOptions.replacements.taintReplacementOptions = {}`. The following example YAML config deletes the `taintReplacementOptions` section.
+
+```yaml
+spec:
+    automationOptions:
+      replacements:
+        enabled: true
+```
+
 ## Enforce Full Replication
 
 The operator only removes ProcessGroups when the cluster has the desired fault tolerance and is available. This is enforced by default in 1.0.0 without disabling.
diff --git a/fdbclient/admin_client.go b/fdbclient/admin_client.go
index c041e933..9f6c7333 100644
--- a/fdbclient/admin_client.go
+++ b/fdbclient/admin_client.go
@@ -728,3 +728,23 @@ func (client *cliAdminClient) GetCoordinatorSet() (map[string]fdbv1beta2.None, e
 func (client *cliAdminClient) SetKnobs(knobs []string) {
 	client.knobs = knobs
 }
+
+// WithValues will update the logger used by the current AdminClient to contain the provided key value pairs. The provided
+// arguments must be even.
+func (client *cliAdminClient) WithValues(keysAndValues ...interface{}) {
+	newLogger := client.log.WithValues(keysAndValues...)
+	client.log = newLogger
+
+	// Update the FDB library client logger
+	realFdbClient, ok := client.fdbLibClient.(*realFdbLibClient)
+	if ok {
+		realFdbClient.logger = newLogger
+	}
+
+	// Update the command runner logger
+	cmdRunner, ok := client.cmdRunner.(*realCommandRunner)
+	if !ok {
+		return
+	}
+	cmdRunner.log = newLogger
+}
diff --git a/internal/removals/remove.go b/internal/removals/remove.go
index e84d3ba2..365f1b6a 100644
--- a/internal/removals/remove.go
+++ b/internal/removals/remove.go
@@ -135,6 +135,7 @@ func GetRemainingMap(logger logr.Logger, adminClient fdbadminclient.AdminClient,
 		if cluster.UseLocalitiesForExclusion() {
 			addresses = append(addresses, fdbv1beta2.ProcessAddress{StringAddress: processGroup.GetExclusionString()})
 		}
+
 		for _, pAddr := range processGroup.Addresses {
 			addresses = append(addresses, fdbv1beta2.ProcessAddress{IPAddress: net.ParseIP(pAddr)})
 		}
diff --git a/internal/replacements/replace_failed_process_groups.go b/internal/replacements/replace_failed_process_groups.go
index b4816eed..a321ad2a 100644
--- a/internal/replacements/replace_failed_process_groups.go
+++ b/internal/replacements/replace_failed_process_groups.go
@@ -45,7 +45,7 @@ func getMaxReplacements(cluster *fdbv1beta2.FoundationDBCluster, maxReplacements
 
 // ReplaceFailedProcessGroups flags failed processes groups for removal and returns an indicator
 // of whether any processes were thus flagged.
-func ReplaceFailedProcessGroups(log logr.Logger, cluster *fdbv1beta2.FoundationDBCluster, hasDesiredFaultTolerance bool) bool {
+func ReplaceFailedProcessGroups(log logr.Logger, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, hasDesiredFaultTolerance bool) bool {
 	// Automatic replacements are disabled, so we don't have to check anything further
 	if !cluster.GetEnableAutomaticReplacements() {
 		return false
@@ -62,6 +62,14 @@ ProcessGroupLoop:
 			continue
 		}
 
+		if processGroupStatus.IsUnderMaintenance(status.Cluster.MaintenanceZone) {
+			log.Info(
+				"Skip process group that is in maintenance zone",
+				"processGroupID", processGroupStatus.ProcessGroupID,
+				"maintenance zone", processGroupStatus.FaultDomain)
+			continue
+		}
+
 		canReplace := maxReplacements > 0
 
 		for _, targets := range crashLoopContainerProcessGroups {
diff --git a/mock-kubernetes-client/client/client.go b/mock-kubernetes-client/client/client.go
index 6cecfeb6..106ec790 100644
--- a/mock-kubernetes-client/client/client.go
+++ b/mock-kubernetes-client/client/client.go
@@ -143,6 +143,7 @@ func (client *MockClient) setNewFakeClient() {
 		})
 	}
 
+	builder.WithScheme(client.scheme)
 	client.fakeClient = builder.Build()
 }
 
diff --git a/pkg/fdbadminclient/admin_client.go b/pkg/fdbadminclient/admin_client.go
index f73ae3e2..8d92b116 100644
--- a/pkg/fdbadminclient/admin_client.go
+++ b/pkg/fdbadminclient/admin_client.go
@@ -27,10 +27,10 @@ import (
 // AdminClient describes an interface for running administrative commands on a
 // cluster
 type AdminClient interface {
-	// GetStatus gets the database's status
+	// GetStatus gets the database's status.
 	GetStatus() (*fdbv1beta2.FoundationDBStatus, error)
 
-	// ConfigureDatabase sets the database configuration
+	// ConfigureDatabase sets the database configuration.
 	ConfigureDatabase(configuration fdbv1beta2.DatabaseConfiguration, newDatabase bool, version string) error
 
 	// ExcludeProcesses starts evacuating processes so that they can be removed
@@ -103,12 +103,16 @@ type AdminClient interface {
 	// SetKnobs sets the Knobs that should be used for the commandline call.
 	SetKnobs([]string)
 
-	// GetMaintenanceZone gets current maintenance zone, if any
+	// GetMaintenanceZone gets current maintenance zone, if any.
 	GetMaintenanceZone() (string, error)
 
-	// SetMaintenanceZone places zone into maintenance mode
+	// SetMaintenanceZone places zone into maintenance mode.
 	SetMaintenanceZone(zone string, timeoutSeconds int) error
 
-	// Reset maintenance mode
+	// ResetMaintenanceMode resets the maintenance mode.
 	ResetMaintenanceMode() error
+
+	// WithValues will update the logger used by the current AdminClient to contain the provided key value pairs. The provided
+	// arguments must be even.
+	WithValues(keysAndValues ...interface{})
 }
diff --git a/pkg/fdbadminclient/mock/admin_client_mock.go b/pkg/fdbadminclient/mock/admin_client_mock.go
index 3d1210a6..b1e0c347 100644
--- a/pkg/fdbadminclient/mock/admin_client_mock.go
+++ b/pkg/fdbadminclient/mock/admin_client_mock.go
@@ -68,6 +68,7 @@ type AdminClient struct {
 	TeamTracker                              []fdbv1beta2.FoundationDBStatusTeamTracker
 	Logs                                     []fdbv1beta2.FoundationDBStatusLogInfo
 	mockError                                error
+	LagInfo                                  map[string]fdbv1beta2.FoundationDBStatusLagInfo
 }
 
 // adminClientCache provides a cache of mock admin clients.
@@ -102,6 +103,7 @@ func NewMockAdminClientUncast(cluster *fdbv1beta2.FoundationDBCluster, kubeClien
 			currentCommandLines:   make(map[string]string),
 			Knobs:                 make(map[string]fdbv1beta2.None),
 			VersionProcessGroups:  make(map[fdbv1beta2.ProcessGroupID]string),
+			LagInfo:               make(map[string]fdbv1beta2.FoundationDBStatusLagInfo),
 		}
 		adminClientCache[cluster.Name] = cachedClient
 		cachedClient.Backups = make(map[string]fdbv1beta2.FoundationDBBackupStatusBackupDetails)
@@ -443,6 +445,23 @@ func (client *AdminClient) GetStatus() (*fdbv1beta2.FoundationDBStatus, error) {
 	}
 	status.Cluster.MaintenanceZone = client.MaintenanceZone
 
+	if len(client.LagInfo) > 0 {
+		limitingDurabilityLag, ok := client.GetLimitingDurabilityLag()
+		if ok {
+			status.Cluster.Qos.LimitingDurabilityLagStorageServer = limitingDurabilityLag
+		}
+
+		worstDataLag, ok := client.GetWorstDataLag()
+		if ok {
+			status.Cluster.Qos.WorstDataLagStorageServer = worstDataLag
+		}
+
+		worstDurabilityLag, ok := client.GetWorstDurabilityLag()
+		if ok {
+			status.Cluster.Qos.WorstDurabilityLagStorageServer = worstDurabilityLag
+		}
+	}
+
 	return status, nil
 }
 
@@ -948,3 +967,58 @@ func (client *AdminClient) MockUptimeSecondsForMaintenanceZone(seconds float64)
 func (client *AdminClient) MockError(err error) {
 	client.mockError = err
 }
+
+// SetLimitingDurabilityLag sets/mocks the limiting durability lag of any storage server in the cluster.
+func (client *AdminClient) SetLimitingDurabilityLag(lagInfo *fdbv1beta2.FoundationDBStatusLagInfo) {
+	adminClientMutex.Lock()
+	defer adminClientMutex.Unlock()
+
+	client.LagInfo["limitingDurabilityLag"] = *lagInfo
+}
+
+// GetLimitingDurabilityLag returns the limiting durability lag of any storage server in the cluster.
+func (client *AdminClient) GetLimitingDurabilityLag() (fdbv1beta2.FoundationDBStatusLagInfo, bool) {
+	adminClientMutex.Lock()
+	defer adminClientMutex.Unlock()
+
+	lagInfo, ok := client.LagInfo["limitingDurabilityLag"]
+	return lagInfo, ok
+}
+
+// SetWorstDataLag sets/mocks the worst data lag of any storage server in the cluster.
+func (client *AdminClient) SetWorstDataLag(lagInfo *fdbv1beta2.FoundationDBStatusLagInfo) {
+	adminClientMutex.Lock()
+	defer adminClientMutex.Unlock()
+
+	client.LagInfo["worstDataLag"] = *lagInfo
+}
+
+// GetWorstDataLag returns the (mocked) worst data lag of any storage server in the cluster.
+func (client *AdminClient) GetWorstDataLag() (fdbv1beta2.FoundationDBStatusLagInfo, bool) {
+	adminClientMutex.Lock()
+	defer adminClientMutex.Unlock()
+
+	lagInfo, ok := client.LagInfo["worstDataLag"]
+	return lagInfo, ok
+}
+
+// SetWorstDurabilityLag sets/mocks the worst durability lag of any storage server in the cluster.
+func (client *AdminClient) SetWorstDurabilityLag(lagInfo *fdbv1beta2.FoundationDBStatusLagInfo) {
+	adminClientMutex.Lock()
+	defer adminClientMutex.Unlock()
+
+	client.LagInfo["worstDurabilityLag"] = *lagInfo
+}
+
+// GetWorstDurabilityLag returns the (mocked) worst durability lag of any storage server in the cluster.
+func (client *AdminClient) GetWorstDurabilityLag() (fdbv1beta2.FoundationDBStatusLagInfo, bool) {
+	adminClientMutex.Lock()
+	defer adminClientMutex.Unlock()
+
+	lagInfo, ok := client.LagInfo["worstDurabilityLag"]
+	return lagInfo, ok
+}
+
+// WithValues will update the logger used by the current AdminClient to contain the provided key value pairs. The provided
+// arguments must be even.
+func (client *AdminClient) WithValues(_ ...interface{}) {}
diff --git a/pkg/fdbstatus/status_checks.go b/pkg/fdbstatus/status_checks.go
index 026ea38a..00b4526e 100644
--- a/pkg/fdbstatus/status_checks.go
+++ b/pkg/fdbstatus/status_checks.go
@@ -48,16 +48,17 @@ type exclusionStatus struct {
 }
 
 // getRemainingAndExcludedFromStatus checks which processes of the input address list are excluded in the cluster and which are not.
-func getRemainingAndExcludedFromStatus(status *fdbv1beta2.FoundationDBStatus, addresses []fdbv1beta2.ProcessAddress) exclusionStatus {
+func getRemainingAndExcludedFromStatus(logger logr.Logger, status *fdbv1beta2.FoundationDBStatus, addresses []fdbv1beta2.ProcessAddress) exclusionStatus {
 	notExcludedAddresses := map[string]fdbv1beta2.None{}
-	fullyExcludedAddresses := map[string]fdbv1beta2.None{}
-	visitedAddresses := map[string]fdbv1beta2.None{}
+	fullyExcludedAddresses := map[string]int{}
+	visitedAddresses := map[string]int{}
 
 	// If there are more than 1 active generations we can not handout any information about excluded processes based on
 	// the cluster status information as only the latest log processes will have the log process role. If we don't check
 	// for the active generations we have the risk to remove a log process that still has mutations on it that must be
 	// popped.
 	if status.Cluster.RecoveryState.ActiveGenerations > 1 {
+		logger.Info("Skipping exclusion check as there are multiple active generations", "activeGenerations", status.Cluster.RecoveryState.ActiveGenerations)
 		return exclusionStatus{
 			inProgress:      nil,
 			fullyExcluded:   nil,
@@ -66,54 +67,63 @@ func getRemainingAndExcludedFromStatus(status *fdbv1beta2.FoundationDBStatus, ad
 		}
 	}
 
+	addressesToVerify := map[string]fdbv1beta2.None{}
 	for _, addr := range addresses {
-		notExcludedAddresses[addr.MachineAddress()] = fdbv1beta2.None{}
+		addressesToVerify[addr.MachineAddress()] = fdbv1beta2.None{}
 	}
 
 	// Check in the status output which processes are already marked for exclusion in the cluster
 	for _, process := range status.Cluster.Processes {
-		if _, ok := notExcludedAddresses[process.Address.MachineAddress()]; !ok {
+		if _, ok := addressesToVerify[process.Address.MachineAddress()]; !ok {
 			continue
 		}
 
-		visitedAddresses[process.Address.MachineAddress()] = fdbv1beta2.None{}
+		visitedAddresses[process.Address.MachineAddress()]++
 		if !process.Excluded {
+			notExcludedAddresses[process.Address.MachineAddress()] = fdbv1beta2.None{}
 			continue
 		}
 
 		if len(process.Roles) == 0 {
-			fullyExcludedAddresses[process.Address.MachineAddress()] = fdbv1beta2.None{}
+			fullyExcludedAddresses[process.Address.MachineAddress()]++
 		}
-
-		delete(notExcludedAddresses, process.Address.MachineAddress())
 	}
 
 	exclusions := exclusionStatus{
-		inProgress:      make([]fdbv1beta2.ProcessAddress, 0, len(addresses)-len(notExcludedAddresses)-len(fullyExcludedAddresses)),
+		inProgress:      make([]fdbv1beta2.ProcessAddress, 0, len(addresses)),
 		fullyExcluded:   make([]fdbv1beta2.ProcessAddress, 0, len(fullyExcludedAddresses)),
 		notExcluded:     make([]fdbv1beta2.ProcessAddress, 0, len(notExcludedAddresses)),
-		missingInStatus: make([]fdbv1beta2.ProcessAddress, 0, len(notExcludedAddresses)),
+		missingInStatus: make([]fdbv1beta2.ProcessAddress, 0, len(addresses)-len(visitedAddresses)),
 	}
 
 	for _, addr := range addresses {
+		machine := addr.MachineAddress()
 		// If we didn't visit that address (absent in the cluster status) we assume it's safe to run the exclude command against it.
 		// We have to run the exclude command against those addresses, to make sure they are not serving any roles.
-		if _, ok := visitedAddresses[addr.MachineAddress()]; !ok {
+		visitedCount, visited := visitedAddresses[machine]
+		if !visited {
 			exclusions.missingInStatus = append(exclusions.missingInStatus, addr)
 			continue
 		}
 
 		// Those addresses are not excluded, so it's not safe to start the exclude command to check if they are fully excluded.
-		if _, ok := notExcludedAddresses[addr.MachineAddress()]; ok {
+		if _, ok := notExcludedAddresses[machine]; ok {
 			exclusions.notExcluded = append(exclusions.notExcluded, addr)
 			continue
 		}
 
 		// Those are the processes that are marked as excluded and are not serving any roles. It's safe to delete Pods
 		// that host those processes.
-		if _, ok := fullyExcludedAddresses[addr.MachineAddress()]; ok {
-			exclusions.fullyExcluded = append(exclusions.fullyExcluded, addr)
-			continue
+		excludedCount, ok := fullyExcludedAddresses[addr.MachineAddress()]
+		if ok {
+			// We have to make sure that we have visited as many processes as we have seen fully excluded. Otherwise we might
+			// return a wrong signal if more than one process is used per Pod. In this case we have to wait for all processes
+			// to be fully excluded.
+			if visitedCount == excludedCount {
+				exclusions.fullyExcluded = append(exclusions.fullyExcluded, addr)
+				continue
+			}
+			logger.Info("found excluded addresses for machine, but not all processes are fully excluded", "visitedCount", visitedCount, "excludedCount", excludedCount, "machine", machine)
 		}
 
 		// Those are the processes that are marked as excluded but still serve at least one role.
@@ -127,7 +137,7 @@ func getRemainingAndExcludedFromStatus(status *fdbv1beta2.FoundationDBStatus, ad
 //
 // The list returned by this method will be the addresses that are *not* safe to remove.
 func CanSafelyRemoveFromStatus(logger logr.Logger, client fdbadminclient.AdminClient, addresses []fdbv1beta2.ProcessAddress, status *fdbv1beta2.FoundationDBStatus) ([]fdbv1beta2.ProcessAddress, error) {
-	exclusions := getRemainingAndExcludedFromStatus(status, addresses)
+	exclusions := getRemainingAndExcludedFromStatus(logger, status, addresses)
 	logger.Info("Filtering excluded processes",
 		"inProgress", exclusions.inProgress,
 		"fullyExcluded", exclusions.fullyExcluded,


Test patch:
diff --git a/api/v1beta2/foundationdb_status_test.go b/api/v1beta2/foundationdb_status_test.go
index 97aca83f..9eefadfe 100644
--- a/api/v1beta2/foundationdb_status_test.go
+++ b/api/v1beta2/foundationdb_status_test.go
@@ -482,6 +482,20 @@ var _ = Describe("FoundationDBStatus", func() {
 							},
 						},
 					},
+					Qos: FoundationDBStatusQosInfo{
+						LimitingDurabilityLagStorageServer: FoundationDBStatusLagInfo{
+							Seconds:  14.1153,
+							Versions: 14115335,
+						},
+						WorstDataLagStorageServer: FoundationDBStatusLagInfo{
+							Seconds:  0,
+							Versions: 0,
+						},
+						WorstDurabilityLagStorageServer: FoundationDBStatusLagInfo{
+							Seconds:  14.115600000000001,
+							Versions: 14115618,
+						},
+					},
 					RecoveryState: RecoveryState{
 						Name: "fully_recovered",
 					},
@@ -865,6 +879,20 @@ var _ = Describe("FoundationDBStatus", func() {
 					SatelliteLogReplicationFactor: 0,
 				},
 			},
+			Qos: FoundationDBStatusQosInfo{
+				LimitingDurabilityLagStorageServer: FoundationDBStatusLagInfo{
+					Seconds:  5.0145299999999997,
+					Versions: 5014530,
+				},
+				WorstDataLagStorageServer: FoundationDBStatusLagInfo{
+					Seconds:  0,
+					Versions: 0,
+				},
+				WorstDurabilityLagStorageServer: FoundationDBStatusLagInfo{
+					Seconds:  5.0150199999999998,
+					Versions: 5015017,
+				},
+			},
 			FaultTolerance: FaultTolerance{
 				MaxZoneFailuresWithoutLosingData:         1,
 				MaxZoneFailuresWithoutLosingAvailability: 1,
diff --git a/controllers/replace_failed_process_groups_test.go b/controllers/replace_failed_process_groups_test.go
index 20f8c8b2..3afec59c 100644
--- a/controllers/replace_failed_process_groups_test.go
+++ b/controllers/replace_failed_process_groups_test.go
@@ -911,6 +911,18 @@ var _ = Describe("replace_failed_process_groups", func() {
 				})
 			})
 		})
+
+		Context("with maintenance mode enabled", func() {
+			BeforeEach(func() {
+				adminClient, err := mock.NewMockAdminClientUncast(cluster, k8sClient)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(adminClient.SetMaintenanceZone("operator-test-1-storage-2", 0)).NotTo(HaveOccurred())
+			})
+
+			It("should not mark the process group for removal", func() {
+				Expect(getRemovedProcessGroupIDs(cluster)).To(BeEmpty())
+			})
+		})
 	})
 
 	Context("with a process that has been missing for a brief time", func() {
diff --git a/e2e/fixtures/factory.go b/e2e/fixtures/factory.go
index e9d5e6e9..5b64b39a 100644
--- a/e2e/fixtures/factory.go
+++ b/e2e/fixtures/factory.go
@@ -36,7 +36,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/duration"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -155,18 +154,18 @@ func (factory *Factory) CreateFdbCluster(
 	config *ClusterConfig,
 	options ...ClusterOption,
 ) *FdbCluster {
-	config.SetDefaults(factory)
-
-	mainOverrides, sidecarOverrides := factory.getContainerOverrides(config.DebugSymbols)
-	spec := factory.createFDBClusterSpec(
-		config.Name,
-		config.Namespace,
-		factory.createProcesses(config),
-		config.CreateDatabaseConfiguration(),
-		config.StorageServerPerPod,
-		mainOverrides,
-		sidecarOverrides)
+	spec := factory.GenerateFDBClusterSpec(config)
+	return factory.CreateFdbClusterFromSpec(spec, config, options...)
+}
 
+// CreateFdbClusterFromSpec creates a FDB cluster. This method can be used in combination with the GenerateFDBClusterSpec method.
+// In general this should only be used for special cases that are not covered by changing the ClusterOptions or the ClusterConfig.
+func (factory *Factory) CreateFdbClusterFromSpec(
+	spec *fdbv1beta2.FoundationDBCluster,
+	config *ClusterConfig,
+	options ...ClusterOption,
+) *FdbCluster {
+	config.SetDefaults(factory)
 	log.Printf("create cluster: %s", ToJSON(spec))
 
 	return factory.startFDBFromClusterSpec(spec, config, options...)
@@ -666,12 +665,11 @@ func (factory *Factory) DumpState(fdbCluster *FdbCluster) {
 		),
 	)
 	// Printout all Pods for this namespace
-	kubeClient := factory.getClient()
-	pods, err := kubeClient.CoreV1().
-		Pods(cluster.Namespace).
-		List(ctx.Background(), metav1.ListOptions{})
+	pods := &corev1.PodList{}
+	err := factory.controllerRuntimeClient.List(ctx.Background(), pods, client.InNamespace(cluster.Namespace))
 	if err != nil {
 		log.Println(err)
+		return
 	}
 
 	buffer.WriteString("---------- Pods ----------")
@@ -695,32 +693,37 @@ func (factory *Factory) DumpState(fdbCluster *FdbCluster) {
 
 	// Printout the logs of the operator Pods for the last 90 seconds.
 	for _, pod := range operatorPods {
-		req := kubeClient.CoreV1().
-			Pods(pod.Namespace).
-			GetLogs(pod.Name, &corev1.PodLogOptions{
-				Container:    "manager",
-				Follow:       false,
-				SinceSeconds: pointer.Int64(300),
-			})
-
-		readCloser, err := req.Stream(ctx.Background())
-		if err != nil {
-			log.Println(err)
-			continue
-		}
+		log.Println(factory.GetLogsForPod(pod, "manager", pointer.Int64(300)))
+	}
+}
 
-		logs, err := io.ReadAll(readCloser)
-		if err != nil {
-			log.Println(err)
-			_ = readCloser.Close()
-			continue
-		}
-		if len(logs) == 0 {
-			continue
-		}
+// GetLogsForPod will fetch the logs for the specified Pod and container since the provided seconds.
+func (factory *Factory) GetLogsForPod(pod corev1.Pod, container string, since *int64) string {
+	req := factory.getClient().CoreV1().
+		Pods(pod.Namespace).
+		GetLogs(pod.Name, &corev1.PodLogOptions{
+			Container:    container,
+			Follow:       false,
+			SinceSeconds: since,
+		})
+
+	readCloser, err := req.Stream(ctx.Background())
+	if err != nil {
+		log.Println(err)
+		return ""
+	}
 
-		log.Println(string(logs))
+	logs, err := io.ReadAll(readCloser)
+	if err != nil {
+		log.Println(err)
+		_ = readCloser.Close()
+		return ""
+	}
+	if len(logs) == 0 {
+		return ""
 	}
+
+	return string(logs)
 }
 
 // DumpStateHaCluster can be used to dump the state of the HA cluster. This includes the Kubernetes custom resource
diff --git a/e2e/fixtures/fdb_cluster_specs.go b/e2e/fixtures/fdb_cluster_specs.go
index d506318a..52c837be 100644
--- a/e2e/fixtures/fdb_cluster_specs.go
+++ b/e2e/fixtures/fdb_cluster_specs.go
@@ -27,6 +27,21 @@ import (
 	"k8s.io/utils/pointer"
 )
 
+// GenerateFDBClusterSpec will generate a *fdbv1beta2.FoundationDBCluster based on the input ClusterConfig.
+func (factory *Factory) GenerateFDBClusterSpec(config *ClusterConfig) *fdbv1beta2.FoundationDBCluster {
+	config.SetDefaults(factory)
+
+	mainOverrides, sidecarOverrides := factory.getContainerOverrides(config.DebugSymbols)
+	return factory.createFDBClusterSpec(
+		config.Name,
+		config.Namespace,
+		factory.createProcesses(config),
+		config.CreateDatabaseConfiguration(),
+		config.StorageServerPerPod,
+		mainOverrides,
+		sidecarOverrides)
+}
+
 // High Level Cluster Spec Supplied by Operator.
 func (factory *Factory) createFDBClusterSpec(
 	clusterName string,
diff --git a/e2e/fixtures/kubernetes_fixtures.go b/e2e/fixtures/kubernetes_fixtures.go
index 98cdb284..eabad466 100644
--- a/e2e/fixtures/kubernetes_fixtures.go
+++ b/e2e/fixtures/kubernetes_fixtures.go
@@ -23,7 +23,6 @@ package fixtures
 import (
 	ctx "context"
 	"log"
-	"regexp"
 
 	"github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
@@ -32,6 +31,10 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+const (
+	namespaceRegEx = `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
+)
+
 // MultipleNamespaces creates multiple namespaces for HA testing.
 func (factory *Factory) MultipleNamespaces(dcIDs []string) []string {
 	res := make([]string, len(dcIDs))
@@ -64,13 +67,8 @@ func (factory *Factory) createNamespace(suffix string) string {
 	namespace := factory.options.namespace
 
 	if namespace == "" {
+		gomega.Expect(factory.singleton.userName).To(gomega.MatchRegexp(namespaceRegEx), "user name contains invalid characters")
 		namespace = factory.singleton.userName + "-" + RandStringRunes(8)
-		matched, err := regexp.Match(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`, []byte(namespace))
-		if !matched {
-			return ""
-		}
-
-		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 	}
 
 	if suffix != "" {
diff --git a/e2e/test_operator_upgrades/operator_upgrades_test.go b/e2e/test_operator_upgrades/operator_upgrades_test.go
index 3bce9701..8e8f8020 100644
--- a/e2e/test_operator_upgrades/operator_upgrades_test.go
+++ b/e2e/test_operator_upgrades/operator_upgrades_test.go
@@ -59,12 +59,10 @@ var _ = AfterSuite(func() {
 	}
 })
 
-func clusterSetup(beforeVersion string, availabilityCheck bool) {
+func clusterSetupWithConfig(beforeVersion string, availabilityCheck bool, config *fixtures.ClusterConfig) {
 	factory.SetBeforeVersion(beforeVersion)
 	fdbCluster = factory.CreateFdbCluster(
-		&fixtures.ClusterConfig{
-			DebugSymbols: false,
-		},
+		config,
 		factory.GetClusterOptions(fixtures.UseVersionBeforeUpgrade)...,
 	)
 
@@ -79,6 +77,12 @@ func clusterSetup(beforeVersion string, availabilityCheck bool) {
 	).ShouldNot(HaveOccurred())
 }
 
+func clusterSetup(beforeVersion string, availabilityCheck bool) {
+	clusterSetupWithConfig(beforeVersion, availabilityCheck, &fixtures.ClusterConfig{
+		DebugSymbols: false,
+	})
+}
+
 // Checks if cluster is running at the expectedVersion. This is done by checking the status of the FoundationDBCluster status.
 // Before that we checked the cluster status json by checking the reported version of all processes. This approach only worked for
 // version compatible upgrades, since incompatible processes won't be part of the cluster anyway. To simplify the check
@@ -530,8 +534,10 @@ var _ = Describe("Operator Upgrades", Label("e2e", "pr"), func() {
 			// The cluster should still be able to upgrade.
 			Expect(fdbCluster.UpgradeCluster(targetVersion, true)).NotTo(HaveOccurred())
 
-			status := fdbCluster.GetStatus()
-			Expect(len(status.Cluster.IncompatibleConnections)).To(Equal(0))
+			// Make sure that the incompatible connections are cleaned up after some time.
+			Eventually(func() []string {
+				return fdbCluster.GetStatus().Cluster.IncompatibleConnections
+			}).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).MustPassRepeatedly(5).Should(BeEmpty())
 		},
 		EntryDescription("Upgrade from %[1]s to %[2]s with one coordinator not being restarted"),
 		fixtures.GenerateUpgradeTableEntries(testOptions),
@@ -1069,4 +1075,64 @@ var _ = Describe("Operator Upgrades", Label("e2e", "pr"), func() {
 		fixtures.GenerateUpgradeTableEntries(testOptions),
 	)
 
+	DescribeTable(
+		"with 2 storage servers per Pod",
+		func(beforeVersion string, targetVersion string) {
+			clusterSetupWithConfig(beforeVersion, true, &fixtures.ClusterConfig{
+				DebugSymbols:        false,
+				StorageServerPerPod: 2,
+			})
+
+			Expect(fdbCluster.UpgradeCluster(targetVersion, false)).NotTo(HaveOccurred())
+			// Make sure the cluster is still running with 2 storage server per Pod.
+			Expect(fdbCluster.GetCluster().Spec.StorageServersPerPod).To(Equal(2))
+
+			if !fixtures.VersionsAreProtocolCompatible(beforeVersion, targetVersion) {
+				// Ensure that the operator is setting the IncorrectConfigMap and IncorrectCommandLine conditions during the upgrade
+				// process.
+				expectedConditions := map[fdbv1beta2.ProcessGroupConditionType]bool{
+					fdbv1beta2.IncorrectConfigMap:   true,
+					fdbv1beta2.IncorrectCommandLine: true,
+				}
+				Eventually(func() bool {
+					cluster := fdbCluster.GetCluster()
+
+					for _, processGroup := range cluster.Status.ProcessGroups {
+						if !processGroup.MatchesConditions(expectedConditions) {
+							return false
+						}
+					}
+
+					return true
+				}).WithTimeout(10 * time.Minute).WithPolling(5 * time.Second).Should(BeTrue())
+			}
+
+			transactionSystemProcessGroups := make(map[fdbv1beta2.ProcessGroupID]fdbv1beta2.None)
+			// Wait until the cluster is upgraded and fully reconciled.
+			Expect(fdbCluster.WaitUntilWithForceReconcile(2, 600, func(cluster *fdbv1beta2.FoundationDBCluster) bool {
+				for _, processGroup := range cluster.Status.ProcessGroups {
+					if processGroup.ProcessClass == fdbv1beta2.ProcessClassStorage {
+						continue
+					}
+
+					transactionSystemProcessGroups[processGroup.ProcessGroupID] = fdbv1beta2.None{}
+				}
+
+				// Allow soft reconciliation and make sure the running version was updated
+				return cluster.Status.Generations.Reconciled == cluster.Generation && cluster.Status.RunningVersion == targetVersion
+			})).NotTo(HaveOccurred())
+
+			// Get the desired process counts based on the current cluster configuration
+			processCounts, err := fdbCluster.GetProcessCounts()
+			Expect(err).NotTo(HaveOccurred())
+
+			// During an upgrade we expect that the transaction system processes are replaced, so we expect to have seen
+			// 2 times the process counts for transaction system processes. Add a small buffer of 5 to allow automatic
+			// replacements during an upgrade.
+			expectedProcessCounts := (processCounts.Total()-processCounts.Storage)*2 + 5
+			Expect(len(transactionSystemProcessGroups)).To(BeNumerically("<=", expectedProcessCounts))
+		},
+		EntryDescription("Upgrade from %[1]s to %[2]s"),
+		fixtures.GenerateUpgradeTableEntries(testOptions),
+	)
 })
diff --git a/pkg/fdbstatus/status_checks_test.go b/pkg/fdbstatus/status_checks_test.go
index 6d85167a..27dc93e8 100644
--- a/pkg/fdbstatus/status_checks_test.go
+++ b/pkg/fdbstatus/status_checks_test.go
@@ -64,10 +64,16 @@ var _ = Describe("status_checks", func() {
 		}
 
 		DescribeTable("fetching the excluded and remaining processes from the status",
-			func(status *fdbv1beta2.FoundationDBStatus, addresses []fdbv1beta2.ProcessAddress, expectedExcluded []fdbv1beta2.ProcessAddress, expectedRemaining []fdbv1beta2.ProcessAddress, expectedFullyExcluded []fdbv1beta2.ProcessAddress, expectedMissing []fdbv1beta2.ProcessAddress) {
-				exclusions := getRemainingAndExcludedFromStatus(status, addresses)
-				Expect(expectedExcluded).To(ConsistOf(exclusions.inProgress))
-				Expect(expectedRemaining).To(ConsistOf(exclusions.notExcluded))
+			func(status *fdbv1beta2.FoundationDBStatus,
+				addresses []fdbv1beta2.ProcessAddress,
+				expectedInProgress []fdbv1beta2.ProcessAddress,
+				expectedNotExcluded []fdbv1beta2.ProcessAddress,
+				expectedFullyExcluded []fdbv1beta2.ProcessAddress,
+				expectedMissing []fdbv1beta2.ProcessAddress) {
+
+				exclusions := getRemainingAndExcludedFromStatus(logr.Discard(), status, addresses)
+				Expect(expectedInProgress).To(ConsistOf(exclusions.inProgress))
+				Expect(expectedNotExcluded).To(ConsistOf(exclusions.notExcluded))
 				Expect(expectedFullyExcluded).To(ConsistOf(exclusions.fullyExcluded))
 				Expect(expectedMissing).To(ConsistOf(exclusions.missingInStatus))
 			},
@@ -146,6 +152,115 @@ var _ = Describe("status_checks", func() {
 				nil,
 				nil,
 			),
+			Entry("when the process group has multiple processes and only one is fully excluded",
+				&fdbv1beta2.FoundationDBStatus{
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						Processes: map[fdbv1beta2.ProcessGroupID]fdbv1beta2.FoundationDBStatusProcessInfo{
+							"1": {
+								Address:  addr1,
+								Excluded: true,
+							},
+							"2": {
+								Address: addr2,
+							},
+							"3": {
+								Address: addr3,
+							},
+							"4-1": {
+								Address:  addr4,
+								Excluded: true,
+								Locality: map[string]string{
+									fdbv1beta2.FDBLocalityProcessIDKey: "4-1",
+								},
+							},
+							"4-2": {
+								Address:  addr4,
+								Excluded: true,
+								Roles: []fdbv1beta2.FoundationDBStatusProcessRoleInfo{
+									{
+										Role: string(fdbv1beta2.ProcessRoleStorage),
+									},
+								},
+								Locality: map[string]string{
+									fdbv1beta2.FDBLocalityProcessIDKey: "4-2",
+								},
+							},
+						},
+					},
+				},
+				[]fdbv1beta2.ProcessAddress{addr4},
+				[]fdbv1beta2.ProcessAddress{addr4},
+				nil,
+				nil,
+				nil,
+			),
+			Entry("when the process group has multiple processes and both are fully excluded",
+				&fdbv1beta2.FoundationDBStatus{
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						Processes: map[fdbv1beta2.ProcessGroupID]fdbv1beta2.FoundationDBStatusProcessInfo{
+							"1": {
+								Address:  addr1,
+								Excluded: true,
+							},
+							"2": {
+								Address: addr2,
+							},
+							"3": {
+								Address: addr3,
+							},
+							"4-1": {
+								Address:  addr4,
+								Excluded: true,
+							},
+							"4-2": {
+								Address:  addr4,
+								Excluded: true,
+							},
+						},
+					},
+				},
+				[]fdbv1beta2.ProcessAddress{addr4},
+				nil,
+				nil,
+				[]fdbv1beta2.ProcessAddress{addr4},
+				nil,
+			),
+			Entry("when the process group has multiple processes and only one is excluded",
+				&fdbv1beta2.FoundationDBStatus{
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						Processes: map[fdbv1beta2.ProcessGroupID]fdbv1beta2.FoundationDBStatusProcessInfo{
+							"1": {
+								Address:  addr1,
+								Excluded: true,
+							},
+							"2": {
+								Address: addr2,
+							},
+							"3": {
+								Address: addr3,
+							},
+							"4-1": {
+								Address:  addr4,
+								Excluded: true,
+							},
+							"4-2": {
+								Address:  addr4,
+								Excluded: false,
+								Roles: []fdbv1beta2.FoundationDBStatusProcessRoleInfo{
+									{
+										Role: string(fdbv1beta2.ProcessRoleStorage),
+									},
+								},
+							},
+						},
+					},
+				},
+				[]fdbv1beta2.ProcessAddress{addr4},
+				nil,
+				[]fdbv1beta2.ProcessAddress{addr4},
+				nil,
+				nil,
+			),
 		)
 	})