Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1660 (2023-06-01 07:22:05)

Add FaultDomain as a typed string
# Description

Adding CRD validation for the fault domain type.

## Type of change

*Please select one of the options below.*

- Bug fix (non-breaking change which fixes an issue)

## Discussion

-

## Testing

Unit tests.

## Documentation

-

## Follow-up

-


foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr-kind on Linux CentOS 7

* Commit ID: 054c776e4ce7ec503d2a36834c7bedb38ff8293a
* Duration 0:06:16
* Result: :x: **FAILED**
* Error: `Error while executing command: make -C e2e kind-setup. Reason: exit status 2`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr-kind/82f60a9d-4a26-4ac6-a1be-436e4ff9a5f3.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net) zip file of the working directory (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: 054c776e4ce7ec503d2a36834c7bedb38ff8293a
* Duration 3:10:56
* Result: :white_check_mark: **SUCCEEDED**
* Error: `N/A`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/aea36c8b-c11c-4766-a0a4-dc0b83d50de2.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/aea36c8b-c11c-4766-a0a4-dc0b83d50de2/workspace.zip) zip file of the working directory (available for 30 days)


Patch:
diff --git a/api/v1beta2/foundationdb_status.go b/api/v1beta2/foundationdb_status.go
index 5bfd512e..ed0156fa 100644
--- a/api/v1beta2/foundationdb_status.go
+++ b/api/v1beta2/foundationdb_status.go
@@ -85,7 +85,7 @@ type FoundationDBStatusClusterInfo struct {
 	Generation int `json:"generation,omitempty"`
 
 	// MaintenanceZone contains current zone under maintenance, if any.
-	MaintenanceZone string `json:"maintenance_zone,omitempty"`
+	MaintenanceZone FaultDomain `json:"maintenance_zone,omitempty"`
 
 	// Clients provides information about clients that are connected to the
 	// database.
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index 4a8d7312..9286ec6e 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -300,8 +300,7 @@ type MaintenanceModeInfo struct {
 	// Deprecated: This setting it not used anymore.
 	StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"`
 	// ZoneID that is placed in maintenance mode
-	// +kubebuilder:validation:MaxLength=512
-	ZoneID string `json:"zoneID,omitempty"`
+	ZoneID FaultDomain `json:"zoneID,omitempty"`
 	// ProcessGroups that are placed in maintenance mode
 	// +kubebuilder:validation:MaxItems=200
 	// Deprecated: This setting it not used anymore.
@@ -334,9 +333,13 @@ type ProcessGroupStatus struct {
 	ProcessGroupConditions []*ProcessGroupCondition `json:"processGroupConditions,omitempty"`
 	// FaultDomain represents the last seen fault domain from the cluster status. This can be used if a Pod or process
 	// is not running and would be missing in the cluster status.
-	FaultDomain string `json:"faultDomain,omitempty"`
+	FaultDomain FaultDomain `json:"faultDomain,omitempty"`
 }
 
+// FaultDomain represents the FaultDomain of a process group
+// +kubebuilder:validation:MaxLength=512
+type FaultDomain string
+
 // ProcessGroupID represents the ID of the process group
 // +kubebuilder:validation:MaxLength=63
 type ProcessGroupID string
@@ -348,7 +351,7 @@ func (processGroupStatus *ProcessGroupStatus) GetExclusionString() string {
 
 // IsExcluded returns if a process group is excluded
 func (processGroupStatus *ProcessGroupStatus) IsExcluded() bool {
-	return (processGroupStatus.ExclusionTimestamp != nil && !processGroupStatus.ExclusionTimestamp.IsZero()) || processGroupStatus.ExclusionSkipped
+	return !processGroupStatus.ExclusionTimestamp.IsZero() || processGroupStatus.ExclusionSkipped
 }
 
 // SetExclude marks a process group as excluded and will reset the process group conditions to only include the ResourcesTerminating
@@ -376,7 +379,7 @@ func (processGroupStatus *ProcessGroupStatus) SetExclude() {
 
 // IsMarkedForRemoval returns if a process group is marked for removal
 func (processGroupStatus *ProcessGroupStatus) IsMarkedForRemoval() bool {
-	return processGroupStatus.RemovalTimestamp != nil && !processGroupStatus.RemovalTimestamp.IsZero()
+	return !processGroupStatus.RemovalTimestamp.IsZero()
 }
 
 // MarkForRemoval marks a process group for removal. If the RemovalTimestamp is already set it won't be changed.
@@ -2287,8 +2290,69 @@ func (cluster *FoundationDBCluster) GetIgnoreTerminatingPodsSeconds() int {
 	return pointer.IntDeref(cluster.Spec.AutomationOptions.IgnoreTerminatingPodsSeconds, int((10 * time.Minute).Seconds()))
 }
 
+// GetProcessGroupsToRemove will returns the list of Process Group IDs that must be added to the ProcessGroupsToRemove
+// it will filter out all Process Group IDs that are already marked for removal to make sure those are clean up. If a
+// provided process group ID doesn't exit it will be ignored.
+func (cluster *FoundationDBCluster) GetProcessGroupsToRemove(processGroupIDs []ProcessGroupID) []ProcessGroupID {
+	currentProcessGroupsToRemove := map[ProcessGroupID]None{}
+
+	for _, id := range cluster.Spec.ProcessGroupsToRemove {
+		currentProcessGroupsToRemove[id] = None{}
+	}
+
+	for _, id := range processGroupIDs {
+		currentProcessGroupsToRemove[id] = None{}
+	}
+
+	filteredProcessGroupsToRemove := make([]ProcessGroupID, 0, len(currentProcessGroupsToRemove))
+	for _, processGroup := range cluster.Status.ProcessGroups {
+		if _, ok := currentProcessGroupsToRemove[processGroup.ProcessGroupID]; !ok {
+			continue
+		}
+
+		if processGroup.IsMarkedForRemoval() {
+			continue
+		}
+
+		filteredProcessGroupsToRemove = append(filteredProcessGroupsToRemove, processGroup.ProcessGroupID)
+	}
+
+	return filteredProcessGroupsToRemove
+}
+
+// GetProcessGroupsToRemoveWithoutExclusion will returns the list of Process Group IDs that must be added to the ProcessGroupsToRemove
+// it will filter out all Process Group IDs that are already marked for removal and are marked as excluded to make sure those are clean up.
+// If a provided process group ID doesn't exit it will be ignored.
+func (cluster *FoundationDBCluster) GetProcessGroupsToRemoveWithoutExclusion(processGroupIDs []ProcessGroupID) []ProcessGroupID {
+	currentProcessGroupsToRemove := map[ProcessGroupID]None{}
+
+	for _, id := range cluster.Spec.ProcessGroupsToRemoveWithoutExclusion {
+		currentProcessGroupsToRemove[id] = None{}
+	}
+
+	for _, id := range processGroupIDs {
+		currentProcessGroupsToRemove[id] = None{}
+	}
+
+	filteredProcessGroupsToRemove := make([]ProcessGroupID, 0, len(currentProcessGroupsToRemove))
+	for _, processGroup := range cluster.Status.ProcessGroups {
+		if _, ok := currentProcessGroupsToRemove[processGroup.ProcessGroupID]; !ok {
+			continue
+		}
+
+		if processGroup.IsMarkedForRemoval() && !processGroup.ExclusionTimestamp.IsZero() {
+			continue
+		}
+
+		filteredProcessGroupsToRemove = append(filteredProcessGroupsToRemove, processGroup.ProcessGroupID)
+	}
+
+	return filteredProcessGroupsToRemove
+}
+
 // AddProcessGroupsToRemovalList adds the provided process group IDs to the remove list.
 // If a process group ID is already present on that list it won't be added a second time.
+// Deprecated: Use GetProcessGroupsToRemove instead and set the cluster.Spec.ProcessGroupsToRemove value to the return value.
 func (cluster *FoundationDBCluster) AddProcessGroupsToRemovalList(processGroupIDs []ProcessGroupID) {
 	removals := map[ProcessGroupID]None{}
 
@@ -2434,6 +2498,7 @@ func (cluster *FoundationDBCluster) RemoveProcessGroupsFromCrashLoopContainerLis
 
 // AddProcessGroupsToRemovalWithoutExclusionList adds the provided process group IDs to the remove without exclusion list.
 // If a process group ID is already present on that list it won't be added a second time.
+// Deprecated: Use GetProcessGroupsToRemoveWithoutExclusion instead and set the cluster.Spec.ProcessGroupsToRemoveWithoutExclusion value to the return value.
 func (cluster *FoundationDBCluster) AddProcessGroupsToRemovalWithoutExclusionList(processGroupIDs []ProcessGroupID) {
 	removals := map[ProcessGroupID]None{}
 
diff --git a/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml b/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
index 5a111508..18713e4d 100644
--- a/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
+++ b/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
@@ -13861,6 +13861,7 @@ spec:
                       format: date-time
                       type: string
                     faultDomain:
+                      maxLength: 512
                       type: string
                     processClass:
                       type: string
diff --git a/controllers/update_status.go b/controllers/update_status.go
index b41e172a..c3d03cee 100644
--- a/controllers/update_status.go
+++ b/controllers/update_status.go
@@ -867,6 +867,6 @@ func updateFaultDomains(logger logr.Logger, processes map[fdbv1beta2.ProcessGrou
 			continue
 		}
 
-		status.ProcessGroups[idx].FaultDomain = faultDomain
+		status.ProcessGroups[idx].FaultDomain = fdbv1beta2.FaultDomain(faultDomain)
 	}
 }
diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
index daebd3d4..f284ccc5 100644
--- a/docs/cluster_spec.md
+++ b/docs/cluster_spec.md
@@ -153,6 +153,12 @@ CrashLoopContainerObject specifies crash-loop target for specific container.
 
 [Back to TOC](#table-of-contents)
 
+## FaultDomain
+
+FaultDomain represents the FaultDomain of a process group
+
+[Back to TOC](#table-of-contents)
+
 ## FoundationDBCluster
 
 FoundationDBCluster is the Schema for the foundationdbclusters API
@@ -350,7 +356,7 @@ MaintenanceModeInfo contains information regarding the zone and process groups t
 | Field | Description | Scheme | Required |
 | ----- | ----------- | ------ | -------- |
 | startTimestamp | StartTimestamp provides the timestamp when this zone is put into maintenance mode **Deprecated: This setting it not used anymore.** | *metav1.Time | false |
-| zoneID | ZoneID that is placed in maintenance mode | string | false |
+| zoneID | ZoneID that is placed in maintenance mode | [FaultDomain](#faultdomain) | false |
 | processGroups | ProcessGroups that are placed in maintenance mode **Deprecated: This setting it not used anymore.** | []string | false |
 
 [Back to TOC](#table-of-contents)
@@ -414,7 +420,7 @@ ProcessGroupStatus represents the status of a ProcessGroup.
 | exclusionTimestamp | ExclusionTimestamp defines when the process group has been fully excluded. This is only used within the reconciliation process, and should not be considered authoritative. | *metav1.Time | false |
 | exclusionSkipped | ExclusionSkipped determines if exclusion has been skipped for a process, which will allow the process group to be removed without exclusion. | bool | false |
 | processGroupConditions | ProcessGroupConditions represents a list of degraded conditions that the process group is in. | []*[ProcessGroupCondition](#processgroupcondition) | false |
-| faultDomain | FaultDomain represents the last seen fault domain from the cluster status. This can be used if a Pod or process is not running and would be missing in the cluster status. | string | false |
+| faultDomain | FaultDomain represents the last seen fault domain from the cluster status. This can be used if a Pod or process is not running and would be missing in the cluster status. | [FaultDomain](#faultdomain) | false |
 
 [Back to TOC](#table-of-contents)
 
diff --git a/fdbclient/admin_client.go b/fdbclient/admin_client.go
index aa79565f..e05730ae 100644
--- a/fdbclient/admin_client.go
+++ b/fdbclient/admin_client.go
@@ -397,7 +397,7 @@ func (client *cliAdminClient) GetMaintenanceZone() (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return status.Cluster.MaintenanceZone, nil
+	return string(status.Cluster.MaintenanceZone), nil
 }
 
 // SetMaintenanceZone places zone into maintenance mode
diff --git a/kubectl-fdb/cmd/analyze.go b/kubectl-fdb/cmd/analyze.go
index 6b84cc87..c0561237 100644
--- a/kubectl-fdb/cmd/analyze.go
+++ b/kubectl-fdb/cmd/analyze.go
@@ -53,10 +53,6 @@ func newAnalyzeCmd(streams genericclioptions.IOStreams) *cobra.Command {
 			if err != nil {
 				return err
 			}
-			sleep, err := cmd.Root().Flags().GetUint16("sleep")
-			if err != nil {
-				return err
-			}
 			autoFix, err := cmd.Flags().GetBool("auto-fix")
 			if err != nil {
 				return err
@@ -141,7 +137,7 @@ func newAnalyzeCmd(streams genericclioptions.IOStreams) *cobra.Command {
 					errs = append(errs, err)
 				}
 
-				err = analyzeCluster(cmd, kubeClient, cluster, autoFix, wait, ignoreConditions, ignoreRemovals, sleep)
+				err = analyzeCluster(cmd, kubeClient, cluster, autoFix, wait, ignoreConditions, ignoreRemovals)
 				if err != nil {
 					errs = append(errs, err)
 				}
@@ -223,7 +219,7 @@ func allConditionsValid(conditions []string) error {
 	return fmt.Errorf(errString.String())
 }
 
-func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, cluster *fdbv1beta2.FoundationDBCluster, autoFix bool, wait bool, ignoreConditions []string, ignoreRemovals bool, sleep uint16) error {
+func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, cluster *fdbv1beta2.FoundationDBCluster, autoFix bool, wait bool, ignoreConditions []string, ignoreRemovals bool) error {
 	var foundIssues bool
 	cmd.Printf("Checking cluster: %s/%s\n", cluster.Namespace, cluster.Name)
 
@@ -394,7 +390,7 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, cluster *fdbv1
 		confirmed := false
 
 		if len(failedProcessGroups) > 0 {
-			err := replaceProcessGroups(kubeClient, cluster.Name, failedProcessGroups, cluster.Namespace, true, wait, false, true, sleep)
+			err := replaceProcessGroups(kubeClient, cluster.Name, failedProcessGroups, cluster.Namespace, true, wait, false, true)
 			if err != nil {
 				return err
 			}
diff --git a/kubectl-fdb/cmd/cordon.go b/kubectl-fdb/cmd/cordon.go
index 6be843a7..db063c8e 100644
--- a/kubectl-fdb/cmd/cordon.go
+++ b/kubectl-fdb/cmd/cordon.go
@@ -47,10 +47,6 @@ func newCordonCmd(streams genericclioptions.IOStreams) *cobra.Command {
 			if err != nil {
 				return err
 			}
-			sleep, err := cmd.Root().Flags().GetUint16("sleep")
-			if err != nil {
-				return err
-			}
 			clusterName, err := cmd.Flags().GetString("fdb-cluster")
 			if err != nil {
 				return err
@@ -88,10 +84,10 @@ func newCordonCmd(streams genericclioptions.IOStreams) *cobra.Command {
 					return err
 				}
 
-				return cordonNode(cmd, kubeClient, clusterName, nodes, namespace, withExclusion, wait, sleep, clusterLabel)
+				return cordonNode(cmd, kubeClient, clusterName, nodes, namespace, withExclusion, wait, clusterLabel)
 			}
 
-			return cordonNode(cmd, kubeClient, clusterName, args, namespace, withExclusion, wait, sleep, clusterLabel)
+			return cordonNode(cmd, kubeClient, clusterName, args, namespace, withExclusion, wait, clusterLabel)
 		},
 		Example: `
 # Evacuate all process groups for a cluster in the current namespace that are hosted on node-1
@@ -130,7 +126,7 @@ kubectl fdb cordon --node-selector machine=a,disk=fast -l fdb-cluster-label
 }
 
 // cordonNode gets all process groups of this cluster that run on the given nodes and add them to the remove list
-func cordonNode(cmd *cobra.Command, kubeClient client.Client, inputClusterName string, nodes []string, namespace string, withExclusion bool, wait bool, sleep uint16, clusterLabel string) error {
+func cordonNode(cmd *cobra.Command, kubeClient client.Client, inputClusterName string, nodes []string, namespace string, withExclusion bool, wait bool, clusterLabel string) error {
 	cmd.Printf("Start to cordon %d nodes\n", len(nodes))
 	if len(nodes) == 0 {
 		return nil
@@ -140,9 +136,9 @@ func cordonNode(cmd *cobra.Command, kubeClient client.Client, inputClusterName s
 	for _, node := range nodes {
 		pods, err := fetchPods(kubeClient, inputClusterName, namespace, node, clusterLabel)
 		if err != nil {
-			error := fmt.Sprintf("Issue fetching Pods running on node: %s. Error: %s\n", node, err)
-			cmd.PrintErr(error)
-			errors = append(errors, error)
+			internalErr := fmt.Sprintf("Issue fetching Pods running on node: %s. Error: %s\n", node, err)
+			cmd.PrintErr(internalErr)
+			errors = append(errors, internalErr)
 			continue
 		}
 
@@ -151,9 +147,9 @@ func cordonNode(cmd *cobra.Command, kubeClient client.Client, inputClusterName s
 			cmd.Printf("Starting operation on %s, node: %s\n", clusterName, node)
 			cluster, err := loadCluster(kubeClient, namespace, clusterName)
 			if err != nil {
-				error := fmt.Sprintf("unable to load cluster: %s, skipping\n", clusterName)
-				errors = append(errors, error)
-				cmd.PrintErr(error)
+				internalErr := fmt.Sprintf("unable to load cluster: %s, skipping\n", clusterName)
+				errors = append(errors, internalErr)
+				cmd.PrintErr(internalErr)
 				continue
 			}
 
@@ -175,11 +171,11 @@ func cordonNode(cmd *cobra.Command, kubeClient client.Client, inputClusterName s
 					processGroups = append(processGroups, processGroup)
 				}
 			}
-			err = replaceProcessGroups(kubeClient, cluster.Name, processGroups, namespace, withExclusion, wait, false, true, sleep)
+			err = replaceProcessGroups(kubeClient, cluster.Name, processGroups, namespace, withExclusion, wait, false, true)
 			if err != nil {
-				error := fmt.Sprintf("unable to cordon all Pods for cluster %s\n", cluster.Name)
-				errors = append(errors, error)
-				cmd.PrintErr(error)
+				internalErr := fmt.Sprintf("unable to cordon all Pods for cluster %s\n", cluster.Name)
+				errors = append(errors, internalErr)
+				cmd.PrintErr(internalErr)
 			}
 		}
 	}
@@ -192,25 +188,26 @@ func cordonNode(cmd *cobra.Command, kubeClient client.Client, inputClusterName s
 func fetchPods(kubeClient client.Client, clusterName string, namespace string, node string, clusterLabel string) (corev1.PodList, error) {
 	var pods corev1.PodList
 	var err error
+
+	var podLabelSelector client.ListOption
+
 	if clusterName == "" {
-		err = kubeClient.List(ctx.Background(), &pods,
-			client.InNamespace(namespace),
-			client.HasLabels([]string{clusterLabel}),
-			client.MatchingFieldsSelector{
-				Selector: fields.OneTermEqualSelector("spec.nodeName", node),
-			})
+		podLabelSelector = client.HasLabels([]string{clusterLabel})
 	} else {
-		cluster, error := loadCluster(kubeClient, namespace, clusterName)
-		if error != nil {
+		cluster, err := loadCluster(kubeClient, namespace, clusterName)
+		if err != nil {
 			return pods, fmt.Errorf("unable to load cluster: %s. Error: %w", clusterName, err)
 		}
-		err = kubeClient.List(ctx.Background(), &pods,
-			client.InNamespace(namespace),
-			client.MatchingLabels(cluster.GetMatchLabels()),
-			client.MatchingFieldsSelector{
-				Selector: fields.OneTermEqualSelector("spec.nodeName", node),
-			})
+
+		podLabelSelector = client.MatchingLabels(cluster.GetMatchLabels())
 	}
+
+	err = kubeClient.List(ctx.Background(), &pods,
+		client.InNamespace(namespace),
+		podLabelSelector,
+		client.MatchingFieldsSelector{
+			Selector: fields.OneTermEqualSelector("spec.nodeName", node),
+		})
 	if err != nil {
 		return pods, fmt.Errorf("unable to fetch pods. Error: %w", err)
 	}
@@ -231,5 +228,6 @@ func getClusterNames(cmd *cobra.Command, clusterName string, pods corev1.PodList
 		}
 		clusterNames[clusterName] = fdbv1beta2.None{}
 	}
+
 	return clusterNames
 }
diff --git a/kubectl-fdb/cmd/remove_process_group.go b/kubectl-fdb/cmd/remove_process_group.go
index d7342dbe..bb04e6d2 100644
--- a/kubectl-fdb/cmd/remove_process_group.go
+++ b/kubectl-fdb/cmd/remove_process_group.go
@@ -23,13 +23,11 @@ package cmd
 import (
 	ctx "context"
 	"fmt"
-	"log"
-	"time"
-
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	"github.com/spf13/cobra"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"log"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -45,10 +43,6 @@ func newRemoveProcessGroupCmd(streams genericclioptions.IOStreams) *cobra.Comman
 			if err != nil {
 				return err
 			}
-			sleep, err := cmd.Root().Flags().GetUint16("sleep")
-			if err != nil {
-				return err
-			}
 			cluster, err := cmd.Flags().GetString("fdb-cluster")
 			if err != nil {
 				return err
@@ -76,7 +70,7 @@ func newRemoveProcessGroupCmd(streams genericclioptions.IOStreams) *cobra.Comman
 				return err
 			}
 
-			return replaceProcessGroups(kubeClient, cluster, args, namespace, withExclusion, wait, removeAllFailed, useProcessGroupID, sleep)
+			return replaceProcessGroups(kubeClient, cluster, args, namespace, withExclusion, wait, removeAllFailed, useProcessGroupID)
 		},
 		Example: `
 # Remove process groups for a cluster in the current namespace
@@ -112,7 +106,7 @@ kubectl fdb -n default remove process-group -c cluster --remove-all-failed
 }
 
 // replaceProcessGroups adds process groups to the removal list of the cluster
-func replaceProcessGroups(kubeClient client.Client, clusterName string, ids []string, namespace string, withExclusion bool, wait bool, removeAllFailed bool, useProcessGroupID bool, sleep uint16) error {
+func replaceProcessGroups(kubeClient client.Client, clusterName string, ids []string, namespace string, withExclusion bool, wait bool, removeAllFailed bool, useProcessGroupID bool) error {
 	if len(ids) == 0 && !removeAllFailed {
 		return nil
 	}
@@ -166,22 +160,11 @@ func replaceProcessGroups(kubeClient client.Client, clusterName string, ids []st
 		}
 	}
 
-	if sleep > 0 {
-		for _, processGroupID := range processGroupIDs {
-			addProcessGroups([]fdbv1beta2.ProcessGroupID{processGroupID}, withExclusion, cluster)
-			time.Sleep(time.Duration(sleep) * time.Second)
-		}
+	if withExclusion {
+		cluster.Spec.ProcessGroupsToRemove = cluster.GetProcessGroupsToRemove(processGroupIDs)
 	} else {
-		addProcessGroups(processGroupIDs, withExclusion, cluster)
+		cluster.Spec.ProcessGroupsToRemoveWithoutExclusion = cluster.GetProcessGroupsToRemoveWithoutExclusion(processGroupIDs)
 	}
 
 	return kubeClient.Patch(ctx.TODO(), cluster, patch)
 }
-
-func addProcessGroups(processGroupIDs []fdbv1beta2.ProcessGroupID, withExclusion bool, cluster *fdbv1beta2.FoundationDBCluster) {
-	if withExclusion {
-		cluster.AddProcessGroupsToRemovalList(processGroupIDs)
-	} else {
-		cluster.AddProcessGroupsToRemovalWithoutExclusionList(processGroupIDs)
-	}
-}
diff --git a/pkg/fdbadminclient/mock/admin_client_mock.go b/pkg/fdbadminclient/mock/admin_client_mock.go
index a1446605..ce3c2ea2 100644
--- a/pkg/fdbadminclient/mock/admin_client_mock.go
+++ b/pkg/fdbadminclient/mock/admin_client_mock.go
@@ -60,7 +60,7 @@ type AdminClient struct {
 	localityInfo                             map[fdbv1beta2.ProcessGroupID]map[string]string
 	MaxZoneFailuresWithoutLosingData         *int
 	MaxZoneFailuresWithoutLosingAvailability *int
-	MaintenanceZone                          string
+	MaintenanceZone                          fdbv1beta2.FaultDomain
 	restoreURL                               string
 	maintenanceZoneStartTimestamp            time.Time
 	uptimeSecondsForMaintenanceZone          float64
@@ -230,7 +230,7 @@ func (client *AdminClient) GetStatus() (*fdbv1beta2.FoundationDBStatus, error) {
 			}
 
 			var uptimeSeconds float64 = 60000
-			if client.MaintenanceZone == locality[fdbv1beta2.FDBLocalityZoneIDKey] || client.MaintenanceZone == "simulation" {
+			if string(client.MaintenanceZone) == locality[fdbv1beta2.FDBLocalityZoneIDKey] || client.MaintenanceZone == "simulation" {
 				if client.uptimeSecondsForMaintenanceZone != 0.0 {
 					uptimeSeconds = client.uptimeSecondsForMaintenanceZone
 				} else {
@@ -287,7 +287,7 @@ func (client *AdminClient) GetStatus() (*fdbv1beta2.FoundationDBStatus, error) {
 		}
 
 		var uptimeSeconds float64 = 60000
-		if client.MaintenanceZone == locality[fdbv1beta2.FDBLocalityZoneIDKey] || client.MaintenanceZone == "simulation" {
+		if string(client.MaintenanceZone) == locality[fdbv1beta2.FDBLocalityZoneIDKey] || client.MaintenanceZone == "simulation" {
 			if client.uptimeSecondsForMaintenanceZone != 0.0 {
 				uptimeSeconds = client.uptimeSecondsForMaintenanceZone
 			} else {
@@ -776,12 +776,12 @@ func (client *AdminClient) SetKnobs(knobs []string) {
 
 // GetMaintenanceZone gets current maintenance zone, if any
 func (client *AdminClient) GetMaintenanceZone() (string, error) {
-	return client.MaintenanceZone, nil
+	return string(client.MaintenanceZone), nil
 }
 
 // SetMaintenanceZone places zone into maintenance mode
 func (client *AdminClient) SetMaintenanceZone(zone string, _ int) error {
-	client.MaintenanceZone = zone
+	client.MaintenanceZone = fdbv1beta2.FaultDomain(zone)
 	client.maintenanceZoneStartTimestamp = time.Now()
 	return nil
 }


Test patch:
diff --git a/api/v1beta2/foundationdbcluster_types_test.go b/api/v1beta2/foundationdbcluster_types_test.go
index 18188b84..6c1e1d9f 100644
--- a/api/v1beta2/foundationdbcluster_types_test.go
+++ b/api/v1beta2/foundationdbcluster_types_test.go
@@ -4459,6 +4459,151 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 		)
 	})
 
+	When("getting the removal list", func() {
+		var cluster *FoundationDBCluster
+		var processGroupsToRemove, result []ProcessGroupID
+
+		BeforeEach(func() {
+			cluster = &FoundationDBCluster{
+				Status: FoundationDBClusterStatus{
+					ProcessGroups: []*ProcessGroupStatus{
+						{
+							ProcessGroupID: "test-1",
+						},
+						{
+							ProcessGroupID: "test-2",
+						},
+						{
+							ProcessGroupID: "test-3",
+						},
+					},
+				},
+			}
+		})
+
+		JustBeforeEach(func() {
+			result = cluster.GetProcessGroupsToRemove(processGroupsToRemove)
+		})
+
+		When("removing a non existing process group ID", func() {
+			BeforeEach(func() {
+				processGroupsToRemove = []ProcessGroupID{"test-4"}
+			})
+
+			It("should return an empty list", func() {
+				Expect(result).To(HaveLen(0))
+			})
+		})
+
+		When("removing an existing process group ID", func() {
+			BeforeEach(func() {
+				processGroupsToRemove = []ProcessGroupID{"test-1"}
+			})
+
+			It("should contain the provided process group", func() {
+				Expect(result).To(ConsistOf(ProcessGroupID("test-1")))
+			})
+
+			When("the process group is already marked for removal", func() {
+				BeforeEach(func() {
+					cluster.Status.ProcessGroups[0].MarkForRemoval()
+				})
+
+				It("should return an empty list", func() {
+					Expect(result).To(HaveLen(0))
+				})
+			})
+
+			When("another process group is already marked for removal", func() {
+				BeforeEach(func() {
+					cluster.Status.ProcessGroups[1].MarkForRemoval()
+				})
+
+				It("should return an empty list", func() {
+					Expect(result).To(ConsistOf(ProcessGroupID("test-1")))
+				})
+			})
+		})
+	})
+
+	When("getting the removal list without exclusion", func() {
+		var cluster *FoundationDBCluster
+		var processGroupsToRemove, result []ProcessGroupID
+
+		BeforeEach(func() {
+			cluster = &FoundationDBCluster{
+				Status: FoundationDBClusterStatus{
+					ProcessGroups: []*ProcessGroupStatus{
+						{
+							ProcessGroupID: "test-1",
+						},
+						{
+							ProcessGroupID: "test-2",
+						},
+						{
+							ProcessGroupID: "test-3",
+						},
+					},
+				},
+			}
+		})
+
+		JustBeforeEach(func() {
+			result = cluster.GetProcessGroupsToRemoveWithoutExclusion(processGroupsToRemove)
+		})
+
+		When("removing a non existing process group ID", func() {
+			BeforeEach(func() {
+				processGroupsToRemove = []ProcessGroupID{"test-4"}
+			})
+
+			It("should return an empty list", func() {
+				Expect(result).To(HaveLen(0))
+			})
+		})
+
+		When("removing an existing process group ID", func() {
+			BeforeEach(func() {
+				processGroupsToRemove = []ProcessGroupID{"test-1"}
+			})
+
+			It("should contain the provided process group", func() {
+				Expect(result).To(ConsistOf(ProcessGroupID("test-1")))
+			})
+
+			When("the process group is already marked for removal", func() {
+				BeforeEach(func() {
+					cluster.Status.ProcessGroups[0].MarkForRemoval()
+				})
+
+				It("should contain the provided process group", func() {
+					Expect(result).To(ConsistOf(ProcessGroupID("test-1")))
+				})
+
+				When("the process group is also excluded", func() {
+					BeforeEach(func() {
+						cluster.Status.ProcessGroups[0].SetExclude()
+					})
+
+					It("should return an empty list", func() {
+						Expect(result).To(HaveLen(0))
+					})
+				})
+			})
+
+			When("another process group is already marked for removal and excluded", func() {
+				BeforeEach(func() {
+					cluster.Status.ProcessGroups[1].MarkForRemoval()
+					cluster.Status.ProcessGroups[1].SetExclude()
+				})
+
+				It("should return an empty list", func() {
+					Expect(result).To(ConsistOf(ProcessGroupID("test-1")))
+				})
+			})
+		})
+	})
+
 	When("adding a process to the removal list", func() {
 		var cluster *FoundationDBCluster
 
diff --git a/controllers/maintenance_mode_checker_test.go b/controllers/maintenance_mode_checker_test.go
index 2f78adbf..e21ed55f 100644
--- a/controllers/maintenance_mode_checker_test.go
+++ b/controllers/maintenance_mode_checker_test.go
@@ -84,7 +84,7 @@ var _ = Describe("maintenance_mode_checker", func() {
 	Context("maintenance mode is on", func() {
 		BeforeEach(func() {
 			cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{
-				ZoneID: targetProcessGroup,
+				ZoneID: fdbv1beta2.FaultDomain(targetProcessGroup),
 			}
 			Expect(k8sClient.Status().Update(context.TODO(), cluster)).NotTo(HaveOccurred())
 		})
diff --git a/controllers/update_status_test.go b/controllers/update_status_test.go
index 5899ae59..c4f80af5 100644
--- a/controllers/update_status_test.go
+++ b/controllers/update_status_test.go
@@ -808,7 +808,7 @@ var _ = Describe("update_status", func() {
 						continue
 					}
 
-					Expect(processGroup.FaultDomain).To(And(HavePrefix(string(processGroup.ProcessGroupID)), HaveSuffix("zone")))
+					Expect(string(processGroup.FaultDomain)).To(And(HavePrefix(string(processGroup.ProcessGroupID)), HaveSuffix("zone")))
 				}
 			})
 		})
@@ -847,7 +847,7 @@ var _ = Describe("update_status", func() {
 						continue
 					}
 
-					Expect(processGroup.FaultDomain).To(And(HavePrefix(string(processGroup.ProcessGroupID)), HaveSuffix("zone")))
+					Expect(string(processGroup.FaultDomain)).To(And(HavePrefix(string(processGroup.ProcessGroupID)), HaveSuffix("zone")))
 				}
 			})
 		})
diff --git a/kubectl-fdb/cmd/analyze_test.go b/kubectl-fdb/cmd/analyze_test.go
index b4522449..7d509342 100644
--- a/kubectl-fdb/cmd/analyze_test.go
+++ b/kubectl-fdb/cmd/analyze_test.go
@@ -107,7 +107,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				inBuffer := bytes.Buffer{}
 
 				cmd := newAnalyzeCmd(genericclioptions.IOStreams{In: &inBuffer, Out: &outBuffer, ErrOut: &errBuffer})
-				err := analyzeCluster(cmd, k8sClient, tc.cluster, tc.AutoFix, tc.NoWait, tc.IgnoredConditions, tc.IgnoreRemovals, 0)
+				err := analyzeCluster(cmd, k8sClient, tc.cluster, tc.AutoFix, tc.NoWait, tc.IgnoredConditions, tc.IgnoreRemovals)
 
 				if err != nil && !tc.HasErrors {
 					Expect(err).To(HaveOccurred())
diff --git a/kubectl-fdb/cmd/cordon_test.go b/kubectl-fdb/cmd/cordon_test.go
index 2655ca59..fe4bf3a5 100644
--- a/kubectl-fdb/cmd/cordon_test.go
+++ b/kubectl-fdb/cmd/cordon_test.go
@@ -65,8 +65,8 @@ var _ = Describe("[plugin] cordon command", func() {
 				errBuffer := bytes.Buffer{}
 				inBuffer := bytes.Buffer{}
 
-				cmd := newAnalyzeCmd(genericclioptions.IOStreams{In: &inBuffer, Out: &outBuffer, ErrOut: &errBuffer})
-				err := cordonNode(cmd, k8sClient, input.clusterName, input.nodes, namespace, input.WithExclusion, false, 0, input.clusterLabel)
+				cmd := newCordonCmd(genericclioptions.IOStreams{In: &inBuffer, Out: &outBuffer, ErrOut: &errBuffer})
+				err := cordonNode(cmd, k8sClient, input.clusterName, input.nodes, namespace, input.WithExclusion, false, input.clusterLabel)
 				Expect(err).NotTo(HaveOccurred())
 
 				clusterNames := []string{clusterName, secondClusterName}
diff --git a/kubectl-fdb/cmd/remove_process_group_test.go b/kubectl-fdb/cmd/remove_process_group_test.go
index 4f521e46..dda374c5 100644
--- a/kubectl-fdb/cmd/remove_process_group_test.go
+++ b/kubectl-fdb/cmd/remove_process_group_test.go
@@ -60,7 +60,7 @@ var _ = Describe("[plugin] remove process groups command", func() {
 
 			DescribeTable("should cordon all targeted processes",
 				func(tc testCase) {
-					err := replaceProcessGroups(k8sClient, clusterName, tc.Instances, namespace, tc.WithExclusion, false, tc.RemoveAllFailed, false, 0)
+					err := replaceProcessGroups(k8sClient, clusterName, tc.Instances, namespace, tc.WithExclusion, false, tc.RemoveAllFailed, false)
 					Expect(err).NotTo(HaveOccurred())
 
 					var resCluster fdbv1beta2.FoundationDBCluster
@@ -117,7 +117,7 @@ var _ = Describe("[plugin] remove process groups command", func() {
 				When("adding the same process group to the removal list without exclusion", func() {
 					It("should add the process group to the removal without exclusion list", func() {
 						removals := []string{"test-storage-1"}
-						err := replaceProcessGroups(k8sClient, clusterName, removals, namespace, false, false, false, false, 0)
+						err := replaceProcessGroups(k8sClient, clusterName, removals, namespace, false, false, false, false)
 						Expect(err).NotTo(HaveOccurred())
 
 						var resCluster fdbv1beta2.FoundationDBCluster
@@ -137,7 +137,7 @@ var _ = Describe("[plugin] remove process groups command", func() {
 				When("adding the same process group to the removal list", func() {
 					It("should add the process group to the removal without exclusion list", func() {
 						removals := []string{"test-storage-1"}
-						err := replaceProcessGroups(k8sClient, clusterName, removals, namespace, true, false, false, false, 0)
+						err := replaceProcessGroups(k8sClient, clusterName, removals, namespace, true, false, false, false)
 						Expect(err).NotTo(HaveOccurred())
 
 						var resCluster fdbv1beta2.FoundationDBCluster
diff --git a/kubectl-fdb/cmd/suite_test.go b/kubectl-fdb/cmd/suite_test.go
index d482594c..aadef718 100644
--- a/kubectl-fdb/cmd/suite_test.go
+++ b/kubectl-fdb/cmd/suite_test.go
@@ -53,5 +53,15 @@ func generateClusterStruct(name string, namespace string) *fdbv1beta2.Foundation
 				Storage: 1,
 			},
 		},
+		Status: fdbv1beta2.FoundationDBClusterStatus{
+			ProcessGroups: []*fdbv1beta2.ProcessGroupStatus{
+				{
+					ProcessGroupID: fdbv1beta2.ProcessGroupID(name + "-instance-1"),
+				},
+				{
+					ProcessGroupID: fdbv1beta2.ProcessGroupID(name + "-instance-2"),
+				},
+			},
+		},
 	}
 }