Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1145 (2022-03-28 19:23:17)

Ensure we delete all resources when a process group is in resource terminating state
# Description

Fixes: https://github.com/FoundationDB/fdb-kubernetes-operator/issues/1144

## Type of change

*Please select one of the options below.*

- Bug fix (non-breaking change which fixes an issue)

## Discussion

-

## Testing

Local + unit test.

## Documentation

-

## Follow-up

-


foundationdb-ci: 
### AWS CodeBuild CI Report for Linux CentOS 7

* CodeBuild project: fdb-kubernetes-operator-pr
* Commit ID: 307e7efff416f1a88ac766a0529d3158ffbeb3f6
* Result: FAILED
* Error: `Error while executing command: make -C tests -kj prOperator. Reason: exit status 2`
* [Build Logs](https://d1e0l78xbkh3xa.cloudfront.net/build-logs/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/12169f8c-07f1-4e30-aae7-de2e26b742dd.gz) (available for 30 days)


Patch:
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 87bb373d..7b502897 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -11,14 +11,14 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        fdbver: [ "6.2.30", "6.3.22" ]
+        fdbver: [ "6.2.30", "6.3.22", "7.1.0-rc3" ]
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
       - name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.17.6
+          go-version: 1.17.8
       - uses: actions/cache@v2
         with:
           path: ~/go/pkg/mod
@@ -40,12 +40,12 @@ jobs:
         # With a new supported Kubernetes version we should adjust the version
         # See https://kubernetes.io/releases for the current releases
         kubever: [ "v1.19.0", "v1.20.0", "v1.21.1", "v1.22.0", "v1.23.0" ]
-        fdbver: ["6.2.30", "6.3.22"]
+        fdbver: [ "6.2.30", "6.3.22", "7.1.0-rc3" ]
     steps:
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.17.6
+        go-version: 1.17.8
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
       with:
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index bc256a5c..a22bea65 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -43,7 +43,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.17.6
+          go-version: 1.17.8
       - name: Release binaries
         run: make release
         env:
diff --git a/Dockerfile b/Dockerfile
index c5f51e88..f4594857 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM docker.io/library/golang:1.17.6 as builder
+FROM docker.io/library/golang:1.17.8 as builder
 
 # Install FDB this version is only required to compile the fdb operator
 ARG FDB_VERSION=6.3.23
diff --git a/README.md b/README.md
index 8a261031..55298e35 100644
--- a/README.md
+++ b/README.md
@@ -44,7 +44,7 @@ For more information on the fields you can define on the cluster resource, see t
 
 1. Install Go on your machine, see the [Getting Started](https://golang.org/doc/install) guide for more information.
 1. Install the required dependencies with `make deps`.
-1. Install the [foundationDB client package](https://www.foundationdb.org/download).
+1. Install the [foundationDB client package](https://github.com/apple/foundationdb/releases).
 
 ### Running Locally
 
diff --git a/api/v1beta2/foundationdb_database_configuration.go b/api/v1beta2/foundationdb_database_configuration.go
index d9a3d78e..239b8b35 100644
--- a/api/v1beta2/foundationdb_database_configuration.go
+++ b/api/v1beta2/foundationdb_database_configuration.go
@@ -1,5 +1,5 @@
 /*
- * foundationdb_labels.go
+ * foundationdb_database_configuration.go
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -39,7 +39,7 @@ type DatabaseConfiguration struct {
 
 	// StorageEngine defines the storage engine the database uses.
 	// +kubebuilder:validation:Optional
-	// +kubebuilder:validation:Enum=ssd;ssd-1;ssd-2;memory;memory-1;memory-2;ssd-redwood-1-experimental;ssd-rocksdb-experimental;memory-radixtree-beta;custom
+	// +kubebuilder:validation:Enum=ssd;ssd-1;ssd-2;memory;memory-1;memory-2;ssd-redwood-1-experimental;ssd-rocksdb-experimental;ssd-rocksdb-v1;memory-radixtree-beta;custom
 	// +kubebuilder:default:=ssd-2
 	StorageEngine StorageEngine `json:"storage_engine,omitempty"`
 
@@ -67,7 +67,7 @@ type Region struct {
 	SatelliteLogs int `json:"satellite_logs,omitempty"`
 
 	// The replication strategy for satellite logs.
-	SatelliteRedundancyMode string `json:"satellite_redundancy_mode,omitempty"`
+	SatelliteRedundancyMode RedundancyMode `json:"satellite_redundancy_mode,omitempty"`
 }
 
 // DataCenter represents a data center in the region configuration
@@ -110,10 +110,12 @@ func (configuration *DatabaseConfiguration) FailOver() DatabaseConfiguration {
 	}
 
 	for _, region := range newConfiguration.Regions {
-		// In order to trigger a fail over we have to change the priority
-		// of the main and remote dc
+		// In order to trigger a fail-over we have to change the priority of the main and remote dc
 		newRegion := Region{}
 		for _, dc := range region.DataCenters {
+			newRegion.SatelliteRedundancyMode = region.SatelliteRedundancyMode
+			newRegion.SatelliteLogs = region.SatelliteLogs
+
 			// Don't change the Satellite config
 			if dc.Satellite == 1 {
 				newRegion.DataCenters = append(newRegion.DataCenters, dc)
@@ -661,6 +663,10 @@ const (
 	RedundancyModeDouble RedundancyMode = "double"
 	// RedundancyModeTriple defines the replication factor 3.
 	RedundancyModeTriple RedundancyMode = "triple"
+	// RedundancyModeOneSatelliteSingle defines the replication factor one_satellite_single.
+	RedundancyModeOneSatelliteSingle RedundancyMode = "one_satellite_single"
+	// RedundancyModeOneSatelliteDouble  defines the replication factor one_satellite_double.
+	RedundancyModeOneSatelliteDouble RedundancyMode = "one_satellite_double"
 	// RedundancyModeUnset defines the replication factor unset.
 	RedundancyModeUnset RedundancyMode = ""
 )
@@ -678,6 +684,10 @@ const (
 	StorageEngineMemory StorageEngine = "memory"
 	// StorageEngineMemory2 defines the storage engine memory-2.
 	StorageEngineMemory2 StorageEngine = "memory-2"
+	// StorageEngineRocksDbExperimental defines the storage engine ssd-rocksdb-experimental.
+	StorageEngineRocksDbExperimental StorageEngine = "ssd-rocksdb-experimental"
+	// StorageEngineRocksDbV1 defines the storage engine ssd-rocksdb-v1.
+	StorageEngineRocksDbV1 StorageEngine = "ssd-rocksdb-v1"
 )
 
 // RoleCounts represents the roles whose counts can be customized.
diff --git a/api/v1beta2/foundationdb_status.go b/api/v1beta2/foundationdb_status.go
index 054ff4e2..07d3d1c6 100644
--- a/api/v1beta2/foundationdb_status.go
+++ b/api/v1beta2/foundationdb_status.go
@@ -133,6 +133,8 @@ type FoundationDBStatusProcessInfo struct {
 type FoundationDBStatusProcessRoleInfo struct {
 	// Role defines the role a process currently has
 	Role string `json:"role,omitempty"`
+	// StoredBytes defines the number of bytes that are currently stored for this process.
+	StoredBytes int `json:"stored_bytes,omitempty"`
 }
 
 // FoundationDBStatusDataStatistics provides information about the data in
diff --git a/api/v1beta2/foundationdb_version.go b/api/v1beta2/foundationdb_version.go
index 2eb45052..2df26cb2 100644
--- a/api/v1beta2/foundationdb_version.go
+++ b/api/v1beta2/foundationdb_version.go
@@ -111,6 +111,12 @@ func (version Version) IsAtLeast(other Version) bool {
 	if version.Patch > other.Patch {
 		return true
 	}
+	if version.ReleaseCandidate == 0 {
+		return true
+	}
+	if other.ReleaseCandidate == 0 {
+		return false
+	}
 	if version.ReleaseCandidate < other.ReleaseCandidate {
 		return false
 	}
@@ -174,6 +180,16 @@ func (version Version) IsSupported() bool {
 	return version.IsAtLeast(Versions.MinimumVersion)
 }
 
+// IsStorageEngineSupported return true if storage engine is supported by FDB version.
+func (version Version) IsStorageEngineSupported(storageEngine StorageEngine) bool {
+	if storageEngine == StorageEngineRocksDbV1 {
+		return version.IsAtLeast(Versions.SupportsRocksDBV1)
+	} else if storageEngine == StorageEngineRocksDbExperimental {
+		return !version.IsAtLeast(Versions.SupportsRocksDBV1)
+	}
+	return true
+}
+
 // IsReleaseCandidate returns true if the version is a release candidate or not
 func (version Version) IsReleaseCandidate() bool {
 	return version.ReleaseCandidate > 0
@@ -185,10 +201,12 @@ var Versions = struct {
 	NextMajorVersion,
 	NextPatchVersion,
 	MinimumVersion,
+	SupportsRocksDBV1,
 	Default Version
 }{
-	Default:          Version{Major: 6, Minor: 2, Patch: 20},
-	NextPatchVersion: Version{Major: 6, Minor: 2, Patch: 21},
-	NextMajorVersion: Version{Major: 7, Minor: 0, Patch: 0},
-	MinimumVersion:   Version{Major: 6, Minor: 2, Patch: 20},
+	Default:           Version{Major: 6, Minor: 2, Patch: 20},
+	NextPatchVersion:  Version{Major: 6, Minor: 2, Patch: 21},
+	NextMajorVersion:  Version{Major: 7, Minor: 0, Patch: 0},
+	MinimumVersion:    Version{Major: 6, Minor: 2, Patch: 20},
+	SupportsRocksDBV1: Version{Major: 7, Minor: 1, Patch: 0, ReleaseCandidate: 4},
 }
diff --git a/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml b/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
index 6e139e2f..5f833a19 100644
--- a/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
+++ b/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
@@ -9463,6 +9463,7 @@ spec:
                         satellite_logs:
                           type: integer
                         satellite_redundancy_mode:
+                          maxLength: 100
                           type: string
                       type: object
                     type: array
@@ -9483,6 +9484,7 @@ spec:
                     - memory-2
                     - ssd-redwood-1-experimental
                     - ssd-rocksdb-experimental
+                    - ssd-rocksdb-v1
                     - memory-radixtree-beta
                     - custom
                     maxLength: 100
@@ -12855,6 +12857,7 @@ spec:
                         satellite_logs:
                           type: integer
                         satellite_redundancy_mode:
+                          maxLength: 100
                           type: string
                       type: object
                     type: array
@@ -12875,6 +12878,7 @@ spec:
                     - memory-2
                     - ssd-redwood-1-experimental
                     - ssd-rocksdb-experimental
+                    - ssd-rocksdb-v1
                     - memory-radixtree-beta
                     - custom
                     maxLength: 100
diff --git a/config/deployment/manager.yaml b/config/deployment/manager.yaml
index 44568f35..e7d1569d 100644
--- a/config/deployment/manager.yaml
+++ b/config/deployment/manager.yaml
@@ -66,7 +66,7 @@ spec:
       containers:
       - command:
         - /manager
-        image: foundationdb/fdb-kubernetes-operator:v1.1.1
+        image: foundationdb/fdb-kubernetes-operator:v1.2.0
         name: manager
         env:
           - name: WATCH_NAMESPACE
diff --git a/config/samples/deployment.yaml b/config/samples/deployment.yaml
index 1887e527..4f454121 100644
--- a/config/samples/deployment.yaml
+++ b/config/samples/deployment.yaml
@@ -150,7 +150,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: foundationdb/fdb-kubernetes-operator:v1.1.1
+        image: foundationdb/fdb-kubernetes-operator:v1.2.0
         name: manager
         ports:
         - containerPort: 8080
diff --git a/controllers/add_pods.go b/controllers/add_pods.go
index b10b5d4d..0c275b41 100644
--- a/controllers/add_pods.go
+++ b/controllers/add_pods.go
@@ -24,9 +24,8 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/FoundationDB/fdb-kubernetes-operator/pkg/podmanager"
-
 	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
+	"github.com/FoundationDB/fdb-kubernetes-operator/pkg/podmanager"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	corev1 "k8s.io/api/core/v1"
@@ -64,8 +63,18 @@ func (a addPods) reconcile(ctx context.Context, r *FoundationDBClusterReconciler
 	podMap := internal.CreatePodMap(cluster, pods)
 
 	for _, processGroup := range cluster.Status.ProcessGroups {
-		_, podExists := podMap[processGroup.ProcessGroupID]
-		if podExists || processGroup.IsMarkedForRemoval() {
+		if _, podExists := podMap[processGroup.ProcessGroupID]; podExists {
+			continue
+		}
+
+		// If this process group is marked for removal, we normally don't want to spin it back up
+		// again. However, in a downscaling scenario, it could be that this is a storage node that
+		// is still draining its data onto another one. Therefore, we only want to leave it off
+		// (by continue'ing) if the cluster says that this process group is fully drained and safe
+		// to delete, which is the case if a previous run of the `removeProcessGroups` subreconciler
+		// has marked it as excluded in the cluster status (it does so only after executing the
+		// `exclude` FDB command and being told that the nodes in question are fully excluded).
+		if processGroup.IsMarkedForRemoval() && processGroup.IsExcluded() {
 			continue
 		}
 
diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go
index e0867a31..7c5e5b65 100644
--- a/controllers/cluster_controller.go
+++ b/controllers/cluster_controller.go
@@ -115,6 +115,15 @@ func (r *FoundationDBClusterReconciler) Reconcile(ctx context.Context, request c
 		return ctrl.Result{}, fmt.Errorf("version %s is not supported", cluster.Spec.Version)
 	}
 
+	storageEngineSupported, err := isStorageEngineSupported(cluster.Spec.Version, cluster.Spec.DatabaseConfiguration.StorageEngine)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	if !storageEngineSupported {
+		r.Recorder.Event(cluster, corev1.EventTypeWarning, "Storage engine not supported", fmt.Sprintf("storage engine %s is not supported on version %s", cluster.Spec.DatabaseConfiguration.StorageEngine, cluster.Spec.Version))
+		return ctrl.Result{}, fmt.Errorf("storage engine %s is not supported on version %s", cluster.Spec.DatabaseConfiguration.StorageEngine, cluster.Spec.Version)
+	}
+
 	subReconcilers := []clusterSubReconciler{
 		updateStatus{},
 		updateLockConfiguration{},
@@ -735,3 +744,11 @@ func (r *FoundationDBClusterReconciler) getCoordinatorSet(cluster *fdbv1beta2.Fo
 
 	return adminClient.GetCoordinatorSet()
 }
+
+func isStorageEngineSupported(versionString string, storageEngine fdbv1beta2.StorageEngine) (bool, error) {
+	version, err := fdbv1beta2.ParseFdbVersion(versionString)
+	if err != nil {
+		return false, err
+	}
+	return version.IsStorageEngineSupported(storageEngine), nil
+}
diff --git a/controllers/remove_process_groups.go b/controllers/remove_process_groups.go
index 0f09b881..64b96c6d 100644
--- a/controllers/remove_process_groups.go
+++ b/controllers/remove_process_groups.go
@@ -135,9 +135,8 @@ func removeProcessGroup(ctx context.Context, r *FoundationDBClusterReconciler, c
 		return err
 	}
 
-	if len(pods) == 1 {
+	if len(pods) == 1 && pods[0].DeletionTimestamp.IsZero() {
 		err = r.PodLifecycleManager.DeletePod(ctx, r, pods[0])
-
 		if err != nil {
 			return err
 		}
@@ -150,7 +149,7 @@ func removeProcessGroup(ctx context.Context, r *FoundationDBClusterReconciler, c
 	if err != nil {
 		return err
 	}
-	if len(pvcs.Items) == 1 {
+	if len(pvcs.Items) == 1 && pvcs.Items[0].DeletionTimestamp.IsZero() {
 		err = r.Delete(ctx, &pvcs.Items[0])
 		if err != nil {
 			return err
@@ -164,7 +163,7 @@ func removeProcessGroup(ctx context.Context, r *FoundationDBClusterReconciler, c
 	if err != nil {
 		return err
 	}
-	if len(services.Items) == 1 {
+	if len(services.Items) == 1 && services.Items[0].DeletionTimestamp.IsZero() {
 		err = r.Delete(ctx, &services.Items[0])
 		if err != nil {
 			return err
@@ -334,10 +333,12 @@ func (r *FoundationDBClusterReconciler) removeProcessGroups(ctx context.Context,
 	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "removeProcessGroups")
 	r.Recorder.Event(cluster, corev1.EventTypeNormal, "RemovingProcesses", fmt.Sprintf("Removing pods: %v", processGroupsToRemove))
 
-	for _, id := range processGroupsToRemove {
+	processGroups := append(processGroupsToRemove, terminatingProcessGroups...)
+
+	for _, id := range processGroups {
 		err := removeProcessGroup(ctx, r, cluster, id)
 		if err != nil {
-			logger.Error(err, "Error during remove Pod", "processGroupID", id)
+			logger.Error(err, "Error during remove process group", "processGroupID", id)
 			continue
 		}
 	}
@@ -345,10 +346,10 @@ func (r *FoundationDBClusterReconciler) removeProcessGroups(ctx context.Context,
 	removedProcessGroups := make(map[string]bool)
 	// We have to check if the currently removed process groups are completely removed.
 	// In addition, we have to check if one of the terminating process groups has been cleaned up.
-	for _, id := range append(processGroupsToRemove, terminatingProcessGroups...) {
+	for _, id := range processGroups {
 		removed, include, err := confirmRemoval(ctx, r, cluster, id)
 		if err != nil {
-			logger.Error(err, "Error during confirm Pod removal", "processGroupID", id)
+			logger.Error(err, "Error during confirm process group removal", "processGroupID", id)
 			continue
 		}
 
diff --git a/distrolesss.dockerfile b/distrolesss.dockerfile
index ff010752..78cf5908 100644
--- a/distrolesss.dockerfile
+++ b/distrolesss.dockerfile
@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM docker.io/library/golang:1.17.6 as builder
+FROM docker.io/library/golang:1.17.8 as builder
 
 # Install FDB this version is only required to compile the fdb operator
 ARG FDB_VERSION=6.3.23
diff --git a/docs/changelog/v1.2.0.md b/docs/changelog/v1.2.0.md
new file mode 100644
index 00000000..d31ad406
--- /dev/null
+++ b/docs/changelog/v1.2.0.md
@@ -0,0 +1,9 @@
+# v1.2.0
+
+## Changes
+
+* Document fail-over with plugin and allow to update config [#1131](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1131).
+* Update go version to 1.17.8 [#1147](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1147).
+* Recreating pods that are not fully excluded [#1136](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1136).
+* Add 7.1.0-rc3 to our CI pipeline [#1128](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1128).
+* Add support for ssd-rocksdb-v1 [#1152](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1152).
diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
index 08bae706..42d023e3 100644
--- a/docs/cluster_spec.md
+++ b/docs/cluster_spec.md
@@ -483,7 +483,7 @@ Region represents a region in the database configuration
 | ----- | ----------- | ------ | -------- |
 | datacenters | The data centers in this region. | [][DataCenter](#datacenter) | false |
 | satellite_logs | The number of satellite logs that we should recruit. | int | false |
-| satellite_redundancy_mode | The replication strategy for satellite logs. | string | false |
+| satellite_redundancy_mode | The replication strategy for satellite logs. | [RedundancyMode](#redundancymode) | false |
 
 [Back to TOC](#table-of-contents)
 
diff --git a/docs/manual/debugging.md b/docs/manual/debugging.md
index a5322ffd..bbc36765 100644
--- a/docs/manual/debugging.md
+++ b/docs/manual/debugging.md
@@ -7,8 +7,8 @@ You can use the [kubectl-fdb](/kubectl-fdb) plugin when investigating issues and
 If a cluster is stuck in a reconciliation you can use the `kubectl-fdb` plugin to analyze the issue:
 
 ```bash
-$ kubectl fdb analyze example-cluster
-Checking cluster: default/example-cluster
+$ kubectl fdb analyze sample-cluster
+Checking cluster: default/sample-cluster
 ✔ Cluster is available
 ✔ Cluster is fully replicated
 ✖ Cluster is not reconciled
@@ -17,19 +17,19 @@ Checking cluster: default/example-cluster
 ✖ ProcessGroup: storage-2 has the following condition: PodFailing since 2021-03-25 18:23:22 +0000 GMT
 ✖ ProcessGroup: storage-2 has the following condition: IncorrectConfigMap since 2021-03-26 19:20:56 +0000 GMT
 ✖ ProcessGroup: transaction-5 has the following condition: MissingProcesses since 2021-04-27 02:57:24 +0100 BST
-✖ Pod default/example-cluster-storage-2 has unexpected Phase Pending with Reason:
-✖ Pod default/example-cluster-storage-2 has an unready container: foundationdb
-✖ Pod default/example-cluster-storage-2 has an unready container: foundationdb-kubernetes-sidecar
-✖ Pod default/example-cluster-storage-2 has an unready container: trace-log-forwarder
+✖ Pod default/sample-cluster-storage-2 has unexpected Phase Pending with Reason:
+✖ Pod default/sample-cluster-storage-2 has an unready container: foundationdb
+✖ Pod default/sample-cluster-storage-2 has an unready container: foundationdb-kubernetes-sidecar
+✖ Pod default/sample-cluster-storage-2 has an unready container: trace-log-forwarder
 Error:
-found issues for cluster example-cluster. Please check them
+found issues for cluster sample-cluster. Please check them
 ```
 
 The plugin can also resolve most of these issues automatically:
 
 ```bash
-$ kubectl fdb analyze example-cluster --auto-fix
-Checking cluster: default/example-cluster
+$ kubectl fdb analyze sample-cluster --auto-fix
+Checking cluster: default/sample-cluster
 ✔ Cluster is available
 ✔ Cluster is fully replicated
 ✖ Cluster is not reconciled
@@ -38,13 +38,13 @@ Checking cluster: default/example-cluster
 ✖ ProcessGroup: storage-2 has the following condition: PodFailing since 2021-03-25 18:23:22 +0000 GMT
 ✖ ProcessGroup: storage-2 has the following condition: IncorrectConfigMap since 2021-03-26 19:20:56 +0000 GMT
 ✖ ProcessGroup: transaction-5 has the following condition: MissingProcesses since 2021-04-27 02:57:24 +0100 BST
-✖ Pod default/example-cluster-storage-2 has unexpected Phase Pending with Reason:
-✖ Pod default/example-cluster-storage-2 has an unready container: foundationdb
-✖ Pod default/example-cluster-storage-2 has an unready container: foundationdb-kubernetes-sidecar
-✖ Pod default/example-cluster-storage-2 has an unready container: trace-log-forwarder
-Replace process groups [stateless-5 storage-2 transaction-5] in cluster default/example-cluster [y/n]: y
+✖ Pod default/sample-cluster-storage-2 has unexpected Phase Pending with Reason:
+✖ Pod default/sample-cluster-storage-2 has an unready container: foundationdb
+✖ Pod default/examsampleple-cluster-storage-2 has an unready container: foundationdb-kubernetes-sidecar
+✖ Pod default/sample-cluster-storage-2 has an unready container: trace-log-forwarder
+Replace process groups [stateless-5 storage-2 transaction-5] in cluster default/sample-cluster [y/n]: y
 Error:
-found issues for cluster example-cluster. Please check them
+found issues for cluster sample-cluster. Please check them
 ```
 
 When using this feature, read carefully what the plugin wants to do and only confirm the dialog when you are sure that you want to do these actions.
@@ -56,11 +56,11 @@ You can do that using a [plugin command](#replacing-pods-with-the-kubectl-plugin
 
 ## Replacing Pods with the kubectl plugin
 
-Let's assume we are working with the cluster `example-cluster`, and the pod `example-cluster-storage-1` is failing to launch.
+Let's assume we are working with the cluster `sample-cluster`, and the pod `sample-cluster-storage-1` is failing to launch.
 
 ```bash
-$ kubectl fdb remove process-groups -c example-cluster example-cluster-storage-1
-Remove [storage-1] from cluster default/example-cluster with exclude: true and shrink: false [y/n]:
+$ kubectl fdb remove process-groups -c sample-cluster sample-cluster-storage-1
+Remove [storage-1] from cluster default/sample-cluster with exclude: true and shrink: false [y/n]:
 # Confirm with 'y' if the excluded Pod is the correct one
 ```
 
@@ -74,8 +74,8 @@ If this happens, you will see an error of the form `Cannot check the exclusion s
 When this happens, the only way to finish the replacement is to skip the exclusion:
 
 ```bash
-$ kubectl fdb remove process-groups --exclusion=false -c example-cluster example-cluster-storage-1
-Remove [storage-1] from cluster default/example-cluster with exclude: false and shrink: false [y/n]:
+$ kubectl fdb remove process-groups --exclusion=false -c sample-cluster sample-cluster-storage-1
+Remove [storage-1] from cluster default/sample-cluster with exclude: false and shrink: false [y/n]:
 # Confirm with 'y' if the excluded Pod is the correct one
 ```
 
@@ -138,7 +138,7 @@ To simplify this process, the kubectl-fdb plugin has a command that encapsulates
 If you want to open up a shell or run a CLI, you can use the [plugin](#kubectl-fdb-plugin):
 
 ```bash
- kubectl fdb exec -c example-cluster -- bash
+ kubectl fdb exec -c sample-cluster -- bash
 ```
 
 Or if you want to check a specific pod:
@@ -151,6 +151,19 @@ This will open up a shell inside the container. You can use this shell to examin
 
 When running the CLI on Kubernetes, you can simply run `fdbcli` with no additional arguments. The shell path, cluster file, TLS certificates, and any other required configuration will be supplied through the environment.
 
+## Get the configuration string
+
+The kubectl plugin supports to generate the configuration string from a FoundationDB cluster spec:
+
+```bash
+kubectl fdb get configuration sample-cluster
+```
+
+When the `--fail-over` flag is provided the resulting configuration string will change the priority for the primary and the remote dc.
+You can also update the config directly by providing the `--update` flag to the command.
+Per default a diff of the new changes will be shown before updating the cluster spec.
+For an HA cluster you have to update all clusters that are managed by the operator with the same command to ensure that all operator instance want to converge to the same configuration. 
+
 ## Next
 
 You can continue on to the [next section](more.md) or go back to the [table of contents](index.md).
diff --git a/docs/manual/technical_design.md b/docs/manual/technical_design.md
index 93f81816..f63f53d5 100644
--- a/docs/manual/technical_design.md
+++ b/docs/manual/technical_design.md
@@ -171,7 +171,7 @@ The `AddPVCs` subreconciler creates any PVCs that are required for the cluster.
 
 ### AddPods
 
-The `AddPods` subreconciler creates any pods that are required for the cluster. Every process group will have one pod created for it. If a process group is flagged for removal, we will not create a pod for it.
+The `AddPods` subreconciler creates any pods that are required for the cluster. Every process group will have one pod created for it. If a process group is flagged for removal and a previous run of `RemoveProcessGroups` has determined (by submitting the `exclude` command to FoundationDB) that it has in fact been fully excluded from the FoundationDB cluster, we will not create a pod for it. However, if we do not know for certain that the process group is fully excluded from FoundationDB, we will bring it back up even if it is flagged for removal - this is to handle a case where a storage node crashes (or is accidentally stopped) while it is draining.
 
 ### GenerateInitialClusterFile
 
diff --git a/fdbclient/admin_client.go b/fdbclient/admin_client.go
index e5d56871..f9ba4c61 100644
--- a/fdbclient/admin_client.go
+++ b/fdbclient/admin_client.go
@@ -242,7 +242,7 @@ func (client *cliAdminClient) GetStatus() (*fdbv1beta2.FoundationDBStatus, error
 		return nil, err
 	}
 	client.log.V(1).Info("Fetched status JSON", "contents", contents)
-	contents, err = removeWarningsInJSON(contents)
+	contents, err = internal.RemoveWarningsInJSON(contents)
 	if err != nil {
 		return nil, err
 	}
@@ -628,7 +628,7 @@ func (client *cliAdminClient) GetBackupStatus() (*fdbv1beta2.FoundationDBLiveBac
 	}
 
 	status := &fdbv1beta2.FoundationDBLiveBackupStatus{}
-	statusString, err = removeWarningsInJSON(statusString)
+	statusString, err = internal.RemoveWarningsInJSON(statusString)
 	if err != nil {
 		return nil, err
 	}
@@ -685,15 +685,6 @@ func (client *cliAdminClient) Close() error {
 	return nil
 }
 
-func removeWarningsInJSON(jsonString string) (string, error) {
-	idx := strings.Index(jsonString, "{")
-	if idx == -1 {
-		return "", fmt.Errorf("the JSON string doesn't contain a starting '{'")
-	}
-
-	return strings.TrimSpace(jsonString[idx:]), nil
-}
-
 // GetCoordinatorSet gets the current coordinators from the status
 func (client *cliAdminClient) GetCoordinatorSet() (map[string]struct{}, error) {
 	status, err := client.GetStatus()
diff --git a/helm/fdb-operator/values.yaml b/helm/fdb-operator/values.yaml
index 87edd297..9f6de5b1 100644
--- a/helm/fdb-operator/values.yaml
+++ b/helm/fdb-operator/values.yaml
@@ -1,7 +1,7 @@
 ---
 image:
   repository: foundationdb/fdb-kubernetes-operator
-  tag: v1.1.1
+  tag: v1.2.0
   pullPolicy: IfNotPresent
 
 initContainers:
diff --git a/internal/fdb_status_helper.go b/internal/fdb_status_helper.go
new file mode 100644
index 00000000..9cacc8c6
--- /dev/null
+++ b/internal/fdb_status_helper.go
@@ -0,0 +1,37 @@
+/*
+ * fdb_status_helper.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package internal
+
+import (
+	"fmt"
+	"strings"
+)
+
+// RemoveWarningsInJSON removes any warning messages that might appear in the status output from the fdbcli and returns
+// the JSON output without the warning message.
+func RemoveWarningsInJSON(jsonString string) (string, error) {
+	idx := strings.Index(jsonString, "{")
+	if idx == -1 {
+		return "", fmt.Errorf("the JSON string doesn't contain a starting '{'")
+	}
+
+	return strings.TrimSpace(jsonString[idx:]), nil
+}
diff --git a/kubectl-fdb/cmd/analyze.go b/kubectl-fdb/cmd/analyze.go
index 4c872cba..2597ebb0 100644
--- a/kubectl-fdb/cmd/analyze.go
+++ b/kubectl-fdb/cmd/analyze.go
@@ -44,7 +44,7 @@ func newAnalyzeCmd(streams genericclioptions.IOStreams) *cobra.Command {
 		Short: "Analyze if the given clusters have any issues",
 		Long:  "Analyze if the given clusters have any issues",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			force, err := cmd.Root().Flags().GetBool("force")
+			wait, err := cmd.Root().Flags().GetBool("wait")
 			if err != nil {
 				return err
 			}
@@ -108,7 +108,7 @@ func newAnalyzeCmd(streams genericclioptions.IOStreams) *cobra.Command {
 
 			var errs []error
 			for _, cluster := range clusters {
-				err := analyzeCluster(cmd, kubeClient, cluster, namespace, autoFix, force, ignoreConditions)
+				err := analyzeCluster(cmd, kubeClient, cluster, namespace, autoFix, wait, ignoreConditions)
 				if err != nil {
 					errs = append(errs, err)
 				}
@@ -208,7 +208,7 @@ func printStatement(cmd *cobra.Command, line string, mesType messageType) {
 	color.Unset()
 }
 
-func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName string, namespace string, autoFix bool, force bool, ignoreConditions []string) error {
+func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName string, namespace string, autoFix bool, wait bool, ignoreConditions []string) error {
 	foundIssues := false
 	cluster, err := loadCluster(kubeClient, namespace, clusterName)
 
@@ -372,14 +372,14 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 		confirmed := false
 
 		if len(failedProcessGroups) > 0 {
-			err := replaceProcessGroups(kubeClient, cluster.Name, failedProcessGroups, namespace, true, false, force, false)
+			err := replaceProcessGroups(kubeClient, cluster.Name, failedProcessGroups, namespace, true, false, wait, false)
 			if err != nil {
 				return err
 			}
 		}
 
 		pods := filterDeletePods(failedProcessGroups, killPods)
-		if !force && len(pods) > 0 {
+		if wait && len(pods) > 0 {
 			podNames := make([]string, 0, len(killPods))
 			for _, pod := range killPods {
 				podNames = append(podNames, pod.Name)
@@ -388,7 +388,7 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 			confirmed = confirmAction(fmt.Sprintf("Delete Pods %v in cluster %s/%s", strings.Join(podNames, ","), namespace, clusterName))
 		}
 
-		if force || confirmed {
+		if !wait || confirmed {
 			for _, pod := range pods {
 				cmd.Printf("Delete Pod: %s/%s\n", pod.Namespace, pod.Name)
 				err := kubeClient.Delete(context.Background(), &pod)
diff --git a/kubectl-fdb/cmd/configuration.go b/kubectl-fdb/cmd/configuration.go
index 126a59d0..e3cda9b8 100644
--- a/kubectl-fdb/cmd/configuration.go
+++ b/kubectl-fdb/cmd/configuration.go
@@ -21,6 +21,9 @@
 package cmd
 
 import (
+	ctx "context"
+	"fmt"
+
 	"github.com/spf13/cobra"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -34,11 +37,21 @@ func newConfigurationCmd(streams genericclioptions.IOStreams) *cobra.Command {
 		Short: "Get the configuration string based on the database configuration of the cluster spec.",
 		Long:  "Get the configuration string based on the database configuration of the cluster spec.",
 		RunE: func(cmd *cobra.Command, args []string) error {
+			wait, err := cmd.Root().Flags().GetBool("wait")
+			if err != nil {
+				return err
+			}
+
 			failOver, err := cmd.Flags().GetBool("fail-over")
 			if err != nil {
 				return err
 			}
 
+			update, err := cmd.Flags().GetBool("update")
+			if err != nil {
+				return err
+			}
+
 			kubeClient, err := getKubeClient(o)
 			if err != nil {
 				return err
@@ -50,6 +63,10 @@ func newConfigurationCmd(streams genericclioptions.IOStreams) *cobra.Command {
 			}
 
 			for _, clusterName := range args {
+				if update {
+					return updateConfig(kubeClient, clusterName, namespace, failOver, wait)
+				}
+
 				configuration, err := getConfigurationString(kubeClient, clusterName, namespace, failOver)
 				if err != nil {
 					return err
@@ -73,6 +90,9 @@ kubectl fdb -n default get configuration c1
 
 # Get the configuration string from cluster c1 and change the priority for an HA cluster
 kubectl fdb get configuration --fail-over c1
+
+# Change the priority for an HA cluster named c1 and apply these changes
+kubectl fdb get configuration --fail-over --update c1
 `,
 	}
 	cmd.SetOut(o.Out)
@@ -80,12 +100,41 @@ kubectl fdb get configuration --fail-over c1
 	cmd.SetIn(o.In)
 
 	cmd.Flags().Bool("fail-over", false, "defines if the configuration should be changed to issue a fail over")
+	cmd.Flags().Bool("update", false, "defines if the configuration should be updated in the cluster spec")
 	o.configFlags.AddFlags(cmd.Flags())
 
 	return cmd
 }
 
-// printConfiguration
+func updateConfig(kubeClient client.Client, clusterName string, namespace string, failOver bool, wait bool) error {
+	cluster, err := loadCluster(kubeClient, namespace, clusterName)
+	if err != nil {
+		return err
+	}
+
+	config := cluster.Spec.DatabaseConfiguration
+	if failOver {
+		config = config.FailOver()
+	}
+
+	if wait {
+		diff, err := getDiff(cluster.Spec.DatabaseConfiguration, config)
+		if err != nil {
+			return err
+		}
+
+		confirmed := confirmAction(fmt.Sprintf("The following changes will be made:\n%s", diff))
+		if !confirmed {
+			return fmt.Errorf("user aborted the change")
+		}
+	}
+
+	cluster.Spec.DatabaseConfiguration = config
+
+	return kubeClient.Update(ctx.Background(), cluster)
+}
+
+// getConfigurationString returns the configuration string
 func getConfigurationString(kubeClient client.Client, clusterName string, namespace string, failOver bool) (string, error) {
 	cluster, err := loadCluster(kubeClient, namespace, clusterName)
 	if err != nil {
diff --git a/kubectl-fdb/cmd/cordon.go b/kubectl-fdb/cmd/cordon.go
index d7f51600..2dd4ea53 100644
--- a/kubectl-fdb/cmd/cordon.go
+++ b/kubectl-fdb/cmd/cordon.go
@@ -47,7 +47,7 @@ func newCordonCmd(streams genericclioptions.IOStreams) *cobra.Command {
 		Short: "Adds all process groups (or multiple) that run on a node to the remove list of the given cluster",
 		Long:  "Adds all process groups (or multiple) that run on a node to the remove list of the given cluster",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			force, err := cmd.Root().Flags().GetBool("force")
+			wait, err := cmd.Root().Flags().GetBool("wait")
 			if err != nil {
 				return err
 			}
@@ -89,10 +89,10 @@ func newCordonCmd(streams genericclioptions.IOStreams) *cobra.Command {
 					return err
 				}
 
-				return cordonNode(kubeClient, cluster, nodes, namespace, withExclusion, force)
+				return cordonNode(kubeClient, cluster, nodes, namespace, withExclusion, wait)
 			}
 
-			return cordonNode(kubeClient, cluster, args, namespace, withExclusion, force)
+			return cordonNode(kubeClient, cluster, args, namespace, withExclusion, wait)
 		},
 		Example: `
 # Evacuate all process groups for a cluster in the current namespace that are hosted on node-1
@@ -123,7 +123,7 @@ kubectl fdb cordon -c cluster --node-selector machine=a,disk=fast
 }
 
 // cordonNode gets all process groups of this cluster that run on the given nodes and add them to the remove list
-func cordonNode(kubeClient client.Client, cluster *fdbv1beta2.FoundationDBCluster, nodes []string, namespace string, withExclusion bool, force bool) error {
+func cordonNode(kubeClient client.Client, cluster *fdbv1beta2.FoundationDBCluster, nodes []string, namespace string, withExclusion bool, wait bool) error {
 	fmt.Printf("Start to cordon %d nodes\n", len(nodes))
 	if len(nodes) == 0 {
 		return nil
@@ -161,5 +161,5 @@ func cordonNode(kubeClient client.Client, cluster *fdbv1beta2.FoundationDBCluste
 		}
 	}
 
-	return replaceProcessGroups(kubeClient, cluster.Name, processGroups, namespace, withExclusion, false, force, false)
+	return replaceProcessGroups(kubeClient, cluster.Name, processGroups, namespace, withExclusion, false, wait, false)
 }
diff --git a/kubectl-fdb/cmd/deprecation.go b/kubectl-fdb/cmd/deprecation.go
index e7ca19e6..fefe1920 100644
--- a/kubectl-fdb/cmd/deprecation.go
+++ b/kubectl-fdb/cmd/deprecation.go
@@ -156,6 +156,9 @@ func checkDeprecation(cmd *cobra.Command, kubeClient client.Client, inputCluster
 		}
 
 		diff := cmp.Diff(originalYAML, normalizedYAML)
+		if err != nil {
+			return err
+		}
 		if diff == "" {
 			if !showClusterSpec {
 				cmd.Printf("Cluster %s has no deprecation\n", cluster.Name)
@@ -182,6 +185,20 @@ func checkDeprecation(cmd *cobra.Command, kubeClient client.Client, inputCluster
 	return nil
 }
 
+func getDiff(objectA interface{}, objectB interface{}) (string, error) {
+	aYaml, err := getMinimalYAML(objectA)
+	if err != nil {
+		return "", err
+	}
+
+	bYaml, err := getMinimalYAML(objectB)
+	if err != nil {
+		return "", err
+	}
+
+	return cmp.Diff(aYaml, bYaml), nil
+}
+
 func getMinimalYAML(object interface{}) ([]byte, error) {
 	rawYAML, err := yaml.Marshal(object)
 	if err != nil {
diff --git a/kubectl-fdb/cmd/exclusion_status.go b/kubectl-fdb/cmd/exclusion_status.go
new file mode 100644
index 00000000..07e1907b
--- /dev/null
+++ b/kubectl-fdb/cmd/exclusion_status.go
@@ -0,0 +1,221 @@
+/*
+ * exclusion_status.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
+
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	"github.com/spf13/cobra"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+func newExclusionStatusCmd(streams genericclioptions.IOStreams) *cobra.Command {
+	o := newFDBOptions(streams)
+
+	cmd := &cobra.Command{
+		Use:   "exclusion-status",
+		Short: "Get the exclusion status for all excluded processes.",
+		Long:  "Get the exclusion status for all excluded processes.",
+		Args:  cobra.ExactValidArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ignoreFullyExcluded, err := cmd.Flags().GetBool("ignore-fully-excluded")
+			if err != nil {
+				return err
+			}
+
+			interval, err := cmd.Flags().GetDuration("interval")
+			if err != nil {
+				return err
+			}
+
+			config, err := o.configFlags.ToRESTConfig()
+			if err != nil {
+				return err
+			}
+
+			clientSet, err := kubernetes.NewForConfig(config)
+			if err != nil {
+				return err
+			}
+
+			kubeClient, err := getKubeClient(o)
+			if err != nil {
+				return err
+			}
+
+			namespace, err := getNamespace(*o.configFlags.Namespace)
+			if err != nil {
+				return err
+			}
+
+			cluster, err := loadCluster(kubeClient, namespace, args[0])
+			if err != nil {
+				return err
+			}
+
+			pods, err := getPodsForCluster(kubeClient, cluster, namespace)
+			if err != nil {
+				return err
+			}
+
+			if len(pods.Items) == 0 {
+				return fmt.Errorf("no running Pods are found for cluster: %s/%s", cluster.Namespace, cluster.Name)
+			}
+
+			// TODO get the pod randomly
+			err = getExclusionStatus(cmd, config, clientSet, pods.Items[0].Name, namespace, ignoreFullyExcluded, interval)
+			if err != nil {
+				return err
+			}
+
+			return nil
+		},
+		Example: `
+Experimental feature!
+
+This command shows the ongoing exclusions for a cluster and how much data must be moved before the exclusion is done.
+
+# Get the exclusion status for cluster c1
+kubectl fdb get exclusion-status c1
+
+
+# Get the exclusion status for cluster c1 and prints out processes that are fully excluded
+kubectl fdb get exclusion-status c1 --ignore-fully-excluded=false
+
+
+# Get the exclusion status for cluster c1 and updates the data every 5 minutes
+kubectl fdb get exclusion-status c1 --interval=5m
+`,
+	}
+	cmd.SetOut(o.Out)
+	cmd.SetErr(o.ErrOut)
+	cmd.SetIn(o.In)
+
+	cmd.Flags().Bool("ignore-fully-excluded", true, "defines if processes that are fully excluded should be ignored.")
+	cmd.Flags().Duration("interval", 1*time.Minute, "defines in which interval new information should be fetched from the cluster.")
+
+	o.configFlags.AddFlags(cmd.Flags())
+
+	return cmd
+}
+
+type exclusionResult struct {
+	id          string
+	storedBytes int
+	estimate    string
+}
+
+func getExclusionStatus(cmd *cobra.Command, restConfig *rest.Config, kubeClient *kubernetes.Clientset, clientPod string, namespace string, ignoreFullyExcluded bool, interval time.Duration) error {
+	timer := time.NewTicker(interval)
+	previousRun := map[string]int{}
+
+	for {
+		out, serr, err := executeCmd(restConfig, kubeClient, clientPod, namespace, "fdbcli --exec 'status json'")
+		if err != nil {
+			// If an error occurs retry
+			cmd.PrintErrln(err)
+			continue
+		}
+
+		if serr.Len() > 0 {
+			cmd.PrintErrln(serr.String())
+		}
+
+		res, err := internal.RemoveWarningsInJSON(out.String())
+		if err != nil {
+			// If an error occurs retry
+			cmd.PrintErrln(err)
+			continue
+		}
+
+		status := &fdbv1beta2.FoundationDBStatus{}
+		err = json.Unmarshal([]byte(res), status)
+		if err != nil {
+			// If an error occurs retry
+			cmd.PrintErrln(err)
+			continue
+		}
+
+		var ongoingExclusions []exclusionResult
+		for _, process := range status.Cluster.Processes {
+			if !process.Excluded {
+				continue
+			}
+
+			if !ignoreFullyExcluded && len(process.Roles) == 0 {
+				cmd.Println(process.Locality["instance_id"], "is fully excluded")
+			}
+
+			instance := process.Locality["instance_id"]
+			// TODO: Add estimate when an exclusion is done
+			// TODO: Add progress bars
+			for _, role := range process.Roles {
+				roleClass := fdbv1beta2.ProcessClass(role.Role)
+				if roleClass.IsStateful() {
+					var estimate string
+
+					previousBytes, ok := previousRun[instance]
+					if ok {
+						// TODO calculate estimates for duration
+						_ = previousBytes
+						estimate = "N/A"
+					}
+
+					// TODO: Check if StoredBytes is the correct value
+					ongoingExclusions = append(ongoingExclusions, exclusionResult{
+						id:          instance,
+						storedBytes: role.StoredBytes,
+						estimate:    estimate,
+					})
+
+					previousRun[instance] = role.StoredBytes
+				}
+			}
+		}
+
+		if len(ongoingExclusions) == 0 {
+			timer.Stop()
+			break
+		}
+
+		sort.SliceStable(ongoingExclusions, func(i, j int) bool {
+			return ongoingExclusions[i].id < ongoingExclusions[j].id
+		})
+
+		for _, exclusion := range ongoingExclusions {
+			cmd.Printf("%s:\t %d bytes are left - estimate: %s\n", exclusion.id, exclusion.storedBytes, exclusion.estimate)
+		}
+
+		cmd.Println("There are", len(ongoingExclusions), "processes that are not fully excluded.")
+		cmd.Println("======================================================================================================")
+		<-timer.C
+	}
+
+	return nil
+}
diff --git a/kubectl-fdb/cmd/get.go b/kubectl-fdb/cmd/get.go
index e14c5139..ba57b1b9 100644
--- a/kubectl-fdb/cmd/get.go
+++ b/kubectl-fdb/cmd/get.go
@@ -49,6 +49,7 @@ kubectl fdb -n default get configuration c1
 	cmd.SetIn(o.In)
 
 	cmd.AddCommand(newConfigurationCmd(streams))
+	cmd.AddCommand(newExclusionStatusCmd(streams))
 	o.configFlags.AddFlags(cmd.Flags())
 
 	return cmd
diff --git a/kubectl-fdb/cmd/k8s_client.go b/kubectl-fdb/cmd/k8s_client.go
index ecb23ef5..f9771a80 100644
--- a/kubectl-fdb/cmd/k8s_client.go
+++ b/kubectl-fdb/cmd/k8s_client.go
@@ -121,14 +121,14 @@ func getPodsForCluster(kubeClient client.Client, cluster *fdbv1beta2.FoundationD
 	return &podList, err
 }
 
-func executeCmd(restConfig *rest.Config, kubeClient *kubernetes.Clientset, name string, namespace string, command string) (*bytes.Buffer, *bytes.Buffer, error) {
+func executeCmd(restConfig *rest.Config, kubeClient *kubernetes.Clientset, podName string, namespace string, command string) (*bytes.Buffer, *bytes.Buffer, error) {
 	cmd := []string{
 		"/bin/bash",
 		"-c",
 		command,
 	}
 	req := kubeClient.CoreV1().RESTClient().Post().
-		Resource("pods").Name(name).
+		Resource("pods").Name(podName).
 		Namespace(namespace).SubResource("exec")
 
 	option := &corev1.PodExecOptions{
diff --git a/kubectl-fdb/cmd/remove_process_group.go b/kubectl-fdb/cmd/remove_process_group.go
index 79cbd747..1511799a 100644
--- a/kubectl-fdb/cmd/remove_process_group.go
+++ b/kubectl-fdb/cmd/remove_process_group.go
@@ -42,7 +42,7 @@ func newRemoveProcessGroupCmd(streams genericclioptions.IOStreams) *cobra.Comman
 		Short: "Adds a process group (or multiple) to the remove list of the given cluster",
 		Long:  "Adds a process group (or multiple) to the remove list field of the given cluster",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			force, err := cmd.Root().Flags().GetBool("force")
+			wait, err := cmd.Root().Flags().GetBool("wait")
 			if err != nil {
 				return err
 			}
@@ -85,21 +85,21 @@ func newRemoveProcessGroupCmd(streams genericclioptions.IOStreams) *cobra.Comman
 				}
 			}
 
-			return replaceProcessGroups(kubeClient, cluster, processGroups, namespace, withExclusion, withShrink, force, removeAllFailed)
+			return replaceProcessGroups(kubeClient, cluster, processGroups, namespace, withExclusion, withShrink, wait, removeAllFailed)
 		},
 		Example: `
 # Remove process groups for a cluster in the current namespace
 kubectl fdb remove process-group -c cluster pod-1 -i pod-2
 
 # Remove process groups for a cluster in the namespace default
-kubectl fdb -n default remove process-group  -c cluster pod-1 pod-2
+kubectl fdb -n default remove process-group -c cluster pod-1 pod-2
 
 # Remove process groups for a cluster with the process group ID.
 # The process group ID of a Pod can be fetched with "kubectl get po -L foundationdb.org/fdb-process-group-id"
 kubectl fdb -n default remove process-group --use-process-group-id -c cluster storage-1 storage-2
 
 # Remove all failed process groups for a cluster (all process groups that have a missing process)
-kubectl fdb -n default remove process-group  -c cluster --remove-all-failed
+kubectl fdb -n default remove process-group -c cluster --remove-all-failed
 `,
 	}
 
@@ -150,7 +150,7 @@ func getProcessGroupIDsFromPod(kubeClient client.Client, clusterName string, pod
 }
 
 // replaceProcessGroups adds process groups to the removal list of the cluster
-func replaceProcessGroups(kubeClient client.Client, clusterName string, processGroups []string, namespace string, withExclusion bool, withShrink bool, force bool, removeAllFailed bool) error {
+func replaceProcessGroups(kubeClient client.Client, clusterName string, processGroups []string, namespace string, withExclusion bool, withShrink bool, wait bool, removeAllFailed bool) error {
 	cluster, err := loadCluster(kubeClient, namespace, clusterName)
 
 	if err != nil {
@@ -207,7 +207,7 @@ func replaceProcessGroups(kubeClient client.Client, clusterName string, processG
 		cluster.Spec.ProcessCounts.DecreaseCount(class, amount)
 	}
 
-	if !force {
+	if wait {
 		confirmed := confirmAction(fmt.Sprintf("Remove %v from cluster %s/%s with exclude: %t and shrink: %t", processGroups, namespace, clusterName, withExclusion, withShrink))
 		if !confirmed {
 			return fmt.Errorf("user aborted the removal")
diff --git a/kubectl-fdb/cmd/restart.go b/kubectl-fdb/cmd/restart.go
index 3306cb5f..b5622f98 100644
--- a/kubectl-fdb/cmd/restart.go
+++ b/kubectl-fdb/cmd/restart.go
@@ -39,7 +39,7 @@ func newRestartCmd(streams genericclioptions.IOStreams) *cobra.Command {
 		Short: "Restarts process(es) in a given FDB cluster.",
 		Long:  "Restarts process(es) in a given FDB cluster.",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			force, err := cmd.Root().Flags().GetBool("force")
+			wait, err := cmd.Root().Flags().GetBool("wait")
 			if err != nil {
 				return err
 			}
@@ -108,7 +108,7 @@ func newRestartCmd(streams genericclioptions.IOStreams) *cobra.Command {
 				processes = args
 			}
 
-			return restartProcesses(cmd, config, clientSet, processes, namespace, clusterName, force)
+			return restartProcesses(cmd, config, clientSet, processes, namespace, clusterName, wait)
 		},
 		Example: `
 # Restart processes for a cluster in the current namespace
@@ -157,8 +157,8 @@ func convertConditions(inputConditions []string) ([]fdbv1beta2.ProcessGroupCondi
 }
 
 //nolint:interfacer // golint has a false-positive here -> `cmd` can be `github.com/hashicorp/go-retryablehttp.Logger`
-func restartProcesses(cmd *cobra.Command, restConfig *rest.Config, kubeClient *kubernetes.Clientset, processes []string, namespace string, clusterName string, force bool) error {
-	if !force {
+func restartProcesses(cmd *cobra.Command, restConfig *rest.Config, kubeClient *kubernetes.Clientset, processes []string, namespace string, clusterName string, wait bool) error {
+	if wait {
 		confirmed := confirmAction(fmt.Sprintf("Restart %v in cluster %s/%s", processes, namespace, clusterName))
 		if !confirmed {
 			return fmt.Errorf("user aborted the removal")
diff --git a/kubectl-fdb/cmd/root.go b/kubectl-fdb/cmd/root.go
index 636b6ace..fb044801 100644
--- a/kubectl-fdb/cmd/root.go
+++ b/kubectl-fdb/cmd/root.go
@@ -67,7 +67,7 @@ func NewRootCmd(streams genericclioptions.IOStreams) *cobra.Command {
 
 	viper.SetDefault("license", "apache 2")
 	cmd.PersistentFlags().StringP("operator-name", "o", "fdb-kubernetes-operator-controller-manager", "Name of the Deployment for the operator.")
-	cmd.PersistentFlags().BoolP("force", "f", false, "Suppress the confirmation dialog")
+	cmd.PersistentFlags().BoolP("wait", "w", true, "If the plugin should wait for confirmation before executing any action")
 	o.configFlags.AddFlags(cmd.Flags())
 
 	cmd.AddCommand(


Test patch:
diff --git a/api/v1beta2/foundationdb_database_configuration_test.go b/api/v1beta2/foundationdb_database_configuration_test.go
new file mode 100644
index 00000000..ad79d498
--- /dev/null
+++ b/api/v1beta2/foundationdb_database_configuration_test.go
@@ -0,0 +1,160 @@
+/*
+ * foundationdb_database_configuration_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package v1beta2
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("DatabaseConfiguration", func() {
+	When("a single dc cluster is provided", func() {
+		var config *DatabaseConfiguration
+
+		BeforeEach(func() {
+			config = &DatabaseConfiguration{
+				StorageEngine:  StorageEngineSSD,
+				RedundancyMode: RedundancyModeTriple,
+				UsableRegions:  1,
+				RoleCounts: RoleCounts{
+					Logs:      3,
+					Storage:   3,
+					Proxies:   3,
+					Resolvers: 1,
+				},
+				Regions: []Region{
+					{
+						DataCenters: []DataCenter{
+							{
+								ID:        "primary",
+								Priority:  1,
+								Satellite: 0,
+							},
+						},
+						SatelliteLogs:           0,
+						SatelliteRedundancyMode: "",
+					},
+				},
+			}
+		})
+
+		When("the configuration string is calculated", func() {
+			It("should print the correct configuration string", func() {
+				Expect(config.GetConfigurationString(Versions.Default.String())).To(Equal("triple ssd usable_regions=1 logs=3 resolvers=1 log_routers=0 remote_logs=0 proxies=3 regions=[{\\\"datacenters\\\":[{\\\"id\\\":\\\"primary\\\",\\\"priority\\\":1}]}]"))
+			})
+		})
+
+		When("the configuration string is calculated with fail over", func() {
+			It("should not change the config and print the correct configuration string", func() {
+				newConfig := config.FailOver()
+				Expect(newConfig).To(Equal(*config))
+				Expect(newConfig.GetConfigurationString(Versions.Default.String())).To(Equal("triple ssd usable_regions=1 logs=3 resolvers=1 log_routers=0 remote_logs=0 proxies=3 regions=[{\\\"datacenters\\\":[{\\\"id\\\":\\\"primary\\\",\\\"priority\\\":1}]}]"))
+			})
+		})
+	})
+
+	When("a multi dc cluster is provided", func() {
+		var config *DatabaseConfiguration
+
+		BeforeEach(func() {
+			config = &DatabaseConfiguration{
+				StorageEngine:  StorageEngineSSD,
+				RedundancyMode: RedundancyModeTriple,
+				UsableRegions:  1,
+				RoleCounts: RoleCounts{
+					Logs:      3,
+					Storage:   3,
+					Proxies:   3,
+					Resolvers: 1,
+				},
+				Regions: []Region{
+					{
+						DataCenters: []DataCenter{
+							{
+								ID:        "primary",
+								Priority:  1,
+								Satellite: 0,
+							},
+							{
+								ID:        "primary-sat",
+								Priority:  1,
+								Satellite: 1,
+							},
+						},
+						SatelliteLogs:           3,
+						SatelliteRedundancyMode: RedundancyModeOneSatelliteSingle,
+					},
+					{
+						DataCenters: []DataCenter{
+							{
+								ID:        "remote",
+								Priority:  0,
+								Satellite: 0,
+							},
+							{
+								ID:        "remote-sat",
+								Priority:  1,
+								Satellite: 1,
+							},
+						},
+						SatelliteLogs:           3,
+						SatelliteRedundancyMode: RedundancyModeOneSatelliteDouble,
+					},
+				},
+			}
+		})
+
+		When("the configuration string is calculated", func() {
+			It("should print the correct configuration string", func() {
+				Expect(config.GetConfigurationString(Versions.Default.String())).To(Equal("triple ssd usable_regions=1 logs=3 resolvers=1 log_routers=0 remote_logs=0 proxies=3 regions=[{\\\"datacenters\\\":[{\\\"id\\\":\\\"primary\\\",\\\"priority\\\":1},{\\\"id\\\":\\\"primary-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1}],\\\"satellite_logs\\\":3,\\\"satellite_redundancy_mode\\\":\\\"one_satellite_single\\\"},{\\\"datacenters\\\":[{\\\"id\\\":\\\"remote\\\"},{\\\"id\\\":\\\"remote-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1}],\\\"satellite_logs\\\":3,\\\"satellite_redundancy_mode\\\":\\\"one_satellite_double\\\"}]"))
+			})
+		})
+
+		When("the configuration string is calculated with fail over", func() {
+			It("should change the priority of the primary and the remote DC and print the new configuration string", func() {
+				newConfig := config.FailOver()
+				Expect(newConfig).NotTo(Equal(*config))
+
+				Expect(len(newConfig.Regions)).To(BeNumerically("==", 2))
+				Expect(newConfig.Regions[0].SatelliteLogs).To(Equal(3))
+				Expect(newConfig.Regions[0].SatelliteRedundancyMode).To(Equal(RedundancyModeOneSatelliteSingle))
+				Expect(len(newConfig.Regions[0].DataCenters)).To(BeNumerically("==", 2))
+				Expect(newConfig.Regions[0].DataCenters[0].Priority).To(Equal(0))
+				Expect(newConfig.Regions[0].DataCenters[0].ID).To(Equal("primary"))
+				Expect(newConfig.Regions[0].DataCenters[0].Satellite).To(Equal(0))
+				Expect(newConfig.Regions[0].DataCenters[1].Priority).To(Equal(1))
+				Expect(newConfig.Regions[0].DataCenters[1].ID).To(Equal("primary-sat"))
+				Expect(newConfig.Regions[0].DataCenters[1].Satellite).To(Equal(1))
+
+				Expect(len(newConfig.Regions[1].DataCenters)).To(BeNumerically("==", 2))
+				Expect(newConfig.Regions[1].SatelliteLogs).To(Equal(3))
+				Expect(newConfig.Regions[1].SatelliteRedundancyMode).To(Equal(RedundancyModeOneSatelliteDouble))
+				Expect(newConfig.Regions[1].DataCenters[0].Priority).To(Equal(1))
+				Expect(newConfig.Regions[1].DataCenters[0].ID).To(Equal("remote"))
+				Expect(newConfig.Regions[1].DataCenters[1].Priority).To(Equal(1))
+				Expect(newConfig.Regions[1].DataCenters[1].ID).To(Equal("remote-sat"))
+				Expect(newConfig.Regions[1].DataCenters[1].Satellite).To(Equal(1))
+
+				Expect(newConfig.GetConfigurationString(Versions.Default.String())).To(Equal("triple ssd usable_regions=1 logs=3 resolvers=1 log_routers=0 remote_logs=0 proxies=3 regions=[{\\\"datacenters\\\":[{\\\"id\\\":\\\"primary\\\"},{\\\"id\\\":\\\"primary-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1}],\\\"satellite_logs\\\":3,\\\"satellite_redundancy_mode\\\":\\\"one_satellite_single\\\"},{\\\"datacenters\\\":[{\\\"id\\\":\\\"remote\\\",\\\"priority\\\":1},{\\\"id\\\":\\\"remote-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1}],\\\"satellite_logs\\\":3,\\\"satellite_redundancy_mode\\\":\\\"one_satellite_double\\\"}]"))
+			})
+		})
+	})
+})
diff --git a/api/v1beta2/foundationdb_version_test.go b/api/v1beta2/foundationdb_version_test.go
index 46c95adc..17d81e54 100644
--- a/api/v1beta2/foundationdb_version_test.go
+++ b/api/v1beta2/foundationdb_version_test.go
@@ -107,6 +107,16 @@ var _ = Describe("[api] FDBVersion", func() {
 			Expect(version.Equal(Version{Major: 6, Minor: 3, Patch: 20})).To(BeFalse())
 			Expect(version.Equal(Version{Major: 6, Minor: 2, Patch: 21})).To(BeFalse())
 		})
+
+		It("should return correct result for IsAtleast", func() {
+			version := Version{Major: 7, Minor: 1, Patch: 0, ReleaseCandidate: 2}
+			Expect(version.IsAtLeast(Version{Major: 7, Minor: 1, Patch: 0})).To(BeFalse())
+			Expect(version.IsAtLeast(Version{Major: 7, Minor: 1, Patch: 0, ReleaseCandidate: 1})).To(BeTrue())
+			Expect(version.IsAtLeast(Version{Major: 7, Minor: 1, Patch: 0, ReleaseCandidate: 3})).To(BeFalse())
+
+			version = Version{Major: 7, Minor: 1, Patch: 0}
+			Expect(version.IsAtLeast(Version{Major: 7, Minor: 1, Patch: 0, ReleaseCandidate: 1})).To(BeTrue())
+		})
 	})
 
 	When("checking if the version has support for non-blocking exclude commands", func() {
diff --git a/controllers/add_pods_test.go b/controllers/add_pods_test.go
index 9f546d03..c0940e7b 100644
--- a/controllers/add_pods_test.go
+++ b/controllers/add_pods_test.go
@@ -38,6 +38,7 @@ var _ = Describe("add_pods", func() {
 	var requeue *requeue
 	var initialPods *corev1.PodList
 	var newPods *corev1.PodList
+	var processGroupWithoutPod *fdbv1beta2.ProcessGroupStatus
 
 	BeforeEach(func() {
 		cluster = internal.CreateDefaultCluster()
@@ -86,7 +87,8 @@ var _ = Describe("add_pods", func() {
 
 	Context("with a storage process group with no pod defined", func() {
 		BeforeEach(func() {
-			cluster.Status.ProcessGroups = append(cluster.Status.ProcessGroups, fdbv1beta2.NewProcessGroupStatus("storage-9", "storage", nil))
+			processGroupWithoutPod = fdbv1beta2.NewProcessGroupStatus("storage-9", "storage", []string{"100.101.102.103"})
+			cluster.Status.ProcessGroups = append(cluster.Status.ProcessGroups, processGroupWithoutPod)
 		})
 
 		It("should not requeue", func() {
@@ -94,26 +96,49 @@ var _ = Describe("add_pods", func() {
 		})
 
 		It("should create an extra pod", func() {
-			Expect(newPods.Items).To(HaveLen(len(initialPods.Items) + 1))
-			lastPod := newPods.Items[len(newPods.Items)-1]
-			Expect(lastPod.Name).To(Equal("operator-test-1-storage-9"))
-			Expect(lastPod.Labels[fdbv1beta2.FDBProcessGroupIDLabel]).To(Equal("storage-9"))
-			Expect(lastPod.Labels[fdbv1beta2.FDBProcessClassLabel]).To(Equal("storage"))
-			Expect(lastPod.OwnerReferences).To(Equal(internal.BuildOwnerReference(cluster.TypeMeta, cluster.ObjectMeta)))
+			expectNewPodToHaveBeenCreated(initialPods, newPods, cluster)
 		})
 
-		Context("when the process group is being removed", func() {
+		When("the process group is being removed", func() {
 			BeforeEach(func() {
-				cluster.Status.ProcessGroups[len(cluster.Status.ProcessGroups)-1].MarkForRemoval()
+				processGroupWithoutPod.MarkForRemoval()
 			})
 
-			It("should not requeue", func() {
-				Expect(requeue).To(BeNil())
+			When("the process group is excluded", func() {
+				BeforeEach(func() {
+					// When the system is actually running, this would previously have been set by
+					// the `removeProcessGroups` subreconciler and saved in the cluster status,
+					// which would then be visible to the next run of the `addPods` subreconciler.
+					processGroupWithoutPod.SetExclude()
+				})
+
+				It("should not requeue", func() {
+					Expect(requeue).To(BeNil())
+				})
+
+				It("should not create any pods", func() {
+					Expect(newPods.Items).To(HaveLen(len(initialPods.Items)))
+				})
 			})
 
-			It("should not create any pods", func() {
-				Expect(newPods.Items).To(HaveLen(len(initialPods.Items)))
+			When("the process group is not excluded", func() {
+				It("should not requeue", func() {
+					Expect(requeue).To(BeNil())
+				})
+
+				It("should create an extra pod", func() {
+					expectNewPodToHaveBeenCreated(initialPods, newPods, cluster)
+				})
 			})
 		})
 	})
 })
+
+func expectNewPodToHaveBeenCreated(initialPods *corev1.PodList, newPods *corev1.PodList, cluster *fdbv1beta2.FoundationDBCluster) {
+	Expect(newPods.Items).To(HaveLen(len(initialPods.Items) + 1))
+	lastPod := newPods.Items[len(newPods.Items)-1]
+	Expect(lastPod.Name).To(Equal("operator-test-1-storage-9"))
+	Expect(lastPod.Labels[fdbv1beta2.FDBProcessGroupIDLabel]).To(Equal("storage-9"))
+	Expect(lastPod.Labels[fdbv1beta2.FDBProcessClassLabel]).To(Equal("storage"))
+	Expect(lastPod.OwnerReferences).To(Equal(internal.BuildOwnerReference(cluster.TypeMeta, cluster.ObjectMeta)))
+}
diff --git a/controllers/cluster_controller_test.go b/controllers/cluster_controller_test.go
index 9af0eda7..60226965 100644
--- a/controllers/cluster_controller_test.go
+++ b/controllers/cluster_controller_test.go
@@ -2535,6 +2535,122 @@ var _ = Describe("cluster_controller", func() {
 			})
 		})
 
+		When("creating a cluster with RocksDB as storage engine", func() {
+			When("using rocksdb-v1 engine", func() {
+				When("using 6.3.26", func() {
+					BeforeEach(func() {
+						cluster.Spec.DatabaseConfiguration.StorageEngine = "ssd-rocksdb-v1"
+						cluster.Spec.Version = "6.3.26"
+						err := k8sClient.Update(context.TODO(), cluster)
+						Expect(err).NotTo(HaveOccurred())
+						shouldCompleteReconciliation = false
+					})
+					It("generations are not matching", func() {
+						generations, err := reloadClusterGenerations(cluster)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(generations.Reconciled).ToNot(Equal(cluster.ObjectMeta.Generation))
+					})
+				})
+				When("using 7.1.0-rc3", func() {
+					BeforeEach(func() {
+						cluster.Spec.DatabaseConfiguration.StorageEngine = "ssd-rocksdb-v1"
+						cluster.Spec.Version = "7.1.0-rc3"
+						err := k8sClient.Update(context.TODO(), cluster)
+						Expect(err).NotTo(HaveOccurred())
+						shouldCompleteReconciliation = false
+					})
+					It("generations are not matching", func() {
+						generations, err := reloadClusterGenerations(cluster)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(generations.Reconciled).ToNot(Equal(cluster.ObjectMeta.Generation))
+					})
+				})
+				When("using 7.1.0", func() {
+					BeforeEach(func() {
+						cluster.Spec.DatabaseConfiguration.StorageEngine = "ssd-rocksdb-v1"
+						cluster.Spec.Version = "7.1.0"
+						err := k8sClient.Update(context.TODO(), cluster)
+						Expect(err).NotTo(HaveOccurred())
+					})
+					It("generations are matching", func() {
+						generations, err := reloadClusterGenerations(cluster)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(generations.Reconciled).To(Equal(cluster.ObjectMeta.Generation))
+					})
+				})
+				When("using 7.1.0-rc4", func() {
+					BeforeEach(func() {
+						cluster.Spec.DatabaseConfiguration.StorageEngine = "ssd-rocksdb-v1"
+						cluster.Spec.Version = "7.1.0-rc4"
+						err := k8sClient.Update(context.TODO(), cluster)
+						Expect(err).NotTo(HaveOccurred())
+					})
+					It("generations are matching", func() {
+						generations, err := reloadClusterGenerations(cluster)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(generations.Reconciled).To(Equal(cluster.ObjectMeta.Generation))
+					})
+				})
+			})
+
+			When("using ssd-rocksdb-experimental", func() {
+				When("using 7.1.0-rc4", func() {
+					BeforeEach(func() {
+						cluster.Spec.DatabaseConfiguration.StorageEngine = "ssd-rocksdb-experimental"
+						cluster.Spec.Version = "7.1.0-rc4"
+						err := k8sClient.Update(context.TODO(), cluster)
+						Expect(err).NotTo(HaveOccurred())
+						shouldCompleteReconciliation = false
+					})
+					It("generations are not matching", func() {
+						generations, err := reloadClusterGenerations(cluster)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(generations.Reconciled).ToNot(Equal(cluster.ObjectMeta.Generation))
+					})
+				})
+				When("using 7.1.0", func() {
+					BeforeEach(func() {
+						cluster.Spec.DatabaseConfiguration.StorageEngine = "ssd-rocksdb-experimental"
+						cluster.Spec.Version = "7.1.0"
+						err := k8sClient.Update(context.TODO(), cluster)
+						Expect(err).NotTo(HaveOccurred())
+						shouldCompleteReconciliation = false
+					})
+					It("generations are not matching", func() {
+						generations, err := reloadClusterGenerations(cluster)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(generations.Reconciled).ToNot(Equal(cluster.ObjectMeta.Generation))
+					})
+				})
+				When("using 7.1.0-rc3", func() {
+					BeforeEach(func() {
+						cluster.Spec.DatabaseConfiguration.StorageEngine = "ssd-rocksdb-experimental"
+						cluster.Spec.Version = "7.1.0-rc3"
+						err := k8sClient.Update(context.TODO(), cluster)
+						Expect(err).NotTo(HaveOccurred())
+					})
+					It("generations are matching", func() {
+						generations, err := reloadClusterGenerations(cluster)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(generations.Reconciled).To(Equal(cluster.ObjectMeta.Generation))
+					})
+				})
+				When("using 6.3.24", func() {
+					BeforeEach(func() {
+						cluster.Spec.DatabaseConfiguration.StorageEngine = "ssd-rocksdb-experimental"
+						cluster.Spec.Version = "6.3.24"
+						err := k8sClient.Update(context.TODO(), cluster)
+						Expect(err).NotTo(HaveOccurred())
+					})
+					It("generations are matching", func() {
+						generations, err := reloadClusterGenerations(cluster)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(generations.Reconciled).To(Equal(cluster.ObjectMeta.Generation))
+					})
+				})
+			})
+		})
+
 		When("When a process have an incorrect commandline", func() {
 			var adminClient *mockAdminClient
 
diff --git a/controllers/remove_process_groups_test.go b/controllers/remove_process_groups_test.go
index 1c64ad5e..6d562bc0 100644
--- a/controllers/remove_process_groups_test.go
+++ b/controllers/remove_process_groups_test.go
@@ -84,8 +84,10 @@ var _ = Describe("remove_process_groups", func() {
 	})
 
 	When("removing a process group", func() {
+		var removedProcessGroup *fdbv1beta2.ProcessGroupStatus
+
 		BeforeEach(func() {
-			removedProcessGroup := cluster.Status.ProcessGroups[0]
+			removedProcessGroup = cluster.Status.ProcessGroups[0]
 			marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, removedProcessGroup.ProcessGroupID, removedProcessGroup.ProcessClass, removedProcessGroup.Addresses[0])
 			Expect(marked).To(BeTrue())
 			Expect(processGroup).To(BeNil())
@@ -99,6 +101,11 @@ var _ = Describe("remove_process_groups", func() {
 			When("the cluster is fully replicated", func() {
 				It("should successfully remove that process group", func() {
 					Expect(result).To(BeNil())
+					// Ensure resources are deleted
+					removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+					Expect(err).To(BeNil())
+					Expect(removed).To(BeTrue())
+					Expect(include).To(BeTrue())
 				})
 			})
 
@@ -112,6 +119,11 @@ var _ = Describe("remove_process_groups", func() {
 				It("should not remove that process group", func() {
 					Expect(result).NotTo(BeNil())
 					Expect(result.message).To(Equal("Removals cannot proceed because cluster has degraded fault tolerance"))
+					// Ensure resources are not deleted
+					removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+					Expect(err).To(BeNil())
+					Expect(removed).To(BeFalse())
+					Expect(include).To(BeFalse())
 				})
 			})
 
@@ -125,6 +137,11 @@ var _ = Describe("remove_process_groups", func() {
 				It("should not remove that process group", func() {
 					Expect(result).NotTo(BeNil())
 					Expect(result.message).To(Equal("Removals cannot proceed because cluster has degraded fault tolerance"))
+					// Ensure resources are not deleted
+					removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+					Expect(err).To(BeNil())
+					Expect(removed).To(BeFalse())
+					Expect(include).To(BeFalse())
 				})
 			})
 
@@ -144,35 +161,50 @@ var _ = Describe("remove_process_groups", func() {
 				It("should not remove that process group", func() {
 					Expect(result).NotTo(BeNil())
 					Expect(result.message).To(Equal("Removals cannot proceed because cluster has degraded fault tolerance"))
+					// Ensure resources are not deleted
+					removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+					Expect(err).To(BeNil())
+					Expect(removed).To(BeFalse())
+					Expect(include).To(BeFalse())
 				})
 			})
 		})
 
 		When("Removing multiple process groups", func() {
 			var initialCnt int
-			var removedProcessGroup *fdbv1beta2.ProcessGroupStatus
+			var secondRemovedProcessGroup *fdbv1beta2.ProcessGroupStatus
 
 			BeforeEach(func() {
 				initialCnt = len(cluster.Status.ProcessGroups)
-				removedProcessGroup = cluster.Status.ProcessGroups[1]
-				marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, removedProcessGroup.ProcessGroupID, removedProcessGroup.ProcessClass, removedProcessGroup.Addresses[0])
+				secondRemovedProcessGroup = cluster.Status.ProcessGroups[1]
+				marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, secondRemovedProcessGroup.ProcessGroupID, secondRemovedProcessGroup.ProcessClass, removedProcessGroup.Addresses[0])
 				Expect(marked).To(BeTrue())
 				Expect(processGroup).To(BeNil())
 				// Exclude the process group
 				adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
 				Expect(err).NotTo(HaveOccurred())
-				adminClient.ExcludedAddresses = append(adminClient.ExcludedAddresses, removedProcessGroup.Addresses...)
+				adminClient.ExcludedAddresses = append(adminClient.ExcludedAddresses, secondRemovedProcessGroup.Addresses...)
 			})
 
 			It("should remove only one process group", func() {
 				Expect(result).To(BeNil())
 				Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 1))
+				// Ensure resources are deleted
+				removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+				Expect(err).To(BeNil())
+				Expect(removed).To(BeTrue())
+				Expect(include).To(BeTrue())
+				// Ensure resources are not deleted
+				removed, include, err = confirmRemoval(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+				Expect(err).To(BeNil())
+				Expect(removed).To(BeFalse())
+				Expect(include).To(BeFalse())
 			})
 
 			When("a process group is marked as terminating and all resources are removed it should be removed", func() {
 				BeforeEach(func() {
-					removedProcessGroup.ProcessGroupConditions = append(removedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
-					err := removeProcessGroup(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+					secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+					err := removeProcessGroup(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
 					Expect(err).NotTo(HaveOccurred())
 					// Sleep here to prevent some timeing issues.
 					time.Sleep(10 * time.Microsecond)
@@ -181,12 +213,45 @@ var _ = Describe("remove_process_groups", func() {
 				It("should remove the process group and the terminated process group", func() {
 					Expect(result).To(BeNil())
 					Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
+					// Ensure resources are deleted
+					removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+					Expect(err).To(BeNil())
+					Expect(removed).To(BeTrue())
+					Expect(include).To(BeTrue())
+					// Ensure resources are deleted
+					removed, include, err = confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+					Expect(err).To(BeNil())
+					Expect(removed).To(BeTrue())
+					Expect(include).To(BeTrue())
+				})
+			})
+
+			When("a process group is marked as terminating and the resources are not removed", func() {
+				BeforeEach(func() {
+					secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+					// Sleep here to prevent some timeing issues.
+					time.Sleep(10 * time.Microsecond)
+				})
+
+				It("should remove the process group and the terminated process group", func() {
+					Expect(result).To(BeNil())
+					Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
+					// Ensure resources are deleted
+					removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+					Expect(err).To(BeNil())
+					Expect(removed).To(BeTrue())
+					Expect(include).To(BeTrue())
+					// Ensure resources are deleted
+					removed, include, err = confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+					Expect(err).To(BeNil())
+					Expect(removed).To(BeTrue())
+					Expect(include).To(BeTrue())
 				})
 			})
 
 			When("a process group is marked as terminating and not fully removed", func() {
 				BeforeEach(func() {
-					removedProcessGroup.ProcessGroupConditions = append(removedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+					secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
 					// Set the wait time to the default value
 					cluster.Spec.AutomationOptions.WaitBetweenRemovalsSeconds = pointer.Int(60)
 				})
@@ -195,6 +260,16 @@ var _ = Describe("remove_process_groups", func() {
 					Expect(result).NotTo(BeNil())
 					Expect(result.message).To(HavePrefix("not allowed to remove process groups, waiting:"))
 					Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 0))
+					// Ensure resources are not deleted
+					removed, include, err := confirmRemoval(context.Background(), clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+					Expect(err).To(BeNil())
+					Expect(removed).To(BeFalse())
+					Expect(include).To(BeFalse())
+					// Ensure resources are deleted
+					removed, include, err = confirmRemoval(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+					Expect(err).To(BeNil())
+					Expect(removed).To(BeFalse())
+					Expect(include).To(BeFalse())
 				})
 			})
 		})
diff --git a/fdbclient/admin_client_test.go b/fdbclient/admin_client_test.go
index 11851e01..a4344441 100644
--- a/fdbclient/admin_client_test.go
+++ b/fdbclient/admin_client_test.go
@@ -21,7 +21,6 @@
 package fdbclient
 
 import (
-	"fmt"
 	"net"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
@@ -102,55 +101,6 @@ var _ = Describe("admin_client_test", func() {
 		})
 	})
 
-	When("Removing warnings in JSON", func() {
-		type testCase struct {
-			input       string
-			expected    string
-			expectedErr error
-		}
-
-		DescribeTable("Test remove warnings in JSON string",
-			func(tc testCase) {
-				result, err := removeWarningsInJSON(tc.input)
-				// We need the if statement to make ginkgo happy:
-				//   Refusing to compare <nil> to <nil>.
-				//   Be explicit and use BeNil() instead.
-				//   This is to avoid mistakes where both sides of an assertion are erroneously uninitialized.
-				// ¯\_(ツ)_/¯
-				if tc.expectedErr == nil {
-					Expect(err).To(BeNil())
-				} else {
-					Expect(err).To(Equal(tc.expectedErr))
-				}
-				Expect(result).To(Equal(tc.expected))
-			},
-			Entry("Valid JSON without warning",
-				testCase{
-					input:       "{}",
-					expected:    "{}",
-					expectedErr: nil,
-				},
-			),
-			Entry("Valid JSON with warning",
-				testCase{
-					input: `
- # Warning Slow response
- 
- {}`,
-					expected:    "{}",
-					expectedErr: nil,
-				},
-			),
-			Entry("Invalid JSON",
-				testCase{
-					input:       "}",
-					expected:    "",
-					expectedErr: fmt.Errorf("the JSON string doesn't contain a starting '{'"),
-				},
-			),
-		)
-	})
-
 	When("getting the excluded and remaining processes", func() {
 		addr1 := fdbv1beta2.NewProcessAddress(net.ParseIP("127.0.0.1"), "", 0, nil)
 		addr2 := fdbv1beta2.NewProcessAddress(net.ParseIP("127.0.0.2"), "", 0, nil)
diff --git a/internal/fdb_status_helper_test.go b/internal/fdb_status_helper_test.go
new file mode 100644
index 00000000..30952de6
--- /dev/null
+++ b/internal/fdb_status_helper_test.go
@@ -0,0 +1,80 @@
+/*
+ * fdb_status_helper_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package internal
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/extensions/table"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("fdb_status_helper_test", func() {
+	When("Removing warnings in JSON", func() {
+		type testCase struct {
+			input       string
+			expected    string
+			expectedErr error
+		}
+
+		DescribeTable("Test remove warnings in JSON string",
+			func(tc testCase) {
+				result, err := RemoveWarningsInJSON(tc.input)
+				// We need the if statement to make ginkgo happy:
+				//   Refusing to compare <nil> to <nil>.
+				//   Be explicit and use BeNil() instead.
+				//   This is to avoid mistakes where both sides of an assertion are erroneously uninitialized.
+				// ¯\_(ツ)_/¯
+				if tc.expectedErr == nil {
+					Expect(err).To(BeNil())
+				} else {
+					Expect(err).To(Equal(tc.expectedErr))
+				}
+				Expect(result).To(Equal(tc.expected))
+			},
+			Entry("Valid JSON without warning",
+				testCase{
+					input:       "{}",
+					expected:    "{}",
+					expectedErr: nil,
+				},
+			),
+			Entry("Valid JSON with warning",
+				testCase{
+					input: `
+ # Warning Slow response
+
+ {}`,
+					expected:    "{}",
+					expectedErr: nil,
+				},
+			),
+			Entry("Invalid JSON",
+				testCase{
+					input:       "}",
+					expected:    "",
+					expectedErr: fmt.Errorf("the JSON string doesn't contain a starting '{'"),
+				},
+			),
+		)
+	})
+})
diff --git a/kubectl-fdb/cmd/analyze_test.go b/kubectl-fdb/cmd/analyze_test.go
index 5024b0ca..54f4f175 100644
--- a/kubectl-fdb/cmd/analyze_test.go
+++ b/kubectl-fdb/cmd/analyze_test.go
@@ -97,7 +97,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 			ExpectedErrMsg    string
 			ExpectedStdouMsg  string
 			AutoFix           bool
-			Force             bool
+			NoWait            bool
 			HasErrors         bool
 			IgnoredConditions []string
 		}
@@ -115,7 +115,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				inBuffer := bytes.Buffer{}
 
 				cmd := newAnalyzeCmd(genericclioptions.IOStreams{In: &inBuffer, Out: &outBuffer, ErrOut: &errBuffer})
-				err := analyzeCluster(cmd, kubeClient, clusterName, namespace, tc.AutoFix, tc.Force, tc.IgnoredConditions)
+				err := analyzeCluster(cmd, kubeClient, clusterName, namespace, tc.AutoFix, tc.NoWait, tc.IgnoredConditions)
 
 				if err != nil && !tc.HasErrors {
 					Expect(err).To(HaveOccurred())
diff --git a/kubectl-fdb/cmd/configuration_test.go b/kubectl-fdb/cmd/configuration_test.go
index 1b4fd1ad..e027c376 100644
--- a/kubectl-fdb/cmd/configuration_test.go
+++ b/kubectl-fdb/cmd/configuration_test.go
@@ -21,12 +21,15 @@
 package cmd
 
 import (
+	ctx "context"
+
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 )
 
@@ -70,7 +73,7 @@ var _ = Describe("[plugin] configuration command", func() {
 				Expect(configuration).To(Equal("  usable_regions=0 logs=0 resolvers=0 log_routers=0 remote_logs=0 proxies=3 regions=[{\\\"datacenters\\\":[{\\\"id\\\":\\\"test\\\",\\\"priority\\\":1}]}]"))
 			})
 
-			It("should return the same configuration string with failover", func() {
+			It("should return the same configuration string with fail-over", func() {
 				scheme := runtime.NewScheme()
 				_ = clientgoscheme.AddToScheme(scheme)
 				_ = fdbv1beta2.AddToScheme(scheme)
@@ -111,6 +114,8 @@ var _ = Describe("[plugin] configuration command", func() {
 											Satellite: 1,
 										},
 									},
+									SatelliteLogs:           3,
+									SatelliteRedundancyMode: fdbv1beta2.RedundancyModeOneSatelliteSingle,
 								},
 								{
 									DataCenters: []fdbv1beta2.DataCenter{
@@ -129,6 +134,8 @@ var _ = Describe("[plugin] configuration command", func() {
 											Satellite: 1,
 										},
 									},
+									SatelliteLogs:           3,
+									SatelliteRedundancyMode: fdbv1beta2.RedundancyModeOneSatelliteDouble,
 								},
 							},
 						},
@@ -146,7 +153,7 @@ var _ = Describe("[plugin] configuration command", func() {
 
 				configuration, err := getConfigurationString(kubeClient, "test", "test", false)
 				Expect(err).NotTo(HaveOccurred())
-				Expect(configuration).To(Equal("  usable_regions=0 logs=0 resolvers=0 log_routers=0 remote_logs=0 proxies=3 regions=[{\\\"datacenters\\\":[{\\\"id\\\":\\\"primary\\\",\\\"priority\\\":1},{\\\"id\\\":\\\"primary-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1},{\\\"id\\\":\\\"remote-sat\\\",\\\"satellite\\\":1}]},{\\\"datacenters\\\":[{\\\"id\\\":\\\"remote\\\"},{\\\"id\\\":\\\"remote-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1},{\\\"id\\\":\\\"primary-sat\\\",\\\"satellite\\\":1}]}]"))
+				Expect(configuration).To(Equal("  usable_regions=0 logs=0 resolvers=0 log_routers=0 remote_logs=0 proxies=3 regions=[{\\\"datacenters\\\":[{\\\"id\\\":\\\"primary\\\",\\\"priority\\\":1},{\\\"id\\\":\\\"primary-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1},{\\\"id\\\":\\\"remote-sat\\\",\\\"satellite\\\":1}],\\\"satellite_logs\\\":3,\\\"satellite_redundancy_mode\\\":\\\"one_satellite_single\\\"},{\\\"datacenters\\\":[{\\\"id\\\":\\\"remote\\\"},{\\\"id\\\":\\\"remote-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1},{\\\"id\\\":\\\"primary-sat\\\",\\\"satellite\\\":1}],\\\"satellite_logs\\\":3,\\\"satellite_redundancy_mode\\\":\\\"one_satellite_double\\\"}]"))
 			})
 
 			It("should return the configuration string with the modified priority", func() {
@@ -157,7 +164,54 @@ var _ = Describe("[plugin] configuration command", func() {
 
 				configuration, err := getConfigurationString(kubeClient, "test", "test", true)
 				Expect(err).NotTo(HaveOccurred())
-				Expect(configuration).To(Equal("  usable_regions=0 logs=0 resolvers=0 log_routers=0 remote_logs=0 proxies=3 regions=[{\\\"datacenters\\\":[{\\\"id\\\":\\\"primary\\\"},{\\\"id\\\":\\\"primary-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1},{\\\"id\\\":\\\"remote-sat\\\",\\\"satellite\\\":1}]},{\\\"datacenters\\\":[{\\\"id\\\":\\\"remote\\\",\\\"priority\\\":1},{\\\"id\\\":\\\"remote-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1},{\\\"id\\\":\\\"primary-sat\\\",\\\"satellite\\\":1}]}]"))
+				Expect(configuration).To(Equal("  usable_regions=0 logs=0 resolvers=0 log_routers=0 remote_logs=0 proxies=3 regions=[{\\\"datacenters\\\":[{\\\"id\\\":\\\"primary\\\"},{\\\"id\\\":\\\"primary-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1},{\\\"id\\\":\\\"remote-sat\\\",\\\"satellite\\\":1}],\\\"satellite_logs\\\":3,\\\"satellite_redundancy_mode\\\":\\\"one_satellite_single\\\"},{\\\"datacenters\\\":[{\\\"id\\\":\\\"remote\\\",\\\"priority\\\":1},{\\\"id\\\":\\\"remote-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1},{\\\"id\\\":\\\"primary-sat\\\",\\\"satellite\\\":1}],\\\"satellite_logs\\\":3,\\\"satellite_redundancy_mode\\\":\\\"one_satellite_double\\\"}]"))
+			})
+
+			When("Updating the config for the cluster", func() {
+				It("should update the config with the modified priority", func() {
+					scheme := runtime.NewScheme()
+					_ = clientgoscheme.AddToScheme(scheme)
+					_ = fdbv1beta2.AddToScheme(scheme)
+					kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
+
+					err := updateConfig(kubeClient, "test", "test", true, false)
+					Expect(err).NotTo(HaveOccurred())
+
+					var resCluster fdbv1beta2.FoundationDBCluster
+					err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+						Namespace: "test",
+						Name:      "test",
+					}, &resCluster)
+					Expect(err).NotTo(HaveOccurred())
+
+					config := resCluster.Spec.DatabaseConfiguration
+					Expect(len(config.Regions)).To(BeNumerically("==", 2))
+
+					Expect(config.Regions[0].SatelliteRedundancyMode).To(Equal(fdbv1beta2.RedundancyModeOneSatelliteSingle))
+					Expect(config.Regions[0].SatelliteLogs).To(Equal(3))
+
+					Expect(config.Regions[0].DataCenters[0].ID).To(Equal("primary"))
+					Expect(config.Regions[0].DataCenters[0].Satellite).To(Equal(0))
+					Expect(config.Regions[0].DataCenters[0].Priority).To(Equal(0))
+					Expect(config.Regions[0].DataCenters[1].ID).To(Equal("primary-sat"))
+					Expect(config.Regions[0].DataCenters[1].Satellite).To(Equal(1))
+					Expect(config.Regions[0].DataCenters[1].Priority).To(Equal(1))
+					Expect(config.Regions[0].DataCenters[2].ID).To(Equal("remote-sat"))
+					Expect(config.Regions[0].DataCenters[2].Satellite).To(Equal(1))
+					Expect(config.Regions[0].DataCenters[2].Priority).To(Equal(0))
+
+					Expect(config.Regions[1].DataCenters[0].ID).To(Equal("remote"))
+					Expect(config.Regions[1].DataCenters[0].Satellite).To(Equal(0))
+					Expect(config.Regions[1].DataCenters[0].Priority).To(Equal(1))
+					Expect(config.Regions[1].DataCenters[1].ID).To(Equal("remote-sat"))
+					Expect(config.Regions[1].DataCenters[1].Satellite).To(Equal(1))
+					Expect(config.Regions[1].DataCenters[1].Priority).To(Equal(1))
+					Expect(config.Regions[1].DataCenters[2].ID).To(Equal("primary-sat"))
+					Expect(config.Regions[1].DataCenters[2].Satellite).To(Equal(1))
+					Expect(config.Regions[1].DataCenters[2].Priority).To(Equal(0))
+
+					Expect(config.GetConfigurationString(fdbv1beta2.Versions.Default.String())).To(Equal("  usable_regions=0 logs=0 resolvers=0 log_routers=0 remote_logs=0 proxies=3 regions=[{\\\"datacenters\\\":[{\\\"id\\\":\\\"primary\\\"},{\\\"id\\\":\\\"primary-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1},{\\\"id\\\":\\\"remote-sat\\\",\\\"satellite\\\":1}],\\\"satellite_logs\\\":3,\\\"satellite_redundancy_mode\\\":\\\"one_satellite_single\\\"},{\\\"datacenters\\\":[{\\\"id\\\":\\\"remote\\\",\\\"priority\\\":1},{\\\"id\\\":\\\"remote-sat\\\",\\\"priority\\\":1,\\\"satellite\\\":1},{\\\"id\\\":\\\"primary-sat\\\",\\\"satellite\\\":1}],\\\"satellite_logs\\\":3,\\\"satellite_redundancy_mode\\\":\\\"one_satellite_double\\\"}]"))
+				})
 			})
 		})
 	})
diff --git a/kubectl-fdb/cmd/cordon_test.go b/kubectl-fdb/cmd/cordon_test.go
index 670fa0ca..6e938663 100644
--- a/kubectl-fdb/cmd/cordon_test.go
+++ b/kubectl-fdb/cmd/cordon_test.go
@@ -112,7 +112,7 @@ var _ = Describe("[plugin] cordon command", func() {
 				_ = fdbv1beta2.AddToScheme(scheme)
 				kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(&cluster, &podList, &nodeList).Build()
 
-				err := cordonNode(kubeClient, &cluster, input.nodes, namespace, input.WithExclusion, true)
+				err := cordonNode(kubeClient, &cluster, input.nodes, namespace, input.WithExclusion, false)
 				Expect(err).NotTo(HaveOccurred())
 
 				var resCluster fdbv1beta2.FoundationDBCluster
diff --git a/kubectl-fdb/cmd/remove_process_group_test.go b/kubectl-fdb/cmd/remove_process_group_test.go
index df46e946..8e0c3074 100644
--- a/kubectl-fdb/cmd/remove_process_group_test.go
+++ b/kubectl-fdb/cmd/remove_process_group_test.go
@@ -171,7 +171,7 @@ var _ = Describe("[plugin] remove instances command", func() {
 					_ = fdbv1beta2.AddToScheme(scheme)
 					kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster, &podList).Build()
 
-					err := replaceProcessGroups(kubeClient, clusterName, tc.Instances, namespace, tc.WithExclusion, tc.WithShrink, true, tc.RemoveAllFailed)
+					err := replaceProcessGroups(kubeClient, clusterName, tc.Instances, namespace, tc.WithExclusion, tc.WithShrink, false, tc.RemoveAllFailed)
 					Expect(err).NotTo(HaveOccurred())
 
 					var resCluster fdbv1beta2.FoundationDBCluster
@@ -262,7 +262,7 @@ var _ = Describe("[plugin] remove instances command", func() {
 				When("adding the same process group to the removal list without exclusion", func() {
 					It("should add the process group to the removal without exclusion list", func() {
 						removals := []string{"instance-1"}
-						err := replaceProcessGroups(kubeClient, clusterName, removals, namespace, false, false, true, false)
+						err := replaceProcessGroups(kubeClient, clusterName, removals, namespace, false, false, false, false)
 						Expect(err).NotTo(HaveOccurred())
 
 						var resCluster fdbv1beta2.FoundationDBCluster
@@ -282,7 +282,7 @@ var _ = Describe("[plugin] remove instances command", func() {
 				When("adding the same process group to the removal list", func() {
 					It("should add the process group to the removal without exclusion list", func() {
 						removals := []string{"instance-1"}
-						err := replaceProcessGroups(kubeClient, clusterName, removals, namespace, true, false, true, false)
+						err := replaceProcessGroups(kubeClient, clusterName, removals, namespace, true, false, false, false)
 						Expect(err).NotTo(HaveOccurred())
 
 						var resCluster fdbv1beta2.FoundationDBCluster