Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1734 (2023-07-11 13:46:06)

Split deployment.yaml config sample RBAC roles
# Description

Because https://github.com/FoundationDB/fdb-kubernetes-operator/issues/1727 the operator does not work if the deployment doesn't have RBAC access to listing nodes. This PR mitigates this issue for users relying on deployment config sample to build their cluster manifests.

The deployment config sample uses a RoleBinding pointing to a ClusterRole. This configuration will not allow to list nodes.

In this PR we split the ClusterRole in two ClusterRoles one to access namespace scoped resources and another to access cluster-wide (nodes) resources. Using a RoleBinding for the first and a ClusterRoleBinding for the former.

## Type of change

*Please select one of the options below.*

- Bug fix (non-breaking change which fixes an issue)

## Discussion

*Are there any design details that you would like to discuss further?*

## Testing

*Please describe the tests that you ran to verify your changes. Unit tests?
Manual testing?*

*Do we need to perform additional testing once this is merged, or perform in a larger testing environment?*

## Documentation

*Did you update relevant documentation within this repository?*

*If this change is adding new functionality, do we need to describe it in our user manual?*

*If this change is adding or removing subreconcilers, have we updated the core technical design doc to reflect that?*

*If this change is adding new safety checks or new potential failure modes, have we documented and how to debug potential issues?*

## Follow-up

*Are there any follow-up issues that we should pursue in the future?*

*Does this introduce new defaults that we should re-evaluate in the future?*


foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: 5e663ff936efb7ee174ed06061bd8ffebd476fc8
* Duration 2:32:42
* Result: :white_check_mark: **SUCCEEDED**
* Error: `N/A`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/69bb42fc-5a35-43ce-9816-e15a308a3124.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/69bb42fc-5a35-43ce-9816-e15a308a3124/workspace.zip) zip file of the working directory (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: 32a54f179b8d092300e8392a8f2cf670ca1c23e2
* Duration 2:29:16
* Result: :white_check_mark: **SUCCEEDED**
* Error: `N/A`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/57fb2abb-f493-45e7-9def-4c02b654fff2.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/57fb2abb-f493-45e7-9def-4c02b654fff2/workspace.zip) zip file of the working directory (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: b820cca7ade416d59825c228ba6167f86bf0cb61
* Duration 2:32:31
* Result: :white_check_mark: **SUCCEEDED**
* Error: `N/A`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/c1cb32c3-0f97-4c38-94fc-7c26db121eb6.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/c1cb32c3-0f97-4c38-94fc-7c26db121eb6/workspace.zip) zip file of the working directory (available for 30 days)


Patch:
diff --git a/.github/workflows/on-push-lint-charts.yml b/.github/workflows/on-push-lint-charts.yml
deleted file mode 100644
index 4e99e0b2..00000000
--- a/.github/workflows/on-push-lint-charts.yml
+++ /dev/null
@@ -1,51 +0,0 @@
-name: Lint and Test Charts
-on:
-  push:
-    paths:
-      - 'charts/**'
-      - '.github/**'
-env:
-  HELM_VERSION: 3.5.2
-  CHECK_VERSION_INCREMENT: false # avoid tests failing when multiple attempts to push chart changes are made
-
-jobs:
-  lint-test:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Set up Helm
-        uses: azure/setup-helm@v1
-        with:
-          version: v${{ env.HELM_VERSION }}
-
-      # Python is a requirement for the chart-testing action below (supports yamllint among other tests)
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.7
-
-      # Those steps can be run manually with:
-      # docker run --rm -it -w /repo -v $(pwd):/repo quay.io/helmpack/chart-testing:v3.3.1 ct lint --all
-      - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.2.1
-
-      - name: Run chart-testing (list-changed)
-        id: list-changed
-        run: |
-          changed=$(ct list-changed)
-          if [[ -n "$(ct list-changed)" ]]; then
-            echo "::set-output name=changed::true"
-          fi
-
-      - name: Run chart-testing (lint)
-        run: ct lint --check-version-increment=${{ env.CHECK_VERSION_INCREMENT }}
-
-      - name: Create kind cluster
-        uses: helm/kind-action@v1.3.0
-        if: steps.list-changed.outputs.changed == 'true'
-
-      - name: Run chart-testing (install)
-        run: ct install
diff --git a/.github/workflows/on-push-main-publish-chart.yml b/.github/workflows/on-push-main-publish-chart.yml
deleted file mode 100644
index b8e5a9cb..00000000
--- a/.github/workflows/on-push-main-publish-chart.yml
+++ /dev/null
@@ -1,36 +0,0 @@
-name: Publish helm chart
-on:
-  push:
-    branches:
-      - main
-    paths:
-      - 'charts/**'
-      - '.github/**'
-env:
-  HELM_VERSION: 3.5.2
-
-jobs:
-  publish-chart:
-
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Configure Git
-        run: |
-          git config user.name "$GITHUB_ACTOR"
-          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
-
-      - name: Install Helm
-        uses: azure/setup-helm@v1
-        with:
-          version: v${{ env.HELM_VERSION }}
-
-      - name: Run chart-releaser
-        uses: helm/chart-releaser-action@v1.2.0
-        env:
-          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 721c4f43..9182a17c 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -18,7 +18,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.19.10
+          go-version: 1.20.6
       - uses: actions/cache@v2
         with:
           path: ~/go/pkg/mod
@@ -44,7 +44,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.19.10
+        go-version: 1.20.6
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
       with:
@@ -111,7 +111,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.19.10
+        go-version: 1.20.6
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
       with:
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 17652331..e3c78f66 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -43,7 +43,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.19.10
+          go-version: 1.20.6
       #  https://github.com/goreleaser/goreleaser/issues/1311
       - name: Get current semver tag
         run: echo "::set-output name=CURRENT_TAG::$(git describe --tags --match "v*" --abbrev=0)"
diff --git a/.golangci.yml b/.golangci.yml
index fddafd66..7436123e 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -22,6 +22,10 @@ issues:
   - linters:
       - stylecheck
     text: "ST1003: const StorageEngineRocksDbV1"
+  # TODO (johscheuer): Fix all go imports to make this check happy, https://github.com/FoundationDB/fdb-kubernetes-operator/issues/1739.
+  - linters:
+      - depguard
+    text: "is not allowed from list 'Main'"
 
 linters:
   disable-all: true
@@ -69,9 +73,9 @@ linters-settings:
     enable:
       - shadow
   staticcheck:
-    go: "1.19"
+    go: "1.20"
   stylecheck:
-    go: "1.19"
+    go: "1.20"
 
 run:
   timeout: 10m
diff --git a/Dockerfile b/Dockerfile
index 77e751d7..622f0908 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,7 +1,7 @@
 ARG BASE_IMAGE=docker.io/debian:bookworm
 
 # Build the manager binary
-FROM docker.io/library/golang:1.19.10 as builder
+FROM docker.io/library/golang:1.20.6 as builder
 
 # Install FDB this version is only required to compile the fdb operator
 ARG FDB_VERSION=6.2.29
diff --git a/Makefile b/Makefile
index 863cddcb..28c4b430 100644
--- a/Makefile
+++ b/Makefile
@@ -33,7 +33,7 @@ CONTROLLER_GEN_PKG?=sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2
 CONTROLLER_GEN=$(GOBIN)/controller-gen
 KUSTOMIZE_PKG?=sigs.k8s.io/kustomize/kustomize/v4@v4.5.2
 KUSTOMIZE=$(GOBIN)/kustomize
-GOLANGCI_LINT_PKG=github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1
+GOLANGCI_LINT_PKG=github.com/golangci/golangci-lint/cmd/golangci-lint@v1.53.3
 GOLANGCI_LINT=$(GOBIN)/golangci-lint
 GORELEASER_PKG=github.com/goreleaser/goreleaser@v1.6.3
 GORELEASER=$(GOBIN)/goreleaser
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index 73e308fe..c9cfe538 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -2233,11 +2233,11 @@ func (cluster *FoundationDBCluster) GetDNSDomain() string {
 
 // GetRemovalMode returns the removal mode of the cluster or default to PodUpdateModeZone if unset.
 func (cluster *FoundationDBCluster) GetRemovalMode() PodUpdateMode {
-	if cluster.Spec.AutomationOptions.DeletionMode == "" {
+	if cluster.Spec.AutomationOptions.RemovalMode == "" {
 		return PodUpdateModeZone
 	}
 
-	return cluster.Spec.AutomationOptions.DeletionMode
+	return cluster.Spec.AutomationOptions.RemovalMode
 }
 
 // GetWaitBetweenRemovalsSeconds returns the WaitDurationBetweenRemovals if set or defaults to 60s.
diff --git a/config/deployment/rbac_role_binding.yaml b/config/deployment/rbac_role_binding.yaml
index 3a76d186..d9dd880d 100644
--- a/config/deployment/rbac_role_binding.yaml
+++ b/config/deployment/rbac_role_binding.yaml
@@ -10,4 +10,18 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: fdb-kubernetes-operator-controller-manager
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  name: manager-clusterrolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: fdb-kubernetes-operator-manager-clusterrole
+subjects:
+- kind: ServiceAccount
+  name: fdb-kubernetes-operator-controller-manager
+  namespace: metadata.namespace
 
diff --git a/config/rbac/cluster_role.yaml b/config/rbac/cluster_role.yaml
new file mode 100644
index 00000000..e7aea3ac
--- /dev/null
+++ b/config/rbac/cluster_role.yaml
@@ -0,0 +1,15 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: manager-clusterrole
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml
index b4dab00b..83f22ee6 100644
--- a/config/rbac/kustomization.yaml
+++ b/config/rbac/kustomization.yaml
@@ -2,3 +2,4 @@ kind: Kustomization
 apiVersion: kustomize.config.k8s.io/v1beta1
 resources:
 - role.yaml
+- cluster_role.yaml
diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
index b5ff077b..957aeb28 100644
--- a/config/rbac/role.yaml
+++ b/config/rbac/role.yaml
@@ -22,14 +22,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
 - apiGroups:
   - apps
   resources:
diff --git a/config/samples/deployment.yaml b/config/samples/deployment.yaml
index 38d8dedd..61d727c5 100644
--- a/config/samples/deployment.yaml
+++ b/config/samples/deployment.yaml
@@ -5,6 +5,21 @@ metadata:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: fdb-kubernetes-operator-manager-clusterrole
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
 metadata:
   creationTimestamp: null
   name: fdb-kubernetes-operator-manager-role
@@ -26,14 +41,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
 - apiGroups:
   - apps
   resources:
@@ -132,6 +139,20 @@ subjects:
 - kind: ServiceAccount
   name: fdb-kubernetes-operator-controller-manager
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  name: fdb-kubernetes-operator-manager-clusterrolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: fdb-kubernetes-operator-manager-clusterrole
+subjects:
+- kind: ServiceAccount
+  name: fdb-kubernetes-operator-controller-manager
+  namespace: metadata.namespace
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
diff --git a/controllers/add_pods.go b/controllers/add_pods.go
index de8278eb..6a239be3 100644
--- a/controllers/add_pods.go
+++ b/controllers/add_pods.go
@@ -24,10 +24,9 @@ import (
 	"context"
 	"fmt"
 
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
 	"github.com/go-logr/logr"
-
-	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
@@ -37,7 +36,7 @@ import (
 type addPods struct{}
 
 // reconcile runs the reconciler's work.
-func (a addPods) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, logger logr.Logger) *requeue {
+func (a addPods) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, _ *fdbv1beta2.FoundationDBStatus, logger logr.Logger) *requeue {
 	configMap, err := internal.GetConfigMap(cluster)
 	if err != nil {
 		return &requeue{curError: err}
diff --git a/controllers/check_client_compatibility.go b/controllers/check_client_compatibility.go
index e51a95d7..c025582d 100644
--- a/controllers/check_client_compatibility.go
+++ b/controllers/check_client_compatibility.go
@@ -38,7 +38,7 @@ import (
 type checkClientCompatibility struct{}
 
 // reconcile runs the reconciler's work.
-func (c checkClientCompatibility) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, logger logr.Logger) *requeue {
+func (c checkClientCompatibility) reconcile(_ context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, logger logr.Logger) *requeue {
 	if !cluster.Status.Configured && !cluster.IsBeingUpgraded() {
 		return nil
 	}
diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go
index 3aab5102..7e1904c4 100644
--- a/controllers/cluster_controller.go
+++ b/controllers/cluster_controller.go
@@ -78,7 +78,6 @@ func NewFoundationDBClusterReconciler(podLifecycleManager podmanager.PodLifecycl
 // +kubebuilder:rbac:groups=apps.foundationdb.org,resources=foundationdbclusters/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups="",resources=pods;configmaps;persistentvolumeclaims;events;secrets;services,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups="coordination.k8s.io",resources=leases,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch
 
 // Reconcile runs the reconciliation logic.
 func (r *FoundationDBClusterReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
diff --git a/controllers/delete_pods_for_buggification.go b/controllers/delete_pods_for_buggification.go
index 20e2b88a..efe630b7 100644
--- a/controllers/delete_pods_for_buggification.go
+++ b/controllers/delete_pods_for_buggification.go
@@ -35,7 +35,7 @@ import (
 type deletePodsForBuggification struct{}
 
 // reconcile runs the reconciler's work.
-func (d deletePodsForBuggification) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, logger logr.Logger) *requeue {
+func (d deletePodsForBuggification) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, _ *fdbv1beta2.FoundationDBStatus, logger logr.Logger) *requeue {
 	crashLoopContainerProcessGroups := cluster.GetCrashLoopContainerProcessGroups()
 
 	noSchedulePods := make(map[fdbv1beta2.ProcessGroupID]fdbv1beta2.None, len(cluster.Spec.Buggify.NoSchedule))
diff --git a/controllers/exclude_processes.go b/controllers/exclude_processes.go
index 23a9dad7..a05c207b 100644
--- a/controllers/exclude_processes.go
+++ b/controllers/exclude_processes.go
@@ -43,7 +43,7 @@ var missingProcessThreshold = 0.8
 type excludeProcesses struct{}
 
 // reconcile runs the reconciler's work.
-func (e excludeProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, logger logr.Logger) *requeue {
+func (e excludeProcesses) reconcile(_ context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, logger logr.Logger) *requeue {
 	adminClient, err := r.getDatabaseClientProvider().GetAdminClient(cluster, r)
 	if err != nil {
 		return &requeue{curError: err}
diff --git a/controllers/remove_services.go b/controllers/remove_services.go
index e6da3edb..54153abf 100644
--- a/controllers/remove_services.go
+++ b/controllers/remove_services.go
@@ -36,7 +36,7 @@ import (
 type removeServices struct{}
 
 // reconcile runs the reconciler's work.
-func (u removeServices) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, logger logr.Logger) *requeue {
+func (u removeServices) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, _ *fdbv1beta2.FoundationDBStatus, logger logr.Logger) *requeue {
 	if cluster.NeedsHeadlessService() {
 		return nil
 	}
diff --git a/controllers/update_metadata.go b/controllers/update_metadata.go
index d7039513..e578eb8b 100644
--- a/controllers/update_metadata.go
+++ b/controllers/update_metadata.go
@@ -36,7 +36,7 @@ import (
 type updateMetadata struct{}
 
 // reconcile runs the reconciler's work.
-func (updateMetadata) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, logger logr.Logger) *requeue {
+func (updateMetadata) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, _ *fdbv1beta2.FoundationDBStatus, logger logr.Logger) *requeue {
 	// TODO(johscheuer): Remove the use of the pvc map and directly make a get request.
 	pvcs := &corev1.PersistentVolumeClaimList{}
 	err := r.List(ctx, pvcs, internal.GetPodListOptions(cluster, "", "")...)
diff --git a/controllers/update_pod_config.go b/controllers/update_pod_config.go
index e73225a6..d5ec0514 100644
--- a/controllers/update_pod_config.go
+++ b/controllers/update_pod_config.go
@@ -41,7 +41,7 @@ import (
 type updatePodConfig struct{}
 
 // reconcile runs the reconciler's work.
-func (updatePodConfig) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, logger logr.Logger) *requeue {
+func (updatePodConfig) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, _ *fdbv1beta2.FoundationDBStatus, logger logr.Logger) *requeue {
 	configMap, err := internal.GetConfigMap(cluster)
 	if err != nil {
 		return &requeue{curError: err}
diff --git a/controllers/update_status.go b/controllers/update_status.go
index 6e9e0dc9..57422f1b 100644
--- a/controllers/update_status.go
+++ b/controllers/update_status.go
@@ -833,8 +833,12 @@ func updateFaultDomains(logger logr.Logger, processes map[fdbv1beta2.ProcessGrou
 	for idx, processGroup := range status.ProcessGroups {
 		process, ok := processes[processGroup.ProcessGroupID]
 		if !ok || len(processes) == 0 {
-			logger.Info("skip updating fault domain for process group with missing process in FoundationDB cluster status", "processGroupID", processGroup.ProcessGroupID)
-			continue
+			// Fallback for multiple storage or log servers, those will contain the process information with the process number as a suffix.
+			process, ok = processes[processGroup.ProcessGroupID+"-1"]
+			if !ok || len(processes) == 0 {
+				logger.Info("skip updating fault domain for process group with missing process in FoundationDB cluster status", "processGroupID", processGroup.ProcessGroupID)
+				continue
+			}
 		}
 
 		faultDomain := getFaultDomainFromProcesses(process)
diff --git a/go.mod b/go.mod
index c28ae027..97a0f41b 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
 module github.com/FoundationDB/fdb-kubernetes-operator
 
-go 1.19
+go 1.20
 
 require (
 	github.com/apple/foundationdb/bindings/go v0.0.0-20201222225940-f3aef311ccfb
diff --git a/mock-kubernetes-client/client/client.go b/mock-kubernetes-client/client/client.go
index 106ec790..7d428fe5 100644
--- a/mock-kubernetes-client/client/client.go
+++ b/mock-kubernetes-client/client/client.go
@@ -24,7 +24,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"k8s.io/apimachinery/pkg/watch"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
@@ -34,6 +33,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/apimachinery/pkg/watch"
 	ctrlClient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 )
diff --git a/pkg/fdbstatus/status_checks.go b/pkg/fdbstatus/status_checks.go
index 00b4526e..4891424f 100644
--- a/pkg/fdbstatus/status_checks.go
+++ b/pkg/fdbstatus/status_checks.go
@@ -306,7 +306,7 @@ func DoLogServerFaultDomainCheckOnStatus(status *fdbv1beta2.FoundationDBStatus)
 
 // DoCoordinatorFaultDomainCheckOnStatus does a coordinator related fault domain check over the given status object.
 // @note an empty function for now. We will revisit this later.
-func DoCoordinatorFaultDomainCheckOnStatus(status *fdbv1beta2.FoundationDBStatus) error {
+func DoCoordinatorFaultDomainCheckOnStatus(_ *fdbv1beta2.FoundationDBStatus) error {
 	// TODO: decide if we need to do coordinator related check.
 	return nil
 }


Test patch:
diff --git a/api/v1beta2/foundationdbcluster_types_test.go b/api/v1beta2/foundationdbcluster_types_test.go
index 20c26999..b606bbb1 100644
--- a/api/v1beta2/foundationdbcluster_types_test.go
+++ b/api/v1beta2/foundationdbcluster_types_test.go
@@ -5501,4 +5501,35 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 			fmt.Errorf("process has missing address in exclusion results: 192.168.0.2"),
 		),
 	)
+
+	DescribeTable("when getting the removal mode", func(cluster *FoundationDBCluster, expected PodUpdateMode) {
+		Expect(cluster.GetRemovalMode()).To(Equal(expected))
+	},
+		Entry("no removal mode defined",
+			&FoundationDBCluster{
+				Spec: FoundationDBClusterSpec{
+					AutomationOptions: FoundationDBClusterAutomationOptions{
+						DeletionMode: PodUpdateModeAll,
+					},
+				},
+			}, PodUpdateModeZone),
+		Entry("removal mode zone defined",
+			&FoundationDBCluster{
+				Spec: FoundationDBClusterSpec{
+					AutomationOptions: FoundationDBClusterAutomationOptions{
+						DeletionMode: PodUpdateModeAll,
+						RemovalMode:  PodUpdateModeZone,
+					},
+				},
+			}, PodUpdateModeZone),
+		Entry("removal mode none defined",
+			&FoundationDBCluster{
+				Spec: FoundationDBClusterSpec{
+					AutomationOptions: FoundationDBClusterAutomationOptions{
+						DeletionMode: PodUpdateModeAll,
+						RemovalMode:  PodUpdateModeNone,
+					},
+				},
+			}, PodUpdateModeNone),
+	)
 })
diff --git a/controllers/remove_process_groups_test.go b/controllers/remove_process_groups_test.go
index 65da4d49..0f2b2371 100644
--- a/controllers/remove_process_groups_test.go
+++ b/controllers/remove_process_groups_test.go
@@ -179,106 +179,204 @@ var _ = Describe("remove_process_groups", func() {
 				var initialCnt int
 				var secondRemovedProcessGroup *fdbv1beta2.ProcessGroupStatus
 
-				BeforeEach(func() {
-					// To allow multiple process groups to be removed we have to use the update mode all
-					cluster.Spec.AutomationOptions.RemovalMode = fdbv1beta2.PodUpdateModeAll
-					err := k8sClient.Update(context.TODO(), cluster)
-					Expect(err).NotTo(HaveOccurred())
-
-					initialCnt = len(cluster.Status.ProcessGroups)
-					secondRemovedProcessGroup = cluster.Status.ProcessGroups[1]
-					marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, secondRemovedProcessGroup.ProcessGroupID, secondRemovedProcessGroup.ProcessClass, removedProcessGroup.Addresses[0])
-					Expect(marked).To(BeTrue())
-					Expect(processGroup).To(BeNil())
-					// Exclude the process group
-					adminClient, err := mock.NewMockAdminClientUncast(cluster, k8sClient)
-					Expect(err).NotTo(HaveOccurred())
-					for _, address := range secondRemovedProcessGroup.Addresses {
-						adminClient.ExcludedAddresses[address] = fdbv1beta2.None{}
-					}
-				})
-
-				// TODO(johscheuer): Fix this flaky test properly, for now retry failing test occurrences with a maximum of 3 retries.
-				It("should remove only one process group", FlakeAttempts(3), func() {
-					Expect(result).To(BeNil())
-					Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 1))
-					// Ensure resources are deleted
-					removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
-					Expect(err).To(BeNil())
-					Expect(removed).To(BeTrue())
-					Expect(include).To(BeTrue())
-					// Ensure resources are not deleted
-					removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
-					Expect(err).To(BeNil())
-					Expect(removed).To(BeFalse())
-					Expect(include).To(BeFalse())
-				})
-
-				When("a process group is marked as terminating and all resources are removed it should be removed", func() {
+				When("the removal mode is the default zone", func() {
 					BeforeEach(func() {
-						secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
-						err := removeProcessGroup(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+						Expect(cluster.Spec.AutomationOptions.RemovalMode).To(BeEmpty())
+						initialCnt = len(cluster.Status.ProcessGroups)
+						secondRemovedProcessGroup = cluster.Status.ProcessGroups[1]
+						marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, secondRemovedProcessGroup.ProcessGroupID, secondRemovedProcessGroup.ProcessClass, removedProcessGroup.Addresses[0])
+						Expect(marked).To(BeTrue())
+						Expect(processGroup).To(BeNil())
+						// Exclude the process group
+						adminClient, err := mock.NewMockAdminClientUncast(cluster, k8sClient)
 						Expect(err).NotTo(HaveOccurred())
+						for _, address := range secondRemovedProcessGroup.Addresses {
+							adminClient.ExcludedAddresses[address] = fdbv1beta2.None{}
+						}
 					})
 
-					It("should remove the process group and the terminated process group", func() {
+					It("should remove only one process group", func() {
 						Expect(result).To(BeNil())
-						Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
-						// Ensure resources are deleted
-						removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+						Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 1))
+						// Check if resources are deleted
+						removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
 						Expect(err).To(BeNil())
-						Expect(removed).To(BeTrue())
-						Expect(include).To(BeTrue())
-						// Ensure resources are deleted
-						removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+						// Check if resources are deleted
+						removedSecondary, includeSecondary, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
 						Expect(err).To(BeNil())
-						Expect(removed).To(BeTrue())
-						Expect(include).To(BeTrue())
+						// Make sure only one of the process groups was deleted.
+						Expect(removed).NotTo(Equal(removedSecondary))
+						Expect(include).NotTo(Equal(includeSecondary))
+					})
+
+					When("a process group is marked as terminating and all resources are removed it should be removed", func() {
+						BeforeEach(func() {
+							secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+							Expect(removeProcessGroup(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)).NotTo(HaveOccurred())
+						})
+
+						It("should remove the process group and the terminated process group", func() {
+							Expect(result).To(BeNil())
+							Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
+							// Ensure resources are deleted
+							removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+							// Ensure resources are deleted
+							removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+						})
+					})
+
+					When("a process group is marked as terminating and the resources are not removed", func() {
+						BeforeEach(func() {
+							secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+						})
+
+						It("should remove the process group and the terminated process group", func() {
+							Expect(result).To(BeNil())
+							Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
+							// Ensure resources are deleted
+							removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+							// Ensure resources are deleted
+							removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+						})
+					})
+
+					When("a process group is marked as terminating and not fully removed", func() {
+						BeforeEach(func() {
+							secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+							// Set the wait time to the default value
+							cluster.Spec.AutomationOptions.WaitBetweenRemovalsSeconds = pointer.Int(60)
+						})
+
+						It("should remove only one process group", func() {
+							Expect(result).NotTo(BeNil())
+							Expect(result.message).To(HavePrefix("not allowed to remove process groups, waiting:"))
+							Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 0))
+							// Ensure resources are not deleted
+							removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeFalse())
+							Expect(include).To(BeFalse())
+							// Ensure resources are deleted
+							removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeFalse())
+							Expect(include).To(BeFalse())
+						})
 					})
 				})
 
-				When("a process group is marked as terminating and the resources are not removed", func() {
+				When("the removal mode is PodUpdateModeAll", func() {
 					BeforeEach(func() {
-						secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+						// To allow multiple process groups to be removed we have to use the update mode all
+						cluster.Spec.AutomationOptions.RemovalMode = fdbv1beta2.PodUpdateModeAll
+						err := k8sClient.Update(context.TODO(), cluster)
+						Expect(err).NotTo(HaveOccurred())
+
+						initialCnt = len(cluster.Status.ProcessGroups)
+						secondRemovedProcessGroup = cluster.Status.ProcessGroups[1]
+						marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, secondRemovedProcessGroup.ProcessGroupID, secondRemovedProcessGroup.ProcessClass, removedProcessGroup.Addresses[0])
+						Expect(marked).To(BeTrue())
+						Expect(processGroup).To(BeNil())
+						// Exclude the process group
+						adminClient, err := mock.NewMockAdminClientUncast(cluster, k8sClient)
+						Expect(err).NotTo(HaveOccurred())
+						for _, address := range secondRemovedProcessGroup.Addresses {
+							adminClient.ExcludedAddresses[address] = fdbv1beta2.None{}
+						}
 					})
 
-					It("should remove the process group and the terminated process group", func() {
+					It("should remove two process groups", func() {
 						Expect(result).To(BeNil())
 						Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
 						// Ensure resources are deleted
-						removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+						removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
 						Expect(err).To(BeNil())
 						Expect(removed).To(BeTrue())
 						Expect(include).To(BeTrue())
-						// Ensure resources are deleted
-						removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+						// Ensure resources are deleted as the RemovalMode is PodUpdateModeAll
+						removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
 						Expect(err).To(BeNil())
 						Expect(removed).To(BeTrue())
 						Expect(include).To(BeTrue())
 					})
-				})
 
-				When("a process group is marked as terminating and not fully removed", func() {
-					BeforeEach(func() {
-						secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
-						// Set the wait time to the default value
-						cluster.Spec.AutomationOptions.WaitBetweenRemovalsSeconds = pointer.Int(60)
+					When("a process group is marked as terminating and all resources are removed it should be removed", func() {
+						BeforeEach(func() {
+							secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+							err := removeProcessGroup(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+							Expect(err).NotTo(HaveOccurred())
+						})
+
+						It("should remove the process group and the terminated process group", func() {
+							Expect(result).To(BeNil())
+							Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
+							// Ensure resources are deleted
+							removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+							// Ensure resources are deleted
+							removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+						})
 					})
 
-					It("should remove only one process group", func() {
-						Expect(result).NotTo(BeNil())
-						Expect(result.message).To(HavePrefix("not allowed to remove process groups, waiting:"))
-						Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 0))
-						// Ensure resources are not deleted
-						removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
-						Expect(err).To(BeNil())
-						Expect(removed).To(BeFalse())
-						Expect(include).To(BeFalse())
-						// Ensure resources are deleted
-						removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
-						Expect(err).To(BeNil())
-						Expect(removed).To(BeFalse())
-						Expect(include).To(BeFalse())
+					When("a process group is marked as terminating and the resources are not removed", func() {
+						BeforeEach(func() {
+							secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+						})
+
+						It("should remove the process group and the terminated process group", func() {
+							Expect(result).To(BeNil())
+							Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
+							// Ensure resources are deleted
+							removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+							// Ensure resources are deleted
+							removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+						})
+					})
+
+					When("a process group is marked as terminating and not fully removed", func() {
+						BeforeEach(func() {
+							secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+							// Set the wait time to the default value
+							cluster.Spec.AutomationOptions.WaitBetweenRemovalsSeconds = pointer.Int(60)
+						})
+
+						It("should remove all process groups", func() {
+							Expect(result).To(BeNil())
+							Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
+							// Ensure resources are not deleted
+							removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+							// Ensure resources are deleted
+							removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+						})
 					})
 				})
 			})
diff --git a/controllers/update_status_test.go b/controllers/update_status_test.go
index bcbb2ae2..ab64c811 100644
--- a/controllers/update_status_test.go
+++ b/controllers/update_status_test.go
@@ -701,6 +701,12 @@ var _ = Describe("update_status", func() {
 			Expect(cluster.Status.Generations.Reconciled).To(Equal(cluster.ObjectMeta.Generation))
 		})
 
+		It("should set the fault domain for all process groups", func() {
+			for _, processGroup := range cluster.Status.ProcessGroups {
+				Expect(processGroup.FaultDomain).NotTo(BeEmpty())
+			}
+		})
+
 		When("disabling an explicit listen address", func() {
 			BeforeEach(func() {
 				result, err := reconcileCluster(cluster)
@@ -716,16 +722,41 @@ var _ = Describe("update_status", func() {
 			})
 		})
 
-		Context("testing maintenance mode functionality", func() {
+		When("testing maintenance mode functionality", func() {
 			When("maintenance mode is on", func() {
 				BeforeEach(func() {
 					Expect(adminClient.SetMaintenanceZone("operator-test-1-storage-4", 0)).NotTo(HaveOccurred())
 				})
+
 				It("status maintenance zone should match", func() {
 					Expect(cluster.Status.MaintenanceModeInfo).To(Equal(fdbv1beta2.MaintenanceModeInfo{ZoneID: "operator-test-1-storage-4"}))
 				})
 			})
 		})
+
+		When("multiple storage server per Pod are used", func() {
+			BeforeEach(func() {
+				cluster.Spec.StorageServersPerPod = 2
+				Expect(k8sClient.Update(context.TODO(), cluster)).NotTo(HaveOccurred())
+
+				result, err := reconcileCluster(cluster)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(result.Requeue).To(BeFalse())
+
+				generation, err := reloadCluster(cluster)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(generation).To(Equal(int64(2)))
+
+				adminClient, err = mock.NewMockAdminClientUncast(cluster, k8sClient)
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			It("should set the fault domain for all process groups", func() {
+				for _, processGroup := range cluster.Status.ProcessGroups {
+					Expect(processGroup.FaultDomain).NotTo(BeEmpty())
+				}
+			})
+		})
 	})
 
 	DescribeTable("when getting the running version from the running processes", func(versionMap map[string]int, fallback string, expected string) {
@@ -846,5 +877,54 @@ var _ = Describe("update_status", func() {
 				}
 			})
 		})
+
+		When("multiple storage servers per Pod are used", func() {
+			BeforeEach(func() {
+				processes = map[fdbv1beta2.ProcessGroupID][]fdbv1beta2.FoundationDBStatusProcessInfo{
+					"storage-1-1": {
+						fdbv1beta2.FoundationDBStatusProcessInfo{
+							Locality: map[string]string{
+								fdbv1beta2.FDBLocalityZoneIDKey: "storage-1-zone",
+							},
+						},
+					},
+					"storage-1-2": {
+						fdbv1beta2.FoundationDBStatusProcessInfo{
+							Locality: map[string]string{
+								fdbv1beta2.FDBLocalityZoneIDKey: "storage-1-zone",
+							},
+						},
+					},
+					"storage-2-1": {
+						fdbv1beta2.FoundationDBStatusProcessInfo{
+							Locality: map[string]string{
+								fdbv1beta2.FDBLocalityZoneIDKey: "storage-2-zone",
+							},
+						},
+					},
+					"storage-2-2": {
+						fdbv1beta2.FoundationDBStatusProcessInfo{
+							Locality: map[string]string{
+								fdbv1beta2.FDBLocalityZoneIDKey: "storage-2-zone",
+							},
+						},
+					},
+				}
+			})
+
+			It("should update the process group status", func() {
+				Expect(status.ProcessGroups).To(HaveLen(3))
+
+				for _, processGroup := range status.ProcessGroups {
+					if processGroup.ProcessGroupID == "storage-3" {
+						Expect(processGroup.FaultDomain).To(BeEmpty())
+						continue
+					}
+
+					Expect(string(processGroup.FaultDomain)).To(And(HavePrefix(string(processGroup.ProcessGroupID)), HaveSuffix("zone")))
+				}
+			})
+		})
+
 	})
 })
diff --git a/e2e/test_operator/operator_test.go b/e2e/test_operator/operator_test.go
index d77caf71..c970bb32 100644
--- a/e2e/test_operator/operator_test.go
+++ b/e2e/test_operator/operator_test.go
@@ -88,6 +88,11 @@ func validateProcessesCount(
 	Eventually(func() int {
 		return fdbCluster.GetProcessCountByProcessClass(fdbv1beta2.ProcessClass(processRole))
 	}).Should(BeNumerically("==", countServer))
+
+	// Make sure that all process group have the fault domain set.
+	for _, processGroup := range fdbCluster.GetCluster().Status.ProcessGroups {
+		Expect(processGroup.FaultDomain).NotTo(BeEmpty())
+	}
 }
 
 func validateStorageClass(processClass fdbv1beta2.ProcessClass, targetStorageClass string) {
@@ -381,6 +386,15 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 		})
 	})
 
+	It("should set the fault domain for all process groups", func() {
+		for _, processGroup := range fdbCluster.GetCluster().Status.ProcessGroups {
+			Expect(processGroup.FaultDomain).NotTo(BeEmpty())
+		}
+
+		// TODO (johscheuer): We should check here further fields in the FoundationDBCluter resource to make sure the
+		// fields that we expect are actually set.
+	})
+
 	When("replacing a Pod", func() {
 		var replacedPod corev1.Pod
 
diff --git a/mock-kubernetes-client/client/client_test.go b/mock-kubernetes-client/client/client_test.go
index 8523f1b3..7a0d274e 100644
--- a/mock-kubernetes-client/client/client_test.go
+++ b/mock-kubernetes-client/client/client_test.go
@@ -23,13 +23,12 @@ package client
 import (
 	"context"
 	"fmt"
-	"k8s.io/utils/pointer"
 	"sort"
 	"time"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
-	"k8s.io/client-go/kubernetes/scheme"
-
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -38,10 +37,9 @@ import (
 	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/utils/pointer"
 	ctrlClient "sigs.k8s.io/controller-runtime/pkg/client"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
 )
 
 func createDummyPod() *corev1.Pod {