Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1751 (2023-07-20 11:04:40)

Fix get removal mode
# Description

Before the `GetRemovalMode` was returning the wrong value.

Fixes: https://github.com/FoundationDB/fdb-kubernetes-operator/issues/1750

## Type of change

*Please select one of the options below.*

- Bug fix (non-breaking change which fixes an issue)

## Discussion

-

## Testing

Added unit tests for the method.

## Documentation

-

## Follow-up

-


foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: b9562e06ea967ae84898ba2a15d9d581df5f690b
* Duration 3:05:35
* Result: :x: **FAILED**
* Error: `Error while executing command: if $fail_test; then exit 1; fi. Reason: exit status 1`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/ec9d8076-e460-4195-8696-0d1a9d498aae.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/ec9d8076-e460-4195-8696-0d1a9d498aae/workspace.zip) zip file of the working directory (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: b9562e06ea967ae84898ba2a15d9d581df5f690b
* Duration 2:28:38
* Result: :white_check_mark: **SUCCEEDED**
* Error: `N/A`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/48a2f27f-16d2-4648-95e5-318be30466e2.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/48a2f27f-16d2-4648-95e5-318be30466e2/workspace.zip) zip file of the working directory (available for 30 days)


Patch:
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index 73e308fe..c9cfe538 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -2233,11 +2233,11 @@ func (cluster *FoundationDBCluster) GetDNSDomain() string {
 
 // GetRemovalMode returns the removal mode of the cluster or default to PodUpdateModeZone if unset.
 func (cluster *FoundationDBCluster) GetRemovalMode() PodUpdateMode {
-	if cluster.Spec.AutomationOptions.DeletionMode == "" {
+	if cluster.Spec.AutomationOptions.RemovalMode == "" {
 		return PodUpdateModeZone
 	}
 
-	return cluster.Spec.AutomationOptions.DeletionMode
+	return cluster.Spec.AutomationOptions.RemovalMode
 }
 
 // GetWaitBetweenRemovalsSeconds returns the WaitDurationBetweenRemovals if set or defaults to 60s.
diff --git a/controllers/update_status.go b/controllers/update_status.go
index 6e9e0dc9..57422f1b 100644
--- a/controllers/update_status.go
+++ b/controllers/update_status.go
@@ -833,8 +833,12 @@ func updateFaultDomains(logger logr.Logger, processes map[fdbv1beta2.ProcessGrou
 	for idx, processGroup := range status.ProcessGroups {
 		process, ok := processes[processGroup.ProcessGroupID]
 		if !ok || len(processes) == 0 {
-			logger.Info("skip updating fault domain for process group with missing process in FoundationDB cluster status", "processGroupID", processGroup.ProcessGroupID)
-			continue
+			// Fallback for multiple storage or log servers, those will contain the process information with the process number as a suffix.
+			process, ok = processes[processGroup.ProcessGroupID+"-1"]
+			if !ok || len(processes) == 0 {
+				logger.Info("skip updating fault domain for process group with missing process in FoundationDB cluster status", "processGroupID", processGroup.ProcessGroupID)
+				continue
+			}
 		}
 
 		faultDomain := getFaultDomainFromProcesses(process)


Test patch:
diff --git a/api/v1beta2/foundationdbcluster_types_test.go b/api/v1beta2/foundationdbcluster_types_test.go
index 20c26999..b606bbb1 100644
--- a/api/v1beta2/foundationdbcluster_types_test.go
+++ b/api/v1beta2/foundationdbcluster_types_test.go
@@ -5501,4 +5501,35 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 			fmt.Errorf("process has missing address in exclusion results: 192.168.0.2"),
 		),
 	)
+
+	DescribeTable("when getting the removal mode", func(cluster *FoundationDBCluster, expected PodUpdateMode) {
+		Expect(cluster.GetRemovalMode()).To(Equal(expected))
+	},
+		Entry("no removal mode defined",
+			&FoundationDBCluster{
+				Spec: FoundationDBClusterSpec{
+					AutomationOptions: FoundationDBClusterAutomationOptions{
+						DeletionMode: PodUpdateModeAll,
+					},
+				},
+			}, PodUpdateModeZone),
+		Entry("removal mode zone defined",
+			&FoundationDBCluster{
+				Spec: FoundationDBClusterSpec{
+					AutomationOptions: FoundationDBClusterAutomationOptions{
+						DeletionMode: PodUpdateModeAll,
+						RemovalMode:  PodUpdateModeZone,
+					},
+				},
+			}, PodUpdateModeZone),
+		Entry("removal mode none defined",
+			&FoundationDBCluster{
+				Spec: FoundationDBClusterSpec{
+					AutomationOptions: FoundationDBClusterAutomationOptions{
+						DeletionMode: PodUpdateModeAll,
+						RemovalMode:  PodUpdateModeNone,
+					},
+				},
+			}, PodUpdateModeNone),
+	)
 })
diff --git a/controllers/remove_process_groups_test.go b/controllers/remove_process_groups_test.go
index 65da4d49..0f2b2371 100644
--- a/controllers/remove_process_groups_test.go
+++ b/controllers/remove_process_groups_test.go
@@ -179,106 +179,204 @@ var _ = Describe("remove_process_groups", func() {
 				var initialCnt int
 				var secondRemovedProcessGroup *fdbv1beta2.ProcessGroupStatus
 
-				BeforeEach(func() {
-					// To allow multiple process groups to be removed we have to use the update mode all
-					cluster.Spec.AutomationOptions.RemovalMode = fdbv1beta2.PodUpdateModeAll
-					err := k8sClient.Update(context.TODO(), cluster)
-					Expect(err).NotTo(HaveOccurred())
-
-					initialCnt = len(cluster.Status.ProcessGroups)
-					secondRemovedProcessGroup = cluster.Status.ProcessGroups[1]
-					marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, secondRemovedProcessGroup.ProcessGroupID, secondRemovedProcessGroup.ProcessClass, removedProcessGroup.Addresses[0])
-					Expect(marked).To(BeTrue())
-					Expect(processGroup).To(BeNil())
-					// Exclude the process group
-					adminClient, err := mock.NewMockAdminClientUncast(cluster, k8sClient)
-					Expect(err).NotTo(HaveOccurred())
-					for _, address := range secondRemovedProcessGroup.Addresses {
-						adminClient.ExcludedAddresses[address] = fdbv1beta2.None{}
-					}
-				})
-
-				// TODO(johscheuer): Fix this flaky test properly, for now retry failing test occurrences with a maximum of 3 retries.
-				It("should remove only one process group", FlakeAttempts(3), func() {
-					Expect(result).To(BeNil())
-					Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 1))
-					// Ensure resources are deleted
-					removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
-					Expect(err).To(BeNil())
-					Expect(removed).To(BeTrue())
-					Expect(include).To(BeTrue())
-					// Ensure resources are not deleted
-					removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
-					Expect(err).To(BeNil())
-					Expect(removed).To(BeFalse())
-					Expect(include).To(BeFalse())
-				})
-
-				When("a process group is marked as terminating and all resources are removed it should be removed", func() {
+				When("the removal mode is the default zone", func() {
 					BeforeEach(func() {
-						secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
-						err := removeProcessGroup(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+						Expect(cluster.Spec.AutomationOptions.RemovalMode).To(BeEmpty())
+						initialCnt = len(cluster.Status.ProcessGroups)
+						secondRemovedProcessGroup = cluster.Status.ProcessGroups[1]
+						marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, secondRemovedProcessGroup.ProcessGroupID, secondRemovedProcessGroup.ProcessClass, removedProcessGroup.Addresses[0])
+						Expect(marked).To(BeTrue())
+						Expect(processGroup).To(BeNil())
+						// Exclude the process group
+						adminClient, err := mock.NewMockAdminClientUncast(cluster, k8sClient)
 						Expect(err).NotTo(HaveOccurred())
+						for _, address := range secondRemovedProcessGroup.Addresses {
+							adminClient.ExcludedAddresses[address] = fdbv1beta2.None{}
+						}
 					})
 
-					It("should remove the process group and the terminated process group", func() {
+					It("should remove only one process group", func() {
 						Expect(result).To(BeNil())
-						Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
-						// Ensure resources are deleted
-						removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+						Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 1))
+						// Check if resources are deleted
+						removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
 						Expect(err).To(BeNil())
-						Expect(removed).To(BeTrue())
-						Expect(include).To(BeTrue())
-						// Ensure resources are deleted
-						removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+						// Check if resources are deleted
+						removedSecondary, includeSecondary, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
 						Expect(err).To(BeNil())
-						Expect(removed).To(BeTrue())
-						Expect(include).To(BeTrue())
+						// Make sure only one of the process groups was deleted.
+						Expect(removed).NotTo(Equal(removedSecondary))
+						Expect(include).NotTo(Equal(includeSecondary))
+					})
+
+					When("a process group is marked as terminating and all resources are removed it should be removed", func() {
+						BeforeEach(func() {
+							secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+							Expect(removeProcessGroup(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)).NotTo(HaveOccurred())
+						})
+
+						It("should remove the process group and the terminated process group", func() {
+							Expect(result).To(BeNil())
+							Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
+							// Ensure resources are deleted
+							removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+							// Ensure resources are deleted
+							removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+						})
+					})
+
+					When("a process group is marked as terminating and the resources are not removed", func() {
+						BeforeEach(func() {
+							secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+						})
+
+						It("should remove the process group and the terminated process group", func() {
+							Expect(result).To(BeNil())
+							Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
+							// Ensure resources are deleted
+							removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+							// Ensure resources are deleted
+							removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+						})
+					})
+
+					When("a process group is marked as terminating and not fully removed", func() {
+						BeforeEach(func() {
+							secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+							// Set the wait time to the default value
+							cluster.Spec.AutomationOptions.WaitBetweenRemovalsSeconds = pointer.Int(60)
+						})
+
+						It("should remove only one process group", func() {
+							Expect(result).NotTo(BeNil())
+							Expect(result.message).To(HavePrefix("not allowed to remove process groups, waiting:"))
+							Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 0))
+							// Ensure resources are not deleted
+							removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeFalse())
+							Expect(include).To(BeFalse())
+							// Ensure resources are deleted
+							removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeFalse())
+							Expect(include).To(BeFalse())
+						})
 					})
 				})
 
-				When("a process group is marked as terminating and the resources are not removed", func() {
+				When("the removal mode is PodUpdateModeAll", func() {
 					BeforeEach(func() {
-						secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+						// To allow multiple process groups to be removed we have to use the update mode all
+						cluster.Spec.AutomationOptions.RemovalMode = fdbv1beta2.PodUpdateModeAll
+						err := k8sClient.Update(context.TODO(), cluster)
+						Expect(err).NotTo(HaveOccurred())
+
+						initialCnt = len(cluster.Status.ProcessGroups)
+						secondRemovedProcessGroup = cluster.Status.ProcessGroups[1]
+						marked, processGroup := fdbv1beta2.MarkProcessGroupForRemoval(cluster.Status.ProcessGroups, secondRemovedProcessGroup.ProcessGroupID, secondRemovedProcessGroup.ProcessClass, removedProcessGroup.Addresses[0])
+						Expect(marked).To(BeTrue())
+						Expect(processGroup).To(BeNil())
+						// Exclude the process group
+						adminClient, err := mock.NewMockAdminClientUncast(cluster, k8sClient)
+						Expect(err).NotTo(HaveOccurred())
+						for _, address := range secondRemovedProcessGroup.Addresses {
+							adminClient.ExcludedAddresses[address] = fdbv1beta2.None{}
+						}
 					})
 
-					It("should remove the process group and the terminated process group", func() {
+					It("should remove two process groups", func() {
 						Expect(result).To(BeNil())
 						Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
 						// Ensure resources are deleted
-						removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+						removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
 						Expect(err).To(BeNil())
 						Expect(removed).To(BeTrue())
 						Expect(include).To(BeTrue())
-						// Ensure resources are deleted
-						removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+						// Ensure resources are deleted as the RemovalMode is PodUpdateModeAll
+						removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
 						Expect(err).To(BeNil())
 						Expect(removed).To(BeTrue())
 						Expect(include).To(BeTrue())
 					})
-				})
 
-				When("a process group is marked as terminating and not fully removed", func() {
-					BeforeEach(func() {
-						secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
-						// Set the wait time to the default value
-						cluster.Spec.AutomationOptions.WaitBetweenRemovalsSeconds = pointer.Int(60)
+					When("a process group is marked as terminating and all resources are removed it should be removed", func() {
+						BeforeEach(func() {
+							secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+							err := removeProcessGroup(context.Background(), clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+							Expect(err).NotTo(HaveOccurred())
+						})
+
+						It("should remove the process group and the terminated process group", func() {
+							Expect(result).To(BeNil())
+							Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
+							// Ensure resources are deleted
+							removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+							// Ensure resources are deleted
+							removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+						})
 					})
 
-					It("should remove only one process group", func() {
-						Expect(result).NotTo(BeNil())
-						Expect(result.message).To(HavePrefix("not allowed to remove process groups, waiting:"))
-						Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 0))
-						// Ensure resources are not deleted
-						removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
-						Expect(err).To(BeNil())
-						Expect(removed).To(BeFalse())
-						Expect(include).To(BeFalse())
-						// Ensure resources are deleted
-						removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
-						Expect(err).To(BeNil())
-						Expect(removed).To(BeFalse())
-						Expect(include).To(BeFalse())
+					When("a process group is marked as terminating and the resources are not removed", func() {
+						BeforeEach(func() {
+							secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+						})
+
+						It("should remove the process group and the terminated process group", func() {
+							Expect(result).To(BeNil())
+							Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
+							// Ensure resources are deleted
+							removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+							// Ensure resources are deleted
+							removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+						})
+					})
+
+					When("a process group is marked as terminating and not fully removed", func() {
+						BeforeEach(func() {
+							secondRemovedProcessGroup.ProcessGroupConditions = append(secondRemovedProcessGroup.ProcessGroupConditions, fdbv1beta2.NewProcessGroupCondition(fdbv1beta2.ResourcesTerminating))
+							// Set the wait time to the default value
+							cluster.Spec.AutomationOptions.WaitBetweenRemovalsSeconds = pointer.Int(60)
+						})
+
+						It("should remove all process groups", func() {
+							Expect(result).To(BeNil())
+							Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
+							// Ensure resources are not deleted
+							removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+							// Ensure resources are deleted
+							removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup.ProcessGroupID)
+							Expect(err).To(BeNil())
+							Expect(removed).To(BeTrue())
+							Expect(include).To(BeTrue())
+						})
 					})
 				})
 			})
diff --git a/controllers/update_status_test.go b/controllers/update_status_test.go
index bcbb2ae2..ab64c811 100644
--- a/controllers/update_status_test.go
+++ b/controllers/update_status_test.go
@@ -701,6 +701,12 @@ var _ = Describe("update_status", func() {
 			Expect(cluster.Status.Generations.Reconciled).To(Equal(cluster.ObjectMeta.Generation))
 		})
 
+		It("should set the fault domain for all process groups", func() {
+			for _, processGroup := range cluster.Status.ProcessGroups {
+				Expect(processGroup.FaultDomain).NotTo(BeEmpty())
+			}
+		})
+
 		When("disabling an explicit listen address", func() {
 			BeforeEach(func() {
 				result, err := reconcileCluster(cluster)
@@ -716,16 +722,41 @@ var _ = Describe("update_status", func() {
 			})
 		})
 
-		Context("testing maintenance mode functionality", func() {
+		When("testing maintenance mode functionality", func() {
 			When("maintenance mode is on", func() {
 				BeforeEach(func() {
 					Expect(adminClient.SetMaintenanceZone("operator-test-1-storage-4", 0)).NotTo(HaveOccurred())
 				})
+
 				It("status maintenance zone should match", func() {
 					Expect(cluster.Status.MaintenanceModeInfo).To(Equal(fdbv1beta2.MaintenanceModeInfo{ZoneID: "operator-test-1-storage-4"}))
 				})
 			})
 		})
+
+		When("multiple storage server per Pod are used", func() {
+			BeforeEach(func() {
+				cluster.Spec.StorageServersPerPod = 2
+				Expect(k8sClient.Update(context.TODO(), cluster)).NotTo(HaveOccurred())
+
+				result, err := reconcileCluster(cluster)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(result.Requeue).To(BeFalse())
+
+				generation, err := reloadCluster(cluster)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(generation).To(Equal(int64(2)))
+
+				adminClient, err = mock.NewMockAdminClientUncast(cluster, k8sClient)
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			It("should set the fault domain for all process groups", func() {
+				for _, processGroup := range cluster.Status.ProcessGroups {
+					Expect(processGroup.FaultDomain).NotTo(BeEmpty())
+				}
+			})
+		})
 	})
 
 	DescribeTable("when getting the running version from the running processes", func(versionMap map[string]int, fallback string, expected string) {
@@ -846,5 +877,54 @@ var _ = Describe("update_status", func() {
 				}
 			})
 		})
+
+		When("multiple storage servers per Pod are used", func() {
+			BeforeEach(func() {
+				processes = map[fdbv1beta2.ProcessGroupID][]fdbv1beta2.FoundationDBStatusProcessInfo{
+					"storage-1-1": {
+						fdbv1beta2.FoundationDBStatusProcessInfo{
+							Locality: map[string]string{
+								fdbv1beta2.FDBLocalityZoneIDKey: "storage-1-zone",
+							},
+						},
+					},
+					"storage-1-2": {
+						fdbv1beta2.FoundationDBStatusProcessInfo{
+							Locality: map[string]string{
+								fdbv1beta2.FDBLocalityZoneIDKey: "storage-1-zone",
+							},
+						},
+					},
+					"storage-2-1": {
+						fdbv1beta2.FoundationDBStatusProcessInfo{
+							Locality: map[string]string{
+								fdbv1beta2.FDBLocalityZoneIDKey: "storage-2-zone",
+							},
+						},
+					},
+					"storage-2-2": {
+						fdbv1beta2.FoundationDBStatusProcessInfo{
+							Locality: map[string]string{
+								fdbv1beta2.FDBLocalityZoneIDKey: "storage-2-zone",
+							},
+						},
+					},
+				}
+			})
+
+			It("should update the process group status", func() {
+				Expect(status.ProcessGroups).To(HaveLen(3))
+
+				for _, processGroup := range status.ProcessGroups {
+					if processGroup.ProcessGroupID == "storage-3" {
+						Expect(processGroup.FaultDomain).To(BeEmpty())
+						continue
+					}
+
+					Expect(string(processGroup.FaultDomain)).To(And(HavePrefix(string(processGroup.ProcessGroupID)), HaveSuffix("zone")))
+				}
+			})
+		})
+
 	})
 })
diff --git a/e2e/test_operator/operator_test.go b/e2e/test_operator/operator_test.go
index d77caf71..c970bb32 100644
--- a/e2e/test_operator/operator_test.go
+++ b/e2e/test_operator/operator_test.go
@@ -88,6 +88,11 @@ func validateProcessesCount(
 	Eventually(func() int {
 		return fdbCluster.GetProcessCountByProcessClass(fdbv1beta2.ProcessClass(processRole))
 	}).Should(BeNumerically("==", countServer))
+
+	// Make sure that all process group have the fault domain set.
+	for _, processGroup := range fdbCluster.GetCluster().Status.ProcessGroups {
+		Expect(processGroup.FaultDomain).NotTo(BeEmpty())
+	}
 }
 
 func validateStorageClass(processClass fdbv1beta2.ProcessClass, targetStorageClass string) {
@@ -381,6 +386,15 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 		})
 	})
 
+	It("should set the fault domain for all process groups", func() {
+		for _, processGroup := range fdbCluster.GetCluster().Status.ProcessGroups {
+			Expect(processGroup.FaultDomain).NotTo(BeEmpty())
+		}
+
+		// TODO (johscheuer): We should check here further fields in the FoundationDBCluter resource to make sure the
+		// fields that we expect are actually set.
+	})
+
 	When("replacing a Pod", func() {
 		var replacedPod corev1.Pod